rUtilAnts 0.1.0.20091014

data/lib/rUtilAnts/URLAccess.rb

@@ -0,0 +1,228 @@
+#--
+# Copyright (c) 2009 Muriel Salvan (murielsalvan@users.sourceforge.net)
+# Licensed under the terms specified in LICENSE file. No warranty is provided.
+#++
+
+module RUtilAnts
+
+  module URLAccess
+
+    # Constants identifying which form is the content returned by URL handlers
+    CONTENT_ERROR = 0
+    CONTENT_REDIRECT = 1
+    CONTENT_STRING = 2
+    CONTENT_LOCALFILENAME = 3
+    CONTENT_LOCALFILENAME_TEMPORARY = 4
+
+    # Exception class handling redirection errors
+    class RedirectionError < RuntimeError
+    end
+
+    # Class
+    class Manager
+
+      # Constructor
+      def initialize
+        # Get the map of plugins to read URLs
+        # map< String, [ list<Regexp>, String ] >
+        # map< PluginName, [ List of matching regexps, Plugin class name ] >
+        @Plugins = {}
+        Dir.glob("#{File.dirname(__FILE__)}/URLHandlers/*.rb").each do |iFileName|
+          begin
+            lPluginName = File.basename(iFileName)[0..-4]
+            require "RUtilAnts/URLHandlers/#{lPluginName}"
+            @Plugins[lPluginName] = [
+              eval("RUtilAnts::URLCache::URLHandlers::#{lPluginName}::getMatchingRegexps"),
+              "RUtilAnts::URLCache::URLHandlers::#{lPluginName}"
+            ]
+          rescue Exception
+            logExc$!, "Error while requiring URLHandler plugin #{iFileName}"
+          end
+        end
+      end
+
+      # Access the content of a URL.
+      # No cache.
+      # It calls a code block with the binary content of the URL (or a local file name if required).
+      #
+      # Parameters:
+      # * *iURL* (_String_): The URL (used to detect cyclic redirections)
+      # * *iParameters* (<em>map<Symbol,Object></em>): Additional parameters:
+      # ** *:FollowRedirections* (_Boolean_): Do we follow redirections ? [optional = true]
+      # ** *:NbrRedirectionsAllowed* (_Integer_): Number of redirections allowed [optional = 10]
+      # ** *:LocalFileAccess* (_Boolean_): Do we need a local file to read the content from ? If not, the content itslef will be given the code block. [optional = false]
+      # ** *:URLHandler* (_Object_): The URL handler, if it has already been instantiated, or nil otherwise [optional = nil]
+      # * _CodeBlock_: The code returning the object corresponding to the content:
+      # ** *iContent* (_String_): File content, or file name if :LocalFileAccess was true
+      # ** *iFileBaseName* (_String_): The base name the file could have. Useful to get file name extensions.
+      # ** Returns:
+      # ** _Exception_: The error encountered, or nil in case of success
+      def accessFile(iURL, iParameters = {})
+        rError = nil
+
+        lFollowRedirections = iParameters[:lFollowRedirections]
+        lNbrRedirectionsAllowed = iParameters[:NbrRedirectionsAllowed]
+        lLocalFileAccess = iParameters[:LocalFileAccess]
+        lURLHandler = iParameters[:URLHandler]
+        if (lFollowRedirections == nil)
+          lFollowRedirections = true
+        end
+        if (lNbrRedirectionsAllowed == nil)
+          lNbrRedirectionsAllowed = 10
+        end
+        if (lLocalFileAccess == nil)
+          lLocalFileAccess = false
+        end
+        if (lURLHandler == nil)
+          lURLHandler = getURLHandler(iURL)
+        end
+        # Get the content from the handler
+        lContentFormat, lContent = lURLHandler.getContent(lFollowRedirections)
+        case (lContentFormat)
+        when CONTENT_ERROR
+          rError = lContent
+        when CONTENT_REDIRECT
+          # Handle too much redirections (cycles)
+          if (lContent.upcase == iURL.upcase)
+            rError = RedirectionError.new("Redirecting to the same URL: #{iURL}")
+          elsif (lNbrRedirectionsAllowed < 0)
+            rError = RedirectionError.new("Too much URL redirections for URL: #{iURL} redirecting to #{lContent}")
+          elsif (lFollowRedirections)
+            # Follow the redirection if we want it
+            lNewParameters = iParameters.clone
+            lNewParameters[:NbrRedirectionsAllowed] = lNbrRedirectionsAllowed - 1
+            # Reset the URL handler for the new parameters.
+            lNewParameters[:URLHandler] = nil
+            rError = accessFile(lContent, lNewParameters) do |iContent, iBaseName|
+              yield(iContent, iBaseName)
+            end
+          else
+            rError = RedirectionError.new("Received invalid redirection for URL: #{iURL}")
+          end
+        when CONTENT_STRING
+          # The content is directly accessible.
+          if (lLocalFileAccess)
+            # Write the content in a local temporary file
+            require 'tmpdir'
+            lBaseName = lURLHandler.getCorrespondingFileBaseName
+            lLocalFileName = "#{Dir.tmpdir}/URLCache/#{lBaseName}"
+            begin
+              require 'fileutils'
+              FileUtils::mkdir_p(File.dirname(lLocalFileName))
+              File.open(lLocalFileName, 'wb') do |oFile|
+                oFile.write(lContent)
+              end
+            rescue Exception
+              rError = $!
+              lContent = nil
+            end
+            if (rError == nil)
+              yield(lLocalFileName, lBaseName)
+              # Delete the temporary file
+              File.unlink(lLocalFileName)
+            end
+          else
+            # Give it to the code block directly
+            yield(lContent, lURLHandler.getCorrespondingFileBaseName)
+          end
+        when CONTENT_LOCALFILENAME, CONTENT_LOCALFILENAME_TEMPORARY
+          lLocalFileName = lContent
+          # The content is a local file name already accessible
+          if (!lLocalFileAccess)
+            # First, read the local file name
+            begin
+              File.open(lLocalFileName, 'rb') do |iFile|
+                # Replace the file name with the real content
+                lContent = iFile.read
+              end
+            rescue Exception
+              rError = $!
+            end
+          end
+          if (rError == nil)
+            yield(lContent, lURLHandler.getCorrespondingFileBaseName)
+          end
+          # If the file was temporary, delete it
+          if (lContentFormat == CONTENT_LOCALFILENAME_TEMPORARY)
+            File.unlink(lLocalFileName)
+          end
+        end
+
+        return rError
+      end
+
+      # Get the URL handler corresponding to this URL
+      #
+      # Parameters:
+      # * *iURL* (_String_): The URL
+      # Return:
+      # * _Object_: The URL handler
+      def getURLHandler(iURL)
+        rURLHandler = nil
+
+        # Try out every regexp unless it matches.
+        # If none matches, assume a local file.
+        @Plugins.each do |iPluginName, iPluginInfo|
+          iRegexps, iPluginClassName = iPluginInfo
+          iRegexps.each do |iRegexp|
+            if (iRegexp.match(iURL) != nil)
+              # Found a matching handler
+              rURLHandler = eval("#{iPluginClassName}.new(iURL)")
+              break
+            end
+          end
+          if (rURLHandler != nil)
+            break
+          end
+        end
+        if (rURLHandler == nil)
+          # Assume a local file
+          rURLHandler = eval("#{@Plugins['LocalFile'][1]}.new(iURL)")
+        end
+
+        return rURLHandler
+      end
+
+    end
+
+    # Initialize a global plugins cache
+    def self.initializeURLAccess
+      $rUtilAnts_URLAccess_Manager = Manager.new
+      Object.module_eval('include RUtilAnts::URLAccess')
+    end
+
+    # Access the content of a URL.
+    # No cache.
+    # It calls a code block with the binary content of the URL (or a local file name if required).
+    #
+    # Parameters:
+    # * *iURL* (_String_): The URL (used to detect cyclic redirections)
+    # * *iParameters* (<em>map<Symbol,Object></em>): Additional parameters:
+    # ** *:FollowRedirections* (_Boolean_): Do we follow redirections ? [optional = true]
+    # ** *:NbrRedirectionsAllowed* (_Integer_): Number of redirections allowed [optional = 10]
+    # ** *:LocalFileAccess* (_Boolean_): Do we need a local file to read the content from ? If not, the content itslef will be given the code block. [optional = false]
+    # ** *:URLHandler* (_Object_): The URL handler, if it has already been instantiated, or nil otherwise [optional = nil]
+    # * _CodeBlock_: The code returning the object corresponding to the content:
+    # ** *iContent* (_String_): File content, or file name if :LocalFileAccess was true
+    # ** *iFileBaseName* (_String_): The base name the file could have. Useful to get file name extensions.
+    # ** Returns:
+    # ** _Exception_: The error encountered, or nil in case of success
+    def accessFile(iURL, iParameters = {})
+      return $rUtilAnts_URLAccess_Manager.accessFile(iURL, iParameters) do |iContent, iBaseName|
+        yield(iContent, iBaseName)
+      end
+    end
+
+    # Get the URL handler corresponding to this URL
+    #
+    # Parameters:
+    # * *iURL* (_String_): The URL
+    # Return:
+    # * _Object_: The URL handler
+    def getURLHandler(iURL)
+      return $rUtilAnts_URLAccess_Manager.getURLHandler(iURL)
+    end
+
+  end
+
+end

data/lib/rUtilAnts/URLCache.rb

@@ -0,0 +1,145 @@
+#--
+# Copyright (c) 2009 Muriel Salvan (murielsalvan@users.sourceforge.net)
+# Licensed under the terms specified in LICENSE file. No warranty is provided.
+#++
+
+module RUtilAnts
+
+  module URLCache
+
+    # Class that caches every access to a URI (local file name, http, data...).
+    # This ensures just that several files are instantiated just once.
+    # For local files, it takes into account the file modification date/time to know if the Wx::Bitmap file has to be refreshed.
+    class URLCache
+
+      # Exception for reporting server down errors.
+      class ServerDownError < RuntimeError
+      end
+
+      # Constructor
+      def initialize
+        # Map of known contents, interpreted in many flavors
+        # map< Integer, [ Integer, Object ] >
+        # map< URL's hash, [ CRC, Content ] >
+        @URLs = {}
+        # Map of hosts down (no need to try again such a host)
+        # map< String >
+        @HostsDown = {}
+      end
+
+      # Get a content from a URL.
+      # Here are the different formats the URL can have:
+      # * Local file name
+      # * http/https/ftp/ftps:// protocols
+      # * data:image URI
+      # * file:// protocol
+      # It also handles redirections or zipped files
+      #
+      # Parameters:
+      # * *iURL* (_String_): The URL
+      # * *iParameters* (<em>map<Symbol,Object></em>): Additional parameters:
+      # ** *:ForceLoad* (_Boolean_): Do we force to refresh the cache ? [optional = false]
+      # ** *:FollowRedirections* (_Boolean_): Do we follow redirections ? [optional = true]
+      # ** *:NbrRedirectionsAllowed* (_Integer_): Number of redirections allowed [optional = 10]
+      # ** *:LocalFileAccess* (_Boolean_): Do we need a local file to read the content from ? If not, the content itslef will be given the code block. [optional = false]
+      # * _CodeBlock_: The code returning the object corresponding to the content:
+      # ** *iContent* (_String_): File content, or file name if :LocalFileAccess was true
+      # ** Returns:
+      # ** _Object_: Object read from the content, or nil in case of error
+      # ** _Exception_: The error encountered, or nil in case of success
+      # Return:
+      # * <em>Object</em>: The corresponding URL content, or nil in case of failure
+      # * _Exception_: The error, or nil in case of success
+      def getURLContent(iURL, iParameters = {})
+        rObject = nil
+        rError = nil
+
+        # Parse parameters
+        lForceLoad = iParameters[:ForceLoad]
+        if (lForceLoad == nil)
+          lForceLoad = false
+        end
+        # Get the URL handler corresponding to this URL
+        lURLHandler = getURLHandler(iURL)
+        lServerID = lURLHandler.getServerID
+        if (@HostsDown.has_key?(lServerID))
+          rError = ServerDownError.new("Server #{iURL} is currently down.")
+        else
+          lURLHash = iURL.hash
+          # Check if it is in the cache, or if we force refresh, or if the URL was invalidated
+          lCurrentCRC = lURLHandler.getCRC
+          if ((@URLs[lURLHash] == nil) or
+              (lForceLoad) or
+              (@URLs[lURLHash][0] != lCurrentCRC))
+            # Load it for real
+            # Reset previous value if it was set
+            @URLs[lURLHash] = nil
+            # Get the object
+            lObject = nil
+            lAccessError = accessFile(iURL, iParameters.merge(:URLHandler => lURLHandler)) do |iContent, iBaseName|
+              lObject, rError = yield(iContent)
+            end
+            if (lAccessError != nil)
+              rError = lAccessError
+            end
+            # Put lObject in the cache if no error was found
+            if (rError == nil)
+              # OK, register it
+              @URLs[lURLHash] = [ lCurrentCRC, lObject ]
+            else
+              if ((defined?(SocketError) != nil) and
+                  (rError.is_a?(SocketError)))
+                # We have a server down
+                @HostsDown[lServerID] = nil
+              end
+            end
+          end
+          # If no error was found (errors can only happen if it was not already in the cache), take it from the cache
+          if (rError == nil)
+            rObject = @URLs[lURLHash][1]
+          end
+        end
+
+        return rObject, rError
+      end
+
+    end
+
+    # Initialize a global cache
+    def self.initializeURLCache
+      $rUtilAnts_URLCache = URLCache.new
+      Object.module_eval('include RUtilAnts::URLCache')
+    end
+
+    # Get a content from a URL.
+    # Here are the different formats the URL can have:
+    # * Local file name
+    # * http/https/ftp/ftps:// protocols
+    # * data:image URI
+    # * file:// protocol
+    # It also handles redirections or zipped files
+    #
+    # Parameters:
+    # * *iURL* (_String_): The URL
+    # * *iParameters* (<em>map<Symbol,Object></em>): Additional parameters:
+    # ** *:ForceLoad* (_Boolean_): Do we force to refresh the cache ? [optional = false]
+    # ** *:FollowRedirections* (_Boolean_): Do we follow redirections ? [optional = true]
+    # ** *:NbrRedirectionsAllowed* (_Integer_): Number of redirections allowed [optional = 10]
+    # ** *:LocalFileAccess* (_Boolean_): Do we need a local file to read the content from ? If not, the content itself will be given the code block. [optional = false]
+    # * _CodeBlock_: The code returning the object corresponding to the content:
+    # ** *iContent* (_String_): File content, or file name if :LocalFileAccess was true
+    # ** Returns:
+    # ** _Object_: Object read from the content, or nil in case of error
+    # ** _Exception_: The error encountered, or nil in case of success
+    # Return:
+    # * <em>Object</em>: The corresponding URL content, or nil in case of failure
+    # * _Exception_: The error, or nil in case of success
+    def getURLContent(iURL, iParameters = {})
+      return $rUtilAnts_URLCache.getURLContent(iURL, iParameters) do |iContent|
+        next yield(iContent)
+      end
+    end
+
+  end
+
+end

data/lib/rUtilAnts/URLHandlers/DataImage.rb

@@ -0,0 +1,104 @@
+#--
+# Copyright (c) 2009 Muriel Salvan (murielsalvan@users.sourceforge.net)
+# Licensed under the terms specified in LICENSE file. No warranty is provided.
+#++
+
+module RUtilAnts
+
+  module URLCache
+
+    module URLHandlers
+
+      # Handler of data:image URIs
+      class DataImage
+
+        # Get a list of regexps matching the URL to get to this handler
+        #
+        # Return:
+        # * <em>list<Regexp></em>: The list of regexps matching URLs from this handler
+        def self.getMatchingRegexps
+          return [
+            /^data:image.*$/
+          ]
+        end
+
+        # Constructor
+        #
+        # Parameters:
+        # * *iURL* (_String_): The URL that this handler will manage
+        def initialize(iURL)
+          @URL = iURL
+          lMatchData = @URL.match(/data:image\/(.*);base64,(.*)/)
+          if (lMatchData == nil)
+            logBug "URL #{iURL[0..23]}... was identified as a data:image like, but it appears to be false."
+          else
+            @Ext = lMatchData[1]
+            if (@Ext == 'x-icon')
+              @Ext = 'ico'
+            end
+            @Data = lMatchData[2]
+          end
+        end
+
+        # Get the server ID
+        #
+        # Return:
+        # * _String_: The server ID
+        def getServerID
+          return nil
+        end
+
+        # Get the current CRC of the URL
+        #
+        # Return:
+        # * _Integer_: The CRC
+        def getCRC
+          # As the content is in the URL, it will be natural to not find it anymore in the cache when it is changed.
+          # Therefore there is no need to return a CRC.
+          return 0
+        end
+
+        # Get a corresponding file base name.
+        # This method has to make sure file extensions are respected, as it can be used for further processing.
+        #
+        # Return:
+        # * _String_: The file name
+        def getCorrespondingFileBaseName
+          return "DataImage.#{@Ext}"
+        end
+
+        # Get the content of the URL
+        #
+        # Parameters:
+        # * *iFollowRedirections* (_Boolean_): Do we follow redirections while accessing the content ?
+        # Return:
+        # * _Integer_: Type of content returned
+        # * _Object_: The content, depending on the type previously returned:
+        # ** _Exception_ if CONTENT_ERROR: The corresponding error
+        # ** _String_ if CONTENT_REDIRECT: The new URL
+        # ** _String_ if CONTENT_STRING: The real content
+        # ** _String_ if CONTENT_LOCALFILENAME: The name of the local file name storing the content
+        # ** _String_ if CONTENT_LOCALFILENAME_TEMPORARY: The name of the temporary local file name storing the content
+        def getContent(iFollowRedirections)
+          rContentFormat = nil
+          rContent = nil
+
+          # Here we unpack the string in a base64 encoding.
+          if (@Data.empty?)
+            rContent = RuntimeError.new("Empty URI to decode: #{@URL}")
+            rContentFormat = CONTENT_ERROR
+          else
+            rContent = @Data.unpack('m')[0]
+            rContentFormat = CONTENT_STRING
+          end
+
+          return rContentFormat, rContent
+        end
+
+      end
+
+    end
+
+  end
+
+end

data/lib/rUtilAnts/URLHandlers/FTP.rb

@@ -0,0 +1,120 @@
+#--
+# Copyright (c) 2009 Muriel Salvan (murielsalvan@users.sourceforge.net)
+# Licensed under the terms specified in LICENSE file. No warranty is provided.
+#++
+
+module RUtilAnts
+
+  module URLCache
+
+    module URLHandlers
+
+      # Handler of FTP URLs
+      class FTP
+
+        # Get a list of regexps matching the URL to get to this handler
+        #
+        # Return:
+        # * <em>list<Regexp></em>: The list of regexps matching URLs from this handler
+        def self.getMatchingRegexps
+          return [
+            /^(ftp|ftps):\/\/.*$/
+          ]
+        end
+
+        # Constructor
+        #
+        # Parameters:
+        # * *iURL* (_String_): The URL that this handler will manage
+        def initialize(iURL)
+          @URL = iURL
+          lURLMatch = iURL.match(/^(ftp|ftps):\/\/([^\/]*)\/(.*)$/)
+          if (lURLMatch == nil)
+            lURLMatch = iURL.match(/^(ftp|ftps):\/\/(.*)$/)
+          end
+          if (lURLMatch == nil)
+            logBug "URL #{iURL} was identified as an ftp like, but it appears to be false."
+          else
+            @URLProtocol, @URLServer, @URLPath = lURLMatch[1..3]
+          end
+        end
+
+        # Get the server ID
+        #
+        # Return:
+        # * _String_: The server ID
+        def getServerID
+          return "#{@URLProtocol}://#{@URLServer}"
+        end
+
+        # Get the current CRC of the URL
+        #
+        # Return:
+        # * _Integer_: The CRC
+        def getCRC
+          # We consider FTP URLs to be definitive: CRCs will never change.
+          return 0
+        end
+
+        # Get a corresponding file base name.
+        # This method has to make sure file extensions are respected, as it can be used for further processing.
+        #
+        # Return:
+        # * _String_: The file name
+        def getCorrespondingFileBaseName
+          lBase = File.basename(@URLPath)
+          lExt = File.extname(@URLPath)
+          lFileName = nil
+          if (lExt.empty?)
+            lFileName = lBase
+          else
+            # Check that extension has no characters following the URL (#, ? and ;)
+            lBase = lBase[0..lBase.size-lExt.size-1]
+            lFileName = "#{lBase}#{lExt.gsub(/^([^#\?;]*).*$/,'\1')}"
+          end
+
+          return getValidFileName(lFileName)
+        end
+
+        # Get the content of the URL
+        #
+        # Parameters:
+        # * *iFollowRedirections* (_Boolean_): Do we follow redirections while accessing the content ?
+        # Return:
+        # * _Integer_: Type of content returned
+        # * _Object_: The content, depending on the type previously returned:
+        # ** _Exception_ if CONTENT_ERROR: The corresponding error
+        # ** _String_ if CONTENT_REDIRECT: The new URL
+        # ** _String_ if CONTENT_STRING: The real content
+        # ** _String_ if CONTENT_LOCALFILENAME: The name of the local file name storing the content
+        # ** _String_ if CONTENT_LOCALFILENAME_TEMPORARY: The name of the temporary local file name storing the content
+        def getContent(iFollowRedirections)
+          rContentFormat = nil
+          rContent = nil
+
+          begin
+            require 'net/ftp'
+            lFTPConnection = Net::FTP.new(@URLServer)
+            lFTPConnection.login
+            lFTPConnection.chdir(File.dirname(@URLPath))
+            rContent = getCorrespondingFileBaseName
+            rContentFormat = CONTENT_LOCALFILENAME_TEMPORARY
+            logDebug "URL #{@URL} => Temporary file #{rContent}"
+            lFTPConnection.getbinaryfile(File.basename(@URLPath), rContent)
+            lFTPConnection.close
+          rescue Exception
+            rContent = $!
+            rContentFormat = CONTENT_ERROR
+            logDebug "Error accessing #{@URL}: #{rContent}"
+          end
+
+          return rContentFormat, rContent
+        end
+
+      end
+
+    end
+
+  end
+
+end