meta_project 0.4.6 → 0.4.7

data/CHANGES

@@ -1,7 +1,18 @@
 = MetaProject Changelog
 
+== Version 0.4.7
+
+This release makes HTTP access more robust and improves the Pathname API.
+
+* More robust HTTP GET with retries and better error messages.
+* Added Pathname.basename and Pathname.parent
+* Small improvements to ScmWeb API.
+* More robust detection of XForge package_id.
+
 == Version 0.4.6
 
+This release updates documentaion, adds iteration support to Pathname and fixes some minor bugs.
+
 * Updated README.
 * Added PathnameIterator mixin that can be used to iterate over Pathname and MetaProject::ScmWeb::Pathname to
   do e.g. egrep or other operations.

data/README

@@ -1,4 +1,4 @@
-= MetaProject
+= MetaProject 0.4.7
 
 MetaProject (formerly XForge) is a library that allows interaction with various project hosting servers, issue trackers, SCMs and SCM browsers through a common API.
 

data/Rakefile

@@ -6,7 +6,7 @@
 # This file is may be distributed under an MIT style license.  See
 # MIT-LICENSE for details.
 
-$:.unshift('lib')
+# $:.unshift('lib')
 require 'meta_project'
 require 'rake/gempackagetask'
 require 'rake/contrib/rubyforgepublisher'
@@ -24,7 +24,7 @@ require 'rake/rdoctask'
 #
 # REMEMBER TO KEEP PKG_VERSION IN SYNC WITH THE CHANGES FILE!
 PKG_NAME = "meta_project"
-PKG_VERSION = "0.4.6"
+PKG_VERSION = "0.4.7"
 PKG_FILE_NAME = "#{PKG_NAME}-#{PKG_VERSION}"
 PKG_FILES = FileList[
   '[A-Z]*',
@@ -34,10 +34,8 @@ PKG_FILES = FileList[
 
 task :default => [:gem]
 
-Rake::TestTask.new do |t|
-  t.libs << "test"
-  t.test_files = FileList['**/*_test.rb']
-  t.verbose = true
+task :spec do
+  ruby 'behaviours/suite.rb'
 end
 
 # Create a task to build the RDOC documentation tree.
@@ -94,7 +92,6 @@ Rake::GemPackageTask.new(spec) do |pkg|
   pkg.need_zip = true
   pkg.need_tar = true
 end
-task :gem => [:test]
 
 # Support Tasks ------------------------------------------------------
 

data/lib/meta_project/core_ext/open_uri.rb

@@ -0,0 +1,22 @@
+require 'open-uri'
+
+# Wrapper around Kernel.open from open-uri
+# that can retry and report better errors
+def better_open(url, retries=2, &block)
+  begin
+    # open-uri's open
+    Kernel.open(url, &block)
+  rescue Errno::ECONNREFUSED, EOFError => e
+    if(retries > 0)
+      STDERR.puts "Connection refused to #{url} - retrying in 1 sec."
+      sleep 1
+      better_open(url, retries-1, &block)
+    else
+      e.message << " (URL:#{url})"
+      raise e
+    end
+  rescue OpenURI::HTTPError => e
+    e.message << " (URL:#{url})"
+    raise e
+  end
+end

data/lib/meta_project/project/base.rb

@@ -3,5 +3,7 @@ module MetaProject
     class Base
       attr_reader :scm, :scm_web, :tracker, :name, :home_page
     end
+    
+    class ProjectException < Exception; end
   end
 end

data/lib/meta_project/project/codehaus/codehaus_project_svn.rb

@@ -5,19 +5,20 @@ module MetaProject
 
         def initialize(unix_name, svn_path, jira_id)
           @unix_name = unix_name
+          @name = unix_name
           @scm = RSCM::Subversion.new("svn://svn.#{unix_name}.codehaus.org/#{unix_name}/scm/#{svn_path}", svn_path)
           @tracker = ::MetaProject::Tracker::Jira::JiraTracker.new("http://jira.codehaus.org", jira_id)
 
-          overview  = "http://svn.#{unix_name}.codehaus.org/#{svn_path}/"
-          history   = "#{overview}\#{path}"
+          dir       = "http://svn.#{unix_name}.codehaus.org/#{svn_path}/\#{path}"
+          history   = dir
           raw       = "#{history}?rev=\#{revision}"
-          html       = "#{raw}&view=markup"
+          html      = "#{raw}&view=markup"
           # http://svn.picocontainer.codehaus.org/java/picocontainer/trunk/container/project.xml?r1=2220&r2=2234&p1=java/picocontainer/trunk/container/project.xml&p2=java/picocontainer/trunk/container/project.xml
           diff      = "#{history}?r1=\#{previous_revision}&r2=\#{revision}&p1=#{svn_path}/\#{path}&p2=#{svn_path}/\#{path}"
           child_dirs_pattern = /<a name="([^"]+)" href="([^"]+)">[\r\n\s]+<img src="\/icons\/small\/dir.gif"/
           child_files_pattern = /<a href="[^"]+\/([^\?]+)\?rev=([\d]+)&view=auto">/
 
-          @scm_web = ScmWeb::Browser.new(overview, history, raw, html, diff, child_dirs_pattern, child_files_pattern)
+          @scm_web = ScmWeb::Browser.new(dir, history, raw, html, diff, child_dirs_pattern, child_files_pattern)
         end
 
         def home_page

data/lib/meta_project/project/trac/trac_project.rb

@@ -10,31 +10,34 @@ module MetaProject
           @tracker = ::MetaProject::Tracker::Trac::TracTracker.new(@trac_base_url)
         end
         
+        TRAC_VERSION_PATTERN = /<strong>Trac ([\d\.]+)\w*<\/strong>/
+        
         def scm_web
           unless @scm_web
-            overview  = "#{@trac_base_url}/browser/#{@svn_path}/"
-            diff      = "#{@trac_base_url}/changeset/\#{revision}"
 
-            front_page = open(@trac_base_url).read
-            if(front_page =~ /<strong>Trac ([\d\.]+)\w*<\/strong>/)
-              
+            front_page = better_open(@trac_base_url).read
+            if(front_page =~ TRAC_VERSION_PATTERN)
               version = $1
               version = "#{version}.0" if version =~ /[\d]+\.[\d]+\.$/              
               version = version.gsub(/\./, "").to_i
 
               if(version >= 90)
-                history   = "#{@trac_base_url}/browser/#{@svn_path}/\#{path}"
                 html      = "#{@trac_base_url}/browser/#{@svn_path}/\#{path}?rev=\#{revision}"
                 raw       = "#{@trac_base_url}/browser/#{@svn_path}/\#{path}?rev=\#{revision}&format=txt"
               else
-                history   = "#{@trac_base_url}/log/#{@svn_path}/\#{path}"
                 html      = "#{@trac_base_url}/file/#{@svn_path}/\#{path}?rev=\#{revision}"
                 raw       = "#{@trac_base_url}/file/#{@svn_path}/\#{path}?rev=\#{revision}&format=txt"
               end
-              child_dirs_pattern = /title="Browse Directory" href="[^"]+">([^<]+)<\/a>/
+
+              dir                 = "#{@trac_base_url}/browser/#{@svn_path}/\#{path}"
+              history             = "#{@trac_base_url}/log/#{@svn_path}/\#{path}"
+              diff                = "#{@trac_base_url}/changeset/\#{revision}"
+              child_dirs_pattern  = /title="Browse Directory" href="[^"]+">([^<]+)<\/a>/
               child_files_pattern = /title="View File" href="[^"]+">([^<]+)<\/a>/
 
-              @scm_web = ScmWeb::Browser.new(overview, history, raw, html, diff, child_dirs_pattern, child_files_pattern)
+              @scm_web = ScmWeb::Browser.new(dir, history, raw, html, diff, child_dirs_pattern, child_files_pattern)
+            else
+              raise ProjectException.new("Couldn't determine the Trac version. Is the URL '#{@trac_base_url}' correct? I was looking for the regexp /#{TRAC_VERSION_PATTERN.source}/ on the page, but couldn't find it.")
             end
           end
 

data/lib/meta_project/project/xforge/ruby_forge.rb

@@ -25,16 +25,16 @@ module MetaProject
           path_cvs_root = "#{mod}/\#{path}#{cvsroot}"
           path_cvs_root_rev = "#{path_cvs_root}&rev=\#{revision}"
 
-          overview = "#{view_cvs}#{mod}/#{cvsroot}"
+          dir      = "#{view_cvs}#{path_cvs_root}"
           history  = "#{view_cvs}#{path_cvs_root}"
           raw      = "#{view_cvs}*checkout*/#{path_cvs_root_rev}"
           html     = "#{view_cvs}#{path_cvs_root_rev}&content-type=text/vnd.viewcvs-markup"
           diff     = "#{view_cvs}#{mod}/\#{path}.diff#{cvsroot}&r1=\#{previous_revision}&r2=\#{revision}"
 
-          child_dirs_pattern = /href="([^\?]*\/)\?cvsroot=#{unix_name}">/
+          child_dirs_pattern = /href="([^\?]*)\/\?cvsroot=#{unix_name}">/
           child_files_pattern = /href="([^\?^\/]*)\?cvsroot=#{unix_name}">/
 
-          ScmWeb::Browser.new(overview, history, raw, html, diff, child_dirs_pattern, child_files_pattern)
+          ScmWeb::Browser.new(dir, history, raw, html, diff, child_dirs_pattern, child_files_pattern)
         end
         
         # Regexp used to find projects' home page

data/lib/meta_project/project/xforge/session.rb

@@ -1,6 +1,3 @@
-require 'net/http'
-require 'open-uri'
-
 module MetaProject
   module Project
     module XForge
@@ -36,9 +33,13 @@ module MetaProject
         def package_id
           unless(@package_id)
             release_uri = "http://#{@host}/frs/admin/?group_id=#{@project.group_id}"
-            release_data = open(release_uri, @headers) { |data| data.read }
-            @package_id = release_data[/[?&]package_id=(\d+)/, 1]
-            raise "Couldn't get package_id" unless @package_id
+            release_data = better_open(release_uri, @headers).read
+            package_id_pattern = /name="package_id" value="(\d+)"/
+            @package_id = release_data[package_id_pattern, 1]
+            unless @package_id
+              File.open("package_id.html", "w") {|io| io.write(release_data)}
+              raise "Couldn't get package_id from #{release_uri}. I was looking for /#{package_id_pattern.source}/. HTML saved to package_id.html for debugging." 
+            end
           end
           @package_id
         end

data/lib/meta_project/project/xforge/source_forge.rb

@@ -25,7 +25,7 @@ module MetaProject
           project_path = "#{unix_name_mod}/\#{path}"
           rev = "rev=\#{revision}"
 
-          overview = "#{view_cvs}#{unix_name_mod}/"
+          dir      = "#{view_cvs}#{project_path}"
           history  = "#{view_cvs}#{project_path}"
           raw      = "#{view_cvs}*checkout*/#{project_path}?#{rev}"
           html     = "#{history}?#{rev}&view=markup"
@@ -35,7 +35,7 @@ module MetaProject
           child_dirs_pattern = /<img src="\/icons\/small\/dir.gif"\s+alt="\(dir\)"\s+border=0\s+width=16\s+height=16>[\r\n\s]*([^\/]+)\/<\/a>/
           child_files_pattern = /href="[^\?]+\/([^\?]+)\?rev=([^&]+)&view=log">/
 
-          ScmWeb::Browser.new(overview, history, raw, html, diff, child_dirs_pattern, child_files_pattern)
+          ScmWeb::Browser.new(dir, history, raw, html, diff, child_dirs_pattern, child_files_pattern)
         end
 
         # Regexp used to find projects' home page

data/lib/meta_project/project/xforge/xforge_base.rb

@@ -1,7 +1,3 @@
-require 'net/http'
-require 'net/https'
-require 'open-uri'
-
 module MetaProject
   module Project
     module XForge
@@ -45,10 +41,10 @@ module MetaProject
         # The group_id of this project
         def group_id
           unless(@group_id)
-            regexp = /project\/memberlist.php\?group_id=(\d+)/
-            html = open(xforge_project_url).read
-            @group_id = html[regexp, 1]
-            raise "Couldn't get group_id from #{xforge_project_url}" unless @group_id
+            html = better_open(xforge_project_url).read
+            group_id_pattern = /project\/memberlist.php\?group_id=(\d+)/
+            @group_id = html[group_id_pattern, 1]
+            raise "Couldn't get group_id from #{xforge_project_url}. I was looking for /#{group_id_pattern.source}/" unless @group_id
           end
           @group_id
         end
@@ -64,9 +60,13 @@ module MetaProject
         # The home page of this project
         def home_page
           unless(@home_page)
-            html = open(xforge_project_url).read
+            html = better_open(xforge_project_url).read
             @home_page = html[home_page_regexp, 1]
-            raise "Couldn't get home_page" unless @home_page
+            unless @home_page
+              STDERR.puts "WARNING: Couldn't get home_page from #{xforge_project_url}. I was looking for /#{home_page_regexp.source}/"
+              STDERR.puts "Will use #{xforge_project_url} as home page instead."
+              @home_page = xforge_project_url
+            end
           end
           @home_page
         end

data/lib/meta_project/project_analyzer.rb

@@ -25,9 +25,9 @@ module MetaProject
       
       # Codehaus SVN
       if(url =~ /http:\/\/svn.(.*).codehaus.org\/(.*)/)
-        svn_id = $1
+        unix_name = $1
         svn_path = $2[-1..-1] == "/" ? $2[0..-2] : $2
-        return Project::Codehaus::CodehausProjectSvn.new(svn_id, svn_path, options[:jira_project_id])
+        return Project::Codehaus::CodehausProjectSvn.new(unix_name, svn_path, options[:jira_project_id])
       end
 
     end

data/lib/meta_project/scm_web/browser.rb

@@ -6,7 +6,7 @@ module MetaProject
     #
     class Browser
 
-      attr_accessor :overview_spec, :history_spec, :raw_spec, :html_spec, :diff_spec
+      attr_accessor :dir_spec, :history_spec, :raw_spec, :html_spec, :diff_spec
 
       # The variables to use in +uri_specs+ are:
       #
@@ -14,25 +14,35 @@ module MetaProject
       # * revision
       # * previous_revision
       #
-      def initialize(overview_spec=nil, history_spec=nil, raw_spec=nil, html_spec=nil, diff_spec=nil, child_dirs_pattern=nil, child_files_pattern=nil)
-        @overview_spec, @history_spec, @raw_spec, @html_spec, @diff_spec, @child_dirs_pattern, @child_files_pattern = overview_spec, history_spec, raw_spec, html_spec, diff_spec, child_dirs_pattern, child_files_pattern
+      def initialize(dir_spec=nil, history_spec=nil, raw_spec=nil, html_spec=nil, diff_spec=nil, child_dirs_pattern=nil, child_files_pattern=nil)
+        @dir_spec, @history_spec, @raw_spec, @html_spec, @diff_spec, @child_dirs_pattern, @child_files_pattern = dir_spec, history_spec, raw_spec, html_spec, diff_spec, child_dirs_pattern, child_files_pattern
+        
+        # TODO: move to setters. maybe create a spec class?
+        validate_presence_of("dir_spec", @dir_spec, "path")
+        validate_presence_of("history_spec", @history_spec, "path")
+        validate_presence_of("raw_spec", @raw_spec, "path")
+        validate_presence_of("raw_spec", @raw_spec, "revision")
+        validate_presence_of("html_spec", @html_spec, "path")
+        validate_presence_of("html_spec", @html_spec, "revision")
+        #validate_presence_of("diff_spec", @diff_spec, "path")
+        validate_presence_of("diff_spec", @diff_spec, "revision")
+        #validate_presence_of("diff_spec", @diff_spec, "previous_revision")
       end
-
-      # TODO: deprecate this and use root.cleanpath instead
-      def overview
-        file_uri(nil, nil, @overview_spec)
+      
+      def dir(path)
+        file_uri(path, nil, @dir_spec, nil)
       end
 
       def history(path)
-        file_uri(path, nil, @history_spec)
+        file_uri(path, nil, @history_spec, nil)
       end
 
       def raw(path, revision)
-        file_uri(path, revision, @raw_spec)
+        file_uri(path, revision, @raw_spec, nil)
       end
 
       def html(path, revision)
-        file_uri(path, revision, @html_spec)
+        file_uri(path, revision, @html_spec, nil)
       end
 
       def diff(path, revision, previous_revision)
@@ -52,19 +62,23 @@ module MetaProject
       # Returns a Pathname representing the root directory of this browser.
       # NOTE: The root of the browser may be at a lower level than
       # the toplevel of the online scm web interface; It depends
-      # on the configuration of this instance (overview).
+      # on the configuration of this instance.
       def root
-        Pathname.new(self, "", nil, true)
+        Pathname.new(self, nil, "", nil, true)
       end
 
     private
 
+      def validate_presence_of(spec_name, spec, var)
+        raise "Missing \#{#{var}} in #{spec_name}: '#{spec}'" unless spec =~ /\#\{#{var}\}/
+        end
+
       # Returns the file URI for +path+. Valid options are:
       #
-      # * :type => ["overview"|"html"|"diff"|"raw"]
+      # * :type => ["dir"|"html"|"diff"|"raw"]
       # * :revision
       # * :previous_revision
-      def file_uri(path="", revision=nil, spec=@overview_spec, previous_revision=nil)
+      def file_uri(path, revision, spec, previous_revision)
         begin
           eval("\"#{spec}\"", binding)
         rescue NameError

data/lib/meta_project/scm_web/pathname.rb

@@ -1,5 +1,3 @@
-require 'open-uri'
-
 module MetaProject
   module ScmWeb
     # A subset of the 
@@ -8,13 +6,18 @@ module MetaProject
     #
     class Pathname
             
-      def initialize(browser, path_from_root, revision_identifier, directory)
-        @browser, @path_from_root, @revision_identifier, @directory = browser, path_from_root, revision_identifier, directory
+      def initialize(browser, parent, basename, revision_identifier, directory)
+        @browser, @parent, @basename, @revision_identifier, @directory = browser, parent, basename, revision_identifier, directory
       end
       
       # The relative path from the root of the browser instance
+      #
+      # ""
+      # "var"
+      # "var/spool"
       def path_from_root
-        @path_from_root
+        parent_path_from_root = @parent ? @parent.path_from_root : ""
+        parent_path_from_root == "" ? @basename : "#{parent_path_from_root}/#{@basename}"
       end
 
       # The revision of this file or nil if this is a directory
@@ -24,47 +27,63 @@ module MetaProject
 
       # Returns a Pathname at the +relative_path+ from this directory.
       # Note: This only works if this instance is a +directory?+
-      def pathname(relative_path, revision_identifier, directory)
-        child_path_from_root = (@path_from_root == "") ? relative_path : "#{@path_from_root}#{relative_path}"
-        Pathname.new(@browser, child_path_from_root, revision_identifier, directory)
+      def child(basename, revision_identifier=nil, directory=true)
+        raise "Can't create children for files" unless @directory
+        Pathname.new(@browser, self, basename, revision_identifier, directory)
       end
 
       # Stlib Pathname methods
+      
+      # The parent of this instance
+      def parent
+        @parent
+      end
 
-      # Returns the full path on the web (to the history page)
+      # The basename of this instance
+      def basename
+        @basename
+      end
+
+      # Returns the full path on the web (to the dir page for directories, history page for files)
       def cleanpath
-        @browser.history(@path_from_root)
+        directory? ? @browser.dir(path_from_root) : @browser.history(path_from_root)
       end
 
       def children
         result = []
-        html = Kernel.open(cleanpath).read
+        html = better_open(cleanpath).read
 
-        html.scan(@browser.child_dirs_pattern) do |child_path_rev_array|
-          relative_child_path = child_path_rev_array[0]
-          child_revision_identifier = child_path_rev_array[1] # nil for most browsers
-#puts "DIR:#{relative_child_path}:#{child_revision_identifier}"
-          result << pathname(relative_child_path, child_revision_identifier, true)
+        html.scan(@browser.child_dirs_pattern) do |child_basename_rev_array|
+          child_basename = child_basename_rev_array[0]
+          child_revision_identifier = child_basename_rev_array[1] # nil for most browsers
+#puts "DIR:#{child_basename}:#{child_revision_identifier}"
+          result << child(child_basename, child_revision_identifier, true)
         end
 
-        html.scan(@browser.child_files_pattern) do |child_path_rev_array|
-          relative_child_path = child_path_rev_array[0]
-          child_revision_identifier = child_path_rev_array[1]
-#puts "FILE:#{relative_child_path}:#{child_revision_identifier}"
-          result << pathname(relative_child_path, child_revision_identifier, false)
+        html.scan(@browser.child_files_pattern) do |child_basename_rev_array|
+          child_basename = child_basename_rev_array[0]
+          child_revision_identifier = child_basename_rev_array[1]
+#puts "FILE:#{child_basename}:#{child_revision_identifier}"
+          result << child(child_basename, child_revision_identifier, false)
         end
         result
       end
       
       def open(&block)
-        url = @browser.raw(@path_from_root, revision_identifier)
-        # open-uri's open
-        Kernel.open(url, &block)
+        url = @browser.raw(path_from_root, revision_identifier)
+        begin
+          better_open(url, &block)
+        rescue Errno::ECONNREFUSED => e
+          # TODO: retry 2-3 times
+          e.message += "(URL:#{url})"
+          raise e
+        end
       end
       
       def directory?
         @directory
       end
+      
     end
   end
 end

data/lib/meta_project/tracker/trac/trac_tracker.rb

@@ -17,7 +17,7 @@ module MetaProject
         def materialize(issue)
           begin
             url = "#{@trac_base_url}/ticket/#{issue.identifier}"
-            html = open(url).read
+            html = better_open(url).read
             summary = html[/Ticket ##{issue.identifier}\s*<\/h1>\s*<h2>([^<]*)<\/h2>/n, 1]
             issue.attributes[:summary] = summary
             issue.attributes[:url] = url
@@ -26,19 +26,6 @@ module MetaProject
           end
         end
         
-      private
-      
-        def XXissue(issue_identifier)
-          issue_uri = "#{@trac_base_url}/ticket/#{issue_identifier}"
-          begin
-            html = open(issue_uri) { |data| data.read }
-            summary = html[/Ticket ##{issue_identifier}\s*<\/h1>\s*<h2>([^<]*)<\/h2>/n, 1]
-            ::MetaProject::Tracker::Issue.new(issue_uri, summary)
-          rescue OpenURI::HTTPError
-            nil
-          end
-        end
-
       end
     end
   end

data/lib/meta_project/tracker/xforge/xforge_tracker.rb

@@ -38,7 +38,7 @@ module MetaProject
           end
 
           def issue(identifier)
-            html = open(@uri) { |data| data.read }
+            html = better_open(@uri).read
 
             issue_summary_regexp = @tracker.issue_summary_regexp(identifier)
             if(html =~ issue_summary_regexp)
@@ -54,7 +54,7 @@ module MetaProject
 
         # The ids of the subtrackers
         def atids
-          html = open(overview) { |data| data.read }
+          html = better_open(overview).read
 
           # TODO: there has to be a better way to extract the atids from the HTML!
           atids = []

data/lib/meta_project.rb

@@ -1,5 +1,6 @@
 require 'meta_project/core_ext/string'
 require 'meta_project/core_ext/pathname'
+require 'meta_project/core_ext/open_uri'
 require 'meta_project/project_analyzer'
 require 'meta_project/patois'
 require 'meta_project/project'

metadata

@@ -3,8 +3,8 @@ rubygems_version: 0.8.11
 specification_version: 1
 name: meta_project
 version: !ruby/object:Gem::Version 
-  version: 0.4.6
-date: 2005-09-01 00:00:00 -04:00
+  version: 0.4.7
+date: 2005-09-06 00:00:00 -04:00
 summary: Ruby based make-like utility.
 require_paths: 
   - lib
@@ -42,6 +42,7 @@ files:
   - lib/meta_project/scm_web.rb
   - lib/meta_project/tracker.rb
   - lib/meta_project/version_parser.rb
+  - lib/meta_project/core_ext/open_uri.rb
   - lib/meta_project/core_ext/pathname.rb
   - lib/meta_project/core_ext/string.rb
   - lib/meta_project/patois/parser.rb