whimsy-asf 0.0.74 → 0.0.75

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6a3cda4938653e84801388a33933a5ff5753ccd8
-  data.tar.gz: cbd46d98523e091ae41001899b2b88c086a8bc73
+  metadata.gz: fbe033a993bc2428dbed396c0a98fece7b33f82b
+  data.tar.gz: 17ca4624c5340b1354ff01f540999338d426f2b0
 SHA512:
-  metadata.gz: e098ec455420ce5293b5457c9056c63ceb44f3187a0242f9d5e0798cd314b1ea8f4ea734f042359ec442e9e4e49578ff4bb948e29742b349db8c99a4e3d52620
-  data.tar.gz: fd7b79fddc1883e8a57248496867ecef4ddd7aaf918d927796df7cacd4c6a9f049906547d74e62420ac749ef90733f5e215248693b5a78673ee4309a1dd662b0
+  metadata.gz: 801153db8122f912ead1cd2df9a56ad65349bb50c95b1719497dd65da332862c2b3ccbf0e26b92f17981dc1ccb5a34474b2bd1eb4266a6267eb5104b97de34cf
+  data.tar.gz: 28230fd4887ed1198b94f5d27bba0d2a38eec271ffeaeaa4d010d4aa9b07129a27ab0bb21b621058df05ed551c2ff621218237cff9a003496c3f5fc316d76824

data/asf.version

@@ -1 +1 @@
-0.0.74
+0.0.75

data/lib/whimsy/asf.rb

@@ -17,4 +17,8 @@ module ASF
     times = sources.map {|source| File.mtime(source)}
     times.max.gmtime
   end
+  def self.library_gitinfo
+    return @info if @info
+    @info = `git show --format="%h  %ci"  -s HEAD`.chomp
+  end
 end

data/lib/whimsy/asf/committee.rb

@@ -59,34 +59,47 @@ module ASF
       @@svn_change = Time.parse(
         `svn info #{file}`[/Last Changed Date: (.*) \(/, 1]).gmtime
 
+      # Split the file on lines starting "* ", i.e. the start of each group in section 3
       info = File.read(file).split(/^\* /)
+      # Extract the text before first entry in section 3 and split on section headers,
+      # keeping sections 1 (COMMITTEES) and 2 (REPORTING).
       head, report = info.shift.split(/^\d\./)[1..2]
+      # Drop lines which could match group entries
       head.gsub! /^\s+NAME\s+CHAIR\s*$/,'' # otherwise could match an entry with no e-mail
 
       # extract the committee chairs (e-mail address is required here)
+      # Note: this includes the non-PMC entries
       head.scan(/^[ \t]+(\w.*?)[ \t][ \t]+(.*)[ \t]+<(.*?)@apache\.org>/).
         each do |committee, name, id|
           list[committee].chairs << {name: name, id: id}
         end
 
       # Extract the non-PMC committees (e-mail address may be absent)
-      @nonpmcs = head.sub(/.*?also has/m,'').
+      # first drop leading text so we only match non-PMCs
+      @nonpmcs = head.sub(/.*?also has /m,'').
         scan(/^[ \t]+(\w.*?)(?:[ \t][ \t]|[ \t]?$)/).flatten.uniq.
         map {|name| list[name]}
 
+      # for each committee in section 3
       info.each do |roster|
+        # extract the committee name and canonicalise
         committee = list[@@namemap.call(roster[/(\w.*?)\s+\(/,1])]
+        # get the start date
         committee.established = roster[/\(est\. (.*?)\)/, 1]
+        # Extract any emeritus members (now probably redundant)
         roster.gsub! /^.*\(\s*emeritus\s*\).*/i do |line|
           committee.emeritus += line.scan(/<(.*?)@apache\.org>/).flatten
           ''
         end
+        # extract the availids (is this used?)
         committee.info = roster.scan(/<(.*?)@apache\.org>/).flatten
+        # drop (chair) markers and extract 0: name, 1: availid, 2: [date], 3: date
         committee.roster = Hash[roster.gsub(/\(\w+\)/, '').
           scan(/^\s*(.*?)\s*<(.*?)@apache\.org>\s+(\[(.*?)\])?/).
           map {|list| [list[1], {name: list[0], date: list[3]}]}]
       end
 
+      # process report section
       report.scan(/^([^\n]+)\n---+\n(.*?)\n\n/m).each do |period, committees|
         committees.scan(/^   \s*(.*)/).each do |committee|
           committee, comment = committee.first.split(/\s+#\s+/,2)
@@ -101,7 +114,7 @@ module ASF
         end
       end
 
-      @committee_info = list.values
+      @committee_info = list.values.uniq
     end
 
     def self.nonpmcs

data/lib/whimsy/asf/config.rb

@@ -9,12 +9,25 @@ module ASF
 
     @config = YAML.load_file("#@home/.whimsy") rescue {}
 
-    # default :svn for backwards compatibility
-    @config[:svn] ||= ['/srv/svn/*', '/home/whimsysvn/svn/*', "#{@home}/svn/*"]
+    # default :svn and :git
+    @config[:svn] ||= '/srv/svn/*'
+    @config[:git] ||= '/srv/git/*'
+
+    @config[:lib] ||= []
+
+    # add gems to lib
+    (@config[:gem] || {}).to_a.reverse.each do |name, version|
+      begin
+        gem = Gem::Specification.find_by_name(name, version)
+        @config[:lib] += Dir[gem.lib_dirs_glob]
+      rescue Gem::LoadError
+      end
+    end
 
     # add libraries to RUBYLIB, load path
     (@config[:lib] || []).reverse.each do |lib|
       next unless File.exist? lib
+      lib = File.realpath(lib)
       ENV['RUBYLIB']=([lib] + ENV['RUBYLIB'].to_s.split(':')).uniq.join(':')
       $LOAD_PATH.unshift lib.untaint unless $LOAD_PATH.include? lib
     end

data/lib/whimsy/asf/ldap.rb

@@ -1,6 +1,38 @@
+#
+# Encapsulate access to LDAP, caching results for performance.  For best
+# performance in applications that access large number of objects, make use
+# of the preload methods to pre-fetch multiple objects in a single LDAP
+# call, and rely on the cache to find the objects later.
+#
+# The cache makes heavy use of Weak References internally to enable garbage
+# collection to reclaim objects; among other things, this ensures that
+# LDAP results don't become too stale.
+#
+# Until garbage collection reclaims an object, calls to find methods for the
+# same name is guaranteed to return the same object.  Holding on to the
+# results of find or preload calls (by assigning it to a variable) is
+# sufficient to prevent reclaiming of objects.
+#
+# To illustrate, the following is likely to return the same id twice, followed
+# by a new id:
+#   puts ASF::Person.find('rubys').__id__
+#   puts ASF::Person.find('rubys').__id__
+#   GC.start
+#   puts ASF::Person.find('rubys').__id__
+#
+# By contrast, the following is guaranteed to produce the same id three times:
+#   rubys1 = ASF::Person.find('rubys')
+#   rubys2 = ASF::Person.find('rubys')
+#   GC.start
+#   rubys3 = ASF::Person.find('rubys')
+#   puts [rubys1.__id__, rubys2.__id__, rubys3.__id__]
+#
+
 require 'wunderbar'
 require 'ldap'
 require 'weakref'
+require 'net/http'
+require 'base64'
 
 module ASF
   module LDAP
@@ -8,41 +40,80 @@ module ASF
      # https://github.com/apache/infrastructure-puppet/blob/deployment/data/common.yaml (ldapserver::slapd_peers)
     HOSTS = %w(
       ldaps://ldap1-us-west.apache.org:636
-      ldaps://ldap1-eu-central.apache.org:636
+      ldaps://ldap1-lw-us.apache.org:636
       ldaps://ldap2-us-west.apache.org:636
-      ldaps://ldap1-us-east.apache.org:636
+      ldaps://ldap1-lw-eu.apache.org:636
       ldaps://snappy5.apache.org:636
+      ldaps://ldap2-lw-us.apache.org:636
+      ldaps://ldap2-lw-eu.apache.org:636
     )
-  end
 
-  # determine where ldap.conf resides
-  if Dir.exist? '/etc/openldap'
-    ETCLDAP = '/etc/openldap'
-  else
-    ETCLDAP = '/etc/ldap'
-  end
+    # fetch configuration from apache/infrastructure-puppet
+    def self.puppet_config
+      return @puppet if @puppet
+      file = '/apache/infrastructure-puppet/deployment/data/common.yaml'
+      http = Net::HTTP.new('raw.githubusercontent.com', 443)
+      http.use_ssl = true
+      @puppet = YAML.load(http.request(Net::HTTP::Get.new(file)).body)
+    end
 
-  # determine whether or not the LDAP API can be used
-  def self.init_ldap
-    @ldap = nil
-    @mtime = Time.now
+    # extract the ldapcert from the puppet configuration
+    def self.puppet_cert
+      puppet_config['ldapclient::ldapcert']
+    end
 
-    host = ASF::LDAP.host
+    # extract the ldap servers from the puppet configuration
+    def self.puppet_ldapservers
+      puppet_config['ldapserver::slapd_peers'].values.
+        map {|host| "ldaps://#{host}:636"}
+    rescue
+      nil
+    end
 
-    Wunderbar.info "Connecting to LDAP server: #{host}"
+    # connect to LDAP
+    def self.connect
+      hosts.shuffle.each do |host|
+        Wunderbar.info "Connecting to LDAP server: #{host}"
+
+        begin
+          # request connection
+          uri = URI.parse(host)
+          if uri.scheme == 'ldaps'
+            ldap = ::LDAP::SSLConn.new(uri.host, uri.port)
+          else
+            ldap = ::LDAP::Conn.new(uri.host, uri.port)
+          end
+
+          # test the connection
+          ldap.bind
+
+          # save the host
+          @host = host
+
+          return ldap
+        rescue ::LDAP::ResultError => re
+          Wunderbar.warn "Error connecting to LDAP server #{host}: " +
+            re.message + " (continuing)"
+        end
 
-    begin
-      uri = URI.parse(host)
-      if uri.scheme == 'ldaps'
-        @ldap = ::LDAP::SSLConn.new(uri.host, uri.port)
-      else
-        @ldap = ::LDAP::Conn.new(uri.host, uri.port)
       end
-    rescue ::LDAP::ResultError=>re
-      Wunderbar.error "Error binding to LDAP server: message: ["+ re.message + "]"
+      Wunderbar.error "Failed to connect to any LDAP host"
+      return nil
     end
   end
 
+  # backwards compatibility for tools that called this interface
+  def self.init_ldap
+    @ldap ||= ASF::LDAP.connect
+  end
+
+  # determine where ldap.conf resides
+  if Dir.exist? '/etc/openldap'
+    ETCLDAP = '/etc/openldap'
+  else
+    ETCLDAP = '/etc/ldap'
+  end
+
   def self.ldap
     @ldap || self.init_ldap
   end
@@ -66,30 +137,34 @@ module ASF
     result
   end
 
-  def self.refresh(symbol)
-    if not @mtime or Time.now - @mtime > 300.0
-      @mtime = Time.now
+  # safely dereference a weakref array attribute.  Block provided is
+  # used when reference is not set or has been reclaimed.
+  def self.dereference_weakref(object, attr, &block)
+    attr = "@#{attr}"
+    value = object.instance_variable_get(attr) || block.call
+    value[0..-1]
+  rescue WeakRef::RefError
+    value = block.call
+  ensure
+    if value and not value.instance_of? WeakRef
+      object.instance_variable_set(attr, WeakRef.new(value))
     end
+  end
 
-    if instance_variable_get("#{symbol}_mtime") != @mtime
-      instance_variable_set("#{symbol}_mtime", @mtime)
-      instance_variable_set(symbol, nil)
-    end
+  def self.weakref(attr, &block)
+    self.dereference_weakref(self, attr, &block)
   end
 
   def self.pmc_chairs
-    refresh(:@pmc_chairs)
-    @pmc_chairs ||= Service.find('pmc-chairs').members
+    weakref(:pmc_chairs) {Service.find('pmc-chairs').members}
   end
 
   def self.committers
-    refresh(:@committers)
-    @committers ||= Group.find('committers').members
+    weakref(:committers) {Group.find('committers').members}
   end
 
   def self.members
-    refresh(:@members)
-    @members ||= Group.find('member').members
+    weakref(:members) {Group.find('member').members}
   end
 
   class Base
@@ -134,6 +209,10 @@ module ASF
       self
     end
 
+    def weakref(attr, &block)
+      ASF.dereference_weakref(self, attr, &block)
+    end
+
     unless Object.respond_to? :id
       def id
         @name
@@ -213,12 +292,16 @@ module ASF
       ASF::Member.status[name] or ASF.members.include? self
     end
 
+    def asf_officer_or_member?
+      asf_member? or ASF.pmc_chairs.include? self
+    end
+
     def asf_committer?
        ASF::Group.new('committers').include? self
     end
 
     def banned?
-      not attrs['loginShell'] or attrs['loginShell'].include? "/usr/bin/false"
+      not attrs['loginShell'] or %w(/usr/bin/false bin/nologin bin/no-cla).any? {|a| attrs['loginShell'].first.include? a}
     end
 
     def mail
@@ -299,9 +382,29 @@ module ASF
       end
     end
 
+    def self.preload
+      Hash[ASF.search_one(base, "cn=*", %w(dn memberUid modifyTimestamp)).map do |results|
+        cn = results['dn'].first[/^cn=(.*?),/, 1]
+        group = ASF::Group.find(cn)
+        group.modifyTimestamp = results['modifyTimestamp'].first # it is returned as an array of 1 entry
+        members = results['memberUid']
+        group.members = members || []
+        [group, members]
+      end]
+    end
+
+    attr_accessor :modifyTimestamp
+
+    def members=(members)
+      @members = WeakRef.new(members)
+    end
+
     def members
-      ASF.search_one(base, "cn=#{name}", 'memberUid').flatten.
-        map {|uid| Person.find(uid)}
+      members = weakref(:members) do
+        ASF.search_one(base, "cn=#{name}", 'memberUid').flatten
+      end
+
+      members.map {|uid| Person.find(uid)}
     end
   end
 
@@ -312,9 +415,29 @@ module ASF
       ASF.search_one(base, filter, 'cn').flatten.map {|cn| Committee.find(cn)}
     end
 
+    def self.preload
+      Hash[ASF.search_one(base, "cn=*", %w(dn member modifyTimestamp)).map do |results|
+        cn = results['dn'].first[/^cn=(.*?),/, 1]
+        committee = ASF::Committee.find(cn)
+        committee.modifyTimestamp = results['modifyTimestamp'].first # it is returned as an array of 1 entry
+        members = results['member']
+        committee.members = members
+        [committee, members]
+      end]
+    end
+
+    attr_accessor :modifyTimestamp
+
+    def members=(members)
+      @members = WeakRef.new(members)
+    end
+
     def members
-      ASF.search_one(base, "cn=#{name}", 'member').flatten.
-        map {|uid| Person.find uid[/uid=(.*?),/,1]}
+      members = weakref(:members) do
+        ASF.search_one(base, "cn=#{name}", 'member').flatten
+      end
+
+      members.map {|uid| Person.find uid[/uid=(.*?),/,1]}
     end
 
     def dn
@@ -354,59 +477,112 @@ module ASF
   module LDAP
     def self.bind(user, password, &block)
       dn = ASF::Person.new(user).dn
+      raise ::LDAP::ResultError.new('Unknown user') unless dn
+
+      ASF.ldap.unbind rescue nil
       if block
         ASF.ldap.bind(dn, password, &block)
+        ASF.init_ldap
       else
         ASF.ldap.bind(dn, password)
       end
-      ASF.init_ldap
     end
 
-    # select LDAP host
-    def self.host
+    # validate HTTP authorization, and optionally invoke a block bound to
+    # that user.
+    def self.http_auth(string, &block)
+      auth = Base64.decode64(string.to_s[/Basic (.*)/, 1] || '')
+      user, password = auth.split(':', 2)
+      return unless password
+
+      if block
+        self.bind(user, password, &block)
+      else
+        begin
+          ASF::LDAP.bind(user, password) {}
+          return ASF::Person.new(user)
+        rescue ::LDAP::ResultError
+          return nil
+        end
+      end
+    end
+
+    # determine what LDAP hosts are available
+    def self.hosts
       # try whimsy config
-      host = ASF::Config.get(:ldap)
+      hosts = Array(ASF::Config.get(:ldap))
 
       # check system configuration
-      unless host
+      if hosts.empty?
         conf = "#{ETCLDAP}/ldap.conf"
         if File.exist? conf
-          host = File.read(conf)[/^uri\s+(ldaps?:\/\/\S+?:\d+)/i, 1]
+          uris = File.read(conf)[/^uri\s+(.*)/i, 1].to_s
+          hosts = uris.scan(/ldaps?:\/\/\S+?:\d+/)
         end
       end
 
-      # if all else fails, pick one at random
-      host = ASF::LDAP::HOSTS.sample unless host
+      # if all else fails, use default list
+      hosts = ASF::LDAP::HOSTS if hosts.empty?
+
+      hosts
+    end
 
-      host
+    # select LDAP host
+    def self.host
+      @host ||= hosts.sample
     end
 
     # query and extract cert from openssl output
-    def self.cert
+    def self.extract_cert
       host = LDAP.host[%r{//(.*?)(/|$)}, 1]
-      query = "openssl s_client -connect #{host} -showcerts"
-      output = `#{query} < /dev/null 2> /dev/null`
-      output[/^-+BEGIN.*?\n-+END[^\n]+\n/m]
+      puts ['openssl', 's_client', '-connect', host, '-showcerts'].join(' ')
+      out, err, rc = Open3.capture3 'openssl', 's_client',
+        '-connect', host, '-showcerts'
+      out[/^-+BEGIN.*?\n-+END[^\n]+\n/m]
     end
 
     # update /etc/ldap.conf. Usage:
+    #
     #   sudo ruby -r whimsy/asf -e "ASF::LDAP.configure"
+    #
     def self.configure
-      if not File.exist? "#{ETCLDAP}/asf-ldap-client.pem"
-        File.write "#{ETCLDAP}/asf-ldap-client.pem", self.cert
+      cert = Dir["#{ETCLDAP}/asf*-ldap-client.pem"].first
+
+      # verify/obtain/write the cert
+      if not cert
+        cert = "#{ETCLDAP}/asf-ldap-client.pem"
+        File.write cert, ASF::LDAP.puppet_cert || self.extract_cert
       end
 
+      # read the current configuration file
       ldap_conf = "#{ETCLDAP}/ldap.conf"
       content = File.read(ldap_conf)
-      unless content.include? 'asf-ldap-client.pem'
-        content.gsub!(/^TLS_CACERT/, '# TLS_CACERT')
-        content.gsub!(/^TLS_REQCERT/, '# TLS_REQCERT')
+
+      # ensure that the right cert is used
+      unless content =~ /asf.*-ldap-client\.pem/
+        content.gsub!(/^TLS_CACERT/i, '# TLS_CACERT')
         content += "TLS_CACERT #{ETCLDAP}/asf-ldap-client.pem\n"
-        content += "uri #{LDAP.host}\n"
+      end
+
+      # provide the URIs of the ldap hosts
+      content.gsub!(/^URI/, '# URI')
+      content += "uri \n" unless content =~ /^uri /
+      content[/uri (.*)\n/, 1] = hosts.join(' ')
+
+      # verify/set the base
+      unless content.include? 'base dc=apache'
+        content.gsub!(/^BASE/i, '# BASE')
         content += "base dc=apache,dc=org\n"
-        content += "TLS_REQCERT allow\n" if ETCLDAP.include? 'openldap'
-        File.write(ldap_conf, content)
       end
+
+      # ensure TLS_REQCERT is allow (Mac OS/X only)
+      if ETCLDAP.include? 'openldap' and not content.include? 'REQCERT allow'
+        content.gsub!(/^TLS_REQCERT/i, '# TLS_REQCERT')
+        content += "TLS_REQCERT allow\n"
+      end
+
+      # write the configuration if there were any changes
+      File.write(ldap_conf, content) unless content == File.read(ldap_conf)
     end
   end
 end

data/lib/whimsy/asf/rack.rb

@@ -21,12 +21,12 @@ module ASF
     def self.decode(env)
       class << env; attr_accessor :user, :password; end
 
-      if env['HTTP_AUTHORIZATION']
+      if env['HTTP_AUTHORIZATION'].to_s.empty?
+        env.user = env['REMOTE_USER'] || ENV['USER'] || Etc.getpwuid.name
+      else
         require 'base64'
         env.user, env.password = Base64.decode64(env['HTTP_AUTHORIZATION'][
-          /^Basic ([A-Za-z0-9+\/=]+)$/,1]).split(':',2)
-      else
-        env.user = env['REMOTE_USER'] || ENV['USER'] || Etc.getpwuid.name
+          /^Basic ([A-Za-z0-9+\/=]+)$/,1].to_s).split(':',2)
       end
 
       env['REMOTE_USER'] ||= env.user
@@ -189,6 +189,7 @@ module ASF
           env['SCRIPT_URI'] += '/'
         end
       end
+
       return  @app.call(env)
     end
   end

data/lib/whimsy/asf/svn.rb

@@ -12,8 +12,9 @@ module ASF
 
     def self.repos
       @semaphore.synchronize do
-        svn = ASF::Config.get(:svn).map {|dir| dir.untaint}
+        svn = Array(ASF::Config.get(:svn)).map {|dir| dir.untaint}
         @repos ||= Hash[Dir[*svn].map { |name| 
+          next unless Dir.exist? name.untaint
           Dir.chdir name.untaint do
             out, err, status = Open3.capture3('svn', 'info')
             if status.success?

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: whimsy-asf
 version: !ruby/object:Gem::Version
-  version: 0.0.74
+  version: 0.0.75
 platform: ruby
 authors:
 - Sam Ruby
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-01-12 00:00:00.000000000 Z
+date: 2016-01-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -104,6 +104,8 @@ extra_rdoc_files: []
 files:
 - asf.gemspec
 - asf.version
+- lib/whimsy/asf.rb
+- lib/whimsy/asf/agenda.rb
 - lib/whimsy/asf/agenda/attachments.rb
 - lib/whimsy/asf/agenda/back.rb
 - lib/whimsy/asf/agenda/committee.rb
@@ -111,7 +113,6 @@ files:
 - lib/whimsy/asf/agenda/front.rb
 - lib/whimsy/asf/agenda/minutes.rb
 - lib/whimsy/asf/agenda/special.rb
-- lib/whimsy/asf/agenda.rb
 - lib/whimsy/asf/auth.rb
 - lib/whimsy/asf/bundler.rb
 - lib/whimsy/asf/committee.rb
@@ -126,7 +127,6 @@ files:
 - lib/whimsy/asf/site.rb
 - lib/whimsy/asf/svn.rb
 - lib/whimsy/asf/watch.rb
-- lib/whimsy/asf.rb
 homepage: https://whimsy.apache.org/
 licenses:
 - Apache License, Version 2.0
@@ -147,7 +147,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.14
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Whimsy 'model' of the ASF