whimsy-asf 0.0.75 → 0.0.76

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fbe033a993bc2428dbed396c0a98fece7b33f82b
-  data.tar.gz: 17ca4624c5340b1354ff01f540999338d426f2b0
+  metadata.gz: 42ee442311d0fc054e4dd8fb6f2bdb4635d03ce1
+  data.tar.gz: d556f03175acf95b7882d59b1abfe0b3332c81b5
 SHA512:
-  metadata.gz: 801153db8122f912ead1cd2df9a56ad65349bb50c95b1719497dd65da332862c2b3ccbf0e26b92f17981dc1ccb5a34474b2bd1eb4266a6267eb5104b97de34cf
-  data.tar.gz: 28230fd4887ed1198b94f5d27bba0d2a38eec271ffeaeaa4d010d4aa9b07129a27ab0bb21b621058df05ed551c2ff621218237cff9a003496c3f5fc316d76824
+  metadata.gz: 5a00c4fa7f49262e06d04cdffb11929b1955e27afb55b3bc5f2224703a1ccd3d6d609a3656e6a6a3d54677bfcd8e7191641e944888aeebc8359028ab33d7c57c
+  data.tar.gz: ed851c6a5edc6fef8325345c96d75d9abfaafa2b1e6753b2de2a8ff3d65ccdb30f35f5bbc74fc0912eb2cdd87227c27d08c4564610b753a06e84721638b2525c

data/asf.version

@@ -1 +1 @@
-0.0.75
+0.0.76

data/lib/whimsy/asf/agenda/attachments.rb

@@ -35,7 +35,7 @@ class ASF::Board::Agenda
       unless @quick
         begin
           committee = ASF::Committee.find(attrs['title'])
-          attrs['chair_email'] = committee.chair.mail.first
+          attrs['chair_email'] = "#{committee.chair.id}@apache.org"
           attrs['mail_list'] = committee.mail_list
           attrs.delete('mail_list') if attrs['mail_list'].include? ' '
 

data/lib/whimsy/asf/committee.rb

@@ -6,11 +6,10 @@ module ASF
   end
 
   class Committee < Base
-    attr_accessor :info, :emeritus, :report, :roster, :established, :chairs,
+    attr_accessor :info, :report, :roster, :established, :chairs,
       :schedule
     def initialize(*args)
       @info = []
-      @emeritus = []
       @chairs = []
       @roster = {}
       super
@@ -49,12 +48,12 @@ module ASF
       file = "#{board}/committee-info.txt"
       return unless File.exist? file
 
-      list = Hash.new {|hash, name| hash[name] = find(name)}
-
       if @committee_info and File.mtime(file) == @committee_mtime
         return @committee_info 
       end
 
+      list = Hash.new {|hash, name| hash[name] = find(name)}
+
       @committee_mtime = File.mtime(file)
       @@svn_change = Time.parse(
         `svn info #{file}`[/Last Changed Date: (.*) \(/, 1]).gmtime
@@ -86,11 +85,6 @@ module ASF
         committee = list[@@namemap.call(roster[/(\w.*?)\s+\(/,1])]
         # get the start date
         committee.established = roster[/\(est\. (.*?)\)/, 1]
-        # Extract any emeritus members (now probably redundant)
-        roster.gsub! /^.*\(\s*emeritus\s*\).*/i do |line|
-          committee.emeritus += line.scan(/<(.*?)@apache\.org>/).flatten
-          ''
-        end
         # extract the availids (is this used?)
         committee.info = roster.scan(/<(.*?)@apache\.org>/).flatten
         # drop (chair) markers and extract 0: name, 1: availid, 2: [date], 3: date

data/lib/whimsy/asf/icla.rb

@@ -1,7 +1,10 @@
 module ASF
 
   class ICLA
-    attr_accessor :id, :legal_name, :name, :email
+    # N.B. only id and name should be considered public
+    # form and claRef may contain details of the legal name beyond that in the public name
+    attr_accessor :id, :legal_name, :name, :email, :form
+    attr_accessor :claRef # cla name or SVN revision info
 
     @@mtime = nil
 
@@ -31,9 +34,15 @@ module ASF
 
     # load ICLA information for every committer
     def self.preload
+      people = []
       each do |icla|
-        ASF::Person.find(icla.id).icla =  icla unless icla.id == 'notinaval'
+        unless icla.id == 'notinaval'
+          person = ASF::Person.find(icla.id)
+          people << person
+          person.icla = icla
+        end
       end
+      people
     end
 
     # find ICLA by ID
@@ -94,12 +103,18 @@ module ASF
       elsif @@name_index and not @@name_index.empty?
         @@name_index.values.each(&block)
       elsif SOURCE and File.exist?(SOURCE)
-        File.read(SOURCE).scan(/^([-\w]+):(.*?):(.*?):(.*?):/).each do |list|
+        File.read(SOURCE).scan(/^([-\w]+):(.*?):(.*?):(.*?):(.*)/).each do |list|
           icla = ICLA.new()
           icla.id = list[0]
           icla.legal_name = list[1]
           icla.name = list[2]
           icla.email = list[3]
+          icla.form = list[4]
+          match = icla.form.match(/^Signed CLA(?:;(\S+)| \((\+=.+)\))/)
+          if match
+            # match either the cla name or the SVN ref (+=...)
+            icla.claRef = match[1] || match[2]
+          end
           block.call(icla)
         end
       end

data/lib/whimsy/asf/ldap.rb

@@ -33,6 +33,7 @@ require 'ldap'
 require 'weakref'
 require 'net/http'
 require 'base64'
+require 'thread'
 
 module ASF
   module LDAP
@@ -48,6 +49,9 @@ module ASF
       ldaps://ldap2-lw-eu.apache.org:636
     )
 
+    CONNECT_LOCK = Mutex.new
+    HOST_QUEUE = Queue.new
+
     # fetch configuration from apache/infrastructure-puppet
     def self.puppet_config
       return @puppet if @puppet
@@ -71,8 +75,13 @@ module ASF
     end
 
     # connect to LDAP
-    def self.connect
-      hosts.shuffle.each do |host|
+    def self.connect(test = true)
+      # Try each host at most once
+      hosts.length.times do
+        # Ensure we use each host in turn
+        hosts.each {|host| HOST_QUEUE.push host} if HOST_QUEUE.empty?
+        host = HOST_QUEUE.shift
+
         Wunderbar.info "Connecting to LDAP server: #{host}"
 
         begin
@@ -85,7 +94,7 @@ module ASF
           end
 
           # test the connection
-          ldap.bind
+          ldap.bind if test
 
           # save the host
           @host = host
@@ -97,14 +106,18 @@ module ASF
         end
 
       end
+
       Wunderbar.error "Failed to connect to any LDAP host"
       return nil
     end
   end
 
-  # backwards compatibility for tools that called this interface
-  def self.init_ldap
-    @ldap ||= ASF::LDAP.connect
+  # public entry point for establishing a connection safely
+  def self.init_ldap(reset = false)
+    ASF::LDAP::CONNECT_LOCK.synchronize do
+      @ldap = nil if reset
+      @ldap ||= ASF::LDAP.connect(!reset)
+    end
   end
 
   # determine where ldap.conf resides
@@ -118,23 +131,39 @@ module ASF
     @ldap || self.init_ldap
   end
 
-  # search with a scope of one
+  # search with a scope of one, with automatic retry/failover
   def self.search_one(base, filter, attrs=nil)
-    init_ldap unless defined? @ldap
-    return [] unless @ldap
 
-    Wunderbar.info "ldapsearch -x -LLL -b #{base} -s one #{filter} " +
+    cmd = "ldapsearch -x -LLL -b #{base} -s one #{filter} " +
       "#{[attrs].flatten.join(' ')}"
-    
+
+    # try once per host, with a minimum of two tries
+    attempts_left = [ASF::LDAP.hosts.length, 2].max
     begin
+      attempts_left -= 1
+      init_ldap unless @ldap
+      return [] unless @ldap
+
+      target = @ldap.get_option(::LDAP::LDAP_OPT_HOST_NAME) rescue '?'
+      Wunderbar.info "[#{target}] #{cmd}"
+
       result = @ldap.search2(base, ::LDAP::LDAP_SCOPE_ONELEVEL, filter, attrs)
-    rescue
-      result = []
+    rescue Exception => re
+      if attempts_left <= 0
+        Wunderbar.error "[#{target}] => #{re.inspect} for #{cmd}"
+        raise
+      else
+        Wunderbar.warn "[#{target}] => #{re.inspect} for #{cmd}, retrying ..."
+        @ldap.unbind if @ldap.bound? rescue nil
+        @ldap = nil # force new connection
+        sleep 1
+        retry
+      end
     end
 
     result.map! {|hash| hash[attrs]} if String === attrs
 
-    result
+    result.compact
   end
 
   # safely dereference a weakref array attribute.  Block provided is
@@ -146,7 +175,9 @@ module ASF
   rescue WeakRef::RefError
     value = block.call
   ensure
-    if value and not value.instance_of? WeakRef
+    if not value or RUBY_VERSION.start_with? '1'
+      object.instance_variable_set(attr, value)
+    elsif value and not value.instance_of? WeakRef
       object.instance_variable_set(attr, WeakRef.new(value))
     end
   end
@@ -313,7 +344,7 @@ module ASF
     end
 
     def pgp_key_fingerprints
-      attrs['asf-pgpKeyFingerprint']
+      attrs['asf-pgpKeyFingerprint'] || []
     end
 
     def urls
@@ -383,17 +414,18 @@ module ASF
     end
 
     def self.preload
-      Hash[ASF.search_one(base, "cn=*", %w(dn memberUid modifyTimestamp)).map do |results|
+      Hash[ASF.search_one(base, "cn=*", %w(dn memberUid modifyTimestamp createTimestamp)).map do |results|
         cn = results['dn'].first[/^cn=(.*?),/, 1]
         group = ASF::Group.find(cn)
         group.modifyTimestamp = results['modifyTimestamp'].first # it is returned as an array of 1 entry
-        members = results['memberUid']
-        group.members = members || []
+        group.createTimestamp = results['createTimestamp'].first # it is returned as an array of 1 entry
+        members = results['memberUid']  || []
+        group.members = members
         [group, members]
       end]
     end
 
-    attr_accessor :modifyTimestamp
+    attr_accessor :modifyTimestamp, :createTimestamp
 
     def members=(members)
       @members = WeakRef.new(members)
@@ -406,6 +438,24 @@ module ASF
 
       members.map {|uid| Person.find(uid)}
     end
+
+    def dn
+      @dn ||= ASF.search_one(base, "cn=#{name}", 'dn').first.first
+    end
+
+    def remove(people)
+      people = Array(people).map(&:id)
+      mod = ::LDAP::Mod.new(::LDAP::LDAP_MOD_DELETE, 'memberUid', people)
+      ASF.ldap.modify(self.dn, [mod])
+      @members = nil
+    end
+
+    def add(people)
+      people = Array(people).map(&:dn)
+      mod = ::LDAP::Mod.new(::LDAP::LDAP_MOD_ADD, 'memberUid', people)
+      ASF.ldap.modify(self.dn, [mod])
+      @members = nil
+    end
   end
 
   class Committee < Base
@@ -416,17 +466,18 @@ module ASF
     end
 
     def self.preload
-      Hash[ASF.search_one(base, "cn=*", %w(dn member modifyTimestamp)).map do |results|
+      Hash[ASF.search_one(base, "cn=*", %w(dn member modifyTimestamp createTimestamp)).map do |results|
         cn = results['dn'].first[/^cn=(.*?),/, 1]
         committee = ASF::Committee.find(cn)
         committee.modifyTimestamp = results['modifyTimestamp'].first # it is returned as an array of 1 entry
-        members = results['member']
+        committee.createTimestamp = results['createTimestamp'].first # it is returned as an array of 1 entry
+        members = results['member'] || []
         committee.members = members
         [committee, members]
       end]
     end
 
-    attr_accessor :modifyTimestamp
+    attr_accessor :modifyTimestamp, :createTimestamp
 
     def members=(members)
       @members = WeakRef.new(members)
@@ -443,6 +494,20 @@ module ASF
     def dn
       @dn ||= ASF.search_one(base, "cn=#{name}", 'dn').first.first
     end
+
+    def remove(people)
+      people = Array(people).map(&:dn)
+      mod = ::LDAP::Mod.new(::LDAP::LDAP_MOD_DELETE, 'member', people)
+      ASF.ldap.modify(self.dn, [mod])
+      @members = nil
+    end
+
+    def add(people)
+      people = Array(people).map(&:dn)
+      mod = ::LDAP::Mod.new(::LDAP::LDAP_MOD_ADD, 'member', people)
+      ASF.ldap.modify(self.dn, [mod])
+      @members = nil
+    end
   end
 
   class Service < Base
@@ -456,21 +521,44 @@ module ASF
       "cn=#{id},#{self.class.base}"
     end
 
+    def self.preload
+      Hash[ASF.search_one(base, "cn=*", %w(dn member modifyTimestamp createTimestamp)).map do |results|
+        cn = results['dn'].first[/^cn=(.*?),/, 1]
+        service = ASF::Service.find(cn)
+        service.modifyTimestamp = results['modifyTimestamp'].first # it is returned as an array of 1 entry
+        service.createTimestamp = results['createTimestamp'].first # it is returned as an array of 1 entry
+        members = results['member'] || []
+        service.members = members
+        [service, members]
+      end]
+    end
+
+    attr_accessor :modifyTimestamp, :createTimestamp
+
+    def members=(members)
+      @members = WeakRef.new(members)
+    end
+
     def members
-      ASF.search_one(base, "cn=#{name}", 'member').flatten.
-        map {|uid| Person.find uid[/uid=(.*?),/,1]}
+      members = weakref(:members) do
+        ASF.search_one(base, "cn=#{name}", 'member').flatten
+      end
+
+      members.map {|uid| Person.find uid[/uid=(.*?),/,1]}
     end
 
     def remove(people)
       people = Array(people).map(&:dn)
       mod = ::LDAP::Mod.new(::LDAP::LDAP_MOD_DELETE, 'member', people)
       ASF.ldap.modify(self.dn, [mod])
+      @members = nil
     end
 
     def add(people)
       people = Array(people).map(&:dn)
       mod = ::LDAP::Mod.new(::LDAP::LDAP_MOD_ADD, 'member', people)
       ASF.ldap.modify(self.dn, [mod])
+      @members = nil
     end
   end
 
@@ -479,12 +567,13 @@ module ASF
       dn = ASF::Person.new(user).dn
       raise ::LDAP::ResultError.new('Unknown user') unless dn
 
-      ASF.ldap.unbind rescue nil
+      ASF.ldap.unbind if ASF.ldap.bound? rescue nil
+      ldap = ASF.init_ldap(true)
       if block
-        ASF.ldap.bind(dn, password, &block)
-        ASF.init_ldap
+        ldap.bind(dn, password, &block)
+        ASF.init_ldap(true)
       else
-        ASF.ldap.bind(dn, password)
+        ldap.bind(dn, password)
       end
     end
 
@@ -509,6 +598,7 @@ module ASF
 
     # determine what LDAP hosts are available
     def self.hosts
+      return @hosts if @hosts # cache the hosts list
       # try whimsy config
       hosts = Array(ASF::Config.get(:ldap))
 
@@ -518,23 +608,24 @@ module ASF
         if File.exist? conf
           uris = File.read(conf)[/^uri\s+(.*)/i, 1].to_s
           hosts = uris.scan(/ldaps?:\/\/\S+?:\d+/)
+          Wunderbar.debug "Using hosts from LDAP config"
         end
+      else
+        Wunderbar.debug "Using hosts from Whimsy config"
       end
 
       # if all else fails, use default list
+      Wunderbar.debug "Using default host list" if hosts.empty?
       hosts = ASF::LDAP::HOSTS if hosts.empty?
 
-      hosts
-    end
-
-    # select LDAP host
-    def self.host
-      @host ||= hosts.sample
+      hosts.shuffle!
+      #Wunderbar.debug "Hosts:\n#{hosts.join(' ')}"
+      @hosts = hosts
     end
 
     # query and extract cert from openssl output
     def self.extract_cert
-      host = LDAP.host[%r{//(.*?)(/|$)}, 1]
+      host = hosts.sample[%r{//(.*?)(/|$)}, 1]
       puts ['openssl', 's_client', '-connect', host, '-showcerts'].join(' ')
       out, err, rc = Open3.capture3 'openssl', 's_client',
         '-connect', host, '-showcerts'

data/lib/whimsy/asf/mail.rb

@@ -1,15 +1,19 @@
+require 'weakref'
 
 module ASF
 
   class Mail
     def self.list
-      return @list if @list
+      begin
+        return Hash[@list.to_a] if @list
+      rescue NoMethodError, WeakRef::RefError
+      end
 
       list = Hash.new
 
       # load info from LDAP
       people = ASF::Person.preload(['mail', 'asf-altEmail'])
-      people.each do |name, person|
+      people.each do |person|
         (person.mail+person.alt_email).each do |mail|
           list[mail.downcase] = person
         end
@@ -28,7 +32,8 @@ module ASF
         list["#{icla.id.downcase}@apache.org"] ||= person
       end
 
-      @list = list
+      @list = WeakRef.new(list)
+      list
     end
 
     def self.lists(public_private= false)

data/lib/whimsy/asf/member.rb

@@ -1,3 +1,5 @@
+require 'weakref'
+
 module ASF
   class Member
     include Enumerable
@@ -15,10 +17,16 @@ module ASF
     end
 
     def self.list
-      result = Hash[self.new(true).map {|name, text| [name, {text: text}]}]
+      result = Hash[self.new.map {|id, text|
+        # extract 1st line and remove any trailing /* comment */
+        name = text[/(.*?)\n/, 1].sub(/\s+\/\*.*/,'')
+        [id, {text: text, name: name}]
+      }]
+
       self.status.each do |name, value|
         result[name]['status'] = value
       end
+
       result
     end
 
@@ -33,7 +41,11 @@ module ASF
     end
 
     def self.status
-      return @status if @status
+      begin
+        return Hash[@status.to_a] if @status
+      rescue NoMethodError, WeakRef::RefError
+      end
+
       status = {}
       foundation = ASF::SVN['private/foundation']
       return status unless foundation
@@ -43,18 +55,15 @@ module ASF
         header.sub!(/s\n=+/,'')
         text.scan(/Avail ID: (.*)/).flatten.each {|id| status[id] = header}
       end
-      @status = status
-    end
 
-    def initialize(full = nil)
-      @full = (full.nil? ? ASF::Person.new($USER).asf_member? : full)
+      @status = WeakRef.new(status)
+      status
     end
 
     def each
       foundation = ASF::SVN['private/foundation']
       File.read("#{foundation}/members.txt").split(/^ \*\) /).each do |section|
         id = section[/Avail ID: (.*)/,1]
-        section = '' unless @full
         yield id, section.sub(/\n.*\n===+\s*?\n(.*\n)+.*/,'').strip if id
       end
       nil
@@ -79,7 +88,7 @@ module ASF
 
   class Person
     def members_txt
-      @members_txt ||= ASF::Member.find_text_by_id(name)
+      @members_txt ||= ASF::Member.find_text_by_id(id)
     end
 
     def member_emails

data/lib/whimsy/asf/nominees.rb

@@ -1,9 +1,14 @@
+require 'weakref'
+
 module ASF
 
   class Person < Base
   
     def self.member_nominees
-      return @member_nominees if @member_nominees
+      begin
+        return Hash[@member_nominees.to_a] if @member_nominees
+      rescue NoMethodError, WeakRef::RefError
+      end
 
       meetings = ASF::SVN['private/foundation/Meetings']
       nominations = Dir["#{meetings}/*/nominated-members.txt"].sort.last.untaint
@@ -22,11 +27,12 @@ module ASF
         nominees[find(id)] = nomination
       end
 
-      @member_nominees = nominees
+      @member_nominees = WeakRef.new(nominees)
+      nominees
     end
 
     def member_nomination
-      Person.member_nominees[self]
+      @member_nomination ||= Person.member_nominees[self]
     end
   end
 end

data/lib/whimsy/asf/rack.rb

@@ -21,12 +21,14 @@ module ASF
     def self.decode(env)
       class << env; attr_accessor :user, :password; end
 
-      if env['HTTP_AUTHORIZATION'].to_s.empty?
+      auth = env['HTTP_AUTHORIZATION'] || ENV['HTTP_AUTHORIZATION']
+
+      if auth.to_s.empty?
         env.user = env['REMOTE_USER'] || ENV['USER'] || Etc.getpwuid.name
       else
         require 'base64'
-        env.user, env.password = Base64.decode64(env['HTTP_AUTHORIZATION'][
-          /^Basic ([A-Za-z0-9+\/=]+)$/,1].to_s).split(':',2)
+        env.user, env.password = Base64.
+          decode64(auth[/^Basic ([A-Za-z0-9+\/=]+)$/,1].to_s).split(':',2)
       end
 
       env['REMOTE_USER'] ||= env.user

data/lib/whimsy/asf/site.rb

@@ -52,6 +52,10 @@ module ASF
     def self.list
       templates = ASF::SVN['asf/infrastructure/site/trunk/content']
       file = "#{templates}/index.html"
+      if not File.exist?(file)
+        Wunderbar.warn "Unable to find 'infrastructure/site/trunk/content'"
+        return {}
+      end
       return @@list if not @@list.empty? and File.mtime(file) == @@mtime
       @@mtime = File.mtime(file)
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: whimsy-asf
 version: !ruby/object:Gem::Version
-  version: 0.0.75
+  version: 0.0.76
 platform: ruby
 authors:
 - Sam Ruby
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-01-25 00:00:00.000000000 Z
+date: 2016-02-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri