basechip 0.0.1

data/lib/base_chip/project.rb

@@ -0,0 +1,472 @@
+# Copyright 2011 Tommy Poulter
+#
+# This file is part of basechip.
+#
+# basechip is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+# 
+# basechip is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with basechip.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'base_chip/block'
+require 'base_chip/cluster_type'
+
+module BaseChip
+  class Project
+    include TaskMaster
+    include Dsl
+    include Base
+    include Hierarchy
+
+    # Sets/gets the shared directory for this project, which is a place to
+    # install tools and IP shared between workareas (usually because of size).
+    # Expects a full path
+    define_setting :shared_directory , [ String ]
+
+    # Sets/gets the order of preference for clusters.  If the user specifies
+    # "-j 500" on the command line, it will be spread across multiple clusters
+    # according to their default submission setting.  This setting sets the
+    # order to fill those clusters.
+    # Expects an array of cluster names
+    # @example use local cluster types first, then lsf cluster types
+    #   cluster_order %w{ local lsf }
+    # @example use local cluster types first, then lsf medium cluster, then lsf low cluster
+    #   cluster_order %w{ local lsf:medium lsf:low }
+    define_setting :cluster_order    , [ Array  ]
+    define_child   :block            , Block      
+    define_child   :subproject       , Project    
+    define_child   :cluster_type     , ClusterType
+
+    attr_accessor :workload
+
+    def initialize()
+      super
+      @project   = self
+      @pwd       = Dir.pwd
+    end
+    # def configure_blocks(block_names)
+    #   block_names.each do |name|
+    #     self.blocks[name.to_sym].configure
+    #   end
+    # end
+    def figure_directory_prefix
+      # if @directory == @pwd
+      #   return nil
+      # elsif @pwd.length > @directory.length
+        configure
+        if @blocks
+          @blocks.each do |bname,block|
+            block.configure
+            if @pwd =~ /^#{block.directory}\b/
+              block.configurations.each do |cname,configuration|
+                configuration.configure
+                return "#{bname}:#{cname}" if @pwd =~ /^#{configuration.directory}\b/
+              end
+              return bname.to_s
+            end
+          end
+        end
+      # end
+      nil
+      # fault "'base_chip' should be run from a block's work directory or the top level of the project."
+    end
+    def block_names
+      return @block_names if @block_names
+      # \@block_names = Dir.glob("#\{@directory}/*")
+      # \@block_names.map!{|path| path.split(/\//).pop}
+      # \@block_names.delete 'settings.yaml'
+      # \@block_names
+    end
+    DEFINED_SUBCOMMANDS = %W{all gate}
+    def ready(targets=nil) #,available_only=false)
+      configure
+      @orig_targets = targets
+      if @directory_prefix
+        # no map! here because @orig_targets needs to stay true to the original command
+        targets = targets.map { |t| "#{@directory_prefix}:#{t}" }
+      end
+      @workload = dereference_workload targets
+      @workload.delete_if {|w|w.nil?}
+      if @workload.empty?
+        if diffgate(targets)
+          puts  "diffgate returned no targets needing to run"
+          exit
+        else
+          fault "Could not determine anything to run based on the targets #{targets}.  Did you mean to run this from another directory?"
+        end
+      end
+
+      w_names = @workload.map{|w|w.name}
+      @workload.each do |w|
+        w.configure
+        if w.depends
+          w.deep_depends.each do |d|
+            if @workload.include? d
+              w.wait_count += 1
+              d.next_tasks << w
+            end
+          end
+        end
+      end
+    end
+    def diffgate(targets)
+      targets.each do |t|
+        return false unless t =~ /\bdiffgate$/
+      end
+      true
+    end
+    def run_all_readied
+      # @report_hash = {}
+      FileUtils.mkdir_p "#{@directory}/reports"
+      @report_name    = "#{@directory}/reports/workload.#{BaseChip.random_string}"
+      @report_file    = File.new(@report_name,'w')
+      if @tasker.clusters
+        @tasker.ready
+      else
+        @foreground = @workload.size == 1
+      end
+      begin
+        @tasker.submit_workload_chain workload_chain
+        @tasker.run
+      rescue Interrupt
+        @tasker.finish
+      end
+    end
+    def workload_chain
+      @workload.each do |w|
+        w.task           = t = Task.new
+        t.task_name      = w.full_name
+        t.foreground     = @foreground
+        t.worker_command = if @tasker.clusters
+                             lambda { |client_id|
+                               # The "@drb_uri ||=" is necessary because Drb.uri can change for some reason, and a command compare happens later
+                               "#{$0} #{@orig_targets.join(' ')} --client-of #{@drb_uri ||= DRb.uri} --client-id #{client_id} --work-dir #{@pwd} #{
+                                 '--modes ' + BaseChip.options.modes                        if BaseChip.options.modes             } #{
+                                 (['--' ]   + BaseChip.options. append_arguments).join(' ') if BaseChip.options. append_arguments } #{
+                                 (['---']   + BaseChip.options.replace_arguments).join(' ') if BaseChip.options.replace_arguments }" }
+                           else
+                             nil
+                           end
+      end
+      @workload.map do |w|
+        t = w.task
+        t.next_tasks = w.next_tasks.map { |nt| nt.task }
+        t.wait_count = w.wait_count
+        w.next_tasks.uniq!
+        t
+      end
+    end
+
+    def block_dereference(name,names,passive)
+      if block = @blocks[name.to_sym]
+        block.configure
+        block.dereference_workload(names,passive)
+      else
+        fault "Could not find block #{name} in project #{@name}" # FIXME say who wanted it, and if shortcut occurred
+      end
+    end
+    def dereference_workload(targets)
+      configure
+      fault "project '#{name}' has no blocks defined" unless @blocks
+      targets ||= ['all']
+      out = []
+
+      workload_hash = Hash.new
+
+      targets.each do |t|
+        case t 
+        when 'gate', 'diffgate', 'all'
+          self.block_names.each do |b|
+            out += block_dereference(b,[t],true)
+          end
+        when /^all:(.*)$/
+          self.block_names.each do |b|
+            out += block_dereference(b,[$1],true)
+          end
+        when /:/
+          tmp = t.split(/:/)
+          b = tmp.shift
+          out += block_dereference(b,[tmp.join(':')],true)
+        else
+          fault "Cannot determine block in order to run #{t}."
+        end
+      end
+      out.uniq!
+      out
+    end
+    def tasker_handle_results(task, result, problem_or_directory=nil, totals={})
+      action = @workload.select{|w|w.full_name == task.task_name}.first
+      # track_results(action, result, problem, totals)
+      if result == 'fail'
+        message = (problem_or_directory || "#{action.class_string} '#{action.name}' failed without an error signature").to_s
+        begin
+          error message
+        rescue
+        end
+        @failing ||= 0
+        @failing  += 1
+        @first_error = {}
+        @first_error[action.class_string] ||= message
+      end
+      @results                                         ||= {}
+      @results[action.class_string]                    ||= {}
+      @results[action.class_string][action.block.name] ||= {'pass' => [], 'fail' => [], :cancel =>[]}
+      @results[action.class_string][action.block.name][result] << action
+      @report_file.puts "#{result} #{action.class_string} #{task.task_name} #{problem_or_directory}"
+    end
+    def results_line(type,block,total)
+      blocks = block == 'all' ? @results[type].keys : [block]
+      if total
+        pass_fail_cancel = [0,0,0,0,0,0,0]
+        blocks.each do |b|
+          res_hash = @results[type][b]
+          pass_fail_cancel[0] += res_hash['pass'].size
+          pass_fail_cancel[2] += res_hash['fail'].size
+          pass_fail_cancel[4] += res_hash[:cancel].size
+        end
+        pass_fail_cancel[6] = pass_fail_cancel[0] + pass_fail_cancel[2] + pass_fail_cancel[4]
+        pass_fail_cancel[1] = (100 * pass_fail_cancel[0]/pass_fail_cancel[6]).round
+        pass_fail_cancel[3] = (100 * pass_fail_cancel[2]/pass_fail_cancel[6]).round
+        pass_fail_cancel[5] = (100 * pass_fail_cancel[4]/pass_fail_cancel[6]).round
+        str  = pass_fail_cancel[0] > 0 ? ' %6d %3d |' : ' %6d %3d |'
+        str += pass_fail_cancel[3] > 0 ? ' %6d %3d |' : ' %6d %3d |'
+        str += pass_fail_cancel[5] > 0 ? ' %6d %3d |' : ' %6d %3d |'
+        return "#{str} %6d" % pass_fail_cancel
+      else
+        fault 'Internal error: detailed result reporting not yet supported'
+      end
+    end
+    def find_action(t_name)
+      address     = t_name.split(/:/)
+
+      block_name  = address.shift.to_sym
+      block       = self.blocks[block_name]           or fault("Couldn't find block '#{block_name}'")
+
+      config_name = address.shift.to_sym
+      config      = block.configurations[config_name] or fault("Couldn't find configuration '#{config_name}' in block '#{block_name}")
+
+      thing       = address.shift.to_sym
+      action      = config.actions[thing]
+      return action if action
+
+      test_list   = config.test_lists[thing]          or fault("Could not find action or test list '#{thing}' in block:configuration '#{block_name}:#{config_name}'")
+      test_name   = address.shift.to_sym
+      test        = test_list.tests[test_name]        or fault("Could not find test '#{test_name}' in block:configuration:list '#{block_name}:#{config_name}:#{thing}'")
+      return test
+    end
+    def banner_line(str)
+      @report_file.puts str.gsub(/\[\d+m/,'')
+                   puts str
+    end
+    def banner
+
+      banner_line      '--------------------------------- ------------ ------------ ------------ --------'
+      banner_line      '                                 | PASS #   % | FAIL #   % | CNCL #   % |  TOTAL '
+      if @results['Action']
+        banner_line  ( ' ACTIONS                         |' + results_line('Action','all',true))
+        @results['Action'].each_key do |block|
+          banner_line(('   %-29s |' % block)                + results_line('Action',block,true))
+        end
+      end
+      if @results['Test']
+        banner_line  ( ' TESTS                           |' + results_line('Test','all',true))
+        @results['Test'].each_key do |block|
+          banner_line(('   %-29s |' % block)                + results_line('Test',block,true))
+        end
+      end
+      banner_line      '--------------------------------- ------------ ------------ ------------ --------'
+      @report_file.close
+
+      puts "Full report available here: #{@report_name}"
+      if @first_error
+        fault("This workload had #{@failing} error(s).\n" + 
+              (@first_error['Action'] ? "  The first action related error encountered was:\n    #{@first_error['Action']}" : '') +
+              (@first_error['Test'  ] ? "  The first test related error encountered was:\n    #{  @first_error['Test'  ]}" : '') ,
+              @first_error['Action'] || @first_error['Test']
+             )
+      end
+    end
+    def tasker_finish
+      banner
+    end
+    def tasker_run_task(t_obj)
+      t = find_action(t_obj.task_name)
+      begin
+        t.foreground = @foreground
+        t.run
+      rescue ReportingError => e
+        # FIXME reuse test error states here
+        t.state = 'fail'
+      rescue Exception => e
+        t.problem = Problem.new
+        t.problem.signature = "#{t.full_name} #{e.class} --- #{e.message}"
+        puts e.backtrace
+        # FIXME reuse test error states here
+        t.state = 'fail'
+      end
+                #track_results(t    ,t.state,t.problem               ,t.totals)
+      @tasker.register_results(t_obj,t.state,t.problem || t.directory,t.totals)
+    end
+    # def enable_tracking(opc,opi)
+    #   # puts "enabled tracking(#{opc},#{opi})"
+    #   init_results_sockets(:client)
+    #   @opc = opc
+    #   @opi = opi
+    # end
+    # def track_complete(state,first_error = nil)
+    #   return unless @opc
+    #   msg = Message.new
+    #   msg.project = 'gandalf'
+    #   msg.type    = 'complete'
+    #   msg.state   = state
+    #   msg.data = {
+    #     'opc'         => @opc  ,
+    #     'opi'         => @opi  ,
+    #     'first_error' => first_error
+    #   }
+    #   send_work_message(msg)
+    # end
+    # def track_results(test_action,state,problem,totals)
+    #   return unless @opc
+    #   msg = Message.new
+    #   msg.project = 'gandalf'
+    #   msg.type    = 'test_action'
+    #   msg.state   = state
+    #   msg.data = {
+    #     'name'        => test_action.bundle_name,
+    #     'block'       => test_action.block.name,
+    #     'opc'         => @opc,
+    #     'opi'         => @opi,
+    #     'first_error' => problem && problem.signature,
+    #     'file'        => problem && problem.file,
+    #     'bundle'      => problem && problem.bundle,
+    #     'totals'      => totals
+    #   }
+    #   # puts "sending totals #{test_action.totals.inspect}"
+    #   puts "sending message #{msg.data.inspect}"
+    #   if test_action.class_string == 'Action'
+    #     msg.data['action'   ] = test_action.name
+    #   else
+    #     msg.data['test'     ] = test_action.name
+    #     msg.data['test_list'] = test_action.test_list.name
+    #   end
+    #   send_work_message(msg)
+    # end
+    def shortcut(name, array)
+      @shortcuts ||= {}
+      @shortcuts[name] = array
+    end
+
+    def discover_subprojects    ; file_glob("#{@directory}/*/base_chip/project.rb"       , /(\w+)\/base_chip\/project\.rb$/              , :subproject    )
+                                  file_glob("#{@directory}/base_chip/subprojects/*.rb"   ,        /base_chip\/subprojects\/(\w+)\.rb$/   , :subproject    ) end
+    def discover_blocks         ; file_glob("#{@directory}/*/base_chip/block.rb"         , /(\w+)\/base_chip\/block\.rb$/                , :block         )
+                                  file_glob("#{@directory}/base_chip/block/*.rb"         ,        /base_chip\/blocks\/(\w+)\.rb$/        , :block         ) end
+    def discover_configurations ; file_glob("#{@directory}/base_chip/configurations/*.rb",        /base_chip\/configurations\/(\w+)\.rb$/, :configuration ) end
+    def discover_cluster_types  ; file_glob("#{@directory}/base_chip/cluster_types/*.rb" ,        /base_chip\/cluster_types\/(\w+)\.rb$/ , :cluster_type  ) end
+    def use_tool(name,version)
+      tool :name do |t|
+        t.select_version version
+      end
+    end
+    attr_reader :registered_modes
+    def register_mode(name)
+      @registered_modes ||= []
+      @registered_modes << name.to_s
+    end
+    def default_modes
+      %w{ coverage gates fast debug profiling }.each { |m| register_mode m }
+    end
+    def configure
+      return if @configured
+      @directory          = BaseChip.root
+      @modes             += BaseChip.options.modes.split(/,/) if BaseChip.options.modes
+      super
+      @modes.each do |name|
+        fault "Attempted to call mode '#{name}', which isn't registered with the project.  Call register_mode(#{name}) in project.rb to allow.  Valid modes are #{project.registered_modes}" unless project.registered_modes.include? name.to_s
+      end
+      if self.blocks
+        @directory_prefix = figure_directory_prefix
+        @block_names      = self.blocks.keys
+      end
+    end
+    def dereference_clusters
+      return nil unless cluster_names = BaseChip.options.jobs
+      configure
+      fault "Cluster submission attempted, but no cluster types are configured." unless @cluster_types
+      @dereferenced = []
+      cluster_names = cluster_names.split(/,/)
+      cluster_names.each do |cn|
+        type     = nil
+        cluster  = nil
+        jobs     = nil
+        cn_orig  = cn
+        clusters = []
+
+        if cn.sub!(/^(\d+)(?::|$)/,'')
+          jobs = $1.to_i
+        end
+        if cn.sub!(/:(\d+)$/,'')
+          fault "Multiple counts found in '#{cn_orig}'.  Separate multiple cluster specifiers with commas." if jobs
+          jobs = $1.to_i
+        end
+
+        case cn
+        when /^([^:]+):([^:]+)$/
+          cluster = $2
+          if $1 == 'all' || $1 == ''
+            cluster_types.each_value do |cluster_type|
+              clusters += cluster_type.dereference(cluster,jobs,true)
+            end
+          elsif cluster_type = @cluster_types[$1.to_sym]
+            clusters += cluster_type.dereference(cluster,jobs,false)
+          else
+            fault "Could not find cluster type '#{$1}' in cluster specifier '#{cn_orig}'." if jobs
+          end
+        when /^([^:]+)$/
+          if cluster_type = @cluster_types[$1.to_sym]
+            clusters += cluster_type.dereference('all',jobs,true)
+          else
+            cluster_types.each_value do |cluster_type|
+              clusters += cluster_type.dereference('all',jobs,true)
+            end
+          end
+        when ''
+          cluster_types.each_value do |cluster_type|
+            clusters += cluster_type.dereference('all',jobs,true)
+          end
+        else
+          fault "Could not parse the cluster specifier '#{cn_orig}'." if jobs
+        end
+        fault "No clusters could be determined for the cluster specifier '#{cn_orig}'." if clusters.empty?
+
+        if cluster_order
+          cluster_order.each do |name|
+            tmp = clusters.delete_if {|c| c.full_name == "#{@name}:#{name}"}
+            try_add_clusters tmp, jobs
+          end
+        end
+        try_add_clusters clusters, jobs
+      end
+      @dereferenced
+    end
+    def try_add_clusters(clusters, jobs)
+      clusters.each do |c|
+        next if @dereferenced.include? c
+        c.configure
+        fault "could not determine number of jobs to run in cluster '#{c.full_name}'.  Consider setting a default for this cluster." unless (c.default_jobs || c.maximum_jobs || jobs)    
+        if    (c.default_jobs || c.maximum_jobs) == nil; c.slots = jobs
+        elsif jobs.nil?                                ; c.slots = c.default_jobs || c.maximum_jobs
+        elsif jobs > c.maximum_jobs                    ; c.slots = c.maximum_jobs
+        else                                             c.slots = jobs
+        end
+        @dereferenced << c
+      end
+    end
+  end
+end

data/lib/base_chip/requirement.rb

@@ -0,0 +1,26 @@
+# Copyright 2011 Tommy Poulter
+#
+# This file is part of basechip.
+#
+# basechip is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+# 
+# basechip is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with basechip.  If not, see <http://www.gnu.org/licenses/>.
+
+module BaseChip
+  class Requirement
+    include Dsl
+    include Base
+
+    def configure
+      @configure = true
+    end
+  end
+end

data/lib/base_chip/runable.rb

@@ -0,0 +1,36 @@
+# Copyright 2011 Tommy Poulter
+#
+# This file is part of basechip.
+#
+# basechip is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+# 
+# basechip is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with basechip.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'base_chip/track_state'
+require 'base_chip/taskable'
+
+module BaseChip
+  module Runable
+    def self.included mod
+      mod.extend Runable::ClassMethods
+      mod.class_eval do
+        include Runable::InstanceMethods
+        include Taskable
+        include TrackState
+        attr_accessor :cl_args
+      end
+    end
+    module ClassMethods
+    end
+    module InstanceMethods
+    end
+  end
+end

data/lib/base_chip/source_language.rb

@@ -0,0 +1,40 @@
+# Copyright 2011 Tommy Poulter
+#
+# This file is part of basechip.
+#
+# basechip is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+# 
+# basechip is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with basechip.  If not, see <http://www.gnu.org/licenses/>.
+
+module BaseChip
+  class SourceLanguage
+    include Dsl
+    include Base
+    define_settings({:directories      => [ Array       ],
+                     :globs            => [ Array       ],
+                     :files            => [ Array       ]})
+
+    def configure
+      return if @configured
+      super
+      if @globs and @directories and not @files
+        @files = []
+        @globs.each do |e|
+          @directories.each do |d|
+            d = absolute_path d
+            @files += Dir.glob("#{d}/#{e}")
+          end
+        end
+        # @files.uniq!
+      end
+    end
+  end
+end

data/lib/base_chip/source_type.rb

@@ -0,0 +1,24 @@
+# Copyright 2011 Tommy Poulter
+#
+# This file is part of basechip.
+#
+# basechip is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+# 
+# basechip is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with basechip.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'base_chip/source_language'
+module BaseChip
+  class SourceType
+    include Dsl
+    include Base
+    define_children({:source_language  => SourceLanguage })
+  end
+end

data/lib/base_chip/statistic.rb

@@ -0,0 +1,27 @@
+# Copyright 2011 Tommy Poulter
+#
+# This file is part of basechip.
+#
+# basechip is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+# 
+# basechip is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with basechip.  If not, see <http://www.gnu.org/licenses/>.
+
+module BaseChip
+  class Statistic
+    include Dsl
+    include Base
+    define_settings({:regex            => [Regexp , Array],
+                     :error_if_missing => [Boolean       ],
+                     :per_state        => [Boolean       ],
+                     :across_states    => [Boolean       ],
+                     :for_states       => [String , Array]})
+  end
+end

data/lib/base_chip/task.rb

@@ -0,0 +1,21 @@
+# Copyright 2011 Tommy Poulter
+#
+# This file is part of basechip.
+#
+# basechip is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+# 
+# basechip is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with basechip.  If not, see <http://www.gnu.org/licenses/>.
+
+module BaseChip
+  class Task
+    include Taskable
+  end
+end