basechip 0.0.1

data/lib/base_chip/task_master.rb

@@ -0,0 +1,50 @@
+# Copyright 2011 Tommy Poulter
+#
+# This file is part of basechip.
+#
+# basechip is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+# 
+# basechip is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with basechip.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'yaml'
+# require 'ipc'
+YAML::ENGINE.yamler = 'psych'
+
+module BaseChip
+  module TaskMaster
+    TASKER = Tasker
+    def self.included mod
+      mod.extend TaskMaster::ClassMethods
+      mod.class_eval do
+        include TaskMaster::InstanceMethods
+        attr_accessor :tasker
+      end
+    end
+    module ClassMethods
+    end
+    module InstanceMethods
+      def initialize(*args)
+        super(*args)
+        @tasker = TASKER.new
+        @tasker.task_master = self
+      end
+      def tasker_run_task
+        raise "Please extend tasker_run_task routine"
+      end
+      def tasker_handle_results(action, result)
+        raise "Please extend tasker_handle_results routine"
+      end
+      def tasker_empty
+        raise "Please extend tasker_run_task routine"
+      end
+    end
+  end
+end

data/lib/base_chip/taskable.rb

@@ -0,0 +1,77 @@
+# Copyright 2011 Tommy Poulter
+#
+# This file is part of basechip.
+#
+# basechip is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+# 
+# basechip is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with basechip.  If not, see <http://www.gnu.org/licenses/>.
+
+module BaseChip
+  module Taskable
+    def self.included mod
+      mod.extend Taskable::ClassMethods
+      mod.class_eval do
+        include Taskable::InstanceMethods
+        attr_accessor :task       # for pointers to real tasks
+        attr_accessor :wait_count
+        attr_accessor :next_tasks
+        attr_accessor :task_name
+        attr_accessor :bundle_name
+        attr_accessor :bundle_tgz
+        attr_accessor :worker_command
+        attr_accessor :foreground
+        attr_accessor :totals
+        attr_accessor :state
+      end
+    end
+    module ClassMethods
+    end
+    module InstanceMethods
+
+      def initialize(*args)
+        super *args
+        initialize_taskable
+      end
+      def initialize_taskable
+        @wait_count = 0
+        @next_tasks = []
+        @totals     = {}
+      end
+      def random_name
+        BaseChip.random_string
+      end
+
+      # nonportable but helpful to be here:
+      def deep_depends
+        # FIXME circular dependencies
+        return @deep_depends if @deep_depends
+        return @deep_depends = [] unless self.depends
+
+        @deep_depends = []
+        self.depends.each do |depend|
+          depend2 = depend.to_s.split(/:/)
+          while depend2.size < 3
+            depend2.unshift nil
+          end
+          depend2[0] ||= self.block        .name.to_s
+          depend2[1] ||= self.configuration.name.to_s
+          @deep_depends += self.project.dereference_workload([depend2.join(':')])
+        end
+        @deep_depends.each do |d|
+          d.deep_configure
+          @deep_depends += d.deep_depends
+        end
+        @deep_depends.uniq!
+        @deep_depends
+      end
+    end
+  end
+end

data/lib/base_chip/tasker.rb

@@ -0,0 +1,260 @@
+# Copyright 2011 Tommy Poulter
+#
+# This file is part of basechip.
+#
+# basechip is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+# 
+# basechip is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with basechip.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'yaml'
+# require 'ipc'
+YAML::ENGINE.yamler = 'psych'
+
+module BaseChip
+  class Tasker
+    include Ipc
+
+    CLEAR = "\e[0K" 
+    attr_accessor :task_master
+
+    attr_accessor :foreground
+
+    attr_accessor :clusters
+
+    attr_accessor :workers
+
+    attr_accessor :pending_tasks
+    attr_accessor :running_tasks
+    attr_accessor :complete_tasks
+
+    attr_accessor :client_of
+    attr_accessor :client_id
+
+    def initialize(*args)
+      @workers = {}
+
+      @pending_tasks  = []; @tasks_pending  = 0
+      @running_tasks  = []; @tasks_running  = 0
+      @complete_tasks = []; @tasks_complete = 0
+      @tasks_passed = 0
+      @tasks_failed = 0
+
+      @mutex = Mutex.new
+
+      @server = self
+    end
+    def tasks_total
+      @tasks_pending + @tasks_running + @tasks_complete
+    end
+
+    def ready
+      init_test_sockets(:server)
+    end
+
+    def run
+      STDOUT.sync = true 
+      if @client_of || @clusters.nil?
+        begin
+          if @client_of
+            init_test_sockets(:client)
+          end
+          while(t = @server.get_task(@client_id))
+            @task_master.tasker_run_task t
+          end
+        rescue DRb::DRbConnError
+          # raise if @options.debug
+        end
+      else
+        DRb.thread.join
+      end
+    end
+
+    def register_results(t, result, *additional)
+      if @client_of
+        @server.register_results(t, result, *additional)
+        return
+      end
+      @mutex.lock
+      inner_register_results(t, result, *additional)
+      @mutex.unlock
+    end
+    def inner_register_results(t, result, *additional)
+      return         if @complete_tasks.select {|t2| t2.task_name == t.task_name}[0]
+      original_task   = (@pending_tasks.select {|t2| t2.task_name == t.task_name}[0])
+      original_task ||= (@running_tasks.select {|t2| t2.task_name == t.task_name}[0])
+      raise "INTERNAL ERROR: couldn't find #{t.task_name}" unless original_task
+
+      if result == :cancel
+           @tasks_pending -= 1; @pending_tasks.delete original_task
+      else @tasks_running -= 1; @running_tasks.delete original_task
+      end
+      @complete_tasks << t
+      @tasks_complete += 1
+
+      if result == 'pass'
+           @tasks_passed += 1
+           original_task.next_tasks.each { |nt| nt.wait_count -= 1                 }
+      else original_task.next_tasks.each { |nt| inner_register_results(nt,:cancel) }
+           @tasks_failed += 1 unless result == :cancel
+      end
+      @task_master.tasker_handle_results(t, result, *additional)
+      status_line
+
+      finish if (BaseChip.options.max_fails  && @tasks_failed >= BaseChip.options.max_fails ) ||
+                (BaseChip.options.max_passes && @tasks_passed >= BaseChip.options.max_passes)
+    end
+    def status_line
+      return if @foreground
+      wr = workers_running
+      wa = workers_alive
+      wp = wa > 0 ? (100*wr/wa).round : 0
+      raise "Workload is empty" if tasks_total == 0
+      print "#{Tasker::CLEAR}complete:#{@tasks_complete}/#{tasks_total}(#{(100*@tasks_complete/tasks_total).round}%) workers:#{wr}/#{wa}(#{wp}%) pass:#{@tasks_passed} fail:#{@tasks_failed} run:#{@tasks_running} pend:#{@tasks_pending}\r"
+    end
+    def available_slots
+      count = 0
+      @clusters.each do |c|
+        count += c.available_slots
+      end
+      count
+    end
+    def spawn_worker(command)
+      @clusters.each do |c|
+        if c.available_slots > 0
+          c.slots_used += 1
+          return c, c.submit(command)
+        end
+      end
+    end
+    @@last_client_id = 0
+    def new_client_id
+      @@last_client_id += 1
+    end
+    def maintain_workers
+      ready = @pending_tasks.select {|t| t.wait_count == 0}
+      if @clusters
+        slots = available_slots
+        foo = ready.size - workers_spawned
+        foo = slots if slots < foo
+        if foo > 0
+          foo.times do |i|
+            client_id = new_client_id
+            cluster, uid = spawn_worker(ready[0].worker_command.call(client_id))
+            raise "spawn_worker response wasn't unique" if @workers[uid]
+            @workers[client_id] = {:uid => uid, :cluster=> cluster, :started=>Time.now, :tasks=>[], :state=>:spawned, :worker_command=>ready[i].worker_command.call(0)}
+            status_line
+          end
+        end
+      else
+        @workers[0] = {:started=>Time.now, :tasks=>[], :state=>:running, :worker_command=>ready[0].worker_command} if ready[0]
+      end
+
+      # puts "ready=#{(ready.*.task_name).inspect}"
+      ready
+    end
+
+    def submit_name(t_name,worker_command=nil)
+      task = Task.new
+      task.task_name      = t_name
+      task.worker_command = worker_command if @clusters
+      submit(task)
+    end
+    def submit(task)
+      task.foreground = self.foreground
+      @pending_tasks << task
+      @tasks_pending += 1
+      maintain_workers
+      status_line
+    end
+    def submit_workload_chain(chain)
+      @pending_tasks = chain
+      @tasks_pending = chain.size
+      maintain_workers
+      status_line
+    end
+    def get_task(client_id)
+      @mutex.lock
+      ready = maintain_workers
+
+      w = @workers[client_id]
+      raise "Could not find worker based on unique id #{client_id}" unless w
+
+      t = nil
+      ready.each do |rt|
+        if rt.worker_command.nil? || rt.worker_command.call(0) == w[:worker_command]
+          t = rt
+          break
+        end
+      end
+
+      if t
+        @pending_tasks.delete t
+        @running_tasks <<     t
+        @tasks_pending -= 1
+        @tasks_running += 1
+
+        w[:tasks] << t
+        w[:state] = :running
+      else
+        w[:state] = :stopped
+        w[:cluster].slots_used -= 1 if w[:cluster]
+      end
+
+      finish if @pending_tasks.empty? && @running_tasks.empty? && ready.empty?
+
+      if @pending_tasks.size > 0 && @running_tasks.empty? && ready.empty?
+        raise("Could not build tasks because their dependencies never cleared: " + (@pending_tasks.map{|t|t.task_name}.join(" ")))
+      end
+
+      status_line
+      @mutex.unlock
+      t
+    end
+
+    def finish
+      return if @finishing
+                @finishing = true
+
+      kill
+      @task_master.tasker_finish
+      DRb.stop_service
+    end
+
+    def kill
+      locked = @mutex.try_lock
+      @workers.each do |id,w|
+        next if w[:stopped]
+        if w[:cluster] && (w[:state] != :stopped)
+          kill_message
+          w[:cluster].kill(w[:uid])
+        end
+      end
+      (@running_tasks + @pending_tasks).each do |t|
+        inner_register_results t, :cancel
+      end
+      @mutex.unlock if locked
+    end
+    def kill_message
+      return if @kill_message
+                @kill_message = true
+      puts "#{Tasker::CLEAR}\rClosing down all workers.  Please be patient."
+    end
+
+    def workers_alive;   c=0; @workers.each_value{|w| c+=1 if w[:state] != :stopped}; c end
+    def workers_running; c=0; @workers.each_value{|w| c+=1 if w[:state] == :running}; c end
+    def workers_spawned; c=0; @workers.each_value{|w| c+=1 if w[:state] == :spawned}; c end
+    def workers_stopped; c=0; @workers.each_value{|w| c+=1 if w[:state] == :stopped}; c end
+
+    def clear_line
+      print "#{Tasker::CLEAR}\r"
+    end
+  end
+end

data/lib/base_chip/test.rb

@@ -0,0 +1,202 @@
+# Copyright 2011 Tommy Poulter
+#
+# This file is part of basechip.
+#
+# basechip is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+# 
+# basechip is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with basechip.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'base_chip/action'
+
+module BaseChip
+  class Test
+    include Taskable
+    include TrackState
+    include Runable
+    include Dsl
+    include Base
+    define_settings({:zip_failing_test   => [ Boolean        ],
+                     :zip_passing_test   => [ Boolean        ],
+                     :clean_passing_test => [ Boolean        ],
+                     :run_test_in_tmp    => [ Boolean        ],
+                     :test_actions       => [ Symbol , Array ]}.merge(Action.settings))
+    define_children(Action.child_types)
+
+    attr_accessor :original_actions
+    attr_accessor :action_extensions
+    # def actions
+    #   parent.actions
+    # end
+
+    def configure
+      return if @configured
+      name_it
+      @directory ||= "#{parent.directory}/#{@name}"
+      super
+    end
+    def deep_configure
+      super
+
+      return if @deep_configured
+                @deep_configured = true
+
+      @test_actions = [@test_actions] unless @test_actions.is_a? Array
+      # @original_actions = @block.dereference_workload([@actions])[0]
+      fault "No test_actions exist for test list '#{@test_list}' to run in '#{@configuration.full_name}'" unless @configuration.actions
+      unless (@original_actions = @configuration.dereference_workload(@test_actions)) && @original_actions.size > 0
+        fault "Test list '#{@test_list.name}' in block:configuration '#{@block.name}:#{@configuration.name}' specified nonexistant action(s) '#{@test_actions}'.  #{@test_list.file}"
+      end
+
+      @action_extensions = @original_actions.map do |a|
+        append_depends a.depends
+        a2 = a.clone
+        a2.clear
+        a2.parent = self # don't think I need to set foster here, because test list is a peer to action
+        a2.deep_configure
+        a2
+      end
+
+      @action_extensions.sort! do |a1,a2|
+        if a1.deep_depends.map{|d|d.name}.include? a2.name
+          1
+        else
+          -1
+        end
+      end
+    end
+    # def find_parent_values
+    #   super
+
+    #   # @original_actions = @block.dereference_workload([@action])[0]
+    #   @original_actions = @block.actions[@action]
+
+    #   @original_actions.         settings.each {|n,s| @children_settings[n] = s if s.value_set and not @children_settings[n].value_set }
+    #   @original_actions.children_settings.each {|n,s| @children_settings[n] = s if s.value_set and not @children_settings[n].value_set }
+
+    #   @action_extensions = Action.new
+    #   
+    #   # FIXME support outfile definition in Test
+    #   @original_actions   .out_files.each do |n,f|
+    #     @action_extensions.out_files[n] = f.dup
+    #   end
+
+    #   @action_extensions.find_parent_values
+
+    #   @action_extensions.name_it
+    #   @name = @action_extensions.name
+    # end
+    # def find_parent_values
+    #   super
+    #   find_parent_values_of_underlying_action
+    # end
+
+    def name_it
+      if BaseChip.options.random_bundle_names
+        @name ||= random_name
+      elsif parent.autoname and not @name
+        if parent.autoname.respond_to? :each
+          parent.autoname.each do |an|
+            if md = /#{an}/.match(@args)
+              @name ||= (md[1] || md.to_s)
+              break
+            end
+          end
+        else
+          if md = /#{parent.autoname}/.match(@args)
+            @name ||= (md[1] || md.to_s)
+          end
+        end
+      end
+    end
+   #def depends(dep=nil)   ;(@action_extensions && @action_extensions.depends            ) || super(dep) end
+    def bundle_name        ; name                                                        end
+    def bundle_tgz         ; "#{@directory}.tgz"                                         end
+    def totals
+      return @totals if @totals 
+      @totals = {}
+      @action_extensions.each do |a|
+        @totals.merge! a.totals
+      end
+    end
+    def run
+      if @run_test_in_tmp
+        bundle_name   = random_string
+        run_directory = "/tmp/#{bundle_name}"
+      else
+        bundle_name   = name
+        run_directory = @directory
+      end
+      FileUtils.rm_rf   run_directory
+      FileUtils.mkdir_p run_directory
+      Dir.pushd         run_directory
+
+      File.open('seed','w') do |f|
+        f.write random_generator.seed
+      end
+      File.open('rerun.yaml','w') do |f|
+        f.write rerun_data.to_yaml
+      end
+
+      @action_extensions.each do |a|
+        a.random_generator = Random.new(random_generator.rand(0x1_0000_0000))
+        a.foreground = @foreground
+        a.inner_run
+        take_state(a)
+        return unless pass?
+      end
+
+      Dir.chdir '..'
+      if (@clean_passing_test && pass? && !@foreground)
+        FileUtils.rm_rf run_directory
+      else
+        bundle_tgz = nil
+        if @zip_failing_test || (@zip_passing_test && pass?)
+           bundle_tgz = run_directory + ".tgz"
+          @bundle_tgz =    @directory + ".tgz"
+          tgz = Zlib::GzipWriter.new(File.open(bundle_tgz, 'wb'))
+          Archive::Tar::Minitar.pack(bundle_name, tgz)
+          FileUtils.rm_rf bundle_name
+        end
+        if @run_test_in_tmp
+          FileUtils.mkdir_p                           "#{@directory}/../"
+          FileUtils.mv((bundle_tgz || run_directory), "#{@directory}/../")
+        end
+      end
+      Dir.popd
+
+      puts "Debug information here: #{@bundle_tgz || @directory}" if @foreground
+    end
+    # SAFE TO DELETE? def configuration_hash
+    # SAFE TO DELETE?   super.merge @action_extensions.configuration_hash
+    # SAFE TO DELETE? end
+
+    def clean
+    end
+    def clobber
+      FileUtil.rm_rf directory
+      if Dir.empty? test_list.directory
+        FileUtil.rm_rf test_list.directory
+      end
+    end
+
+    # one random generator per test, all subordinate actions use it
+    def random_generator
+      @random_generator ||= BaseChip.new_random_generator
+    end
+
+    def rerun_data
+      { 'full_name'         => full_name                          ,
+        'seed'              => random_generator.seed              ,
+         'append_arguments' => BaseChip.options. append_arguments ,
+        'replace_arguments' => BaseChip.options.replace_arguments }
+    end
+  end
+end

data/lib/base_chip/test_list.rb

@@ -0,0 +1,120 @@
+# Copyright 2011 Tommy Poulter
+#
+# This file is part of basechip.
+#
+# basechip is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+# 
+# basechip is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with basechip.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'base_chip/test'
+require 'base_chip/permute'
+require 'base_chip/requirement'
+require 'base_chip/post'
+module BaseChip
+  class TestList
+    include Dsl
+    include Base
+    define_settings({:autoname => [ Regexp , Array ]})
+    define_children({:test          => Test       ,
+                     :permute       => Permute    ,
+                     :requirement   => Requirement,
+                     :post          => Post       })
+
+    def initialize
+      super
+      @num_tests = 0
+    end
+    # def actions
+    #   parent.actions
+    # end
+    alias old_requirement requirement
+    alias old_post        post
+    def requirement(name=:no_name,&blk); old_requirement(name,&blk) end
+    def post(       name=:no_name,&blk); old_post(       name,&blk) end
+    def configure
+      return if @configured
+      @directory ||= "#{configuration.out_directory}/#{name}"
+      super
+      return unless @tests
+      return unless @permutes
+      tests = @tests.values
+      @permutes.each_value do |permute|
+        permute.configure
+        next unless permute.permutations
+        tests.map! do |t|
+          permute.permutations.values.map do |permutation|
+            test = t.dup
+            test.name = "#{test.name}_#{permute.name}#{permutation.name}".to_sym
+            test.blks += permutation.blks
+            test
+          end
+        end
+        tests.flatten!
+      end
+
+      if @posts
+        tests.each do |t|
+          @posts.each_value do |p|
+            p.blks.each do |b|
+              b.call(t)
+            end
+          end
+        end
+      end
+      if @requirements
+        new_tests = []
+        tests.each do |t|
+          t.configure
+          ditch = false
+          @requirements.each_value do |m|
+            m.blks.each do |b|
+              ditch = !(b.call t)
+              break if ditch
+            end
+            break if ditch
+          end
+          next if ditch
+          new_tests << t
+        end
+        tests = new_tests
+      end
+      @tests = {}
+      tests.each do |t|
+        name = t.name
+        i = 0
+        while @tests[name]
+          name = "#{t.name}_#{i+=1}"
+        end
+        t.name = name.to_sym
+        @tests[name] = t
+      end
+    end
+    def dereference_workload(tests = nil,passive = false)
+      configure
+      tests ||= ['all']
+      out = []
+      return out unless @tests
+      tests.each do |t|
+        if t == 'all'
+          out += @tests.values
+        else
+          if foo = @tests[t.to_sym]
+            out << foo
+          else
+            fault "Could not find test #{t.inspect} in list #{@name.inspect} in block #{@block.name.inspect}" unless passive
+          end
+        end
+      end
+      out
+    end
+
+  end
+end