basechip 0.0.1

data/lib/base_chip/action.rb

@@ -0,0 +1,309 @@
+# Copyright 2011 Tommy Poulter
+#
+# This file is part of basechip.
+#
+# basechip is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+# 
+# basechip is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with basechip.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'zlib'
+require 'base_chip/runable'
+require 'base_chip/statistic'
+require 'base_chip/out_file'
+
+module BaseChip
+  class Action
+    include Taskable
+    include TrackState
+    include Runable
+    include Dsl
+    include Base
+
+    define_settings({:zip_failing_action   => [ Boolean                 ],
+                     :zip_passing_action   => [ Boolean                 ],
+                     :clean_passing_action => [ Boolean                 ],
+                     :run_action_in_tmp    => [ Boolean                 ],
+                     :depends              => [ String  , Array, Symbol ],
+                     :before               => [           Proc          ],
+                     :after                => [           Proc          ],
+                     :environment          => [                  Hash   ],
+                     :command              => [ String                  ],
+                     :command_prefix       => [ String                  ],
+                     :command_suffix       => [ String                  ],
+                     :pre_commands         => [           Array         ],
+                     :post_commands        => [           Array         ],
+                     :name_regex           => [ Regexp                  ],
+                     :seed_options         => [ String  , Array         ],
+                     :args                 => [ String                  ],
+                     :log_to               => [ String                  ],
+                     :stdout_to            => [ String                  ],
+                     :stderr_to            => [ String                  ],
+                     :echo_stdin           => [ Boolean                 ],
+                     :needs_clobber        => [ Boolean                 ],
+                     :clean_proc           => [ Proc                    ]})
+                                           
+    define_children({:out_file             => OutFile     })
+
+    def configure
+      return if @configured
+      unless @test
+        @bundle_name  ||= if BaseChip.options.random_bundle_names
+                            random_string
+                          else
+                            @name.to_s
+                          end
+        @directory    ||= @configuration.out_directory + "/#{@bundle_name}"
+      end
+      super
+    end
+    def run
+      if @run_action_in_tmp
+        bundle_name   = random_string
+        run_directory = "/tmp/#{bundle_name}"
+      else
+        bundle_name   = @bundle_name
+        run_directory = @directory
+      end
+      FileUtils.rm_rf   run_directory
+      FileUtils.mkdir_p run_directory
+      Dir.pushd         run_directory
+
+      File.open('seed','w') do |f|
+        f.write random_generator.seed
+      end
+      File.open('rerun.yaml','w') do |f|
+        f.write rerun_data.to_yaml
+      end
+
+      # Where the magic happens
+      inner_run
+
+      FileUtils.touch "#{run_directory}/.fail_detected" unless self.pass?
+
+      Dir.chdir '..'
+      if (@clean_passing_action && pass? && !@foreground)
+        FileUtils.rm_rf run_directory
+      else
+        bundle_tgz = nil
+        if @zip_failing_action || (@zip_passing_action && pass?)
+           bundle_tgz = run_directory + ".tgz"
+          @bundle_tgz =    @directory + ".tgz"
+          tgz = Zlib::GzipWriter.new(File.open(bundle_tgz, 'wb'))
+          Archive::Tar::Minitar.pack(bundle_name, tgz)
+          FileUtils.rm_rf bundle_name
+        end
+        if @run_action_in_tmp
+          FileUtils.mkdir_p                           "#{@directory}/.."
+          FileUtils.mv((bundle_tgz || run_directory), "#{@directory}/..")
+        end
+      end
+
+      puts "Debug information here: #{@bundle_tgz || @directory}" if @foreground
+
+      # p @totals if @foreground
+      Dir.popd
+    end
+    def inner_run
+      # puts "#{full_name} started"
+      script_file = "#{@name}.bash"
+      script = File.open(script_file,'w')
+      script.puts "#!/bin/bash"
+      # $environment.each do |n,v|
+      #   script.puts "export #{n}=#{v}"
+      # end
+      if @environment
+        @environment.each do |n,v|
+          script.puts "export #{n}=#{v}"
+        end
+      end
+
+      script.puts "NAME=#{@name}"
+      script.puts "BLOCK=#{@block.name}"
+      if @args
+        script.puts "ARGS='#{@args.gsub(/'/,"\\'")}'"
+      end
+
+      # FIXME fault if commands and command both exist
+      if            @commands
+        script.puts @commands
+      elsif         @command 
+        script.puts @pre_commands  if @pre_commands
+        script.puts "#{@command_prefix} #{@command} #{@command_suffix} $*"
+        script.puts @post_commands if @post_commands
+      else
+        fault "Action #{@name} has no command(s) specified to run"
+      end
+
+      script.chmod 0777
+      script.close
+      process_status = nil
+      errmsg         = ''
+
+      STDOUT.sync = true
+      STDERR.sync = true
+
+      @before.call if @before
+
+      script_call = (@foreground || @log_to || @stdout_to) ? script.path : "#{script.path} > /dev/null"
+
+         log_storage = (   @log_to                         ) ? File.open(   @log_to, 'w') : nil
+      stdout_storage = (@stdout_to && @stdout_to != @log_to) ? File.open(@stdout_to, 'w') : nil
+      stderr_storage = (@stderr_to && @stderr_to != @log_to) ? File.open(@stderr_to, 'w') : nil
+
+      # stdin, stdout, stderr, wait_thr = Open3.popen3([env,] cmd... [, opts])
+      Open3.popen3(script_call) do |stdin, stdout, stderr, wait_thr|
+       
+        to = Thread.new(stdout) do |pipe|
+          out_data = nil
+          loop do
+            begin
+              out_data = pipe.readpartial(4096)
+            rescue EOFError, IOError
+              break
+            end
+                    STDOUT.print out_data if @foreground
+               log_storage.print out_data if    log_storage
+            stderr_storage.print out_data if stderr_storage
+          end
+        end
+        ti = Thread.new(STDIN) do |pipe|
+          in_data = nil
+          loop do
+            begin
+              in_data = pipe.readpartial(4096)
+            rescue EOFError, IOError
+              break
+            end
+                       stdin.print in_data if @foreground && !stdin.closed?
+            if @echo_stdin
+                      STDOUT.print in_data if @foreground
+                 log_storage.print in_data if    log_storage
+              stderr_storage.print in_data if stdout_storage
+            end
+          end
+        end
+        te = Thread.new(stderr) do |pipe|
+          err_data = nil
+          loop do
+            begin
+              err_data = pipe.readpartial(4096)
+            rescue EOFError, IOError
+              break
+            end
+            errmsg += err_data
+                    STDERR.print err_data if @foreground
+               log_storage.print err_data if    log_storage
+            stderr_storage.print err_data if stderr_storage
+          end
+        end
+
+        begin
+          wait_thr.join
+          if @foreground
+            ti.kill
+            to.kill unless te.join(10)
+          end
+          te  .kill unless te.join(10)
+        rescue Interrupt
+          if @foreground #  && @int_for_shell
+            Process.kill 'INT', wait_thr.pid
+            retry
+          else
+            raise
+          end
+        end
+        process_status = wait_thr.value # Process::Status object returned.
+      end
+
+         log_storage.close if    log_storage
+      stdout_storage.close if stdout_storage
+      stderr_storage.close if stderr_storage
+
+      @after.call if @after
+
+      statistics = {}
+      totals     = {}
+      if @out_files
+        @out_files.each_value do |f|
+          f.go(@name)
+         
+          take_state(f)
+
+          statistics.merge! f.statistics if f.statistics
+          totals    .merge! f.totals     if f.totals
+
+          # warning!(pf,ps) if f.warning?
+          # error!(  pf,ps) if f.error?
+          # fault!(  pf,ps) if f.fault?
+        end
+      end
+      error!(nil, errmsg) if !process_status.success? || errmsg.size > 0
+
+      @state ||= 'pass'
+      @state = 'fail' unless self.pass? # state is error, warning or fault up to this point
+
+      # puts "log file produced #{totals}"
+      totals.each do |k,v|
+        if statistics[k].for_states.include? @state
+          if statistics[k].across_states
+            @totals[k] = v
+          end
+          if statistics[k].per_state
+            @totals["#{@state}_#{k}"] = v
+          end
+          # FIXME error_if_missing
+        end
+      end
+      # puts "we collected #{@totals}"
+
+      unless pass?
+        @problem.bundle=(@bundle_tgz || @directory)
+        # error(@problem.to_s)
+      end
+      # puts "#{full_name} ended"
+    end
+
+    def clean
+      return if @needs_clobber
+      clobber
+    end
+    def clobber
+      @clean_proc.call(self) if @clean_proc
+      FileUtils.rm_rf    @directory
+      FileUtils.rm_rf "#{@directory}.tgz"
+    end
+
+    attr_writer :random_generator
+    def random_generator
+      @random_generator ||= BaseChip.new_random_generator
+    end
+    def generate_seed(format, bits=32)
+      case format
+      when :num
+         @random_generator.rand(2 ** bits)
+      when :hex
+        (@random_generator.rand(2 ** bits)).to_hex
+      when :dec
+        (@random_generator.rand(2 ** bits)).to_s
+      when :alphanumeric
+        random_string(bits)
+      end
+    end
+
+    def rerun_data
+      { 'full_name'         => full_name                          ,
+        'seed'              => random_generator.seed              ,
+        'modes'             => modes                              ,
+         'append_arguments' => BaseChip.options. append_arguments ,
+        'replace_arguments' => BaseChip.options.replace_arguments }
+    end
+  end
+end

data/lib/base_chip/base.rb

@@ -0,0 +1,105 @@
+# Copyright 2011 Tommy Poulter
+#
+# This file is part of basechip.
+#
+# basechip is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+# 
+# basechip is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with basechip.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'reporting'
+module BaseChip
+  module Base
+    def self.included mod
+      mod.extend Base::ClassMethods
+      mod.class_eval do
+        include Reporting
+        include Base::InstanceMethods
+
+        define_setting :directory, [ String ]
+
+        # chain together projects, blocks, and actions
+        attr_reader :project
+        attr_reader :block
+        attr_reader :configuration
+        attr_reader :action
+        attr_reader :test_list
+        attr_reader :test
+        attr_reader :tool
+        attr_reader :version
+        attr_reader :source_type
+        attr_reader :source_language
+      end
+    end
+    module ClassMethods
+    end
+    module InstanceMethods
+      def parent=(parent); @parent = parent; update_chaining; end
+      def foster=(parent); @foster = parent; update_chaining; end
+      def update_chaining
+        # chain blocks, projects, and actions together
+        @project         ||= (self.is_a? BaseChip::Project       ) ? self : ((parent.project         if parent) || (foster.project         if foster))
+        @block           ||= (self.is_a? BaseChip::Block         ) ? self : ((parent.block           if parent) || (foster.block           if foster))
+        @configuration   ||= (self.is_a? BaseChip::Configuration ) ? self : ((parent.configuration   if parent) || (foster.configuration   if foster))
+        @action          ||= (self.is_a? BaseChip::Action        ) ? self : ((parent.action          if parent) || (foster.action          if foster))
+        @test_list       ||= (self.is_a? BaseChip::TestList      ) ? self : ((parent.test_list       if parent) || (foster.test_list       if foster))
+        @test            ||= (self.is_a? BaseChip::Test          ) ? self : ((parent.test            if parent) || (foster.test            if foster))
+        @tool            ||= (self.is_a? BaseChip::Tool          ) ? self : ((parent.tool            if parent) || (foster.tool            if foster))
+        @tool_version    ||= (self.is_a? BaseChip::ToolVersion   ) ? self : ((parent.tool_version    if parent) || (foster.tool_version    if foster))
+        @source_type     ||= (self.is_a? BaseChip::SourceType    ) ? self : ((parent.source_type     if parent) || (foster.source_type     if foster))
+        @source_language ||= (self.is_a? BaseChip::SourceLanguage) ? self : ((parent.source_language if parent) || (foster.source_language if foster))
+      end                                                                                                                                  
+
+      def configure
+        return  if @configured
+        super
+        # if self.class.children_types.keys.include?(:tool) && self.tools
+        #   self.tools.each_value do |t|
+        #     t.configure
+        #     t.selected_version.configure
+        #     t.selected_version.actions.each do |n,a|
+        #       self.actions ||= {}
+        #       if self.actions[n]
+        #         self.actions[n].blks     = a.blks + self.actions[n].blks
+        #         self.actions[n].foster ||= a.parent
+        #       else
+        #         self.actions[n]          = a.clone
+        #         self.actions[n].blks     =          self.actions[n].blks.dup
+        #         self.actions[n].foster   = a.parent
+        #         self.actions[n].parent   = self
+        #       end
+        #     end
+        #   end
+        # end
+      end
+
+      def discover_configurations
+        Dir.glob("#{BaseChip.root}/*/base_chip/configuration.rb").each do |f|
+          if f =~ /(\w+)\/base_chip\/configuration\.rb$/
+            self.block($1.to_sym) {|b| b.file f}
+          end
+        end
+      end
+      def file_glob(glob, regex = nil, type = nil) 
+        Dir.glob(glob).each do |f|
+          if regex
+            if f =~ regex
+              self.send(type, $1.to_sym) { |t| t.file(f) }
+            else
+              fault "fileglob '#{glob}' did not match regex /#{regex}/ in #{self.name}"
+            end
+          else
+            self.file(f) 
+          end
+        end
+      end
+    end
+  end
+end

data/lib/base_chip/block.rb

@@ -0,0 +1,75 @@
+# Copyright 2011 Tommy Poulter
+#
+# This file is part of basechip.
+#
+# basechip is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+# 
+# basechip is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with basechip.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'base_chip/configuration'
+
+module BaseChip
+  class Block
+    include Dsl
+    include Base
+    include Hierarchy
+    define_children({:configuration => Configuration})
+
+    def configuration_dereference(name,names,passive)
+      if configuration = @configurations[name]
+        configuration.configure
+        configuration.dereference_workload(names,passive)
+      elsif passive
+        []
+      else
+        fault "Could not find configuration #{name} in block #{@name}" # FIXME say who wanted it, and if shortcut occurred
+      end
+    end
+    def dereference_workload(names = nil,passive=false)
+      configure
+      fault "No configurations specified for block #{@name}" unless @configurations # || passive || names.nil?
+      names ||= ['all']
+      out     = Array.new
+      names.each do |n|
+        n = n.to_s
+        case n
+        when 'all', 'gate', 'upgate', 'downgate', 'diffgate' # FIXME , 'alltests', 'allactions'
+          next unless @configurations
+          @configurations.keys.each { |k| out += configuration_dereference(k,[n],true) }
+        when /^all:(.*)$/
+          next unless @configurations
+          tmp = $1
+          @configurations.keys.each { |k| out += configuration_dereference(k,[tmp],true) }
+        when /^([^:]*?):(.*)$/
+          out += configuration_dereference($1.to_sym,[$2],passive)
+        else
+          fault "No configurations specified for block #{@name}" unless @configurations || passive
+          if @configurations[:default] 
+            out += configuration_dereference(:default,[n],passive)
+          elsif @configurations.size == 1
+            out += configuration_dereference(@configurations.keys[0],[n],passive)
+          end
+        end
+      end
+      out.uniq!
+      out
+    end
+
+    def shortcut(name, array)
+      @shortcuts ||= {}
+      @shortcuts[name] = array
+    end
+
+    def discover_configurations ; file_glob("#{@directory}/*/base_chip/configuration.rb" , /(\w+)\/base_chip\/configuration\.rb$/        , :configuration)
+                                  file_glob("#{@directory}/base_chip/configurations/*.rb",        /base_chip\/configurations\/(\w+)\.rb$/, :configuration) end
+
+  end
+end

data/lib/base_chip/bom.rb

@@ -0,0 +1,75 @@
+# Copyright 2011 Tommy Poulter
+#
+# This file is part of basechip.
+#
+# basechip is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+# 
+# basechip is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with basechip.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'base_chip/bom_file'
+
+class Hash
+  def deep_values
+    self.values.map do |v|
+      if v.respond_to? :values
+        v.values
+      else
+        v
+      end
+    end
+  end
+end
+
+module BaseChip
+  class Bom
+    attr_reader :hash
+    attr_reader :name1
+    attr_reader :name2
+    def initialize(hash)
+      @hash  = hash.with_indifferent_access
+      @name1 = 'all'
+      @name2 = 'all'
+    end
+    def verilog   ; @verilog   ||= trim(2,:verilog  ) end
+    def classes   ; @classes   ||= trim(2,:classes  ) end
+    def vhdl      ; @vhdl      ||= trim(2,:vhdl     ) end
+    def systemc   ; @systemc   ||= trim(2,:systemc  ) end
+    def headers   ; @headers   ||= trim(2,:headers  ) end
+
+    def rtl       ; @rtl       ||= trim(1,:rtl      ) end
+    def gates     ; @gates     ||= trim(1,:gates    ) end
+    def testbench ; @testbench ||= trim(1,:testbench) end
+    def stimulus  ; @stimulus  ||= trim(1,:stimulus ) end
+
+    def to_a ; @to_a ||= @hash.deep_values.flatten; end
+    def list ; @list ||= to_a.join(' ')           ; end
+    def to_s ; @to_s ||= list                     ; end
+
+    def +(bom)
+      Bom.new(@hash.merge bom.hash)
+    end
+
+    protected
+    def trim(depth,key)
+      hash = HashWithIndifferentAccess.new
+      @hash.each do |k1,l1|
+        case depth 
+        when 1; hash[k1] = l1 if k1.to_s == key.to_s
+        when 2; hash[k1] = tmp = HashWithIndifferentAccess.new
+          l1.each do |k2,l2| 
+            tmp[k2] = l2 if k2.to_s == key.to_s
+          end
+        end
+      end
+      Bom.new(hash)
+    end
+  end
+end

data/lib/base_chip/bom_file.rb

@@ -0,0 +1,45 @@
+# Copyright 2011 Tommy Poulter
+#
+# This file is part of basechip.
+#
+# basechip is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+# 
+# basechip is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with basechip.  If not, see <http://www.gnu.org/licenses/>.
+
+module BaseChip
+  class BomType
+    def initialize(bom,type)
+      @bom    = bom
+      @type   = type
+    end
+    def to_s
+      "#{@bom.name1}.#{@bom.name2}.#{@type}"
+    end
+    def make
+      f = File.open(to_s,'w')
+      case @type
+      when :text ; f.puts @bom.to_a.join("\n")
+      when :tcl  ; fault "TCL formatted bom files not yet implemented"
+      else       ; fault "#{@type.inspect} was not understood by #{self.class}"
+      end
+      f.close
+    end
+  end
+
+  class Bom
+    def text
+      @text ||= BomType.new(self,:text)
+    end
+    def tcl
+      @tcl  ||= BomType.new(self,:tcl )
+    end
+  end
+end