origen 0.34.3 → 0.52.0

data/lib/origen/generator/comparator.rb

@@ -51,7 +51,8 @@ module Origen
       end
 
       def relative_path_to(file)
-        p = Pathname(file).relative_path_from(Pathname.pwd).to_s
+        file = Pathname.new(file) unless file.is_a?(Pathname)
+        p = file.expand_path.relative_path_from(Pathname.pwd).to_s
         p.gsub!('/', '\\') if Origen.running_on_windows?
         p
       end

data/lib/origen/generator/flow.rb

@@ -31,7 +31,9 @@ module Origen
           end
           interface.close(flow: true, sub_flow: true)
         else
-          Origen.log.info "Generating... #{Origen.file_handler.current_file.basename}"
+          unless tester.try(:sim?)
+            Origen.log.info "Generating... #{Origen.file_handler.current_file.basename}"
+          end
           interface = Origen.reset_interface(options)
           Origen.interface.set_top_level_flow
           Origen.interface.flow_generator.set_flow_description(Origen.interface.consume_comments)

data/lib/origen/generator/job.rb

@@ -121,18 +121,34 @@ module Origen
         end
       end
 
+      def strip_dir_and_ext(name)
+        Pathname.new(name).basename('.*').basename('.*').to_s
+      end
+
       def run
         Origen.app.current_jobs << self
         begin
           if @options[:compile]
+            Origen.log.start_job(strip_dir_and_ext(@requested_pattern), :compiler)
             Origen.generator.compiler.compile(@requested_pattern, @options)
           elsif @options[:job_type] == :merge
+            Origen.log.start_job(strip_dir_and_ext(@requested_pattern), :merger)
             Origen.generator.compiler.merge(@requested_pattern)
           elsif @options[:action] == :program
+            if Origen.running_simulation?
+              Origen.log.start_job(strip_dir_and_ext(@requested_pattern), :simulator)
+            else
+              Origen.log.start_job(strip_dir_and_ext(@requested_pattern), :program_generator)
+            end
             Origen.flow.reset
             Origen.resources.reset
             OrigenTesters::Generator.execute_source(@pattern)
           else
+            if Origen.running_simulation?
+              Origen.log.start_job(strip_dir_and_ext(@requested_pattern), :simulator)
+            else
+              Origen.log.start_job(strip_dir_and_ext(@requested_pattern), :pattern_generator)
+            end
             Origen.generator.pattern.reset      # Resets the pattern controller ready for a new pattern
             # Give the app a chance to handle pattern dispatch
             skip = false
@@ -140,30 +156,58 @@ module Origen
               skip ||= !listener.before_pattern_lookup(@requested_pattern)
             end
             unless skip
-              @pattern = Origen.generator.pattern_finder.find(@requested_pattern, @options)
-              if @pattern.is_a?(Hash)
-                @output_file_body = @pattern[:output]
-                @pattern = @pattern[:pattern]
+              if @options[:sequence]
+                @pattern = @requested_pattern
+                Origen.pattern.sequence do |seq|
+                  # This splits the pattern name by "_" then removes all values that are common to all patterns
+                  # and then rejoins what is left.
+                  # The goal is to keep the thread ID concise for the log and rather than using the whole pattern
+                  # name only focussing on what is different.
+                  # e.g. if you combined patterns flash_read_ckbd_ip1_max.rb and flash_read_ckbd_ip2_max.rb into
+                  # a concurrent sequence then the two threads would be called 'ip1' and 'ip2'.
+                  ids = @options[:patterns].map do |pat|
+                    Pathname.new(pat).basename('.*').to_s.split('_')
+                  end
+                  ids = ids.map { |id| id.reject { |i| ids.all? { |id| id.include?(i) } }.join('_') }
+
+                  @options[:patterns].each_with_index do |pat, i|
+                    id = ids[i]
+                    id = i.to_s if id.empty?
+                    seq.in_parallel id do
+                      seq.run pat
+                    end
+                  end
+                end
+              else
+                @pattern = Origen.generator.pattern_finder.find(@requested_pattern, @options)
+                if @pattern.is_a?(Hash)
+                  @output_file_body = @pattern[:output]
+                  @pattern = @pattern[:pattern]
+                end
+                load @pattern unless @pattern == :skip  # Run the pattern
               end
-              load @pattern unless @pattern == :skip  # Run the pattern
             end
           end
         rescue Exception => e
-          if @options[:continue] || Origen.running_remotely?
-            Origen.log.error "FAILED - #{@requested_pattern} (for target #{Origen.target.name})"
-            Origen.log.error e.message
-            e.backtrace.each do |l|
-              Origen.log.error l
-            end
-            if @options[:compile]
-              Origen.app.stats.failed_files += 1
+          # Whoever has aborted the job is responsible for cleaning it up
+          unless e.is_a?(Origen::Generator::AbortError)
+            if @options[:continue] || Origen.running_remotely?
+              Origen.log.error "FAILED - #{@requested_pattern} (for target #{Origen.target.name})"
+              Origen.log.error e.message
+              e.backtrace.each do |l|
+                Origen.log.error l
+              end
+              if @options[:compile]
+                Origen.app.stats.failed_files += 1
+              else
+                Origen.app.stats.failed_patterns += 1
+              end
             else
-              Origen.app.stats.failed_patterns += 1
+              raise
             end
-          else
-            raise
           end
         end
+        Origen.log.stop_job
         Origen.app.current_jobs.pop
       end
     end

data/lib/origen/generator/pattern.rb

@@ -59,87 +59,144 @@ module Origen
         pattern_close(options)
       end
 
-      def create(options = {})
+      def sequence(options = {}, &block)
         @create_options = options
         unless Origen.tester
           puts 'The current target has not instantiated a tester and pattern generation cannot run.'
-          puts 'Add something like this to your target file:'
-          puts ''
-          puts '  $tester = Origen::Tester::J750.new'
-          puts ''
+          puts 'Add something like this to an environment file:'
+          puts
+          puts '  Origen::Tester::J750.new'
+          puts
+          puts
+          puts 'Then select it by running:  origen e <environment name>'
           exit 1
         end
         Origen.tester.generating = :pattern
 
         job.output_file_body = options.delete(:name).to_s if options[:name]
 
-        # Order the iterators by the order that their enable keys appear in the options, pad
-        # any missing iterators with a dummy function...
-        iterators = options.map do |key, _val|
-          Origen.app.pattern_iterators.find { |iterator| iterator.key == key }
-        end.compact
-        iterators << DummyIterator.new while iterators.size < 10
-
-        args = []
-
-        # Couldn't get this to work fully dynamically, so hard-coded for 10 custom
-        # iterators for now, should be plenty for any application in the meantime.
-        # Should revisit this when time allows and remove this limitation by changing
-        # this to a recursive structure.
-        iterators[0].invoke(options) do |arg0|
-          args[0] = arg0
-          iterators[1].invoke(options) do |arg1|
-            args[1] = arg1
-            iterators[2].invoke(options) do |arg2|
-              args[2] = arg2
-              iterators[3].invoke(options) do |arg3|
-                args[3] = arg3
-                iterators[4].invoke(options) do |arg4|
-                  args[4] = arg4
-                  iterators[5].invoke(options) do |arg5|
-                    args[5] = arg5
-                    iterators[6].invoke(options) do |arg6|
-                      args[6] = arg6
-                      iterators[7].invoke(options) do |arg7|
-                        args[7] = arg7
-                        iterators[8].invoke(options) do |arg8|
-                          args[8] = arg8
-                          iterators[9].invoke(options) do |arg9|
-                            args[9] = arg9
-                            # Refresh the target to start all settings from scratch each time
-                            # This is an easy way to reset all registered values
-                            Origen.app.reload_target!(skip_first_time: true)
-
-                            # Final call back to the project to allow it to make any pattern name specific
-                            # configuration changes
-                            Origen.app.listeners_for(:before_pattern).each do |listener|
-                              listener.before_pattern(job.output_pattern_filename)
-                            end
+        # Refresh the target to start all settings from scratch each time
+        # This is an easy way to reset all registered values
+        Origen.app.reload_target!(skip_first_time: true)
 
-                            # Work out the final pattern name based on the current iteration
-                            job.reset_output_pattern_filename
-                            iterators.each_with_index do |iterator, i|
-                              if iterator.enabled?(options)
-                                job.output_pattern_filename =
-                                  iterator.pattern_name.call(job.output_pattern_filename, args[i])
-                              end
-                            end
+        # Final call back to the project to allow it to make any pattern name specific
+        # configuration changes
+        Origen.app.listeners_for(:before_pattern).each do |listener|
+          listener.before_pattern(job.output_pattern_filename)
+        end
 
-                            # Allow custom pattern postfix
-                            unless options[:pat_postfix].to_s.empty?
-                              job.output_pattern_filename = job.output_pattern_filename.sub(job.output_postfix + job.output_extension, "_#{options[:pat_postfix]}" + job.output_postfix + job.output_extension)
-                            end
+        ## Allow custom pattern postfix
+        # unless options[:pat_postfix].to_s.empty?
+        #  job.output_pattern_filename = job.output_pattern_filename.sub(job.output_postfix + job.output_extension, "_#{options[:pat_postfix]}" + job.output_postfix + job.output_extension)
+        # end
+
+        @pattern_sequence = true
+        pattern_wrapper([], [], options) do
+          PatternSequencer.send(:active=, true)
+          @pattern_sequence = PatternSequence.new(job.output_pattern_filename, block)
+          @pattern_sequence.send(:execute)
+          PatternSequencer.send(:active=, false)
+        end
+        @pattern_sequence = false
+        @create_options = nil
+      end
+
+      def create(options = {})
+        if @pattern_sequence
+          yield
+        else
+          @create_options = options
+          unless Origen.tester
+            puts 'The current target has not instantiated a tester and pattern generation cannot run.'
+            puts 'Add something like this to an environment file:'
+            puts
+            puts '  Origen::Tester::J750.new'
+            puts
+            puts
+            puts 'Then select it by running:  origen e <environment name>'
+            exit 1
+          end
+          Origen.tester.generating = :pattern
+
+          job.output_file_body = options.delete(:name).to_s if options[:name]
+
+          # Order the iterators by the order that their enable keys appear in the options, pad
+          # any missing iterators with a dummy function...
+          iterators = options.map do |key, _val|
+            Origen.app.pattern_iterators.find { |iterator| iterator.key == key }
+          end.compact
+          iterators << DummyIterator.new while iterators.size < 10
+
+          args = []
+
+          # Couldn't get this to work fully dynamically, so hard-coded for 10 custom
+          # iterators for now, should be plenty for any application in the meantime.
+          # Should revisit this when time allows and remove this limitation by changing
+          # this to a recursive structure.
+          iterators[0].invoke(options) do |arg0|
+            args[0] = arg0
+            iterators[1].invoke(options) do |arg1|
+              args[1] = arg1
+              iterators[2].invoke(options) do |arg2|
+                args[2] = arg2
+                iterators[3].invoke(options) do |arg3|
+                  args[3] = arg3
+                  iterators[4].invoke(options) do |arg4|
+                    args[4] = arg4
+                    iterators[5].invoke(options) do |arg5|
+                      args[5] = arg5
+                      iterators[6].invoke(options) do |arg6|
+                        args[6] = arg6
+                        iterators[7].invoke(options) do |arg7|
+                          args[7] = arg7
+                          iterators[8].invoke(options) do |arg8|
+                            args[8] = arg8
+                            iterators[9].invoke(options) do |arg9|
+                              args[9] = arg9
+                              # Refresh the target to start all settings from scratch each time
+                              # This is an easy way to reset all registered values
+                              Origen.app.reload_target!(skip_first_time: true)
+
+                              # Final call back to the project to allow it to make any pattern name specific
+                              # configuration changes
+                              Origen.app.listeners_for(:before_pattern).each do |listener|
+                                listener.before_pattern(job.output_pattern_filename)
+                              end
 
-                            pattern_wrapper(iterators, args, options) do
-                              # Call iterator setups, whatever these return are passed to the pattern
-                              yield_items = []
+                              # Work out the final pattern name based on the current iteration
+                              job.reset_output_pattern_filename
                               iterators.each_with_index do |iterator, i|
                                 if iterator.enabled?(options)
-                                  yield_items << iterator.setup.call(args[i])
+                                  job.output_pattern_filename =
+                                    iterator.pattern_name.call(job.output_pattern_filename, args[i])
+                                end
+                              end
+
+                              # Allow custom pattern prefix
+                              unless options[:pat_prefix].to_s.empty?
+                                if job.output_prefix.empty?
+                                  job.output_pattern_filename = "#{options[:pat_prefix]}_" + job.output_pattern_filename
+                                else
+                                  job.output_pattern_filename = job.output_pattern_filename.sub(job.output_prefix, job.output_prefix + "#{options[:pat_prefix]}_")
                                 end
                               end
 
-                              yield(*yield_items)
+                              # Allow custom pattern postfix
+                              unless options[:pat_postfix].to_s.empty?
+                                job.output_pattern_filename = job.output_pattern_filename.sub(job.output_postfix + job.output_extension, "_#{options[:pat_postfix]}" + job.output_postfix + job.output_extension)
+                              end
+
+                              pattern_wrapper(iterators, args, options) do
+                                # Call iterator setups, whatever these return are passed to the pattern
+                                yield_items = []
+                                iterators.each_with_index do |iterator, i|
+                                  if iterator.enabled?(options)
+                                    yield_items << iterator.setup.call(args[i])
+                                  end
+                                end
+
+                                yield(*yield_items)
+                              end
                             end
                           end
                         end
@@ -363,9 +420,7 @@ module Origen
           unless job.test?
             File.delete(job.output_pattern) if File.exist?(job.output_pattern)
 
-            if options[:inhibit]
-              log.info "Generating...  #{job.output_pattern_directory}/#{job.output_pattern_filename}".ljust(50)
-            else
+            unless tester.try(:sim?)
               log.info "Generating...  #{job.output_pattern_directory}/#{job.output_pattern_filename}".ljust(50)
             end
           end
@@ -459,11 +514,16 @@ module Origen
             end
           end
 
-          log.info ' '
-          log.info "Pattern vectors: #{stats.number_of_vectors_for(job.output_pattern).to_s.ljust(10)}"
-          log.info 'Execution time'.ljust(15) + ': %.6f' % stats.execution_time_for(job.output_pattern)
-          log.info '----------------------------------------------------------------------'
-          check_for_changes(job.output_pattern, job.reference_pattern) unless tester.try(:disable_pattern_diffs)
+          unless tester.try(:sim?)
+            log.info ' '
+            log.info "Pattern vectors: #{stats.number_of_vectors_for(job.output_pattern).to_s.ljust(10)}"
+            log.info 'Execution time'.ljust(15) + ': %.6f' % stats.execution_time_for(job.output_pattern)
+            log.info '----------------------------------------------------------------------'
+            check_for_changes(job.output_pattern, job.reference_pattern) unless tester.try(:disable_pattern_diffs)
+          end
+          if @pattern_sequence
+            @pattern_sequence.send(:log_execution_profile)
+          end
           stats.record_pattern_completion(job.output_pattern)
         end
 

data/lib/origen/generator/pattern_finder.rb

@@ -95,10 +95,10 @@ module Origen
           matches = Dir.glob("#{current_plugin_pattern_path}/**/#{name}.rb").sort
           # If the current plugin does  not include the pattern then look into the current app
           if matches.size == 0
-            matches = Dir.glob("#{pattern_directory}/**/#{name}.rb").sort # <= this does not include symlinks
+            matches = Dir.glob(["#{pattern_directory}/**/#{name}.rb", "#{Origen.root}/app/patterns/**/#{name}.rb"]).sort # <= this does not include symlinks
           end
         else
-          matches = Dir.glob("#{pattern_directory}/**/#{name}.rb").sort # <= this does not include symlinks
+          matches = Dir.glob(["#{pattern_directory}/**/#{name}.rb", "#{Origen.root}/app/patterns/**/#{name}.rb"]).sort # <= this does not include symlinks
         end
 
         matches
@@ -114,7 +114,7 @@ module Origen
 
       def all_matches(name)
         name = name.gsub(/\..*$/, '')
-        matches = Dir.glob("#{pattern_directory}/**{,/*/**}/#{name}.rb").sort # Takes symlinks into consideration
+        matches = Dir.glob(["#{pattern_directory}/**{,/*/**}/#{name}.rb", "#{Origen.root}/app/patterns/**{,/*/**}/#{name}.rb"]).sort # Takes symlinks into consideration
         matches.flatten.uniq
       end
 

data/lib/origen/generator/pattern_sequence.rb

@@ -0,0 +1,201 @@
+require 'io/console'
+
+module Origen
+  class Generator
+    # Manages a single pattern sequence, i.e. an instance of PatternSequence is
+    # created for every Pattern.sequence do ... end block
+    class PatternSequence
+      def initialize(name, block)
+        @number_of_threads = 1
+        @name = name
+        @running_thread_ids = { main: true }
+        # The contents of the main Pattern.sequence block will be executed as a thread and treated
+        # like any other parallel block
+        thread = PatternThread.new(:main, self, block, true)
+        threads << thread
+        active_threads << thread
+      end
+
+      # Execute the given pattern
+      def run(pattern_name)
+        pattern = Origen.generator.pattern_finder.find(pattern_name.to_s, {})
+        pattern = pattern[:pattern] if pattern.is_a?(Hash)
+        load pattern
+      end
+      alias_method :call, :run
+
+      # Execute the given block in a new concurrent thread
+      def thread(id = nil, &block)
+        @number_of_threads += 1
+        id ||= "thread#{@number_of_threads}".to_sym
+        # Just stage the request for now, it will be started at the end of the current execute loop
+        @parallel_blocks_waiting_to_start ||= []
+        @parallel_blocks_waiting_to_start << [id, block]
+        @running_thread_ids[id] = true
+      end
+      alias_method :in_parallel, :thread
+
+      def wait_for_threads(*ids)
+        completed = false
+        blocked = false
+        ids = ids.map(&:to_sym)
+        all = ids.empty? || ids.include?(:all)
+        until completed
+          if all
+            limit = current_thread.id == :main ? 1 : 2
+            if @running_thread_ids.size > limit
+              current_thread.waiting_for_thread(blocked)
+              blocked = true
+            else
+              current_thread.record_active if blocked
+              completed = true
+            end
+          else
+            if ids.any? { |id| @running_thread_ids[id] }
+              current_thread.waiting_for_thread(blocked)
+              blocked = true
+            else
+              current_thread.record_active if blocked
+              completed = true
+            end
+          end
+        end
+      end
+      alias_method :wait_for_thread, :wait_for_threads
+
+      private
+
+      def thread_running?(id)
+        @running_thread_ids[id]
+      end
+
+      def current_thread
+        PatSeq.thread
+      end
+
+      def log_execution_profile
+        if threads.size > 1
+          thread_id_size = threads.map { |t| t.id.to_s.size }.max
+          line_size = IO.console.winsize[1] - 35 - thread_id_size
+          line_size -= 16 if tester.try(:sim?)
+          cycles_per_tick = (@cycle_count_stop / (line_size * 1.0)).ceil
+          if tester.try(:sim?)
+            execution_time = tester.execution_time_in_ns / 1_000_000_000.0
+          else
+            execution_time = Origen.app.stats.execution_time_for(Origen.app.current_job.output_pattern)
+          end
+          Origen.log.info ''
+          tick_time = execution_time / line_size
+
+          Origen.log.info "Concurrent execution profile (#{pretty_time(tick_time)}/increment):"
+          Origen.log.info
+
+          number_of_ticks = @cycle_count_stop / cycles_per_tick
+
+          ticks_per_step = 0
+          step_size = 0.1.us
+
+          while ticks_per_step < 10
+            step_size = step_size * 10
+            ticks_per_step = step_size / tick_time
+          end
+
+          ticks_per_step = ticks_per_step.ceil
+          step_size = tick_time * ticks_per_step
+
+          if tester.try(:sim?)
+            padding = '.' + (' ' * (thread_id_size + 1))
+          else
+            padding = ' ' * (thread_id_size + 2)
+          end
+          scale_step = '|' + ('-' * (ticks_per_step - 1))
+          number_of_steps = (number_of_ticks / ticks_per_step) + 1
+          scale = scale_step * number_of_steps
+          scale = scale[0, number_of_ticks]
+          Origen.log.info padding + scale
+
+          scale = ''
+          number_of_steps.times do |i|
+            scale += pretty_time(i * step_size, 1).ljust(ticks_per_step)
+          end
+          scale = scale[0, number_of_ticks]
+          Origen.log.info padding + scale
+
+          threads.each do |thread|
+            line = thread.execution_profile(0, @cycle_count_stop, cycles_per_tick)
+            Origen.log.info ''
+            Origen.log.info "#{thread.id}: ".ljust(thread_id_size + 2) + line
+          end
+          Origen.log.info ''
+        end
+      end
+
+      def pretty_time(time, number_decimal_places = 0)
+        return '0' if time == 0
+        if time < 1.us
+          "%.#{number_decimal_places}fns" % (time * 1_000_000_000)
+        elsif time < 1.ms
+          "%.#{number_decimal_places}fus" % (time * 1_000_000)
+        elsif time < 1.s
+          "%.#{number_decimal_places}fms" % (time * 1_000)
+        else
+          "%.#{number_decimal_places}fs" % tick_time
+        end
+      end
+
+      def thread_completed(thread)
+        @running_thread_ids.delete(thread.id)
+        active_threads.delete(thread)
+      end
+
+      def threads
+        @threads ||= []
+      end
+
+      def active_threads
+        @active_threads ||= []
+      end
+
+      def threads_waiting_to_start?
+        @parallel_blocks_waiting_to_start
+      end
+
+      def execute
+        active_threads.first.start
+        until active_threads.empty?
+          # Advance all threads to their next cycle point in sequential order. Keeping tight control of
+          # when threads are running in this way ensures that the output is deterministic no matter what
+          # computer it is running on, and ensures that the application code does not have to worry about
+          # race conditions.
+          cycs = active_threads.map do |t|
+            t.advance
+            t.pending_cycles
+          end.compact.min
+
+          if cycs
+            # Now generate the required number of cycles which is defined by the thread that has the least
+            # amount of cycles ready to go.
+            # Since tester.cycle is being called by the master process here it will generate as normal (as
+            # opposed to when called from a thread in which case it causes the thread to wait).
+            cycs.cycles
+
+            # Now let each thread know how many cycles we just generated, so they can decide whether they
+            # need to wait for more cycles or if they can start preparing the next one
+            active_threads.each { |t| t.executed_cycles(cycs) }
+          end
+
+          if @parallel_blocks_waiting_to_start
+            @parallel_blocks_waiting_to_start.each do |id, block|
+              thread = PatternThread.new(id, self, block)
+              threads << thread
+              active_threads << thread
+              thread.start
+            end
+            @parallel_blocks_waiting_to_start = nil
+          end
+        end
+        @cycle_count_stop = threads.first.current_cycle_count
+      end
+    end
+  end
+end