origen 0.0.1 → 0.0.2

data/lib/origen/application/runner.rb

@@ -0,0 +1,246 @@
+require 'fileutils'
+module Origen
+  class Application
+    autoload :Statistics,    'origen/application/statistics'
+
+    # The Runner is responsible for co-ordinating all compile and generate
+    # requests from the command line
+    class Runner
+      attr_accessor :options
+
+      # Launch Origen, any command which generates an output file should launch from here
+      # as it gives a common point for listeners to hook in and to establish output
+      # directories and so on.
+      #
+      # Originally this method was called generate but that is now deprecated in favour
+      # of the more generic 'launch' as the Origen feature set has expanded.
+      def launch(options = {})
+        # Clean up the input from legacy code
+        options[:action] = extract_action(options)
+        options[:files] = extract_files(options)
+        @options = options
+        prepare_and_validate_workspace(options)
+        if options[:lsf]
+          record_invocation(options) do
+            prepare_for_lsf
+            Origen.app.listeners_for(:before_lsf_submission).each(&:before_lsf_submission)
+            expand_lists_and_directories(options[:files], options).each do |file|
+              Origen.app.lsf_manager.submit_origen_job(file, options)
+            end
+          end
+          Origen.log.info ''
+          Origen.log.info 'Monitor status of remote jobs via:'
+          Origen.log.info '  origen l'
+        else
+          Origen.log.info '*' * 70 unless options[:quiet]
+          Origen.app.listeners_for(:before_generate).each do |listener|
+            if listener.class.instance_method(:before_generate).arity == 0
+              listener.before_generate
+            else
+              listener.before_generate(options)
+            end
+          end
+          if Origen.running_remotely?
+            Origen.app.listeners_for(:before_generate_remote).each do |listener|
+              if listener.class.instance_method(:before_generate_remote).arity == 0
+                listener.before_generate_remote
+              else
+                listener.before_generate_remote(options)
+              end
+            end
+          else
+            Origen.app.listeners_for(:before_generate_local).each do |listener|
+              if listener.class.instance_method(:before_generate_local).arity == 0
+                listener.before_generate_local
+              else
+                listener.before_generate_local(options)
+              end
+            end
+          end
+
+          record_invocation(options) do
+            case options[:action]
+            when :forecast_test_time
+              Origen.time.forecast_test_time(options)
+            else
+              if options[:action] == :program
+                Origen.generator.generate_program(expand_lists_and_directories(options[:files], options), options)
+              else
+                temporary_plugin_from_options = options[:current_plugin]
+                expand_lists_and_directories(options[:files], options).each do |file|
+                  if temporary_plugin_from_options
+                    Origen.current_plugin.temporary = temporary_plugin_from_options
+                  end
+                  case options[:action]
+                  when :compile
+                    Origen.generator.compile_file_or_directory(file, options)
+                  when :merge
+                    Origen.generator.merge_file_or_directory(file, options)
+                  when :import_test_time
+                    Origen.time.import_test_time(file, options)
+                  when :import_test_flow
+                    Origen.time.import_test_flow(file, options)
+                  else
+                    Origen.generator.generate_pattern(file, options)
+                  end
+                  if temporary_plugin_from_options
+                    Origen.current_plugin.default
+                  end
+                end
+              end
+            end
+          end
+
+          unless options[:quiet]
+            Origen.log.info '*' * 70
+            stats.print_summary unless options[:action] == :merge
+          end
+        end
+      end
+      alias_method :generate, :launch
+
+      def prepare_and_validate_workspace(options = {})
+        confirm_production_ready(options)
+        prepare_directories(options)
+      end
+
+      # Post an invocation to the Origen server for usage statistics tracking.
+      #
+      # Posting an invocation was found to add ~0.5s to all command times,
+      # so here we run it in a separate thread to try and hide it behind
+      # the user's command.
+      #
+      # @api private
+      def record_invocation(options)
+        record_invocation = false
+        begin
+          # Only record user invocations at this time, also bypass windows since it seems
+          # that threads can't be trusted not to block
+          unless Origen.running_remotely? # || Origen.running_on_windows?
+            record_invocation = Thread.new do
+              Origen.client.record_invocation(options[:action]) if options[:action]
+            end
+          end
+        rescue
+          # Don't allow this to kill an origen command
+        end
+        yield
+        begin
+          unless Origen.running_remotely?
+            # Wait for a server response, ideally would like to not wait here, but it seems if not
+            # then invocation postings can be dropped, especially on windows
+            Origen.profile 'waiting for recording invocation' do
+              record_invocation.value
+            end
+          end
+        rescue
+          # Don't allow this to kill an origen command
+        end
+      end
+
+      # The action to take should be set by the action option, but legacy code will pass
+      # things like :compile => true, the extract_action method handles the old code
+      def extract_action(options)
+        return options[:action] if options[:action]
+        if options[:compile]
+          :compile
+        elsif options[:program]
+          :program
+        elsif options[:job_type] == :merge
+          :merge
+        else
+          :pattern
+        end
+      end
+
+      # Legacy file references can be input via :pattern, :patterns, etc. this
+      # cleans it up and forces them all to be in an array assigned to options[:files]
+      def extract_files(options)
+        files = [options[:pattern]] + [options[:patterns]] + [options[:file]] + [options[:files]]
+        files.flatten!
+        files.compact!
+        files
+      end
+
+      def shutdown
+        if Origen.app.stats.failed_files > 0 ||
+           Origen.app.stats.failed_patterns > 0
+          exit 1
+        end
+      end
+
+      # Expands any list references in the supplied pattern array and
+      # returns an array of pattern names. No guarantee is made to
+      # whether the pattern names are valid at this stage.
+      # Any duplicates will be removed.
+      def expand_lists_and_directories(files, options = {})
+        Origen.file_handler.expand_list(files, options)
+      end
+
+      def statistics
+        @statistics ||= Statistics.new(options)
+      end
+      alias_method :stats, :statistics
+
+      def prepare_for_lsf
+        if options[:lsf]
+          # Build an options string for saving with the LSF job that represents this runtime environment
+          str = "-t #{Origen.target.file.basename}"
+          if Origen.environment.file
+            str += " --environment #{Origen.environment.file.basename}"
+          end
+          if options[:output]
+            str += " -o #{options[:output]}"
+          end
+          if options[:reference]
+            str += " -r #{options[:reference]}"
+          end
+          options[:lsf_option_string] = str
+          # Clear the LSF manager job list if specifically requested or if that is the default action and
+          # no specific action has been requested
+          if options[:lsf_action]
+            if options[:lsf_action] == :clear
+              Origen.app.lsf_manager.clear_all
+            end
+          elsif Origen.config.default_lsf_action == :clear
+            Origen.app.lsf_manager.clear_all
+          end
+        end
+      end
+
+      def prepare_directories(options = {})
+        # When running remotely on the LSF never create directories to
+        # prevent race conditions as multiple processes run concurrently,
+        # instead assume they were already created by the runner who
+        # submitted the job.
+        Origen.file_handler.set_output_directory(options.merge(create: Origen.running_locally?))
+        Origen.file_handler.set_reference_directory(options.merge(create: Origen.running_locally?))
+        tmp = "#{Origen.root}/tmp"
+        FileUtils.mkdir(tmp) unless File.exist?(tmp)
+        if Origen.running_locally?
+          mkdir Origen::Log.log_file_directory
+          mkdir "#{Origen.root}/.lsf"
+        end
+        if options[:lsf]
+          mkdir Origen.app.lsf_manager.log_file_directory
+        end
+      end
+
+      # Make the given directory if it doesn't exist, must be a full path
+      def mkdir(dir)
+        unless File.exist?(dir)
+          FileUtils.mkdir_p(dir)
+        end
+      end
+
+      def confirm_production_ready(_options = {})
+        # The caller would have already verified the status before submission
+        if Origen.running_locally?
+          if Origen.mode.production?
+            Origen.app.cm.ensure_workspace_unmodified!
+          end
+        end
+      end
+    end
+  end
+end

data/lib/origen/application/statistics.rb

@@ -0,0 +1,191 @@
+module Origen
+  class Application
+    # Responsible for keeping track of all stats collected during a run
+    class Statistics
+      attr_accessor :completed_files, :failed_files, :missing_files,
+                    :new_files, :changed_files
+      attr_accessor :completed_patterns, :failed_patterns, :missing_patterns,
+                    :new_patterns, :changed_patterns
+      attr_accessor :total_vectors, :total_cycles, :total_duration, :errors
+
+      class Pattern
+        attr_accessor :vectors, :cycles, :duration
+        def initialize
+          @vectors = 0
+          @cycles = 0
+          @duration = 0
+        end
+      end
+
+      def initialize(options)
+        @options = options
+        @patterns = {}
+        reset_global_stats
+      end
+
+      def reset_global_stats
+        @completed_files = 0
+        @failed_files = 0
+        @missing_files = 0
+        @new_files = 0
+        @changed_files = 0
+
+        @completed_patterns = 0
+        @failed_patterns = 0
+        @missing_patterns = 0
+        @new_patterns = 0
+        @changed_patterns = 0
+
+        @total_vectors = 0
+        @total_cycles = 0
+        @total_duration = 0
+
+        @errors = 0
+      end
+
+      def reset_pattern_stats
+      end
+
+      def print_summary
+        method = clean_run? ? :success : :info
+        if @completed_patterns > 0 || @failed_patterns > 0
+          Origen.log.send method, "Total patterns:   #{@completed_patterns}"
+          Origen.log.send method, "Total vectors:    #{@total_vectors}"
+          Origen.log.send method, 'Total duration:   %.6f' % @total_duration
+          Origen.log.send method, "New patterns:     #{@new_patterns}"
+          if @changed_patterns > 0
+            Origen.log.warn "Changed patterns: #{@changed_patterns}"
+          else
+            Origen.log.send method, "Changed patterns: #{@changed_patterns}"
+          end
+          Origen.log.error "FAILED patterns:  #{@failed_patterns}" if @failed_patterns > 0
+          Origen.log.info
+        end
+        if @completed_files > 0 || @failed_files > 0
+          Origen.log.send method, "Total files:      #{@completed_files}"
+          Origen.log.send method, "New files:        #{@new_files}"
+          Origen.log.send method, "Changed files:    #{@changed_files}"
+          Origen.log.error "FAILED files:     #{@failed_files}" if @failed_files > 0
+          Origen.log.info
+        end
+        if @errors > 0
+          Origen.log.error "ERRORS:           #{@errors}"
+        end
+
+        if @changed_files > 0 || @changed_patterns > 0
+          changes = true
+          Origen.log.info 'To accept all of these changes run:'
+          Origen.log.info '  origen save changed'
+        end
+        if @new_files > 0 || @new_patterns > 0
+          news = true
+          Origen.log.info 'To save all of these new files as the reference version run:'
+          Origen.log.info '  origen save new'
+        end
+        if changes && news
+          Origen.log.info 'To save both new and changed files run:'
+          Origen.log.info '  origen save all'
+        end
+        Origen.log.info '**********************************************************************'
+      end
+
+      def summary_text
+        <<-END
+    Total patterns:   #{@completed_patterns}
+    New patterns:     #{@new_patterns}
+    Changed patterns: #{@changed_patterns}
+    FAILED patterns:  #{@failed_patterns}
+
+    Total files:      #{@completed_files}
+    New files:        #{@new_files}
+    Changed files:    #{@changed_files}
+    FAILED files:     #{@failed_files}
+
+    ERRORS:           #{@errors}
+        END
+      end
+
+      def clean_run?
+        @changed_files == 0 && @changed_patterns == 0 &&
+          @new_files == 0 && @new_patterns == 0 &&
+          @failed_files == 0 && @failed_patterns == 0 &&
+          @errors == 0
+      end
+
+      def record_failed_pattern
+        @failed_patterns += 1
+      end
+
+      def record_missing_pattern
+        @missing_patterns += 1
+      end
+
+      def add_vector(x = 1)
+        current_pattern.vectors += x
+      end
+
+      def add_cycle(x = 1)
+        current_pattern.cycles += x
+      end
+
+      def add_time_in_ns(x)
+        current_pattern.duration += x
+      end
+
+      def collect_for_pattern(key)
+        @pattern_key = key
+        yield
+        @pattern_key = nil
+      end
+
+      def current_pattern
+        pattern(@pattern_key)
+      end
+
+      def pattern(key)
+        @patterns[key] ||= Pattern.new
+      end
+
+      def number_of_vectors_for(key)
+        pattern(key).vectors
+      end
+
+      def number_of_cycles_for(key)
+        pattern(key).vectors
+      end
+
+      def execution_time_for(key)
+        pattern(key).duration.to_f / 1_000_000_000
+      end
+
+      def record_pattern_completion(key)
+        @completed_patterns += 1
+        @total_vectors += number_of_vectors_for(key)
+        @total_cycles += number_of_cycles_for(key)
+        @total_duration += execution_time_for(key)
+      end
+
+      def report_pass
+        Origen.log.success ''
+        Origen.log.success '    PPPPP      AA        SSSS    SSSS'
+        Origen.log.success '    PP  PP    AAAA      SS  SS  SS  SS'
+        Origen.log.success '    PPPPP    AA  AA      SS      SS'
+        Origen.log.success '    PP      AAAAAAAA       SS      SS'
+        Origen.log.success '    PP     AA      AA   SS  SS  SS  SS'
+        Origen.log.success '    PP    AA        AA   SSSS    SSSS'
+        Origen.log.success ''
+      end
+
+      def report_fail
+        Origen.log.error ''
+        Origen.log.error '    FFFFFF     AA       II   LL'
+        Origen.log.error '    FF        AAAA      II   LL'
+        Origen.log.error '    FFFFF    AA  AA     II   LL'
+        Origen.log.error '    FF      AAAAAAAA    II   LL'
+        Origen.log.error '    FF     AA      AA   II   LL'
+        Origen.log.error '    FF    AA        AA  II   LLLLLL'
+        Origen.log.error ''
+      end
+    end
+  end
+end