origen 0.0.1 → 0.0.2

data/lib/origen/tester/ultraflex/parser/descriptions.rb

@@ -0,0 +1,342 @@
+module Origen
+  module Tester
+    class Ultraflex
+      class Parser
+        # Extracts embedded test and flow descriptions (comments) from test
+        # program source files
+        class Descriptions
+          require 'fileutils'
+
+          attr_accessor :source_directories, :template_directories, :parser
+
+          SCRATCH_DIR = "#{Origen.root}/.j750_scratch"
+
+          # All descriptions are stored in this lookup table
+          def lookup
+            return @lookup if @lookup
+            # Use the one from the interface if present, program generation will
+            # automatically push descriptions in here
+            if Origen.interface_present?
+              @lookup = Origen.interface.descriptions
+            else
+              @lookup = Origen::Tester::Parser::DescriptionLookup.new
+            end
+          end
+
+          def initialize(options = {})
+            @parser = options[:parser]
+            FileUtils.rm_rf(SCRATCH_DIR) if File.exist?(SCRATCH_DIR)
+            parse_program
+            true
+          end
+
+          # Returns the description for the given flow
+          def flow_summary(options = {})
+            lookup.for_flow(options[:file])
+          end
+
+          # Returns the description of the given test from the test
+          # instance sheet declaration
+          def test_instance(options = {})
+            lookup.for_test_definition(options[:name])
+          end
+
+          # Returns the description of the given test from the test
+          # flow
+          def flow_line(options = {})
+            lookup.for_test_usage(options[:name], options[:flow])
+          end
+
+          def parse_program
+            Origen.file_handler.preserve_state do
+              generate_program_files
+              # Comments must be extracted manually for any compiled files, for
+              # generated files the comments will already be in the lookup
+              extract_flow_summaries
+              extract_test_instance_descriptions
+              extract_flow_line_descriptions
+            end
+          end
+
+          def source_directories
+            [@source_directories, Origen.config.test_program_source_directory].compact.flatten
+          end
+
+          def template_directories
+            [@template_directories, Origen.config.test_program_template_directory].compact.flatten
+          end
+
+          def extract_flow_summaries
+            Origen.file_handler.resolve_files(compiled_dir) do |file|
+              if flow_file?(file)
+                lookup.add_for_flow(file, parse_flow_summary(file))
+              end
+            end
+          end
+
+          # Parses a compiled template for marked up comments
+          def extract_test_instance_descriptions
+            Origen.file_handler.resolve_files(compiled_dir) do |file|
+              if instance_file?(file)
+                comments = []
+                File.readlines(file).each do |line|
+                  if line =~ /^<comment>(.*)/
+                    comments << Regexp.last_match[1].gsub("\r", '')
+                  else
+                    fields = line.split("\t")
+                    unless ['Test Instances', '', 'Test Name'].include? fields[1]
+                      lookup.add_for_test_definition(fields[1], comments)
+                    end
+                    comments = []
+                  end
+                end
+              end
+            end
+          end
+
+          def extract_flow_line_descriptions
+            Origen.file_handler.resolve_files(compiled_dir) do |file|
+              if flow_file?(file)
+                f = file.basename('.txt').to_s
+                comments = []
+                header_line = true
+                File.readlines(file).each do |line|
+                  if header_line
+                    header_line = false if line =~ /^\s*Label/
+                  else
+                    if line =~ /^<comment>(.*)/
+                      comments << Regexp.last_match[1].gsub("\r", '')
+                    else
+                      t = FlowLine.extract_test(line)
+                      if t
+                        lookup.add_for_test_usage(t, file, comments)
+                      end
+                      comments = []
+                    end
+                  end
+                end
+              end
+            end
+          end
+
+          def generate_program_files
+            a = generate_program
+            b = compile_program
+            unless a || b
+              fail 'No source or template files declared from which to parse descriptions!'
+            end
+          end
+
+          # Parses the given flow file for summary text and returns it, summary
+          # text must be the very first thing in the file.
+          # Returns an array of strings each representing a line of text.
+          def parse_flow_summary(file)
+            desc = []
+            File.readlines(file).each do |line|
+              if line =~ /%?\s*<comment>(.*)/
+                desc << Regexp.last_match[1].gsub("\r", '')
+              else
+                break
+              end
+            end
+            desc
+          end
+
+          # Generate a scratch version of the program for parsing
+          def generate_program
+            if source_directories.size > 0
+              unless @program_generated
+                Origen.log.info ''
+                Origen.log.info 'Extracting embedded comments:'
+                Origen.log.info ''
+                copy_source_files_to_scratch
+                markup_source_file_comments
+                # Compile the flow file, with Ruby comments now preserved and marked up
+                desc = Origen.app.runner.generate(program: true, patterns: ungenerated_dir, output: generated_dir,
+                                                check_for_changes: false, collect_stats: false, quiet: true,
+                                                collect_descriptions: true)
+                Origen.log.info ''
+              end
+              @program_generated = true
+            else
+              false
+            end
+          end
+
+          # Compile a scratch version of the program for parsing
+          def compile_program
+            if template_directories.size > 0
+              unless @program_compiled
+                Origen.log.info ''
+                Origen.log.info 'Extracting embedded comments:'
+                Origen.log.info ''
+                copy_templates_to_scratch
+                markup_template_comments
+                # Compile the flow file, with Ruby comments now preserved and marked up
+                Origen.app.runner.generate(compile: true, patterns: uncompiled_dir, output: compiled_dir,
+                                         check_for_changes: false, collect_stats: false, quiet: true)
+                Origen.log.info ''
+              end
+              @program_compiled = true
+            else
+              false
+            end
+          end
+
+          # Copy all flow and instance template files to the scratch dir
+          def copy_templates_to_scratch
+            uncompiled_dir(true)
+            template_directories.each do |dir|
+              Origen.file_handler.resolve_files(dir) do |file|
+                subdir = file.relative_path_from(Pathname.new(dir)).dirname.to_s
+                cpydir = "#{uncompiled_dir}/#{subdir}"
+                FileUtils.mkdir_p(cpydir) unless File.exist?(cpydir)
+                FileUtils.copy(file, cpydir) if flow_or_instance_file?(file)
+              end
+            end
+            `chmod -R 777 #{uncompiled_dir}/*` unless Dir["#{uncompiled_dir}/*"].empty?
+          end
+
+          # Copy all flow and instance source files to the scratch dir
+          def copy_source_files_to_scratch
+            source_directories.each do |dir|
+              Origen.file_handler.resolve_files(dir) do |file|
+                subdir = file.relative_path_from(Pathname.new(dir)).dirname.to_s
+                cpydir = "#{ungenerated_dir}/#{subdir}"
+                FileUtils.mkdir_p(cpydir) unless File.exist?(cpydir)
+                FileUtils.copy(file, cpydir)
+              end
+            end
+          end
+
+          def uncompiled_dir(force_make = false)
+            @uncompiled_dir ||= "#{SCRATCH_DIR}/uncompiled"
+            if force_make
+              FileUtils.rm_rf(@uncompiled_dir) if File.exist?(@uncompiled_dir)
+              @uncompiled_dir_created = false
+            end
+            unless @uncompiled_dir_created
+              FileUtils.mkdir_p(@uncompiled_dir) unless File.exist?(@uncompiled_dir)
+              @uncompiled_dir_created = true
+            end
+            @uncompiled_dir
+          end
+
+          def ungenerated_dir
+            @ungenerated_dir ||= "#{SCRATCH_DIR}/ungenerated"
+            unless @ungenerated_dir_created
+              FileUtils.mkdir_p(@ungenerated_dir) unless File.exist?(@ungenerated_dir)
+              @ungenerated_dir_created = true
+            end
+            @ungenerated_dir
+          end
+
+          def compiled_dir
+            @compiled_dir ||= "#{SCRATCH_DIR}/compiled"
+            unless @compiled_dir_created
+              FileUtils.mkdir_p(@compiled_dir) unless File.exist?(@compiled_dir)
+              @compiled_dir_created = true
+            end
+            @compiled_dir
+          end
+
+          def generated_dir
+            @generated_dir ||= "#{SCRATCH_DIR}/generated"
+            unless @generated_dir_created
+              FileUtils.mkdir_p(@generated_dir) unless File.exist?(@generated_dir)
+              @generated_dir_created = true
+            end
+            @generated_dir
+          end
+
+          # Returns true if the given file looks like a Ultraflex flow file, works for
+          # templates to
+          def flow_or_instance_file?(file, options = {})
+            options = { flow:     true,
+                        instance: true
+            }.merge(options)
+            if options[:flow] && options[:instance]
+              match = 'Flow|Instances'
+            elsif options[:flow]
+              match = 'Flow'
+            else
+              match = 'Instances'
+            end
+            # Not sure the best way to determine the file type of a partial, just
+            # return true for now to play it safe
+            return true if file.basename.to_s =~ /^_/
+            File.readlines(file).each do |line|
+              begin
+                unless line =~ /^%/ || line =~ /^\s*<comment>/
+                  return !!(line =~ /#{match}/)
+                end
+              rescue Exception => e
+                if e.is_a?(ArgumentError) && e.message =~ /invalid byte sequence/
+                  return false
+                else
+                  puts e.message
+                  puts e.backtrace
+                  exit 1
+                end
+              end
+            end
+          end
+
+          def flow_file?(file)
+            flow_or_instance_file?(file, instance: false)
+          end
+
+          def instance_file?(file)
+            flow_or_instance_file?(file, flow: false)
+          end
+
+          # Substitute Ruby line comments so they are preserved by compilation
+          def markup_template_comments
+            Origen.file_handler.resolve_files(uncompiled_dir) do |file|
+              lines = File.readlines(file)
+              File.open(file, 'w') do |f|
+                lines.each do |line|
+                  if line =~ /^%\s*#\s?(.*)/ # Remove single leading whitespace from comment if it exists
+                    comment = Regexp.last_match[1]
+                    # If comment starts with a '#-' it should be removed by compilation
+                    if line =~ /^%\s*#-.*/
+                      f.write line
+                    # Otherwise preserve it
+                    else
+                      f.write "<comment>#{comment}\n"
+                    end
+                  else
+                    f.write line
+                  end
+                end
+              end
+            end
+          end
+
+          # Substitute Ruby line comments so they are preserved by generation
+          def markup_source_file_comments
+            Origen.file_handler.resolve_files(ungenerated_dir) do |file|
+              lines = File.readlines(file)
+              File.open(file, 'w') do |f|
+                lines.each do |line|
+                  if line =~ /^\s*#\s?(.*)/ # Remove single leading whitespace from comment if it exists
+                    comment = Regexp.last_match[1]
+                    # If comment starts with a '#-' it should be removed by generation
+                    if line =~ /^\s*#-.*/
+                      f.write line
+                    # Otherwise preserve it
+                    else
+                      f.write "Origen.interface.comment '#{comment}'\n"
+                    end
+                  else
+                    f.write line
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/origen/tester/ultraflex/parser/flow.rb

@@ -0,0 +1,111 @@
+module Origen
+  module Tester
+    class Ultraflex
+      class Parser
+        class Flow < Origen::Tester::Parser::SearchableArray
+          require 'pathname'
+
+          attr_accessor :parser, :file
+
+          def initialize(file, options = {})  # :nodoc:
+            @parser = options[:parser]
+            @file = Pathname.new(file)
+            parse
+          end
+
+          def description
+            @parser.descriptions.flow_summary(file: file)
+          end
+          alias_method :summary, :description
+
+          # Returns the filename of the sheet that contained the current flow
+          def filename
+            @file.basename.to_s
+          end
+          alias_method :name, :filename
+
+          # Returns all flow lines that are tests, optionally supply a context to
+          # have only the test that will execute in that context returned
+          #
+          #   $tester.flow.first.tests.size
+          #     => 20
+          #   $tester.flow.first.tests(:job => "P1").size
+          #     => 10
+          #   $tester.flow.first.tests(:job => "P1", :enable => "data_collection").size
+          #     => 15
+          def tests(context = {})
+            run_context(context)
+          end
+
+          # Returns all tests in the current flow, regardless of context
+          def all_tests
+            where(opcode: %w(Test characterize), exact: true)
+          end
+
+          def run_context(context)  # :nodoc:
+            capture = true
+            waiting_for_label = false
+            select do |line|
+              if capture
+                if !waiting_for_label || waiting_for_label == line.label
+                  waiting_for_label = false
+                  case line.type
+                  when 'Test', 'characterize'
+                    line.executes_under_context?(context)
+                  when 'set-device', 'stop'
+                    capture = false if line.executes_under_context?(context)
+                    false
+                  when 'enable-flow-word'
+                    if line.executes_under_context?(context)
+                      context[:enable] = [context[:enable]].flatten
+                      context[:enable] << line.parameter
+                    end
+                    false
+                  when 'flag-true'
+                    if line.executes_under_context?(context)
+                      context[:true_flags] = [context[:true_flags]].flatten
+                      context[:true_flags] << line.parameter
+                    end
+                    false
+                  when 'flag-false'
+                    if line.executes_under_context?(context)
+                      context[:false_flags] = [context[:false_flags]].flatten
+                      context[:false_flags] << line.parameter
+                    end
+                    false
+                  when 'disable-flow-word'
+                    if line.executes_under_context?(context)
+                      context[:enable] = [context[:enable]].flatten
+                      context[:enable].delete(line.parameter)
+                    end
+                    false
+                  when 'logprint', 'nop', 'print'
+                    false
+                  when 'goto'
+                    waiting_for_label = line.parameter
+                    false
+                  else
+                    fail "Don't know how to process: #{line.type}, in file #{filename}"
+                  end
+                else
+                  false
+                end
+              end
+            end
+          end
+
+          def parse  # :nodoc:
+            File.readlines(@file).each do |line|
+              l = FlowLine.new(line, parser: parser, flow: self)
+              self << l if l.valid?
+            end
+          end
+
+          def inspect  # :nodoc:
+            "<TestFlow: #{filename}, Lines: #{size}>"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/origen/tester/ultraflex/parser/flow_line.rb

@@ -0,0 +1,207 @@
+module Origen
+  module Tester
+    class Ultraflex
+      class Parser
+        class FlowLine
+          attr_accessor :parser, :flow, :line
+
+          TYPES = %w(
+            Test characterize defaults enable-flow-word disable-flow-word error-print goto
+            goto-on-all-done goto-on-all-lastfail goto-on-all-lastfaildoall logprint modify
+            nop print reset set-device set-device-new set-error-bin set-retest-bin skip
+            stop assign-integer create-integer delete-integer create-site-var assign-site-var
+            flag-clear flag-clear-all flag-false flag-false-all flag-true flag-true-all
+            state-clear-all state-false-all state-true-all
+          )
+
+          ATTRS = %w(
+            label enable job part env opcode parameter
+            tname tnum bin_pass bin_fail sort_pass sort_fail result flag_pass
+            flag_fail state group_specifier group_sense group_condition group_name
+            device_sense device_condition device_name debug_assume debug_sites
+            comment
+          )
+
+          ALIASES = {
+            bin:         :bin_fail,
+            softbin:     :sort_fail,
+            soft_bin:    :sort_fail,
+            name:        :tname,
+            number:      :tnum,
+            test_number: :tnum,
+            test_num:    :tnum,
+            type:        :opcode
+          }
+
+          # Make readers for each low level attribute
+          ATTRS.each do |attr|
+            attr_reader attr
+          end
+
+          # And the aliases
+          ALIASES.each do |_alias, attr|
+            define_method("#{_alias}") do
+              send(attr)
+            end
+          end
+
+          # Returns the test instance called by the given line or nil
+          def self.extract_test(line)
+            l = new(line)
+            if l.valid? && l.test?
+              l.test_instance_name
+            end
+          end
+
+          def initialize(line, options = {})
+            @parser = options[:parser]
+            @flow = options[:flow]
+            @line = line
+            parse
+            if valid?
+              ATTRS.each_with_index do |attr, i|
+                instance_variable_set("@#{attr}", components[i + 1])
+              end
+            end
+          end
+
+          def inspect  # :nodoc:
+            "<FlowLine: #{type}, Parameter: #{parameter}>"
+          end
+
+          def description
+            from_instance = test_instance ? test_instance.description : ''
+            from_flow = parser.descriptions.flow_line(name: test_instance_name, flow: flow.file)
+            if !from_instance.empty? && !from_flow.empty?
+              [from_instance, "\n", from_flow].flatten
+            elsif from_instance.empty?
+              from_flow
+            else
+              from_instance
+            end
+          end
+
+          def parse
+            @components = @line.split("\t") unless @line.strip.empty?
+          end
+
+          def valid?
+            components[6] && TYPES.include?(components[6])
+          end
+
+          def components
+            @components ||= []
+          end
+
+          def test?
+            %w(Test characterize).include? opcode
+          end
+
+          def executes_under_context?(context)
+            enable_conditions_met?(context) &&
+              job_conditions_met?(context) &&
+              part_conditions_met?(context) &&
+              env_conditions_met?(context)
+          end
+
+          def enable_conditions_met?(context)
+            conditions_met?(enable, context[:enable])
+          end
+
+          def job_conditions_met?(context)
+            conditions_met?(job, context[:job])
+          end
+
+          def part_conditions_met?(context)
+            conditions_met?(part, context[:part])
+          end
+
+          def env_conditions_met?(context)
+            conditions_met?(env, context[:env])
+          end
+
+          def conditions_met?(conditions, values)
+            if conditions.empty?
+              true
+            else
+              values = [values].flatten
+              conditions = conditions.split(',').map(&:strip)
+              not_conditions = conditions.select { |c| c =~ /^!/ }
+              conditions = conditions - not_conditions
+              # Make sure all -ve conditions are not met
+              if not_conditions.all? do |c|
+                   c =~ /^!(.*)/
+                   c = Regexp.last_match[1]
+                   !values.include?(c)
+                 end
+                # And then any +ve conditions
+                if conditions.empty?
+                  true
+                else
+                  values.any? { |v| conditions.include?(v) }
+                end
+              else
+                false
+              end
+            end
+          end
+
+          def test_instance_name
+            parameter
+          end
+          alias_method :instance_name, :test_instance_name
+
+          def test_instance
+            instances = parser.test_instances.where(name: parameter, exact: true)
+            if instances.size > 1
+              puts "Warning multiple instances of #{name} found, using the first one"
+            end
+            if instances.size == 0
+              nil
+            else
+              instances.first
+            end
+          end
+          alias_method :instance, :test_instance
+
+          # Returns an array of patterns used by the given test, if there are none
+          # an empty array is returned.
+          # Optionally supply patterns to exclude if you want to ignore common subroutine
+          # patterns for example.
+          def patterns(options = {})
+            i = test_instance
+            if i
+              pats = i.patterns
+              if options[:ignore] && pats
+                pats.reject { |p| [options[:ignore]].flatten.include?(p) }
+              else
+                []
+              end
+            else
+              []
+            end
+          end
+          alias_method :pattern, :patterns
+
+          # Returns a string summarizing any conditions (enable words, jobs, etc.) that
+          # gate the execution of this line
+          def conditions
+            c = []
+            c << "Enable: #{enable}" unless enable.empty?
+            c << "Job: #{job}" unless job.empty?
+            c << "Part: #{part}" unless part.empty?
+            c << "Env: #{env}" unless env.empty?
+            c.join('; ')
+          end
+
+          def vdd
+            i = test_instance
+            if i
+              i.vdd
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/origen/tester/ultraflex/parser/flows.rb

@@ -0,0 +1,23 @@
+module Origen
+  module Tester
+    class Ultraflex
+      class Parser
+        class Flows < Origen::Tester::Parser::SearchableArray
+          attr_accessor :parser
+
+          def initialize(options = {})
+            @parser = options[:parser]
+          end
+
+          def import(file)
+            self << Flow.new(file, parser: parser)
+          end
+
+          def inspect
+            "<TestFlows: #{size}>"
+          end
+        end
+      end
+    end
+  end
+end