origen_testers 0.4.0

data/lib/origen_testers/igxl_based_tester/j750_hpt/test_instance_group.rb

@@ -0,0 +1,9 @@
+module OrigenTesters
+  module IGXLBasedTester
+    class J750_HPT
+      require 'origen_testers/igxl_based_tester/j750/test_instance_group'
+      class TestInstanceGroup < J750::TestInstanceGroup
+      end
+    end
+  end
+end

data/lib/origen_testers/igxl_based_tester/j750_hpt/test_instances.rb

@@ -0,0 +1,9 @@
+module OrigenTesters
+  module IGXLBasedTester
+    class J750_HPT
+      require 'origen_testers/igxl_based_tester/j750/test_instances'
+      class TestInstances < J750::TestInstances
+      end
+    end
+  end
+end

data/lib/origen_testers/igxl_based_tester/parser.rb

@@ -0,0 +1,102 @@
+module OrigenTesters
+  class IGXLBasedTester
+    class Parser
+      autoload :Flows,   'origen_testers/igxl_based_tester/parser/flows'
+      autoload :Flow,   'origen_testers/igxl_based_tester/parser/flow'
+      autoload :FlowLine,   'origen_testers/igxl_based_tester/parser/flow_line'
+      autoload :TestInstances,   'origen_testers/igxl_based_tester/parser/test_instances'
+      autoload :TestInstance,   'origen_testers/igxl_based_tester/parser/test_instance'
+      autoload :PatternSets,   'origen_testers/igxl_based_tester/parser/pattern_sets'
+      autoload :PatternSet,   'origen_testers/igxl_based_tester/parser/pattern_set'
+      autoload :DCSpecs,   'origen_testers/igxl_based_tester/parser/dc_specs'
+      autoload :DCSpec,   'origen_testers/igxl_based_tester/parser/dc_spec'
+      autoload :ACSpecs,   'origen_testers/igxl_based_tester/parser/ac_specs'
+      autoload :ACSpec,   'origen_testers/igxl_based_tester/parser/ac_spec'
+      autoload :Descriptions,   'origen_testers/igxl_based_tester/parser/descriptions'
+
+      def reset
+        @flows = nil
+        @test_instances = nil
+        @pattern_sets = nil
+        @dc_specs = nil
+        @ac_specs = nil
+      end
+
+      def descriptions
+        @descriptions ||= Descriptions.new(parser: self)
+      end
+
+      # Returns an array of test flows
+      def flows
+        @flows ||= Flows.new(parser: self)
+      end
+
+      def test_instances
+        @test_instances ||= TestInstances.new(parser: self)
+      end
+      alias_method :instances, :test_instances
+
+      def pattern_sets
+        @pattern_sets ||= PatternSets.new(parser: self)
+      end
+      alias_method :patsets, :pattern_sets
+      alias_method :pat_sets, :pattern_sets
+
+      def dc_specs
+        @dc_specs ||= DCSpecs.new(parser: self)
+      end
+
+      def ac_specs
+        @ac_specs ||= ACSpecs.new(parser: self)
+      end
+
+      def inspect
+        "<Parsed Program: Flows: #{flows.size}>"
+      end
+
+      # Parse a file, array of files, or a directory.
+      #
+      # This can be called multiple times to add new files to the
+      # program model.
+      def parse(file)
+        Origen.log.info ''
+        Origen.log.info "Parsing J750 test program from: #{file}"
+        Origen.log.info ''
+        reset
+        # Note use of local file handler here, this should be how it is
+        # done globally, otherwise we can run into hard to debug problems
+        # due to state/reference dir changes in the single Origen.file_handler
+        Origen::FileHandler.new.resolve_files(file) do |f|
+          parse_file(f)
+        end
+        Origen.log.info ''
+        self
+      end
+
+      def parse_file(file)
+        line = File.readlines(file).first
+        begin
+          if line =~ /Flow Table/
+            flows.import(file)
+          elsif line =~ /Instances/
+            test_instances.import(file)
+          elsif line =~ /Pattern Sets/
+            patsets.import(file)
+          elsif line =~ /DC Spec/
+            dc_specs.import(file)
+          else
+            puts "Skipped (un-supported file type): #{file}"
+          end
+        rescue Exception => e
+          if e.is_a?(ArgumentError) && e.message =~ /invalid byte sequence/
+            puts "Skipped (not ASCII): #{file}"
+          else
+            puts e.message
+            puts e.backtrace
+            exit 1
+          end
+        end
+      end
+    end
+  end
+end

data/lib/origen_testers/igxl_based_tester/parser/ac_spec.rb

@@ -0,0 +1,9 @@
+module OrigenTesters
+  class IGXLBasedTester
+    class Parser
+      class ACSpec
+        attr_accessor :collection
+      end
+    end
+  end
+end

data/lib/origen_testers/igxl_based_tester/parser/dc_spec.rb

@@ -0,0 +1,33 @@
+module OrigenTesters
+  class IGXLBasedTester
+    class Parser
+      class DCSpec
+        attr_accessor :parser
+        attr_accessor :name, :categories
+        alias_method :symbol, :name
+
+        def initialize(name, categories, options = {})
+          @parser = options[:parser]
+          @name = name
+          @categories = categories
+          @values = {}
+        end
+
+        def add_values(components)
+          @categories.each_with_index do |category, i|
+            @values[category] ||= {}
+            @values[category]['Typ'] ||= components[5 + (i * 3) + 0]
+            @values[category]['Min'] ||= components[5 + (i * 3) + 1]
+            @values[category]['Max'] ||= components[5 + (i * 3) + 2]
+          end
+        end
+
+        def lookup(category, selector)
+          if v == @values[category]
+            v[selector]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/origen_testers/igxl_based_tester/parser/dc_specs.rb

@@ -0,0 +1,48 @@
+module OrigenTesters
+  class IGXLBasedTester
+    class Parser
+      class DCSpecs < ::OrigenTesters::Parser::SearchableHash
+        attr_accessor :parser
+
+        def initialize(options = {})
+          @parser = options[:parser]
+        end
+
+        def import(file)
+          @categories = []
+          File.readlines(file).each do |line|
+            unless line.strip.empty?
+              components = line.split("\t")
+              if components[3] == 'Selector'
+                extract_categories(components)
+              else
+                unless components[1] == 'DC Specs' || components[1] == 'Symbol'
+                  extract_spec(components)
+                end
+              end
+            end
+          end
+        end
+
+        def inspect
+          "<DCSpecs: #{size}>"
+        end
+
+        def extract_categories(components)
+          components.each_with_index do |val, i|
+            if i > 4
+              @categories << val unless val.strip.empty?
+            end
+          end
+          @categories.uniq!
+        end
+
+        def extract_spec(components)
+          name = components[1]
+          self[name] ||= DCSpec.new(name, @categories, parser: parser)
+          self[name].add_values(components)
+        end
+      end
+    end
+  end
+end

data/lib/origen_testers/igxl_based_tester/parser/descriptions.rb

@@ -0,0 +1,339 @@
+module OrigenTesters
+  class IGXLBasedTester
+    class Parser
+      # Extracts embedded test and flow descriptions (comments) from test
+      # program source files
+      class Descriptions
+        require 'fileutils'
+
+        attr_accessor :source_directories, :template_directories, :parser
+
+        SCRATCH_DIR = "#{Origen.root}/.j750_scratch"
+
+        # All descriptions are stored in this lookup table
+        def lookup
+          return @lookup if @lookup
+          # Use the one from the interface if present, program generation will
+          # automatically push descriptions in here
+          if Origen.interface_present?
+            @lookup = Origen.interface.descriptions
+          else
+            @lookup = ::OrigenTesters::Parser::DescriptionLookup.new
+          end
+        end
+
+        def initialize(options = {})
+          @parser = options[:parser]
+          FileUtils.rm_rf(SCRATCH_DIR) if File.exist?(SCRATCH_DIR)
+          parse_program
+          true
+        end
+
+        # Returns the description for the given flow
+        def flow_summary(options = {})
+          lookup.for_flow(options[:file])
+        end
+
+        # Returns the description of the given test from the test
+        # instance sheet declaration
+        def test_instance(options = {})
+          lookup.for_test_definition(options[:name])
+        end
+
+        # Returns the description of the given test from the test
+        # flow
+        def flow_line(options = {})
+          lookup.for_test_usage(options[:name], options[:flow])
+        end
+
+        def parse_program
+          Origen.file_handler.preserve_state do
+            generate_program_files
+            # Comments must be extracted manually for any compiled files, for
+            # generated files the comments will already be in the lookup
+            extract_flow_summaries
+            extract_test_instance_descriptions
+            extract_flow_line_descriptions
+          end
+        end
+
+        def source_directories
+          [@source_directories, Origen.config.test_program_source_directory].compact.flatten
+        end
+
+        def template_directories
+          [@template_directories, Origen.config.test_program_template_directory].compact.flatten
+        end
+
+        def extract_flow_summaries
+          Origen.file_handler.resolve_files(compiled_dir) do |file|
+            if flow_file?(file)
+              lookup.add_for_flow(file, parse_flow_summary(file))
+            end
+          end
+        end
+
+        # Parses a compiled template for marked up comments
+        def extract_test_instance_descriptions
+          Origen.file_handler.resolve_files(compiled_dir) do |file|
+            if instance_file?(file)
+              comments = []
+              File.readlines(file).each do |line|
+                if line =~ /^<comment>(.*)/
+                  comments << Regexp.last_match[1].gsub("\r", '')
+                else
+                  fields = line.split("\t")
+                  unless ['Test Instances', '', 'Test Name'].include? fields[1]
+                    lookup.add_for_test_definition(fields[1], comments)
+                  end
+                  comments = []
+                end
+              end
+            end
+          end
+        end
+
+        def extract_flow_line_descriptions
+          Origen.file_handler.resolve_files(compiled_dir) do |file|
+            if flow_file?(file)
+              f = file.basename('.txt').to_s
+              comments = []
+              header_line = true
+              File.readlines(file).each do |line|
+                if header_line
+                  header_line = false if line =~ /^\s*Label/
+                else
+                  if line =~ /^<comment>(.*)/
+                    comments << Regexp.last_match[1].gsub("\r", '')
+                  else
+                    if t = FlowLine.extract_test(line)
+                      lookup.add_for_test_usage(t, file, comments)
+                    end
+                    comments = []
+                  end
+                end
+              end
+            end
+          end
+        end
+
+        def generate_program_files
+          a = generate_program
+          b = compile_program
+          unless a || b
+            fail 'No source or template files declared from which to parse descriptions!'
+          end
+        end
+
+        # Parses the given flow file for summary text and returns it, summary
+        # text must be the very first thing in the file.
+        # Returns an array of strings each representing a line of text.
+        def parse_flow_summary(file)
+          desc = []
+          File.readlines(file).each do |line|
+            if line =~ /%?\s*<comment>(.*)/
+              desc << Regexp.last_match[1].gsub("\r", '')
+            else
+              break
+            end
+          end
+          desc
+        end
+
+        # Generate a scratch version of the program for parsing
+        def generate_program
+          if source_directories.size > 0
+            unless @program_generated
+              Origen.log.info ''
+              Origen.log.info 'Extracting embedded comments:'
+              Origen.log.info ''
+              copy_source_files_to_scratch
+              markup_source_file_comments
+              # Compile the flow file, with Ruby comments now preserved and marked up
+              desc = Origen.app.runner.generate(program: true, patterns: ungenerated_dir, output: generated_dir,
+                                              check_for_changes: false, collect_stats: false, quiet: true,
+                                              collect_descriptions: true)
+              Origen.log.info ''
+            end
+            @program_generated = true
+          else
+            false
+          end
+        end
+
+        # Compile a scratch version of the program for parsing
+        def compile_program
+          if template_directories.size > 0
+            unless @program_compiled
+              Origen.log.info ''
+              Origen.log.info 'Extracting embedded comments:'
+              Origen.log.info ''
+              copy_templates_to_scratch
+              markup_template_comments
+              # Compile the flow file, with Ruby comments now preserved and marked up
+              Origen.app.runner.generate(compile: true, patterns: uncompiled_dir, output: compiled_dir,
+                                       check_for_changes: false, collect_stats: false, quiet: true)
+              Origen.log.info ''
+            end
+            @program_compiled = true
+          else
+            false
+          end
+        end
+
+        # Copy all flow and instance template files to the scratch dir
+        def copy_templates_to_scratch
+          uncompiled_dir(true)
+          template_directories.each do |dir|
+            Origen.file_handler.resolve_files(dir) do |file|
+              subdir = file.relative_path_from(Pathname.new(dir)).dirname.to_s
+              cpydir = "#{uncompiled_dir}/#{subdir}"
+              FileUtils.mkdir_p(cpydir) unless File.exist?(cpydir)
+              FileUtils.copy(file, cpydir) if flow_or_instance_file?(file)
+            end
+          end
+          `chmod -R 777 #{uncompiled_dir}/*` unless Dir["#{uncompiled_dir}/*"].empty?
+        end
+
+        # Copy all flow and instance source files to the scratch dir
+        def copy_source_files_to_scratch
+          source_directories.each do |dir|
+            Origen.file_handler.resolve_files(dir) do |file|
+              subdir = file.relative_path_from(Pathname.new(dir)).dirname.to_s
+              cpydir = "#{ungenerated_dir}/#{subdir}"
+              FileUtils.mkdir_p(cpydir) unless File.exist?(cpydir)
+              FileUtils.copy(file, cpydir)
+            end
+          end
+        end
+
+        def uncompiled_dir(force_make = false)
+          @uncompiled_dir ||= "#{SCRATCH_DIR}/uncompiled"
+          if force_make
+            FileUtils.rm_rf(@uncompiled_dir) if File.exist?(@uncompiled_dir)
+            @uncompiled_dir_created = false
+          end
+          unless @uncompiled_dir_created
+            FileUtils.mkdir_p(@uncompiled_dir) unless File.exist?(@uncompiled_dir)
+            @uncompiled_dir_created = true
+          end
+          @uncompiled_dir
+        end
+
+        def ungenerated_dir
+          @ungenerated_dir ||= "#{SCRATCH_DIR}/ungenerated"
+          unless @ungenerated_dir_created
+            FileUtils.mkdir_p(@ungenerated_dir) unless File.exist?(@ungenerated_dir)
+            @ungenerated_dir_created = true
+          end
+          @ungenerated_dir
+        end
+
+        def compiled_dir
+          @compiled_dir ||= "#{SCRATCH_DIR}/compiled"
+          unless @compiled_dir_created
+            FileUtils.mkdir_p(@compiled_dir) unless File.exist?(@compiled_dir)
+            @compiled_dir_created = true
+          end
+          @compiled_dir
+        end
+
+        def generated_dir
+          @generated_dir ||= "#{SCRATCH_DIR}/generated"
+          unless @generated_dir_created
+            FileUtils.mkdir_p(@generated_dir) unless File.exist?(@generated_dir)
+            @generated_dir_created = true
+          end
+          @generated_dir
+        end
+
+        # Returns true if the given file looks like a J750 flow file, works for
+        # templates to
+        def flow_or_instance_file?(file, options = {})
+          options = { flow:     true,
+                      instance: true
+          }.merge(options)
+          if options[:flow] && options[:instance]
+            match = 'Flow|Instances'
+          elsif options[:flow]
+            match = 'Flow'
+          else
+            match = 'Instances'
+          end
+          # Not sure the best way to determine the file type of a partial, just
+          # return true for now to play it safe
+          return true if file.basename.to_s =~ /^_/
+          File.readlines(file).each do |line|
+            begin
+              unless line =~ /^%/ || line =~ /^\s*<comment>/
+                return !!(line =~ /#{match}/)
+              end
+            rescue Exception => e
+              if e.is_a?(ArgumentError) && e.message =~ /invalid byte sequence/
+                return false
+              else
+                puts e.message
+                puts e.backtrace
+                exit 1
+              end
+            end
+          end
+        end
+
+        def flow_file?(file)
+          flow_or_instance_file?(file, instance: false)
+        end
+
+        def instance_file?(file)
+          flow_or_instance_file?(file, flow: false)
+        end
+
+        # Substitute Ruby line comments so they are preserved by compilation
+        def markup_template_comments
+          Origen.file_handler.resolve_files(uncompiled_dir) do |file|
+            lines = File.readlines(file)
+            File.open(file, 'w') do |f|
+              lines.each do |line|
+                if line =~ /^%\s*#\s?(.*)/ # Remove single leading whitespace from comment if it exists
+                  comment = Regexp.last_match[1]
+                  # If comment starts with a '#-' it should be removed by compilation
+                  if line =~ /^%\s*#-.*/
+                    f.write line
+                  # Otherwise preserve it
+                  else
+                    f.write "<comment>#{comment}\n"
+                  end
+                else
+                  f.write line
+                end
+              end
+            end
+          end
+        end
+
+        # Substitute Ruby line comments so they are preserved by generation
+        def markup_source_file_comments
+          Origen.file_handler.resolve_files(ungenerated_dir) do |file|
+            lines = File.readlines(file)
+            File.open(file, 'w') do |f|
+              lines.each do |line|
+                if line =~ /^\s*#\s?(.*)/ # Remove single leading whitespace from comment if it exists
+                  comment = Regexp.last_match[1]
+                  # If comment starts with a '#-' it should be removed by generation
+                  if line =~ /^\s*#-.*/
+                    f.write line
+                  # Otherwise preserve it
+                  else
+                    f.write "Origen.interface.comment '#{comment}'\n"
+                  end
+                else
+                  f.write line
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end