origen 0.34.3 → 0.52.0

data/lib/origen/site_config.rb

@@ -1,13 +1,21 @@
 module Origen
   class SiteConfig
+    # NOTE: Gems are not allowed to be required here, only Ruby stlibs
     require 'pathname'
     require 'yaml'
     require 'etc'
+    require 'erb'
+    require_relative 'site_config/config'
+
+    # Require some useful methods from Origen without requiring the entire module
+    require_relative 'boot/api'
 
     TRUE_VALUES = ['true', 'TRUE', '1', 1]
     FALSE_VALUES = ['false', 'FALSE', '0', 0]
 
-    # Define a couple of site configs variables that need a bit of processing
+    # Adding parameters to this array will prevent them from being converted to booleans if
+    # they are assigned one of the values in the TRUE_VALUES/FALSE_VALUES arrays
+    NON_BOOLEAN_PARAMETERS = [:lsf_cores, :centralized_site_config_refresh]
 
     # Gets the gem_intall_dir. This is either site_config.home_dir/gems or the site configs gem_install_dir
     def gem_install_dir
@@ -72,11 +80,15 @@ module Origen
       path
     end
 
+    def centralized_site_config_cache_dir
+      File.expand_path(find_val('centralized_site_config_cache_dir'))
+    end
+
     # Dynamically remove the highest instance of :var
     def remove_highest(var)
-      @configs.each do |c|
-        if c.key?(var)
-          return c.delete(var)
+      configs.each do |c|
+        if c.has_var?(var)
+          return c.remove_var(var)
         end
       end
 
@@ -91,9 +103,9 @@ module Origen
       # from lowest priority to highest.
       # If [] is returned, it implies that there was no instancs of :var to be removed.
       ret = []
-      @configs.each do |c|
-        if c.key?(var)
-          ret << c.delete(var)
+      configs.each do |c|
+        if c.has_var?(var)
+          ret << c.remove_var(var)
         end
       end
       ret
@@ -104,7 +116,8 @@ module Origen
     def add_as_highest(var, value)
       # Don't want to override anything, so just shift in a dummy site config instance at the highest level and
       # set the value there.
-      configs.prepend(var.to_s => value)
+      c = Config.new(path: :runtime, parent: self, values: { var.to_s => value })
+      configs.prepend(Config.new(path: :runtime, parent: self, values: { var.to_s => value }))
     end
     alias_method :[]=, :add_as_highest
 
@@ -113,17 +126,19 @@ module Origen
     def add_as_lowest(var, value)
       # Don't want to override anything, so just shift in a dummy site config at the lowest level and
       # set the value there.
-      configs.append(var.to_s => value)
+      configs.append(Config.new(path: :runtime, parent: self, values: { var.to_s => value }))
     end
 
     # Adds a new site config file as the highest priority
     def add_site_config_as_highest(site_config_file)
-      configs.prepend YAML.load_file(File.expand_path('../../../origen_site_config.yml', __FILE__))
+      # configs.prepend YAML.load_file(File.expand_path('../../../origen_site_config.yml', __FILE__))
+      configs.prepend(Config.new(path: site_config_file, parent: self))
     end
 
     # Adds a new site config file as the highest priority
     def add_site_config_as_lowest(site_config_file)
-      configs.append YAML.load_file(File.expand_path('../../../origen_site_config.yml', __FILE__))
+      # configs.append YAML.load_file(File.expand_path('../../../origen_site_config.yml', __FILE__))
+      configs.append(Config.new(path: site_config_file, parent: self))
     end
 
     def method_missing(method, *args, &block)
@@ -144,15 +159,10 @@ module Origen
       val
     end
 
-    def get(val)
-      find_val(val)
-    end
-    alias_method :[], :get
-
     def get_all(val)
       ret = []
-      @configs.each do |c|
-        if c.key?(val)
+      configs.each do |c|
+        if c.has_var?(val)
           ret << c[val]
         end
       end
@@ -160,86 +170,267 @@ module Origen
     end
 
     def clear
-      @configs.clear
+      configs.clear
     end
 
     def rebuild!
       configs!
     end
 
-    private
+    def refresh
+      configs.each(&:refresh)
+    end
+
+    def pretty_print_configs
+      puts 'The following config files are ordered from last-encountered (highest priority) first-encountered (lowest priority)'
+      puts
+      configs.each_with_index do |config, i|
+        puts "#{i}: #{config.path} (#{config.type})"
+      end
+    end
+    alias_method :pp_configs, :pretty_print_configs
+
+    def all_vars
+      vars = {}
+      configs.each do |c|
+        vars = c.values.merge(vars)
+      end
+      vars
+    end
+    alias_method :env, :all_vars
+
+    # Gets all config variables as a hash, but the hash's values are the Config instances which defines the highest
+    # priority of each var, instead of the var's value itself.
+    def vars_by_configs
+      vars = {}
+      configs.each do |c|
+        vars = c.values.map { |k, v| [k, c] }.to_h.merge(vars)
+      end
+      vars
+    end
+    alias_method :vars_by_config, :vars_by_configs
+
+    def pretty_print_env(*vars)
+      puts
+      spacing = ' ' * 2
+      r = vars.empty? ? nil : Regexp.union(vars.map { |v| Regexp.new(v) })
+      all_vars.each do |var, val|
+        if !r.nil? && !(var.match r)
+          next
+        end
+
+        if val.is_a?(Array)
+          puts "#{var}: ["
+          val.each { |v| puts "#{spacing} #{v}" }
+          puts ']'
+        elsif val.is_a?(Hash)
+          puts "#{var}: {"
+          val.each { |v| puts "#{spacing} #{v}" }
+          puts '}'
+        else
+          puts "#{var}: #{val}"
+        end
+      end
+      puts
+    end
+    alias_method :pp_env, :pretty_print_env
+
+    def to_env(val)
+      "ORIGEN_#{val.upcase}"
+    end
+
+    def env_contains?(val)
+      ENV.key?(val)
+    end
+
+    def env(val)
+      if env_contains?(val)
+        ENV[val]
+      end
+    end
+
+    def inspect_variable(*vars)
+      vars.each do |var|
+        puts "Inspecting Variable: #{var}"
+        if env_contains?(to_env(var))
+          puts "Environment Variable (#{to_env(var)}): #{env(to_env(var))}"
+        else
+          puts "(No enviornment variable #{to_env(var)} defined)"
+        end
+        configs.each do |c|
+          if c.has_var?(var)
+            puts "#{c.path} (#{c.type}): #{c[var]}"
+          end
+        end
+        puts
+      end
+    end
+    alias_method :inspect_var, :inspect_variable
+    alias_method :inspect_variables, :inspect_variable
+    alias_method :inspect_vars, :inspect_variable
+
+    # Inspects the config(s) at the incex given.
+    def inspect_configs(*config_indexes)
+      config_indexes.each do |i|
+        if i.to_i > configs.size
+          puts red("Origen::SiteConfig: index #{i} is out of range of the available configs! Total configs: #{configs.size}.")
+        elsif i.to_i < 0
+          puts red("Origen::SiteConfig: index #{i} is less than 0. This index is ignored.")
+        else
+          c = configs[i.to_i]
+          puts "Inspecting config \##{i}"
+          puts "Type: #{c.type}"
+          puts "Path: #{c.path}"
+          if c.centralized?
+            # Add a safeguard in case something happened being bootup and now and the cache is no longer present
+            puts "Cached At: #{c.cache_file}" if c.cache_file
+            puts "Cached On: #{c.cache_file.ctime}" if c.cache_file
+          end
+
+          puts
+          puts 'Values from this config:'
+          spacing = ' ' * 2
+          c.values.each do |var, val|
+            if val.is_a?(Array)
+              puts "#{var}: ["
+              val.each { |v| puts "#{spacing} #{v}" }
+              puts ']'
+            elsif val.is_a?(Hash)
+              puts "#{var}: {"
+              val.each { |v| puts "#{spacing} #{v}" }
+              puts '}'
+            else
+              puts "#{var}: #{val}"
+            end
+          end
+
+          puts
+          puts 'Active (highest-level) values from this config:'
+          spacing = ' ' * 2
+          vars_by_config.select { |k, v| v == c }.map { |k, v| [k, v[k]] }.to_h.each do |var, val|
+            if val.is_a?(Array)
+              puts "#{var}: ["
+              val.each { |v| puts "#{spacing} #{v}" }
+              puts ']'
+            elsif val.is_a?(Hash)
+              puts "#{var}: {"
+              val.each { |v| puts "#{spacing} #{v}" }
+              puts '}'
+            else
+              puts "#{var}: #{val}"
+            end
+          end
+          puts
+        end
+      end
+    end
+    alias_method :inspect_config, :inspect_configs
 
     def find_val(val, options = {})
       env = "ORIGEN_#{val.upcase}"
       if ENV.key?(env)
         value = ENV[env]
-        value
       else
-        config = configs.find { |c| c.key?(val) }
-        value = config ? config[val] : nil
+        config = configs.find { |c| c.has_var?(val) }
+        value = config ? config.find_val(val) : nil
       end
 
-      if TRUE_VALUES.include?(value)
-        return true
-      elsif FALSE_VALUES.include?(value)
-        return false
+      unless NON_BOOLEAN_PARAMETERS.include?(val.to_s.downcase.to_sym)
+        if TRUE_VALUES.include?(value)
+          return true
+        elsif FALSE_VALUES.include?(value)
+          return false
+        end
       end
       value
     end
+    alias_method :get, :find_val
+    alias_method :[], :find_val
+
+    private
+
+    def red(message)
+      "\e[0;31;49m#{message}\e[0m"
+    end
 
     def configs
       @configs ||= configs!
     end
 
+    # Searches a directory and returns an array of config objects (from lowest to highest priority) that were found
+    # @note This includes searching for <code>./config/</code> in <code>dir</code>. In other words, this searches both
+    #  <code>dir</code> and <code>dir/config</code>.
+    def load_directory(dir, prepend: false)
+      [
+        File.join(dir, 'config', 'origen_site_config.yml'),
+        File.join(dir, 'config', 'origen_site_config.yml.erb'),
+        File.join(dir, 'origen_site_config.yml'),
+        File.join(dir, 'origen_site_config.yml.erb')
+      ].each do |f|
+        if File.exist?(f)
+          if prepend
+            configs.unshift(Config.new(path: f, parent: self))
+          else
+            configs << Config.new(path: f, parent: self)
+          end
+        end
+      end
+    end
+
     # Forces a reparse of the site configs.
-    # This will set the @configs along the current path first,
+    # This will set the configs along the current path first,
     # then, using those values, will add a site config at the home directory.
     def configs!
-      @configs = begin
-        # This global is set when Origen is first required, it generally means that what is considered
-        # to be the pwd for the purposes of looking for a site_config file is the place from where the
-        # user invoked Origen. Otherwise if the running app switches the PWD it can lead to confusing
-        # behavior - this was a particular problem when testing the new app generator which switches the
-        # pwd to /tmp to build the new app
-        path = $_origen_invocation_pwd
-        configs = []
-        # Add any site_configs from where we are currently running from, i.e. the application
-        # directory area
-        until path.root?
-          file = File.join(path, 'config', 'origen_site_config.yml')
-          configs << YAML.load_file(file) if File.exist?(file) && YAML.load_file(file)
-          file = File.join(path, 'origen_site_config.yml')
-          configs << YAML.load_file(file) if File.exist?(file) && YAML.load_file(file)
-          path = path.parent
-        end
-
-        # Add and any site_configs from the directory hierarchy where Ruby is installed
-        path = Pathname.new($LOAD_PATH.last)
-        until path.root?
-          file = File.join(path, 'origen_site_config.yml')
-          configs << YAML.load_file(file) if File.exist?(file) && YAML.load_file(file)
-          path = path.parent
-        end
+      # This global is set when Origen is first required, it generally means that what is considered
+      # to be the pwd for the purposes of looking for a site_config file is the place from where the
+      # user invoked Origen. Otherwise if the running app switches the PWD it can lead to confusing
+      # behavior - this was a particular problem when testing the new app generator which switches the
+      # pwd to /tmp to build the new app
+      path = $_origen_invocation_pwd
+      @configs = []
+
+      # Add any site_configs from where we are currently running from, i.e. the application
+      # directory area
+      until path.root?
+        load_directory(path)
+        path = path.parent
+      end
 
-        # Add the one from the Origen core as the lowest priority, this one defines
-        # the default values
-        configs << YAML.load_file(File.expand_path('../../../origen_site_config.yml', __FILE__))
-        configs
+      # Add and any site_configs from the directory hierarchy where Ruby is installed
+      path = Pathname.new($LOAD_PATH.last)
+      until path.root?
+        load_directory(path)
+        path = path.parent
       end
 
+      # Add the one from the Origen core as the lowest priority, this one defines
+      # the default values
+      load_directory(File.expand_path('../../../', __FILE__))
+
       # Add the site_config from the user's home directory as highest priority, if it exists
       # But, make sure we take the site installation's setup into account.
       # That is, if user's home directories are somewhere else, make sure we use that directory to the find
       # the user's overwrite file. The user can then override that if they want."
-      user_config = File.join(File.expand_path(user_install_dir), 'origen_site_config.yml')
-      if File.exist?(user_config)
-        @configs.unshift(YAML.load_file(user_config)) if YAML.load_file(user_config)
+      load_directory(File.expand_path(user_install_dir), prepend: true)
+
+      # Load any centralized site configs now.
+      centralized_site_config = find_val('centralized_site_config')
+      if centralized_site_config
+        # We know the last two site configs will exists (they are in Origen core) and that they contain the default
+        # values. We want the centralized config to load right after those.
+        @configs.insert(-3, Config.new(path: centralized_site_config, parent: self))
       end
 
+      # After all configs have been populated, see if the centralized needs refreshing
+      @configs.each { |c| c.refresh if c.needs_refresh? }
+
       @configs
     end
   end
 
+  # Blow this cache whenever this file is re-loaded
+  @site_config = nil
+
   def self.site_config
     @site_config ||= SiteConfig.new
   end

data/lib/origen/site_config/config.rb

@@ -0,0 +1,217 @@
+module Origen
+  class SiteConfig
+    class Config
+      attr_reader :path
+      attr_reader :parent
+      attr_reader :type
+      attr_reader :values
+
+      RESTRICTED_FROM_CENTRALIZED_VARIABLES = %w(centralized_site_config centralized_site_config_cache_dir centralized_site_config_verify_ssl)
+
+      def initialize(path:, parent:, values: nil)
+        @parent = parent
+        if path == :runtime
+          path = "runtime_#{object_id}"
+          @type = :runtime
+        elsif path.start_with?('http')
+          @path = path
+          @type = :centralized
+        else
+          @path = path
+          @type = :local
+        end
+        @contains_centralized = false
+        @loaded = false
+
+        if values
+          @values = values
+          @loaded = true
+        else
+          @values = nil
+          load
+        end
+      end
+
+      def needs_refresh?
+        if centralized?
+          if refresh_time < 0
+            false
+          elsif cached?
+            # If the refresh time is 0, this will always be true
+            # Note the difference of time objects below will give the difference in seconds.
+            (Time.now - cached_file.ctime) / 3600.0 > refresh_time
+          else
+            # If the cached file cannot be found, force a new fetch
+            true
+          end
+        else
+          false
+        end
+      end
+
+      def refresh_time
+        parent.find_val('centralized_site_config_refresh')
+      end
+
+      def cached_file
+        @cached_file ||= Pathname(parent.centralized_site_config_cache_dir).join('cached_config')
+      end
+
+      def cached?
+        File.exist?(cached_file)
+      end
+
+      def fetch
+        def inform_user_of_cached_file
+          if cached?
+            puts yellow('Origen: Site Config: Found previously cached site config. Using the older site config.')
+          else
+            puts yellow('Origen: Site Config: No cached file found. An empty site config will be used in its place.')
+          end
+          puts
+        end
+
+        if centralized?
+          puts "Pulling centralized site config from: #{path}"
+
+          begin
+            # TODO: needs to be replaced with a net/http equivalent, can't use gems here. The reference
+            # to HTTParty will raise an error until that is done, but it will be handled gracefully below.
+            text = HTTParty.get(path, verify: parent.find_val('centralized_site_config_verify_ssl'))
+            puts "Caching centralized site config to: #{cached_file}"
+
+            unless Dir.exist?(cached_file.dirname)
+              FileUtils.mkdir_p(cached_file.dirname)
+            end
+            File.open(cached_file, 'w+') do |f|
+              f.write(text)
+            end
+
+          rescue SocketError => e
+            puts red("Origen: Site Config: Unable to connect to #{path}")
+            puts red('Origen: Site Config: Failed to retrieve centralized site config!')
+            puts red("Error from exception: #{e.message}")
+
+            inform_user_of_cached_file
+          rescue OpenSSL::SSL::SSLError => e
+            puts red("Origen: Site Config: Unable to connect to #{path}")
+            puts red('Origen: Site Config: Failed to retrieve centralized site config!')
+            puts red("Error from exception: #{e.message}")
+            puts red('It looks like the error is related to SSL certification. If this is a trusted server, you can use')
+            puts red("the site config setting 'centralized_site_config_verify_ssl' to disable verifying the SSL certificate.")
+
+            inform_user_of_cached_file
+          rescue Exception => e
+            # Rescue anything else to avoid any un-caught exceptions causing Origen not to boot.
+            # Print lots of red so that the users are aware that there's a problem, but don't ultimately want this
+            # to render Origen un-bootable
+            puts red("Origen: Site Config: Unexpected exception ocurred trying to either retrieve or cache the site config at #{path}")
+            puts red('Origen: Site Config: Failed to retrieve centralized site config!')
+            puts red("Class of exception:   #{e.class}")
+            puts red("Error from exception: #{e.message}")
+
+            inform_user_of_cached_file
+          end
+          text
+        end
+      end
+      alias_method :refresh, :fetch
+
+      # Loads the site config into memory.
+      # Process the site config as an ERB, if indicated to do so (.erb file extension)
+      # After the initial load, any centralized site configs will be retreived (if needed), cached, and loaded.
+      def load
+        def read_erb(erb)
+          ERB.new(File.read(erb), 0, '%<>')
+        end
+
+        if centralized?
+          if !cached?
+            if fetch
+              erb = read_erb(cached_file)
+            else
+              # There was a problem fetching the config. Just use an empty string.
+              # Warning message will come from #fetch
+              erb = ERB.new('')
+            end
+          else
+            erb = read_erb(cached_file)
+          end
+
+          @values = (YAML.load(erb.result) || {})
+        else
+          if File.extname(path) == '.erb'
+            erb = read_erb(path)
+            @values = (YAML.load(erb.result) || {})
+          else
+            @values = (YAML.load_file(path) || {})
+          end
+        end
+
+        unless @values.is_a?(Hash)
+          puts red("Origen: Site Config: The config at #{path} was not parsed as a Hash, but as a #{@values.class}")
+          puts red('                     Please review the format of the this file.')
+          puts red('                     Alternatively, an error condition may have been received from the server.')
+          puts red("                     This site config is available at: #{cached_file}")
+          puts red('                     This config will not be loaded and will be replaced with an empty config.')
+          puts
+          @values = {}
+        end
+
+        if centralized?
+          # check for restricted centralized config values
+          RESTRICTED_FROM_CENTRALIZED_VARIABLES.each do |var|
+            if @values.key?(var)
+              val = @values.delete(var)
+              puts red("Origen: Site Config: config variable #{var} is not allowed in the centralized site config and will be removed.")
+              puts red("                     Value #{val} will not be applied!")
+            end
+          end
+        end
+
+        @loaded = true
+        @values
+      end
+
+      def remove_var(var)
+        @values.delete(var)
+      end
+
+      def has_var?(var)
+        @values.key?(var)
+      end
+
+      # Finds the value from this config, or from one of its centralized configs (if applicable)
+      def find_val(val)
+        @values[val]
+      end
+      alias_method :[], :find_val
+
+      def loaded?
+        @loaded
+      end
+
+      def local?
+        type == :local
+      end
+
+      def centralized?
+        type == :centralized
+      end
+
+      def runtime?
+        type == :runtime
+      end
+
+      private
+
+      def red(message)
+        "\e[0;31;49m#{message}\e[0m"
+      end
+
+      def yellow(message)
+        "\e[0;33;49m#{message}\e[0m"
+      end
+    end
+  end
+end