bcome 1.4.0 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7ace465761e84db321e1de7eea860a3ade94ecda5d5e091e8784740b1cb07ea4
-  data.tar.gz: 996329541843570edfef238c8a1dd46830402fd04586530b66810478b303b59c
+  metadata.gz: 6487e49ca4d5c08aa0ffa0debb3fa7451db2a2e5cafa03c4962e40bf8790f3dc
+  data.tar.gz: 8a0878e1514dd2207939474d30998e4ac777de4ef68da8eff0570fe776d54602
 SHA512:
-  metadata.gz: db096f8fdde0c83e57519c5d09089d3d8e2bf26a4e4f07c675cf18aafb42a279ca7788114378dd7737e84f4279bbc09e141a1612669de6bea792370bd419fffd
-  data.tar.gz: 4ed639e3378da6b8cefef9727c640d8462ff3fe01f35f63d30b28ef4cd3c50c8975cf7226f4b74f789068223238e860dd7f8300cc4861b18f25b6dec4149b9a7
+  metadata.gz: fccfbb6428abfc99e58865369574f04860833015d7dbe4062f93533dcf3b8362e051603371db25b396012ee94c51e565d0666c8766bd139b52577897da18e3ff
+  data.tar.gz: 920284c1ba38dfeb1c688fb78d262f7d5762a3350aa61fa6145045cde303599e391fe60076f2207bf2d90ccabafc38c7c0e38cda9a99dcc65bfad309ab37ebe1

data/lib/bcome.rb

@@ -7,6 +7,7 @@ require 'pmap'
 require 'singleton'
 require 'require_all'
 require 'tty-cursor'
+require 'strings-ansi'
 require 'pry'
 
 require_all "#{File.dirname(__FILE__)}/../patches"
@@ -15,3 +16,9 @@ require_all "#{File.dirname(__FILE__)}/../lib/objects"
 # Load in any user defined orchestration
 path_to_bcome_orchestration_configs = "#{Dir.getwd}/bcome/orchestration"
 require_all path_to_bcome_orchestration_configs if File.directory?(path_to_bcome_orchestration_configs)
+
+# Load in any patches
+# Note: previously patches could have (and were placed) anywhere in the orchestration directory above. This just makes it more explicit, and keeps
+# things tidier.
+path_to_user_monkey_patches = "#{Dir.getwd}/bcome/patches"
+require_all path_to_user_monkey_patches if File.directory?(path_to_user_monkey_patches)

data/lib/objects/bcome/version.rb

@@ -7,15 +7,15 @@ module Bcome
     end
 
     def self.release
-      '1.4.0'
+      '2.0.0'
     end
 
     def self.release_name
-      'multicloud'
+      'Multicloud & Hybrid'
     end
 
     def self.display
-      "#{name} #{release} #{release_name}"
+      "#{name} v#{release} - #{release_name}"
     end
   end
 end

data/lib/objects/bootup.rb

@@ -29,8 +29,7 @@ module Bcome
 
     def init_context(context)
       if @spawn_into_console
-        puts "\n\n"
-        ::Bcome::Workspace.instance.set(context: context)
+        ::Bcome::Workspace.instance.set(context: context, show_welcome: true)
       else
         context
       end
@@ -44,11 +43,13 @@ module Bcome
         starting_context.load_nodes if starting_context.inventory? && !starting_context.nodes_loaded?
 
         # Attempt to load our next context resource
-        next_context = starting_context.resource_for_identifier(crumb)
+        next_context = starting_context.resources.active.first if crumb == 'first'
+        next_context ||= starting_context.resource_for_identifier(crumb)
 
         # Our current breadcrumb is not a node, and so we'll attempt to invoke a method call on the previous
         # e.g. given resource:foo, then invoke 'foo' on 'resource'
         unless next_context
+          puts "\n" # clean any trailing loading bars
           starting_context.invoke(crumb, @arguments)
           return
         end

data/lib/objects/driver/base.rb

@@ -5,7 +5,7 @@ module Bcome::Driver
     class << self
       def create_from_config(config, node)
         raise Bcome::Exception::InvalidNetworkDriverType, 'Your network configurtion is invalid' unless config.is_a?(Hash)
-        raise Bcome::Exception::InvalidNetworkDriverType, "Missing config parameter 'type'" unless config[:type]
+        raise Bcome::Exception::InvalidNetworkDriverType, "Missing config parameter 'type' for namespace '#{config.inspect}'" unless config[:type]
 
         config_klass_key = config[:type].to_sym
         driver_klass = klass_for_type[config_klass_key]
@@ -17,7 +17,6 @@ module Bcome::Driver
 
       def klass_for_type
         {
-          static: ::Bcome::Driver::Static,
           ec2: ::Bcome::Driver::Ec2,
           gcp: ::Bcome::Driver::Gcp
         }
@@ -58,5 +57,20 @@ module Bcome::Driver
     def config
       @params
     end
+
+    ## Spoof-fetch. Used with the network-socket linkup POC.
+    def spoof_fetch_server_list(monkey_patched_inventory)
+      if @node.nodes_loaded?
+        monkey_patched_inventory.set_static_servers
+      else
+        wrap_indicator type: :basic, title: loader_title, completed_title: loader_completed_title do
+          fake_delay_milliseconds = rand(1..400).to_f / 1000
+          sleep fake_delay_milliseconds
+          monkey_patched_inventory.set_static_servers
+          signal_success
+        end
+        @node.nodes_loaded!
+      end
+    end
   end
 end

data/lib/objects/driver/ec2.rb

@@ -4,11 +4,14 @@ require 'fog/aws'
 
 module Bcome::Driver
   class Ec2 < Bcome::Driver::Base
-    PATH_TO_FOG_CREDENTIALS = "#{ENV['HOME']}/.fog"
+    PATH_TO_FOG_CREDENTIALS = '.aws/keys'
 
     def initialize(*params)
       super
       raise Bcome::Exception::Ec2DriverMissingProvisioningRegion, params.inspect unless provisioning_region
+      raise ::Bcome::Exception::Ec2DriverMissingAuthorizationKeys, PATH_TO_FOG_CREDENTIALS unless File.exist?(PATH_TO_FOG_CREDENTIALS)
+
+      ENV['FOG_RC'] = PATH_TO_FOG_CREDENTIALS
     end
 
     def pretty_provider_name
@@ -23,7 +26,12 @@ module Bcome::Driver
       @fog_client ||= get_fog_client
     end
 
-    def fetch_server_list(filters)
+    def fetch_server_list(legacy_ec2_filters)
+      # Filters should be defined within a namespace's :network element. Pre 2.0 the expection for AWS was
+      # to define filters at the root level of the namespace. Here we move :filters into :network, yet retain
+      # ec2_filters at the root level for backwards compaibility with pre 2.0 versions.
+      filters = config.key?(:filters) ? config[:filters] : legacy_ec2_filters
+
       wrap_indicator type: :basic, title: loader_title, completed_title: loader_completed_title do
         begin
           @servers = unfiltered_server_list.all(filters)
@@ -72,6 +80,7 @@ module Bcome::Driver
 
     def get_fog_client
       ::Fog.credential = credentials_key
+
       client = ::Fog::Compute.new(
         provider: 'AWS',
         region: provisioning_region

data/lib/objects/driver/gcp.rb

@@ -39,8 +39,16 @@ module Bcome::Driver
     end
 
     def do_fetch_server_list(_filters)
-      gcp_service.list_instances(@params[:project], @params[:zone])
-    rescue Google::Apis::AuthorizationError
+      # Network filter key now called :filter. retained :list_filter for backwards compatibility.
+      # Fallback is ""
+      filters = (
+        @params[:filters] || (
+          @params[:list_filter] || ''
+        )
+      )
+
+      gcp_service.list_instances(@params[:project], @params[:zone], filter: filters)
+    rescue Google::Apis::AuthorizationError => e
       raise ::Bcome::Exception::CannotAuthenticateToGcp
     rescue Google::Apis::ClientError => e
       raise ::Bcome::Exception::Generic, "Namespace #{@node.namespace} / #{e.message}"
@@ -77,8 +85,8 @@ module Bcome::Driver
     def auth_schemes
       {
         oauth: ::Bcome::Driver::Gcp::Authentication::Oauth,
-        serviceaccount: ::Bcome::Driver::Gcp::Authentication::ServiceAccount,
-        api_key: ::Bcome::Driver::Gcp::Authentication::ApiKey
+        service_account: ::Bcome::Driver::Gcp::Authentication::ServiceAccount
+        # api_key: ::Bcome::Driver::Gcp::Authentication::ApiKey
       }
     end
 
@@ -94,7 +102,43 @@ module Bcome::Driver
     def authentication_scheme
       # Service scopes are specified directly from the network config
       # A minumum scope of https://www.googleapis.com/auth/compute.readonly is required in order to list resources.
-      @authentication_scheme ||= auth_scheme.new(self, compute_service, service_scopes, @node, @params[:secrets_path])
+
+      auth_scheme_key = @params[:authentication_scheme].to_sym
+      auth_scheme = auth_schemes[auth_scheme_key]
+      raise ::Bcome::Exception::InvalidGcpAuthenticationScheme, "Invalid GCP authentication scheme '#{auth_scheme_key}' for node #{@node.namespace}" unless auth_scheme
+
+      case auth_scheme_key
+      when :oauth
+
+        client_config = ::Bcome::Driver::Gcp::Authentication::OauthClientConfig.new(service_scopes, oauth_filename)
+
+        # Prevent second oauth flow during same session with same credentials, different inventory.
+        # If we already have an outh authentication scheme for the same scopes & oauth credentials, then we'll return that one
+
+        # If the scheme is set, return it
+        return @authentication_scheme if @authentication_scheme
+
+        # Look to see if we have an existing oauth scheme setup for the same scopes & credentials file
+        if @authentication_scheme = ::Bcome::Driver::Gcp::Authentication::OauthSessionStore.instance.in_memory_session_for(client_config)
+          @compute_service = @authentication_scheme.service
+
+          return @authentication_scheme
+        end
+
+        # Otherwise, we'll create a new outh scheme and register it with the session store
+        @authentication_scheme = auth_scheme.new(self, compute_service, client_config, @node)
+        ::Bcome::Driver::Gcp::Authentication::OauthSessionStore.instance << @authentication_scheme
+        @authentication_scheme
+
+      when :service_account
+        @authentication_scheme ||= auth_scheme.new(compute_service, service_scopes, @node, @params[:service_account_credentials], self)
+      else
+        raise ::Bcome::Exception::InvalidGcpAuthenticationScheme, "Invalid GCP authentication scheme '#{auth_scheme_key}' for node #{@node.namespace}"
+      end
+    end
+
+    def oauth_filename
+      @params[:secrets_path] || @params[:secrets_filename]
     end
 
     def gcp_service

data/lib/objects/driver/gcp/authentication/base.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Bcome::Driver::Gcp::Authentication
+  class Base
+    include ::Bcome::LoadingBar::Handler
+
+    ## Overrides
+    def authorized?
+      raise 'Should be overidden'
+    end
+
+    ## Loading bar -
+
+    def loader_title
+      'Authenticating' + "\s#{@driver.pretty_provider_name.bc_blue.bold}\s#{@driver.pretty_resource_location.underline}".bc_green
+    end
+
+    ## Credential helpers --
+
+    def credential_directory
+      '.gauth'
+    end
+
+    def full_path_to_credential_file
+      "#{credential_directory}/#{credential_file}"
+    end
+
+    def credential_file
+      "#{@node.keyed_namespace}:#{credential_file_suffix}"
+    end
+
+    def ensure_credential_directory
+      `mkdir -p #{credential_directory}`
+    end
+  end
+end

data/lib/objects/driver/gcp/authentication/oauth.rb

@@ -5,19 +5,24 @@ require 'google/api_client/auth/storages/file_store'
 require 'google/api_client/client_secrets'
 
 module Bcome::Driver::Gcp::Authentication
-  class Oauth
+  class Oauth < Base
+    include Utilities
+
     credential_directory = '.gauth'
-    credential_file_suffix = 'oauth2.json'
 
-    include ::Bcome::LoadingBar::Handler
+    attr_reader :scopes, :secrets_filename, :service, :client_config
 
-    def initialize(driver, service, scopes, node, path_to_secrets)
+    def initialize(driver, service, client_config, node)
       @service = service
-      @scopes = scopes
+      @scopes = client_config.scopes
       @node = node
       @driver = driver
+      @client_config = client_config
+      @secrets_filename = client_config.secrets_filename
+      @path_to_secrets = "#{credential_directory}/#{@secrets_filename}"
+
+      raise ::Bcome::Exception::Generic, "Missing OAuth 2.0 client secrets file from GCP network configuration. Cannot find '#{@path_to_secrets}'" unless File.exist?(@path_to_secrets) && File.file?(@path_to_secrets)
 
-      @path_to_secrets = "#{credential_directory}/#{path_to_secrets}"
       # All credentials are held in .gauth
       ensure_credential_directory
     end
@@ -26,26 +31,10 @@ module Bcome::Driver::Gcp::Authentication
       storage && !@storage.authorization.nil?
     end
 
-    def storage
-      @storage ||= ::Google::APIClient::Storage.new(Google::APIClient::FileStore.new(full_path_to_credential_file))
-    end
-
-    def credential_directory
-      '.gauth'
-    end
-
     def credential_file_suffix
       'oauth2.json'
     end
 
-    def full_path_to_credential_file
-      "#{credential_directory}/#{credential_file}"
-    end
-
-    def credential_file
-      "#{@node.keyed_namespace}:#{credential_file_suffix}"
-    end
-
     def authorize!
       @service.authorization = storage.authorize
     end
@@ -60,8 +49,14 @@ module Bcome::Driver::Gcp::Authentication
       raise ::Bcome::Exception::MissingOrInvalidClientSecrets, "#{@path_to_secrets}. Gcp exception: #{e.class} #{e.message}"
     end
 
-    def loader_title
-      'Authenticating' + "\s#{@driver.pretty_provider_name.bc_blue.bold}\s#{@driver.pretty_resource_location.underline}".bc_green
+    def storage
+      @storage ||= ::Google::APIClient::Storage.new(Google::APIClient::FileStore.new(full_path_to_credential_file))
+    end
+
+    def credential_file
+      # If an authorization has the same scopes & secrets file, it is the same authorization. Hence we store the resulting oauth2 access credentials as the same file. This allows
+      # re-use of authorizations and prevents multiple oauth loops.
+      "#{@client_config.checksum}:#{credential_file_suffix}"
     end
 
     def do!
@@ -70,12 +65,16 @@ module Bcome::Driver::Gcp::Authentication
         # Total bloat from google here. Thanks google... requiring at last possible moment.
         require 'google/api_client/auth/installed_app'
 
-        wrap_indicator type: :basic, title: loader_title, completed_title: '' do
+          wrap_indicator type: :basic, title: loader_title, completed_title: '' do
           flow = Google::APIClient::InstalledAppFlow.new(
             client_id: client_secrets.client_id,
             client_secret: client_secrets.client_secret,
             scope: @scopes
           )
+ 
+          ## Override the redirected-to screen so that clearer instruction can be given          
+          flow.class.send(:remove_const,'RESPONSE_BODY') if flow.class.const_defined?('RESPONSE_BODY')
+          flow.class.send(:const_set,'RESPONSE_BODY', oauth_redirect_html)
 
           begin
              @service.authorization = flow.authorize(storage)
@@ -93,9 +92,5 @@ module Bcome::Driver::Gcp::Authentication
     def notify_success
       print "[\s" + "Credentials file written to\s" + full_path_to_credential_file + "\s]" + "\n"
     end
-
-    def ensure_credential_directory
-      `mkdir -p #{credential_directory}`
-    end
   end
 end

data/lib/objects/driver/gcp/authentication/oauth_client_config.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require 'digest/md5'
+
+module Bcome::Driver::Gcp::Authentication
+  class OauthClientConfig
+    attr_reader :scopes, :secrets_filename
+
+    def initialize(scopes, secrets_filename)
+      @scopes = scopes
+      @secrets_filename = secrets_filename
+    end
+
+    def ==(other)
+      checksum == other.checksum
+    end
+
+    def checksum
+      @checksum ||= ::Digest::MD5.hexdigest(Marshal.dump("#{@scopes}-#{@secrets_filename}"))
+    end
+  end
+end

data/lib/objects/driver/gcp/authentication/oauth_session_store.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Bcome::Driver::Gcp::Authentication
+  class OauthSessionStore
+    include Singleton
+
+    def initialize
+      @oauth_sessions = []
+    end
+
+    def in_memory_session_for(oauth_client_config)
+      existing_session = @oauth_sessions.detect do |session|
+        session.client_config == oauth_client_config
+      end
+      existing_session
+    end
+
+    def <<(session)
+      @oauth_sessions << session
+    end
+  end
+end

data/lib/objects/driver/gcp/authentication/service_account.rb

@@ -1,7 +1,62 @@
 # frozen_string_literal: true
 
 module Bcome::Driver::Gcp::Authentication
-  class ServiceAccount
-    def initialize(service, scopes, node, path_to_secrets); end
+  class ServiceAccount < Base
+    def initialize(service, scopes, node, credentials_file_name, driver)
+      @service = service
+      @scopes = scopes
+      @node = node
+      @driver = driver
+      @credentials_file_name = credentials_file_name
+      ensure_credential_directory
+      ensure_credentials_file
+    end
+
+    def do!
+      @service.authorization = service_account
+    end
+
+    def authorized?
+      !@service.authorization.nil?
+    end
+
+    def service_account
+      @service_account ||= ::Bcome::Driver::Gcp::Authentication::SignetServiceAccountClient.new(@scopes, credentials_file_path)
+    end
+
+    def credentials_file_path
+      has_namespaced_keyed_filename? ? namespaced_keyed_filename : defined_credentials_files
+    end
+
+    def ensure_credentials_file
+      return if has_namespaced_keyed_filename?
+      raise ::Bcome::Exception::MissingGcpServiceAccountCredentialsFilename unless @credentials_file_name
+    end
+
+    ## New implementation - we take a defined file name for the service account credentials
+    ## Clean & may be re-used
+    def defined_credentials_files
+      "#{credential_directory}/#{@credentials_file_name}"
+    end
+
+    def has_namespaced_keyed_filename?
+      @has_namespaced_keyed_filename ||= File.exist?(namespaced_keyed_filename)
+    end
+
+    ## Older implementation - we infer the credentials file from the namespace
+    ## Retained to provide backwards compatibility
+    def namespaced_keyed_filename
+      full_path_to_credential_file
+    end
+
+    def credential_file_suffix
+      'service-account.json'
+    end
+    #######################################
+
+    def authorize!
+      storage.authorize
+      @service.authorization = storage.authorization
+    end
   end
 end