kpm 0.7.0 → 0.8.2

data/lib/kpm/blob.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module KPM
+  class Blob
+    def initialize(value, tmp_dir)
+      @tmp_dir = tmp_dir
+      @blob_file = @tmp_dir + File::SEPARATOR + rand.to_s
+      store_value(value)
+    end
+
+    # On Macos systems, this will require defining a `secure_file_priv` config:
+    #
+    # e.g /usr/local/etc/my.cnf :
+    # [mysqld]
+    # ...
+    # secure_file_priv=""
+    def value
+      "LOAD_FILE(\"#{@blob_file}\")"
+    end
+
+    private
+
+    def store_value(value)
+      File.open(@blob_file, 'wb') do |file|
+        file.write(value)
+      end
+    end
+  end
+end

data/lib/kpm/cli.rb

@@ -1,7 +1,9 @@
+# frozen_string_literal: true
+
 require 'thor'
 
 module KPM
   class Cli < Thor
     include KPM::Tasks
   end
-end
+end

data/lib/kpm/coordinates.rb

@@ -1,9 +1,8 @@
-module KPM
+# frozen_string_literal: true
 
+module KPM
   class Coordinates
-
     class << self
-
       def build_coordinates(coordinate_map)
         group_id = coordinate_map[:group_id]
         artifact_id = coordinate_map[:artifact_id]
@@ -26,15 +25,13 @@ module KPM
         parts = entry.split(':')
         length = parts.size
         if length == 3
-          {:group_id => parts[0], :artifact_id => parts[1], :packaging => parts[2]}
+          { group_id: parts[0], artifact_id: parts[1], packaging: parts[2] }
         elsif length == 4
-          {:group_id => parts[0], :artifact_id => parts[1], :packaging => parts[2], :version => parts[3]}
+          { group_id: parts[0], artifact_id: parts[1], packaging: parts[2], version: parts[3] }
         elsif length == 5
-          {:group_id => parts[0], :artifact_id => parts[1], :packaging => parts[2],  :classifier => parts[3], :version => parts[4]}
+          { group_id: parts[0], artifact_id: parts[1], packaging: parts[2], classifier: parts[3], version: parts[4] }
         end
       end
-
     end
-
   end
-end
+end

data/lib/kpm/database.rb

@@ -1,131 +1,120 @@
+# frozen_string_literal: true
+
 require 'tmpdir'
 
 module KPM
-
   class Database
-    class << self
-
-      # Mysql Information functions
-      LAST_INSERTED_ID = 'SELECT LAST_INSERT_ID();'
-      ROWS_UPDATED = 'SELECT ROW_COUNT();'
-
-      # Destination database
-      DATABASE = ENV['DATABASE'] || 'killbill'
-      USERNAME = ENV['USERNAME'] || 'root'
-      PASSWORD = ENV['PASSWORD'] || 'root'
-      HOST = ENV['HOST'] || 'localhost'
-      PORT = ENV['PORT'] || '3306'
-
-      COLUMN_NAME_POS = 3
-      
-      STATEMENT_TMP_FILE = Dir.mktmpdir('statement') + File::SEPARATOR + 'statement.sql'
-
-      MYSQL_COMMAND_LINE = "mysql #{DATABASE} --user=#{USERNAME} --password=#{PASSWORD} "
-
-      @@mysql_command_line = MYSQL_COMMAND_LINE
-      @@username = USERNAME
-      @@password = PASSWORD
-      @@database = DATABASE
-      @@host = HOST
-      @@port = PORT
-
-      def set_logger(logger)
-        @@logger = logger
-      end
-
-      def set_credentials(user = nil, password = nil)
-        @@username = user
-        @@password = password
-      end
-
-      def set_host(host)
-        @@host = host
-      end
-
-      def set_port(port)
-        @@port = port
-      end
+    # Mysql Information functions
+    LAST_INSERTED_ID = 'SELECT LAST_INSERT_ID();'
+    ROWS_UPDATED = 'SELECT ROW_COUNT();'
+
+    # Destination database
+    DATABASE = ENV['DATABASE'] || 'killbill'
+    USERNAME = ENV['USERNAME'] || 'root'
+    PASSWORD = ENV['PASSWORD'] || 'root'
+    HOST = ENV['HOST'] || 'localhost'
+    PORT = ENV['PORT'] || '3306'
+
+    COLUMN_NAME_POS = 3
+
+    STATEMENT_TMP_FILE = Dir.mktmpdir('statement') + File::SEPARATOR + 'statement.sql'
+
+    def initialize(database_name, host, port, username, password, logger)
+      @database_name = database_name || DATABASE
+      @host = host || HOST
+      @port = port || PORT
+      @username = username || USERNAME
+      @password = password || PASSWORD
+      @mysql_command_line = "mysql --max_allowed_packet=128M #{@database_name} --host=#{@host} --port=#{@port} --user=#{@username} --password=#{@password} "
+
+      @logger = logger
+    end
 
-      def set_database_name(database_name = nil)
-        @@database = database_name
-      end
+    def execute_insert_statement(table_name, query, qty_to_insert, _table_data, record_id = nil)
+      query = "set #{record_id[:variable]}=#{record_id[:value]}; #{query}" unless record_id.nil?
+      query = "SET sql_mode = ''; SET autocommit=0; #{query} COMMIT; SHOW WARNINGS;"
 
-      def set_mysql_command_line
-        @@mysql_command_line = "mysql #{@@database} --host=#{@@host} --port=#{@@port} --user=#{@@username} --password=#{@@password} "
+      File.open(STATEMENT_TMP_FILE, 'w') do |s|
+        s.puts query
       end
 
-      def execute_insert_statement(table_name, query, qty_to_insert, table_data, record_id = nil)
-
-        unless record_id.nil?
-          query = "set #{record_id[:variable]}=#{record_id[:value]}; #{query}"
-        end
-        query = "SET autocommit=0; #{query} COMMIT;"
-        
-        File.open(STATEMENT_TMP_FILE,'w') do |s|
-          s.puts query
-        end
+      response = `#{@mysql_command_line} < "#{STATEMENT_TMP_FILE}" 2>&1`
 
-        response = `#{@@mysql_command_line} < "#{STATEMENT_TMP_FILE}" 2>&1`
-
-        if response.include? 'ERROR'
-          @@logger.error "\e[91;1mTransaction that fails to be executed\e[0m"
-          @@logger.error "\e[91m#{query}\e[0m"
+      if response.include? 'ERROR'
+        @logger.error "\e[91;1mTransaction that fails to be executed (first 1,000 chars)\e[0m"
+        # Queries can be really big (bulk imports)
+        @logger.error "\e[91m#{query[0..1000]}\e[0m"
+        if response.include?('Table') && response.include?('doesn\'t exist')
+          @logger.warn "Skipping unknown table #{table_name}...."
+        else
           raise Interrupt, "Importing table #{table_name}...... \e[91;1m#{response}\e[0m"
         end
+      end
 
-        if response.include? 'LAST_INSERT_ID'
-          @@logger.info "\e[32mImporting table #{table_name}...... Row 1 of #{qty_to_insert} success\e[0m"
-
-          return response.split("\n")[1]
-        end
+      if response.include? 'LAST_INSERT_ID'
+        @logger.info "\e[32mImporting table #{table_name}...... Row 1 of #{qty_to_insert} success\e[0m"
 
-        if response.include? 'ROW_COUNT'
-          response_msg = response.split("\n")
-          row_count_inserted = response_msg[response_msg.size - 1]
-          @@logger.info "\e[32mImporting table #{table_name}...... Row #{ row_count_inserted || 1} of #{qty_to_insert} success\e[0m"
+        return response.split("\n")[1]
+      end
 
-          return true
+      if response.include? 'ROW_COUNT'
+        # Typically, something like: "mysql: [Warning] Using a password on the command line interface can be insecure.\nROW_COUNT()\n3\n"
+        # With warning: "mysql: [Warning] Using a password on the command line interface can be insecure.\nROW_COUNT()\n1743\nLevel\tCode\tMessage\nWarning\t1264\tOut of range value for column 'amount' at row 582\n"
+        response_msg = response.split("\n")
+        idx_row_count_inserted = response_msg.index('ROW_COUNT()') + 1
+        row_count_inserted = response_msg[idx_row_count_inserted]
+        @logger.info "\e[32mImporting table #{table_name}...... Row #{row_count_inserted || 1} of #{qty_to_insert} success\e[0m"
+        if idx_row_count_inserted < response_msg.size - 1
+          warning_msg = response_msg[response_msg.size - 1]
+          @logger.warn "\e[91m#{warning_msg}\e[0m"
         end
-
-        return true
       end
 
-      def generate_insert_statement(tables)
-
-        statements = []
-        @@logger.info "\e[32mGenerating statements\e[0m"
-
-        tables.each_key do |table_name|
-          table = tables[table_name]
-          if !table[:rows].nil? && table[:rows].size > 0
-            columns_names = table[:col_names].join(",").gsub(/'/,'')
+      true
+    end
 
-            rows = []
-            table[:rows].each do |row|
-              rows << row.map{|value| value.is_a?(Symbol) ? value.to_s : "'#{value.to_s.gsub(/['"]/, "'" => "\\'", '"' => '\\"')}'" }.join(",")
+    def generate_insert_statement(tables)
+      statements = []
+      @logger.info "\e[32mGenerating statements\e[0m"
+
+      tables.each_key do |table_name|
+        table = tables[table_name]
+        next unless !table[:rows].nil? && !table[:rows].empty?
+
+        columns_names = table[:col_names].join(',').gsub(/'/, '')
+
+        rows = []
+        table[:rows].each do |row|
+          rows << row.map do |value|
+            if value.is_a?(Symbol)
+              value.to_s
+            elsif value.is_a?(Blob)
+              value.value
+            else
+              escaped_value = value.to_s.gsub(/['"]/, "'" => "\\'", '"' => '\\"')
+                                   .gsub('\N{LINE FEED}', "\n")
+                                   .gsub('\N{VERTICAL LINE}', '|')
+              "'#{escaped_value}'"
             end
-
-            value_data = rows.map{|row| "(#{row})" }.join(",")
-
-            statements << {:query => get_insert_statement(table_name,columns_names,value_data, rows.size),
-                                      :qty_to_insert => rows.size, :table_name => table_name, :table_data => table}
-
-          end
-
+          end.join(',')
         end
 
-        statements
+        # Break the insert statement into small chunks to avoid timeouts
+        rows.each_slice(1000).each do |subset_of_rows|
+          value_data = subset_of_rows.map { |row| "(#{row})" }.join(',')
 
+          statements << { query: build_insert_statement(table_name, columns_names, value_data, subset_of_rows.size),
+                          qty_to_insert: subset_of_rows.size, table_name: table_name, table_data: table }
+        end
       end
 
-      private
+      statements
+    end
 
-        def get_insert_statement(table_name, columns_names, values, rows_qty)
-          return "INSERT INTO #{table_name} ( #{columns_names} ) VALUES #{values}; #{rows_qty == 1 ? LAST_INSERTED_ID : ROWS_UPDATED}"
-        end
+    private
 
+    def build_insert_statement(table_name, columns_names, values, rows_qty)
+      "INSERT INTO #{table_name} ( #{columns_names} ) VALUES #{values}; #{rows_qty == 1 ? LAST_INSERTED_ID : ROWS_UPDATED}"
     end
-
   end
-
-end
+end

data/lib/kpm/diagnostic_file.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'yaml'
 require 'tmpdir'
 require 'zip'
@@ -5,195 +7,173 @@ require 'json'
 require 'fileutils'
 require 'date'
 
-
 module KPM
+  class DiagnosticFile
+    # Temporary directory
+    TMP_DIR_PREFIX = 'killbill-diagnostics-'
+    TMP_DIR = Dir.mktmpdir(TMP_DIR_PREFIX)
+    TMP_LOGS_DIR = TMP_DIR + File::Separator + 'logs'
+
+    TENANT_FILE = 'tenant_config.data'
+    SYSTEM_FILE = 'system_configuration.data'
+    ACCOUNT_FILE = 'account.data'
+
+    TODAY_DATE = Date.today.strftime('%m-%d-%y')
+    ZIP_FILE = 'killbill-diagnostics-' + TODAY_DATE + '.zip'
+    ZIP_LOG_FILE = 'logs.zip'
+
+    def initialize(config_file = nil, killbill_api_credentials = nil, killbill_credentials = nil, killbill_url = nil,
+                   database_name = nil, database_credentials = nil, database_host = nil, database_port = nil, kaui_web_path = nil,
+                   killbill_web_path = nil, bundles_dir = nil, logger = nil)
+      @killbill_api_credentials = killbill_api_credentials
+      @killbill_credentials = killbill_credentials
+      @killbill_url = killbill_url
+      @database_name = database_name
+      @database_credentials = database_credentials
+      @database_host = database_host
+      @database_port = database_port
+      @config_file = config_file
+      @kaui_web_path = kaui_web_path
+      @killbill_web_path = killbill_web_path
+      @logger = logger
+      @original_logger_level = logger.level
+      @catalina_base = nil
+      @bundles_dir = bundles_dir
+    end
 
-    class DiagnosticFile
-
-      # Temporary directory
-      TMP_DIR_PREFIX  = 'killbill-diagnostics-'
-      TMP_DIR         = Dir.mktmpdir(TMP_DIR_PREFIX)
-      TMP_LOGS_DIR    = TMP_DIR + File::Separator + 'logs'
-
-      TENANT_FILE     = 'tenant_config.data'
-      SYSTEM_FILE     = 'system_configuration.data'
-      ACCOUNT_FILE    = 'account.data'
-
-      TODAY_DATE      = Date.today.strftime('%m-%d-%y')
-      ZIP_FILE        = 'killbill-diagnostics-' + TODAY_DATE + '.zip'
-      ZIP_LOG_FILE    = 'logs.zip'
-
-      def initialize(config_file = nil, killbill_api_credentials = nil, killbill_credentials = nil, killbill_url = nil,
-                     database_name = nil, database_credentials = nil, database_host = nil, database_port = nil, kaui_web_path = nil,
-                     killbill_web_path = nil, bundles_dir = nil, logger = nil)
-        @killbill_api_credentials = killbill_api_credentials
-        @killbill_credentials = killbill_credentials
-        @killbill_url = killbill_url
-        @database_name = database_name
-        @database_credentials = database_credentials
-        @database_host = database_host
-        @database_port = database_port
-        @config_file = config_file
-        @kaui_web_path = kaui_web_path;
-        @killbill_web_path = killbill_web_path;
-        @logger = logger
-        @original_logger_level = logger.level;
-        @catalina_base = nil
-        @bundles_dir = bundles_dir
-      end
-
-      def export_data(account_id = nil, log_dir = nil)
-        set_config(@config_file)
-
-        tenant_export_file  = get_tenant_config
-        system_export_file  = get_system_config
-        account_export_file = get_account_data(account_id) unless account_id.nil?
-        log_files           = get_log_files(log_dir)
-
-        if File.exist?(system_export_file) && File.exist?(tenant_export_file)
-
-
-          zip_file_name = TMP_DIR + File::Separator + ZIP_FILE
-
-          Zip::File.open(zip_file_name, Zip::File::CREATE) do |zipFile|
-
-            zipFile.add(TENANT_FILE,  tenant_export_file)
-            zipFile.add(SYSTEM_FILE,  system_export_file)
-            zipFile.add(ACCOUNT_FILE, account_export_file) unless account_id.nil?
-            zipFile.add(ZIP_LOG_FILE, log_files) unless log_files.nil?
+    def export_data(account_id = nil, log_dir = nil)
+      self.config = @config_file
 
-          end
+      tenant_export_file = retrieve_tenant_config
+      system_export_file = retrieve_system_config
+      account_export_file = retrieve_account_data(account_id) unless account_id.nil?
+      log_files = retrieve_log_files(log_dir)
 
-          @logger.info "\e[32mDiagnostic data exported under #{zip_file_name} \e[0m"
+      raise Interrupt, 'Account id or configuration file not found' unless File.exist?(system_export_file) && File.exist?(tenant_export_file)
 
-        else
-          raise Interrupt, 'Account id or configuration file not found'
-        end
+      zip_file_name = TMP_DIR + File::Separator + ZIP_FILE
 
+      Zip::File.open(zip_file_name, Zip::File::CREATE) do |zip_file|
+        zip_file.add(TENANT_FILE, tenant_export_file)
+        zip_file.add(SYSTEM_FILE, system_export_file)
+        zip_file.add(ACCOUNT_FILE, account_export_file) unless account_id.nil?
+        zip_file.add(ZIP_LOG_FILE, log_files) unless log_files.nil?
       end
 
-      # Private methods
-
-      private
+      @logger.info "\e[32mDiagnostic data exported under #{zip_file_name} \e[0m"
 
-      def get_tenant_config
+      zip_file_name
+    end
 
-        @logger.info 'Retrieving tenant configuration'
-        # this suppress the message of where it put the account file, this is to avoid confusion
-        @logger.level = Logger::WARN
+    # Private methods
 
-        @killbill_api_credentials ||= [get_config('killbill', 'api_key'), get_config('killbill','api_secret')] unless @config_file.nil?
-        @killbill_credentials ||= [get_config('killbill', 'user'), get_config('killbill','password')] unless @config_file.nil?
-        @killbill_url ||= 'http://' + get_config('killbill', 'host').to_s + ':' + get_config('killbill','port').to_s unless @config_file.nil?
+    private
 
-        tenant_config = KPM::TenantConfig.new(@killbill_api_credentials,
-                                              @killbill_credentials, @killbill_url, @logger)
-        export_file = tenant_config.export
+    def retrieve_tenant_config
+      @logger.info 'Retrieving tenant configuration'
+      # this suppress the message of where it put the account file, this is to avoid confusion
+      @logger.level = Logger::WARN
 
-        final = TMP_DIR + File::Separator + TENANT_FILE
-        FileUtils.move(export_file, final)
-        @logger.level = @original_logger_level
+      @killbill_api_credentials ||= [retrieve_config('killbill', 'api_key'), retrieve_config('killbill', 'api_secret')] unless @config_file.nil?
+      @killbill_credentials ||= [retrieve_config('killbill', 'user'), retrieve_config('killbill', 'password')] unless @config_file.nil?
+      @killbill_url ||= 'http://' + retrieve_config('killbill', 'host').to_s + ':' + retrieve_config('killbill', 'port').to_s unless @config_file.nil?
 
-        final
-      end
+      tenant_config = KPM::TenantConfig.new(@killbill_api_credentials,
+                                            @killbill_credentials,
+                                            @killbill_url,
+                                            @logger)
+      export_file = tenant_config.export
 
-      def get_system_config
+      final = TMP_DIR + File::Separator + TENANT_FILE
+      FileUtils.move(export_file, final)
+      @logger.level = @original_logger_level
 
-        @logger.info 'Retrieving system configuration'
-        system = KPM::System.new
-        export_data = system.information(@bundles_dir, true, @config_file, @kaui_web_path, @killbill_web_path)
+      final
+    end
 
-        get_system_catalina_base(export_data)
+    def retrieve_system_config
+      @logger.info 'Retrieving system configuration'
+      system = KPM::System.new(@logger)
+      export_data = system.information(@bundles_dir, true, @config_file, @kaui_web_path, @killbill_web_path)
 
-        export_file = TMP_DIR + File::SEPARATOR + SYSTEM_FILE
-        File.open(export_file, 'w') { |io| io.puts export_data }
-        export_file
-      end
+      system_catalina_base(export_data)
 
+      export_file = TMP_DIR + File::SEPARATOR + SYSTEM_FILE
+      File.open(export_file, 'w') { |io| io.puts export_data }
+      export_file
+    end
 
-      def get_account_data(account_id)
+    def retrieve_account_data(account_id)
+      @logger.info 'Retrieving account data for id: ' + account_id
+      # this suppress the message of where it put the account file, this is to avoid confusion
+      @logger.level = Logger::WARN
 
-        @logger.info 'Retrieving account data for id: ' + account_id
-        # this suppress the message of where it put the account file, this is to avoid confusion
-        @logger.level = Logger::WARN
+      account = KPM::Account.new(@config_file, @killbill_api_credentials, @killbill_credentials,
+                                 @killbill_url, @database_name,
+                                 @database_credentials, @database_host, @database_port, nil, @logger)
+      export_file = account.export_data(account_id)
 
-        account = KPM::Account.new(@config_file, @killbill_api_credentials, @killbill_credentials,
-                                   @killbill_url, @database_name,
-                                   @database_credentials,@database_host, @database_port, nil, @logger)
-        export_file = account.export_data(account_id)
+      final = TMP_DIR + File::Separator + ACCOUNT_FILE
+      FileUtils.move(export_file, final)
+      @logger.level = @original_logger_level
+      final
+    end
 
-        final  = TMP_DIR + File::Separator + ACCOUNT_FILE
-        FileUtils.move(export_file, final)
-        @logger.level = @original_logger_level
-        final
+    def retrieve_log_files(log_dir)
+      if @catalina_base.nil? && log_dir.nil?
+        @logger.warn "\e[91;1mUnable to find Tomcat process, logs won't be collected: make sure to run kpm using the same user as the Tomcat process or pass the option --log-dir\e[0m"
+        return nil
       end
 
-      def get_log_files(log_dir)
+      @logger.info 'Collecting log files'
+      log_base = log_dir || (@catalina_base + File::Separator + 'logs')
+      log_items = Dir.glob(log_base + File::Separator + '*')
 
-        @logger.info 'Collecting log files'
+      zip_file_name = TMP_DIR + File::Separator + ZIP_LOG_FILE
 
-        if @catalina_base.nil? && log_dir.nil?
-          @logger.warn 'Unable to find Tomcat process, make sure to run kpm using the same user as the Tomcat process.'
-          return nil
+      Zip::File.open(zip_file_name, Zip::File::CREATE) do |zip_file|
+        log_items.each do |file|
+          name = file.split('/').last
+          zip_file.add(name, file)
         end
-
-        log_base = log_dir || (@catalina_base + File::Separator + 'logs')
-        log_items = Dir.glob(log_base + File::Separator + '*')
-
-        zip_file_name = TMP_DIR + File::Separator + ZIP_LOG_FILE
-
-        Zip::File.open(zip_file_name, Zip::File::CREATE) do |zipFile|
-
-          log_items.each do |file|
-            name = file.split('/').last
-            zipFile.add(name, file)
-          end
-
-        end
-
-        zip_file_name
       end
 
-      # Helpers
-
-      def get_system_catalina_base(export_data)
-        @catalina_base = nil
-        system_json = JSON.parse(export_data)
+      zip_file_name
+    end
 
-        return if system_json['java_system_information']['catalina.base'].nil?
+    # Helpers
 
-        @catalina_base = system_json['java_system_information']['catalina.base']['value']
+    def system_catalina_base(export_data)
+      @catalina_base = nil
+      system_json = JSON.parse(export_data)
 
-      end
+      return if system_json['java_system_information']['catalina.base'].nil?
 
-      # Utils
+      @catalina_base = system_json['java_system_information']['catalina.base']['value']
+    end
 
-      def get_config(parent, child)
-        item = nil;
+    # Utils
 
-        if not @config.nil?
+    def retrieve_config(parent, child)
+      item = nil
 
-          config_parent = @config[parent]
+      unless @config.nil?
 
-          if not config_parent.nil?
-            item =config_parent[child]
-          end
+        config_parent = @config[parent]
 
-        end
+        item = config_parent[child] unless config_parent.nil?
 
-        item
       end
 
-      def set_config(config_file = nil)
-        @config = nil
+      item
+    end
 
-        if not config_file.nil?
-          if not Dir[config_file][0].nil?
-            @config = YAML::load_file(config_file)
-          end
-        end
+    def config=(config_file = nil)
+      @config = nil
 
-      end
+      return if config_file.nil?
 
+      @config = YAML.load_file(config_file) unless Dir[config_file][0].nil?
     end
-end
+  end
+end