egis 1.1.1 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d232fad45575196b2a8a2a65ccca79529a2fd72121bf5883a9c3b037a6c40850
-  data.tar.gz: d269d52126bf32592f3b3425abc76b1664b119684674db5e618d9cbacbc0d3f3
+  metadata.gz: bc78871d415888b51b651537cf0716a553d44b20627e60f8e6110d8c1929791c
+  data.tar.gz: 96919f1a0ab277b6895dbea4db17c26544d21995010c53a87f1f9515d48eddfd
 SHA512:
-  metadata.gz: 37c73df4ff2e1bf1f280816f0a882046a02447932bfdb8c77893b27d2e106f263d22a0d7cd0951861898aef882b2ea9ddca737ef980b9bfc2e342609a2716845
-  data.tar.gz: 93e325830f6382c683430e7dac987b01dc9bc93890155f0c28f19373742610d8f24195e7c074684f813821f508a956a7ad936c7e4297a0738af52ec285445805
+  metadata.gz: e8c30d0705a76438655608af048138ba54cc710a1d1811fe44513551d63c6e1a9fd0a8650a25f2dbba2d828eaaa8c87c75e76562ccb433c720d59bca1a2b655f
+  data.tar.gz: 6b61e9a3adcea83fc6e31c04ef64eae21398a7b1ea4facf1f6b34d4c518f2cfb1722594b308cc48209181dc5a6d2b3a35d76542d0f23ff24b778cbfc97bbed58

data/egis.gemspec

@@ -16,7 +16,7 @@ Gem::Specification.new do |spec|
 
   spec.metadata['homepage_uri'] = spec.homepage
   spec.metadata['source_code_uri'] = spec.homepage
-  spec.metadata['changelog_uri'] = 'https://github.com/u2i/egis/blob/master/docs/CHANGELOG.md'
+  spec.metadata['changelog_uri'] = 'https://u2i.github.io/egis/file.CHANGELOG.html'
 
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.

data/lib/egis/client.rb

@@ -65,14 +65,16 @@ module Egis
     #   by workgroup.
     # @return [Egis::QueryStatus]
 
-    def execute_query(query, work_group: nil, database: nil, output_location: nil, async: true)
-      query_execution_id = aws_athena_client.start_query_execution(
+    def execute_query(query, work_group: nil, database: nil, output_location: nil, async: true, system_execution: false)
+      query_id = aws_athena_client.start_query_execution(
         query_execution_params(query, work_group, database, output_location)
       ).query_execution_id
 
-      return query_status(query_execution_id) if Egis.mode.async(async)
+      log_query_execution(query, query_id, system_execution)
 
-      query_status = wait_for_query_to_finish(query_execution_id)
+      return query_status(query_id) if Egis.mode.async(async)
+
+      query_status = wait_for_query_to_finish(query_id)
 
       raise Egis::Errors::QueryExecutionError, query_status.message unless query_status.finished?
 
@@ -89,10 +91,13 @@ module Egis
       resp = aws_athena_client.get_query_execution(query_execution_id: query_id)
 
       query_execution = resp.query_execution
+      query_status = query_execution.status.state
+
+      Egis.logger.debug { "Checking query status (#{query_id}): #{query_status}" }
 
       Egis::QueryStatus.new(
         query_execution.query_execution_id,
-        QUERY_STATUS_MAPPING.fetch(query_execution.status.state),
+        QUERY_STATUS_MAPPING.fetch(query_status),
         query_execution.status.state_change_reason,
         parse_output_location(query_execution)
       )
@@ -112,11 +117,20 @@ module Egis
       params
     end
 
-    def wait_for_query_to_finish(query_execution_id)
+    def log_query_execution(query, query_id, system_execution)
+      if system_execution
+        Egis.logger.debug { "Executing system query (#{query_id}): #{query.gsub(/\s+/, ' ')}" }
+      else
+        Egis.logger.info { "Executing query (#{query_id}): #{query.gsub(/\s+/, ' ')}" }
+      end
+    end
+
+    def wait_for_query_to_finish(query_id)
       attempt = 1
       loop do
         sleep(query_status_backoff.call(attempt))
-        status = query_status(query_execution_id)
+        status = query_status(query_id)
+
         return status unless status.queued? || status.running?
 
         attempt += 1

data/lib/egis/configuration.rb

@@ -4,6 +4,10 @@ module Egis
   # @!visibility private
   class Configuration
     attr_accessor :work_group, :aws_region, :aws_access_key_id, :aws_secret_access_key, :aws_profile,
-                  :query_status_backoff, :testing_s3_bucket
+                  :query_status_backoff, :testing_s3_bucket, :logger
+
+    def initialize
+      @logger = Logger.new(STDOUT, level: :info)
+    end
   end
 end

data/lib/egis/database.rb

@@ -10,13 +10,18 @@ module Egis
   #
   # It is recommended to create database objects using {Egis::Client#database} method.
   #
+  # @!attribute [r] name
+  #   @return [String] Athena database name
+  #
   class Database
-    def initialize(database_name, client: Egis::Client.new, output_downloader: Egis::OutputDownloader.new)
+    def initialize(name, client: Egis::Client.new, output_downloader: Egis::OutputDownloader.new)
       @client = client
-      @database_name = database_name
+      @name = name
       @output_downloader = output_downloader
     end
 
+    attr_reader :name
+
     ##
     # Creates {Egis::Table} object. Executing it doesn't create Athena table yet.
     #
@@ -36,7 +41,10 @@ module Egis
     # @return [void]
 
     def create
-      client.execute_query("CREATE DATABASE IF NOT EXISTS #{translate_name(database_name)};", async: false)
+      log_database_creation
+
+      client.execute_query("CREATE DATABASE IF NOT EXISTS #{translate_name(name)};", async: false,
+                                                                                     system_execution: true)
     end
 
     ##
@@ -45,7 +53,9 @@ module Egis
     # @return [void]
 
     def create!
-      client.execute_query("CREATE DATABASE #{translate_name(database_name)};", async: false)
+      log_database_creation
+
+      client.execute_query("CREATE DATABASE #{translate_name(name)};", async: false, system_execution: true)
     end
 
     ##
@@ -54,7 +64,10 @@ module Egis
     # @return [void]
 
     def drop
-      client.execute_query("DROP DATABASE IF EXISTS #{translate_name(database_name)} CASCADE;", async: false)
+      log_database_removal
+
+      client.execute_query("DROP DATABASE IF EXISTS #{translate_name(name)} CASCADE;", async: false,
+                                                                                       system_execution: true)
     end
 
     ##
@@ -63,14 +76,16 @@ module Egis
     # @return [void]
 
     def drop!
-      client.execute_query("DROP DATABASE #{translate_name(database_name)} CASCADE;", async: false)
+      log_database_removal
+
+      client.execute_query("DROP DATABASE #{translate_name(name)} CASCADE;", async: false, system_execution: true)
     end
 
     ##
     # (see Egis::Client#execute_query)
 
     def execute_query(query, **options)
-      client.execute_query(query, **{database: database_name, **options})
+      client.execute_query(query, **{database: name, **options})
     end
 
     ##
@@ -86,14 +101,22 @@ module Egis
     # @return [Boolean]
 
     def exists?
-      query_status = client.execute_query("SHOW DATABASES LIKE '#{database_name}';", async: false)
+      query_status = client.execute_query("SHOW DATABASES LIKE '#{name}';", async: false, system_execution: true)
       parsed_result = output_downloader.download(query_status.output_location)
-      parsed_result.flatten.include?(database_name)
+      parsed_result.flatten.include?(name)
     end
 
     private
 
-    attr_reader :client, :database_name, :output_downloader
+    attr_reader :client, :output_downloader
+
+    def log_database_creation
+      Egis.logger.info { "Creating database #{name}" }
+    end
+
+    def log_database_removal
+      Egis.logger.info { "Removing database #{name}" }
+    end
 
     def translate_name(name)
       Egis.mode.database_name(name)

data/lib/egis/table.rb

@@ -9,7 +9,7 @@ module Egis
   # @!attribute [r] database
   #   @return [Egis::Database]
   # @!attribute [r] name
-  #   @return [String] Athena database name
+  #   @return [String] Athena table name
   # @!attribute [r] schema
   #   @return [Egis::TableSchema] table's schema object
   #
@@ -42,8 +42,10 @@ module Egis
     # @return [void]
 
     def create
+      log_table_creation
+
       create_table_sql = table_ddl_generator.create_table_sql(self, permissive: true)
-      database.execute_query(create_table_sql, async: false)
+      database.execute_query(create_table_sql, async: false, system_execution: true)
     end
 
     ##
@@ -52,8 +54,10 @@ module Egis
     # @return [void]
 
     def create!
+      log_table_creation
+
       create_table_sql = table_ddl_generator.create_table_sql(self, permissive: false)
-      database.execute_query(create_table_sql, async: false)
+      database.execute_query(create_table_sql, async: false, system_execution: true)
     end
 
     ##
@@ -67,7 +71,7 @@ module Egis
 
     def add_partitions(partitions)
       load_partitions_query = partitions_generator.to_sql(name, partitions, permissive: true)
-      database.execute_query(load_partitions_query, async: false)
+      database.execute_query(load_partitions_query, async: false, system_execution: true)
     end
 
     ##
@@ -76,7 +80,7 @@ module Egis
 
     def add_partitions!(partitions)
       load_partitions_query = partitions_generator.to_sql(name, partitions, permissive: false)
-      database.execute_query(load_partitions_query, async: false)
+      database.execute_query(load_partitions_query, async: false, system_execution: true)
     end
 
     ##
@@ -87,7 +91,7 @@ module Egis
     # @return [void]
 
     def discover_partitions
-      database.execute_query("MSCK REPAIR TABLE #{name};", async: false)
+      database.execute_query("MSCK REPAIR TABLE #{name};", async: false, system_execution: true)
     end
 
     ##
@@ -98,7 +102,7 @@ module Egis
 
     def upload_data(rows)
       query = data_insert_query(rows)
-      database.execute_query(query, async: false)
+      database.execute_query(query, async: false, system_execution: true)
     end
 
     ##
@@ -107,7 +111,7 @@ module Egis
     # @return [Array] Array of arrays with row values.
 
     def download_data
-      result = database.execute_query("SELECT * FROM #{name};", async: false)
+      result = database.execute_query("SELECT * FROM #{name};", async: false, system_execution: true)
       content = output_downloader.download(result.output_location)
       output_parser.parse(content, column_types)
     end
@@ -141,6 +145,10 @@ module Egis
     attr_reader :options, :partitions_generator, :table_ddl_generator, :output_downloader, :output_parser,
                 :table_data_wiper
 
+    def log_table_creation
+      Egis.logger.info { "Creating table #{database.name}.#{name} located in #{location}" }
+    end
+
     def column_serializers
       @column_serializers ||= column_types.map { |type| Egis::Types.serializer(type) }
     end

data/lib/egis/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Egis
-  VERSION = '1.1.1'
+  VERSION = '1.2.0'
 end

data/lib/egis.rb

@@ -40,6 +40,11 @@ require 'egis/s3_location_parser'
 #     config.aws_profile = 'my-profile'
 #   end
 #
+# @example Configure logger
+#   Egis.configure do |config|
+#     config.logger = Logger.new('athena.log', level: :debug)
+#   end
+#
 # @yield [Egis::Configuration]
 # @return [void]
 #
@@ -58,5 +63,10 @@ module Egis
     def mode
       @mode ||= Egis::StandardMode.new
     end
+
+    # @!visibility private
+    def logger
+      @configuration.logger
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: egis
 version: !ruby/object:Gem::Version
-  version: 1.1.1
+  version: 1.2.0
 platform: ruby
 authors:
 - Agnieszka Czereba
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-05-18 00:00:00.000000000 Z
+date: 2020-05-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-athena
@@ -83,7 +83,7 @@ licenses:
 metadata:
   homepage_uri: https://github.com/u2i/egis
   source_code_uri: https://github.com/u2i/egis
-  changelog_uri: https://github.com/u2i/egis/blob/master/docs/CHANGELOG.md
+  changelog_uri: https://u2i.github.io/egis/file.CHANGELOG.html
 post_install_message: 
 rdoc_options: []
 require_paths: