egis 1.1.0 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1d2255a76aef464d0d8faee0c39be753e1a928133a25de5c4a86f553e51e92ca
-  data.tar.gz: f18ae28053651576ccf941bd42f5a9bf40db32fed3c1c2025e1bc987844199f8
+  metadata.gz: 4fa3323e3c02d8537df83f33799f580fb196c255315c9923eb70c0e87bd1f50e
+  data.tar.gz: 9b4054a2201d98f501603cc137c1c7df93fccf105c448b6d4261515c3f53d959
 SHA512:
-  metadata.gz: 9b143dbc650f8c02ba39f1404a2d9b4ebe36c49182a36b947a1d091079b1c94839320c2d05dd9a0db1516d82c9b03694244e4fa467898302e888addb234e1291
-  data.tar.gz: a1577d30cbfd63632dbd55052adf0f863f7a5f571e8a8f8a210649446acc786764d8bec6a15be7cd91742f4f7681bfc1d379a9099934f62b8a30379d12a4bd77
+  metadata.gz: 3da9b098de6948b584244db63eda8ef4ba3d26ff29806be0e1a3b3469a321bb72d0761bc4bcb415759646a2397c71d32221477cc2dabeb9233dd8cad22300cef
+  data.tar.gz: 5c50463ef9585563b3aa6fefbb89b4df101f93693b7e5d5f423e98fa803e614d7541639db9afe06fc1a4544205d6e3b0cbf3c166112f556f58b8c798b528571d

data/egis.gemspec

@@ -16,7 +16,7 @@ Gem::Specification.new do |spec|
 
   spec.metadata['homepage_uri'] = spec.homepage
   spec.metadata['source_code_uri'] = spec.homepage
-  spec.metadata['changelog_uri'] = 'https://github.com/u2i/egis/blob/master/CHANGELOG.md'
+  spec.metadata['changelog_uri'] = 'https://u2i.github.io/egis/file.CHANGELOG.html'
 
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.

data/lib/egis.rb

@@ -40,6 +40,11 @@ require 'egis/s3_location_parser'
 #     config.aws_profile = 'my-profile'
 #   end
 #
+# @example Configure logger
+#   Egis.configure do |config|
+#     config.logger = Logger.new('athena.log', level: :debug)
+#   end
+#
 # @yield [Egis::Configuration]
 # @return [void]
 #
@@ -58,5 +63,10 @@ module Egis
     def mode
       @mode ||= Egis::StandardMode.new
     end
+
+    # @!visibility private
+    def logger
+      @configuration.logger
+    end
   end
 end

data/lib/egis/client.rb

@@ -65,14 +65,16 @@ module Egis
     #   by workgroup.
     # @return [Egis::QueryStatus]
 
-    def execute_query(query, work_group: nil, database: nil, output_location: nil, async: true)
-      query_execution_id = aws_athena_client.start_query_execution(
+    def execute_query(query, work_group: nil, database: nil, output_location: nil, async: true, system_execution: false)
+      query_id = aws_athena_client.start_query_execution(
         query_execution_params(query, work_group, database, output_location)
       ).query_execution_id
 
-      return query_status(query_execution_id) if Egis.mode.async(async)
+      log_query_execution(query, query_id, system_execution)
 
-      query_status = wait_for_query_to_finish(query_execution_id)
+      return query_status(query_id) if Egis.mode.async(async)
+
+      query_status = wait_for_query_to_finish(query_id)
 
       raise Egis::Errors::QueryExecutionError, query_status.message unless query_status.finished?
 
@@ -89,10 +91,13 @@ module Egis
       resp = aws_athena_client.get_query_execution(query_execution_id: query_id)
 
       query_execution = resp.query_execution
+      query_status = query_execution.status.state
+
+      Egis.logger.debug { "Checking query status (#{query_id}): #{query_status}" }
 
       Egis::QueryStatus.new(
         query_execution.query_execution_id,
-        QUERY_STATUS_MAPPING.fetch(query_execution.status.state),
+        QUERY_STATUS_MAPPING.fetch(query_status),
         query_execution.status.state_change_reason,
         parse_output_location(query_execution)
       )
@@ -112,11 +117,20 @@ module Egis
       params
     end
 
-    def wait_for_query_to_finish(query_execution_id)
+    def log_query_execution(query, query_id, system_execution)
+      if system_execution
+        Egis.logger.debug { "Executing system query (#{query_id}): #{query.gsub(/\s+/, ' ')}" }
+      else
+        Egis.logger.info { "Executing query (#{query_id}): #{query.gsub(/\s+/, ' ')}" }
+      end
+    end
+
+    def wait_for_query_to_finish(query_id)
       attempt = 1
       loop do
         sleep(query_status_backoff.call(attempt))
-        status = query_status(query_execution_id)
+        status = query_status(query_id)
+
         return status unless status.queued? || status.running?
 
         attempt += 1

data/lib/egis/configuration.rb

@@ -4,6 +4,10 @@ module Egis
   # @!visibility private
   class Configuration
     attr_accessor :work_group, :aws_region, :aws_access_key_id, :aws_secret_access_key, :aws_profile,
-                  :query_status_backoff, :testing_s3_bucket
+                  :query_status_backoff, :testing_s3_bucket, :logger
+
+    def initialize
+      @logger = Logger.new(STDOUT, level: :info)
+    end
   end
 end

data/lib/egis/database.rb

@@ -10,20 +10,26 @@ module Egis
   #
   # It is recommended to create database objects using {Egis::Client#database} method.
   #
+  # @!attribute [r] name
+  #   @return [String] Athena database name
+  #
   class Database
-    def initialize(database_name, client: Egis::Client.new, output_downloader: Egis::OutputDownloader.new)
+    def initialize(name, client: Egis::Client.new, output_downloader: Egis::OutputDownloader.new)
       @client = client
-      @database_name = database_name
+      @name = name
       @output_downloader = output_downloader
     end
 
+    attr_reader :name
+
     ##
     # Creates {Egis::Table} object. Executing it doesn't create Athena table yet.
     #
     # @param [String] table_name
     # @param [Egis::TableSchema] table_schema
     # @param [String] table_location S3 URL with table location (e.g. `s3://s3_bucket/table/location/`)
-    # @param [:tsv, :csv, :orc] format Table format (defaults to :tsv)
+    # @param [:tsv, :csv, :orc, {serde: 'SerdeClass', serde_properties: {property: value}}] format Table format
+    # (defaults to :tsv)
     # @return [Egis::Table]
 
     def table(table_name, table_schema, table_location, **options)
@@ -36,7 +42,10 @@ module Egis
     # @return [void]
 
     def create
-      client.execute_query("CREATE DATABASE IF NOT EXISTS #{translate_name(database_name)};", async: false)
+      log_database_creation
+
+      client.execute_query("CREATE DATABASE IF NOT EXISTS #{translate_name(name)};", async: false,
+                                                                                     system_execution: true)
     end
 
     ##
@@ -45,7 +54,9 @@ module Egis
     # @return [void]
 
     def create!
-      client.execute_query("CREATE DATABASE #{translate_name(database_name)};", async: false)
+      log_database_creation
+
+      client.execute_query("CREATE DATABASE #{translate_name(name)};", async: false, system_execution: true)
     end
 
     ##
@@ -54,7 +65,10 @@ module Egis
     # @return [void]
 
     def drop
-      client.execute_query("DROP DATABASE IF EXISTS #{translate_name(database_name)} CASCADE;", async: false)
+      log_database_removal
+
+      client.execute_query("DROP DATABASE IF EXISTS #{translate_name(name)} CASCADE;", async: false,
+                                                                                       system_execution: true)
     end
 
     ##
@@ -63,14 +77,16 @@ module Egis
     # @return [void]
 
     def drop!
-      client.execute_query("DROP DATABASE #{translate_name(database_name)} CASCADE;", async: false)
+      log_database_removal
+
+      client.execute_query("DROP DATABASE #{translate_name(name)} CASCADE;", async: false, system_execution: true)
     end
 
     ##
     # (see Egis::Client#execute_query)
 
     def execute_query(query, **options)
-      client.execute_query(query, **{database: database_name, **options})
+      client.execute_query(query, **{database: name, **options})
     end
 
     ##
@@ -86,14 +102,22 @@ module Egis
     # @return [Boolean]
 
     def exists?
-      query_status = client.execute_query("SHOW DATABASES LIKE '#{database_name}';", async: false)
+      query_status = client.execute_query("SHOW DATABASES LIKE '#{name}';", async: false, system_execution: true)
       parsed_result = output_downloader.download(query_status.output_location)
-      parsed_result.flatten.include?(database_name)
+      parsed_result.flatten.include?(name)
     end
 
     private
 
-    attr_reader :client, :database_name, :output_downloader
+    attr_reader :client, :output_downloader
+
+    def log_database_creation
+      Egis.logger.info { "Creating database #{name}" }
+    end
+
+    def log_database_removal
+      Egis.logger.info { "Removing database #{name}" }
+    end
 
     def translate_name(name)
       Egis.mode.database_name(name)

data/lib/egis/table.rb

@@ -9,7 +9,7 @@ module Egis
   # @!attribute [r] database
   #   @return [Egis::Database]
   # @!attribute [r] name
-  #   @return [String] Athena database name
+  #   @return [String] Athena table name
   # @!attribute [r] schema
   #   @return [Egis::TableSchema] table's schema object
   #
@@ -42,8 +42,10 @@ module Egis
     # @return [void]
 
     def create
+      log_table_creation
+
       create_table_sql = table_ddl_generator.create_table_sql(self, permissive: true)
-      database.execute_query(create_table_sql, async: false)
+      database.execute_query(create_table_sql, async: false, system_execution: true)
     end
 
     ##
@@ -52,8 +54,10 @@ module Egis
     # @return [void]
 
     def create!
+      log_table_creation
+
       create_table_sql = table_ddl_generator.create_table_sql(self, permissive: false)
-      database.execute_query(create_table_sql, async: false)
+      database.execute_query(create_table_sql, async: false, system_execution: true)
     end
 
     ##
@@ -67,7 +71,7 @@ module Egis
 
     def add_partitions(partitions)
       load_partitions_query = partitions_generator.to_sql(name, partitions, permissive: true)
-      database.execute_query(load_partitions_query, async: false)
+      database.execute_query(load_partitions_query, async: false, system_execution: true)
     end
 
     ##
@@ -76,7 +80,7 @@ module Egis
 
     def add_partitions!(partitions)
       load_partitions_query = partitions_generator.to_sql(name, partitions, permissive: false)
-      database.execute_query(load_partitions_query, async: false)
+      database.execute_query(load_partitions_query, async: false, system_execution: true)
     end
 
     ##
@@ -87,18 +91,30 @@ module Egis
     # @return [void]
 
     def discover_partitions
-      database.execute_query("MSCK REPAIR TABLE #{name};", async: false)
+      database.execute_query("MSCK REPAIR TABLE #{name};", async: false, system_execution: true)
     end
 
     ##
     # Insert data into the table. Mostly useful for testing purposes.
     #
-    # @param [Array] rows Array of arrays with row values
+    # @example Insert with array of arrays
+    #   table.upload_data([
+    #       ['hello world', 'mx', 1],
+    #       ['hello again', 'us', 2]
+    #   ])
+    #
+    # @example Insert with array of hashes
+    #   table.upload_data([
+    #       {message: 'hello world', country: 'mx', type: 1},
+    #       {message: 'hello again', country: 'us', type: 2}
+    #   ])
+    #
+    # @param [Array] rows Array of arrays or hashes with row values
     # @return [void]
 
     def upload_data(rows)
       query = data_insert_query(rows)
-      database.execute_query(query, async: false)
+      database.execute_query(query, async: false, system_execution: true)
     end
 
     ##
@@ -107,7 +123,7 @@ module Egis
     # @return [Array] Array of arrays with row values.
 
     def download_data
-      result = database.execute_query("SELECT * FROM #{name};", async: false)
+      result = database.execute_query("SELECT * FROM #{name};", async: false, system_execution: true)
       content = output_downloader.download(result.output_location)
       output_parser.parse(content, column_types)
     end
@@ -141,23 +157,37 @@ module Egis
     attr_reader :options, :partitions_generator, :table_ddl_generator, :output_downloader, :output_parser,
                 :table_data_wiper
 
-    def column_serializers
-      @column_serializers ||= column_types.map { |type| Egis::Types.serializer(type) }
+    def log_table_creation
+      Egis.logger.info { "Creating table #{database.name}.#{name} located in #{location}" }
     end
 
     def column_types
-      (schema.columns + schema.partitions).map(&:type)
+      all_columns.map(&:type)
+    end
+
+    def all_columns
+      schema.columns + schema.partitions
     end
 
     def data_insert_query(rows)
+      insert_values = rows.map { |row| row_literal_values(row) }
+      row_clause = insert_values.map { |row| row_values_statement(row) }.join(",\n")
+
       <<~SQL
         INSERT INTO #{name} VALUES
-        #{rows.map { |row| row_values_statement(row) }.join(",\n")};
+        #{row_clause}
       SQL
     end
 
+    def row_literal_values(row)
+      all_columns.map.with_index do |column, index|
+        value = row.is_a?(Hash) ? row[column.name] : row[index]
+        Egis::Types.serializer(column.type).literal(value)
+      end
+    end
+
     def row_values_statement(row)
-      "(#{row.zip(column_serializers).map { |value, serializer| serializer.literal(value) }.join(', ')})"
+      "(#{row.join(', ')})"
     end
   end
 end

data/lib/egis/table_ddl_generator.rb

@@ -9,7 +9,7 @@ module Egis
           #{column_definition_sql(table.schema.columns)}
         )
         #{partition_statement(table.schema)}
-        #{format_statement(table.format)}
+        #{row_format_statement(table.format)}
         LOCATION '#{table.location}';
       SQL
     end
@@ -34,7 +34,30 @@ module Egis
       columns.map { |column| "`#{column.name}` #{column.type}" }.join(",\n")
     end
 
-    def format_statement(format)
+    def serde?(format)
+      format.is_a?(Hash) && format.key?(:serde)
+    end
+
+    def row_format_statement(format)
+      return serde_row_format_statement(format) if serde?(format)
+
+      delimited_row_format_statement(format)
+    end
+
+    def serde_row_format_statement(format)
+      row_format = "ROW FORMAT SERDE '#{format[:serde]}'"
+      return row_format unless format.key?(:serde_properties)
+
+      serde_properties = format[:serde_properties].map { |property, value| "'#{property}' = '#{value}'" }
+      <<-SQL
+        #{row_format}
+        WITH SERDEPROPERTIES (
+          #{serde_properties.join(",\n")}
+        )
+      SQL
+    end
+
+    def delimited_row_format_statement(format)
       case format
       when :csv
         "ROW FORMAT DELIMITED FIELDS TERMINATED BY ','"

data/lib/egis/testing.rb

@@ -43,6 +43,6 @@ module Egis # rubocop:disable Style/Documentation
     yield
   ensure
     @mode = previous_mode
-    test_mode.cleanup
+    test_mode&.cleanup
   end
 end

data/lib/egis/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Egis
-  VERSION = '1.1.0'
+  VERSION = '1.4.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: egis
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.4.0
 platform: ruby
 authors:
 - Agnieszka Czereba
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-05-15 00:00:00.000000000 Z
+date: 2021-03-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-athena
@@ -83,7 +83,7 @@ licenses:
 metadata:
   homepage_uri: https://github.com/u2i/egis
   source_code_uri: https://github.com/u2i/egis
-  changelog_uri: https://github.com/u2i/egis/blob/master/CHANGELOG.md
+  changelog_uri: https://u2i.github.io/egis/file.CHANGELOG.html
 post_install_message: 
 rdoc_options: []
 require_paths:
@@ -99,7 +99,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: A handy wrapper for AWS Athena Ruby SDK.