athens 0.3.3 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fa391fbff6c5016740bc28cfb0692fcc79ee2e1361d31541a635691e6cbbc8a7
-  data.tar.gz: ef1fbb6307221f616576e844ced0ed9398ff9e6db436fde0142e58c89be8c9f5
+  metadata.gz: cb4922b50ef935d722a54d54d4f3d9a9e47b587b67592454b806bf11e00f7be1
+  data.tar.gz: 2fc14349880958e3212444e8285a0005c76f9e1bcb219ac8423cb1f1cdacfb57
 SHA512:
-  metadata.gz: e37d29e7d93b1aa61d6266b5feccc3789087f697c1751830893adc4c795761f587bafdaa44f34de6b5786712d8983e66511d3d0f10fa1b0922ec6921e4201e66
-  data.tar.gz: 0ccaf8bbb67bceb9136806a93da4447ace996ab9709e82d8f6861249e6a62246680c3a618a1479481b931127e568e966038569da2d1814540ef64b7ab3dd155e
+  metadata.gz: 6c8dc6d78814f5b947d08434b8f2ccf13998439fca59465495401cc9e73f7e7e30efe3899679d545fe17184704b81fab33ddf5ffc5d2eb833e6ea2ef67dc6f2e
+  data.tar.gz: 4e7d3d6f0cd1bdd5da7644c16b8802bc810d8f15f12e05fcd746acb743d48fe5c0ff740928ae117047aa2d6c2417daf33b86cab630cfbe0f42e87650c58e7909

data/.gitignore

@@ -8,4 +8,5 @@
 /tmp/
 
 /.vagrant/
-/ubuntu-xenial-16.04-cloudimg-console.log
+/ubuntu-xenial-16.04-cloudimg-console.log
+/vendor/

data/CHANGELOG.md

@@ -1,3 +1,57 @@
+## 0.4.0 / 2021-12-03
+
+* Added automatic parsing of JSON columns (thanks [darkside](https://github.com/darkside))
+
+#### Potentially breaking change
+
+If you were previously querying JSON type data and parsing it from a string in your own code, you'll want to remove that before upgrading.
+
+For example, with previous versions you could do this (even though it would generate a warning):
+
+```ruby
+> query = conn.execute("SELECT JSON_PARSE('{\"a\": 1, \"b\": 2}');")
+> query.wait
+> JSON.parse(query.to_a(header_row: false).first[0])
+WARNING: Unsupported type: json, defaulting to string
+ => {"a"=>1, "b"=>2}
+```
+
+After upgrading, that same code will give you an error:
+```ruby
+> JSON.parse(query.to_a(header_row: false).first[0])
+Traceback (most recent call last):
+        5: from bin/console:14:in `<main>'
+        4: from (irb):12:in `<main>'
+        3: from /home/vagrant/.rvm/rubies/ruby-3.0.0/lib/ruby/3.0.0/json/common.rb:216:in `parse'
+        2: from /home/vagrant/.rvm/rubies/ruby-3.0.0/lib/ruby/3.0.0/json/common.rb:216:in `new'
+        1: from /home/vagrant/.rvm/rubies/ruby-3.0.0/lib/ruby/3.0.0/json/common.rb:216:in `initialize'
+TypeError (no implicit conversion of Hash into String)
+```
+
+Instead just remove your json parsing since Athens handles it now:
+```ruby
+> query = conn.execute("SELECT JSON_PARSE('{\"a\": 1, \"b\": 2}');")
+> query.wait
+> query.to_a(header_row: false).first[0]
+ => {:a=>1, :b=>2}
+```
+
+
+## 0.3.6 / 2021-11-16
+
+* Addition of :result_encryption as a configuration option to change encryption options for query results (https://github.com/getletterpress/athens/issues/12)
+* Bumped development gem versions to latest releases
+
+## 0.3.5 / 2021-05-19
+
+* Addition of :aws_profile as a configuration option for credentials (thanks [oisin](https://github.com/oisin))
+* Fix for BigDecimal in Ruby 3.x (thanks [mediafinger](https://github.com/mediafinger))
+* Bumped development gem versions to latest releases
+
+## 0.3.4 / 2021-03-02
+
+* Added configurable polling period (thanks [jesseproudman](https://github.com/jesseproudman))
+
 ## 0.3.3 / 2021-01-12
 
 * Added support for Ruby 3.0 (thanks [blackjiro](https://github.com/blackjiro))

data/Gemfile.lock

@@ -1,27 +1,29 @@
 PATH
   remote: .
   specs:
-    athens (0.3.3)
+    athens (0.4.0)
       aws-sdk-athena (~> 1)
+      multi_json (~> 1.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    aws-eventstream (1.1.0)
-    aws-partitions (1.416.0)
-    aws-sdk-athena (1.33.0)
-      aws-sdk-core (~> 3, >= 3.109.0)
+    aws-eventstream (1.2.0)
+    aws-partitions (1.539.0)
+    aws-sdk-athena (1.44.0)
+      aws-sdk-core (~> 3, >= 3.122.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-core (3.111.0)
+    aws-sdk-core (3.124.0)
       aws-eventstream (~> 1, >= 1.0.2)
-      aws-partitions (~> 1, >= 1.239.0)
+      aws-partitions (~> 1, >= 1.525.0)
       aws-sigv4 (~> 1.1)
       jmespath (~> 1.0)
-    aws-sigv4 (1.2.2)
+    aws-sigv4 (1.4.0)
       aws-eventstream (~> 1, >= 1.0.2)
     jmespath (1.4.0)
-    rake (13.0.1)
-    rexml (3.2.4)
+    multi_json (1.15.0)
+    rake (13.0.6)
+    rexml (3.2.5)
 
 PLATFORMS
   ruby

data/README.md

@@ -44,8 +44,8 @@ When your query is done, grab the results as an array:
 ```ruby
 results = query.to_a
 # [
-#   ['column_1', 'column_2', 'column_3'], 
-#   [15, 'data', true], 
+#   ['column_1', 'column_2', 'column_3'],
+#   [15, 'data', true],
 #   [20, 'foo', false],
 #   ...
 # ]
@@ -55,7 +55,7 @@ Or as a hash (which is really an array where each row is a hash):
 ```ruby
 results = query.to_h
 # [
-#   {'column_1': 15, 'column_2': 'data', 'column_3': true}, 
+#   {'column_1': 15, 'column_2': 'data', 'column_3': true},
 #   {'column_1': 20, 'column_2': 'foo', 'column_3': false},
 #   ...
 # ]
@@ -87,9 +87,12 @@ Configure your AWS settings in an `Athens.configure` block (in rails put this in
 ```ruby
 Athens.configure do |config|
   config.output_location = "s3://my-bucket/my-folder/athena/results/"  # Required
-  config.aws_access_key = 'access'     # Optional
-  config.aws_secret_key = 'secret'     # Optional
-  config.aws_region     = 'us-east-1'  # Optional
+  config.aws_access_key      = 'access'     # Optional
+  config.aws_secret_key      = 'secret'     # Optional
+  config.aws_profile         = 'myprofile'  # Optional
+  config.aws_region          = 'us-east-1'  # Optional
+  config.wait_polling_period = 0.25         # Optional - What period should we poll for the complete query?
+  config.result_encryption   = nil          # Optional, see below
 end
 ```
 
@@ -103,6 +106,14 @@ conn = Athens::Connection.new(aws_client_override: {})
 
 Take a look at the [AWS Athena SDK](https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/Athena/Client.html#initialize-instance_method) for a list of all the available options.
 
+The `result_encryption` option controls how the Athens results will be encrypted at the `output_location`.  By default it's set to use the Amazon SSE encryption if you don't set it at all:
+
+```ruby
+{ encryption_option: "SSE_S3" }
+```
+
+If you set it to `nil`, it'll default to the bucket encryption settings.  You can also use a customer kms key, see https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/Athena/Types/EncryptionConfiguration.html for the correct format.
+
 ### Advanced Usage
 
 Providing a database name to the connection is optional, if you omit the name you'll have to specify it in your query:
@@ -172,3 +183,4 @@ The gem is available as open source under the terms of the [WTFPL License](http:
 ## Code of Conduct
 
 Everyone interacting in the Athens project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/getletterpress/athens/blob/master/CODE_OF_CONDUCT.md).
+

data/Vagrantfile

@@ -4,13 +4,11 @@
 # Provisioning script
 $script = <<SCRIPT
   echo "*** Updating packages"
-
-  sudo DEBIAN_FRONTEND=noninteractive apt-get install -y aptitude
-  sudo DEBIAN_FRONTEND=noninteractive aptitude update
-  sudo DEBIAN_FRONTEND=noninteractive aptitude -y safe-upgrade
+  sudo DEBIAN_FRONTEND=noninteractive apt-get update
+  sudo DEBIAN_FRONTEND=noninteractive apt-get upgrade
 
   echo "*** Installing new packages"
-  sudo DEBIAN_FRONTEND=noninteractive aptitude install -y curl git-core vim
+  sudo DEBIAN_FRONTEND=noninteractive apt-get install -y curl git-core vim
 
   if rvm -v 2>/dev/null; then
     echo "*** rvm already installed, skipping"
@@ -30,7 +28,7 @@ SCRIPT
 
 
 Vagrant.configure('2') do |config|
-  config.vm.box      = "ubuntu/xenial64"
+  config.vm.box      = "ubuntu/focal64"
   config.vm.hostname = 'athens-dev'
 
   # Provision the machine with the shell script above

data/athens.gemspec

@@ -35,6 +35,7 @@ Gem::Specification.new do |spec|
   spec.required_ruby_version = '>= 2.4'
 
   spec.add_dependency "aws-sdk-athena", "~> 1"
+  spec.add_dependency 'multi_json', '~> 1.0'
 
   spec.add_development_dependency "bundler", ">= 1.17"
   spec.add_development_dependency "rake", "~> 13.0"

data/lib/athens/configuration.rb

@@ -1,15 +1,21 @@
 module Athens
   class Configuration
-    attr_accessor :aws_access_key, 
+    attr_accessor :aws_access_key,
                   :aws_secret_key,
                   :aws_region,
-                  :output_location
+                  :output_location,
+                  :wait_polling_period,
+                  :aws_profile,
+                  :result_encryption
 
     def initialize
       @aws_access_key = nil
       @aws_secret_key = nil
       @aws_region = nil
       @output_location = nil
+      @wait_polling_period = 0.25
+      @aws_profile = nil
+      @result_encryption = { encryption_option: "SSE_S3" }
     end
   end
-end
+end

data/lib/athens/connection.rb

@@ -11,7 +11,8 @@ module Athens
       client_config = {
         access_key_id: Athens.configuration.aws_access_key,
         secret_access_key: Athens.configuration.aws_secret_key,
-        region: Athens.configuration.aws_region
+        region: Athens.configuration.aws_region,
+        profile: Athens.configuration.aws_profile
       }.merge(aws_client_override).compact
 
       @client = Aws::Athena::Client.new(client_config)
@@ -47,9 +48,7 @@ module Athens
       def result_config
         Aws::Athena::Types::ResultConfiguration.new(
           output_location: Athens.configuration.output_location,
-          encryption_configuration: {
-            encryption_option: "SSE_S3"
-          }
+          encryption_configuration: Athens.configuration.result_encryption
         )
       end
 

data/lib/athens/query.rb

@@ -14,6 +14,7 @@ module Athens
 
       version = RUBY_VERSION.split('.').map {|v| v.to_i}
       @decimal_without_new = (version[0] >= 2 && version[1] >= 5)
+      @decimal_without_new = (version[0] == 2 && version[1] >= 5) || (version[0] >= 3)
     end
 
     def state
@@ -49,7 +50,7 @@ module Athens
         end
 
         # Wait a bit and check again
-        sleep(0.25)
+        sleep(Athens.configuration.wait_polling_period.to_f)
       end
     end
 
@@ -140,7 +141,7 @@ module Athens
         metadata.column_info.each_with_index do |col, index|
           data = row.data[index].var_char_value
           nullable = ["UNKNOWN", "NULLABLE"].include?(col.nullable)
-
+          
           if nullable && data.nil?
             mapped << data
           elsif !nullable && data.nil?
@@ -165,6 +166,8 @@ module Athens
               mapped << Date.parse(data)
             when 'boolean'
               mapped << (data == "true")
+            when 'json'
+              mapped << MultiJson.load(data, symbolize_keys: true)
             else
               puts "WARNING: Unsupported type: #{col.type}, defaulting to string"
               mapped << data
@@ -173,9 +176,8 @@ module Athens
         end
 
         return mapped
-      end    
+      end
 
 
   end
 end
-

data/lib/athens/version.rb

@@ -1,3 +1,3 @@
 module Athens
-  VERSION = "0.3.3"
+  VERSION = "0.4.0"
 end

data/lib/athens.rb

@@ -5,6 +5,7 @@ require "athens/connection"
 require "athens/query"
 
 require 'aws-sdk-athena'
+require 'multi_json'
 
 module Athens
   class << self

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: athens
 version: !ruby/object:Gem::Version
-  version: 0.3.3
+  version: 0.4.0
 platform: ruby
 authors:
 - Chris Schulte
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-01-12 00:00:00.000000000 Z
+date: 2021-12-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-athena
@@ -24,6 +24,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1'
+- !ruby/object:Gem::Dependency
+  name: multi_json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -113,7 +127,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.0.9
 signing_key:
 specification_version: 4
 summary: Run simple SQL queries in AWS Athena