athens 0.2.0 → 0.3.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d1fb8ee7934d4e7019f1f54dbd3b3eb4368536e3974a00b35442898e4e8b267f
-  data.tar.gz: 1685787b3a1c68f850cc7ca01523a0f3a4feaeaae26ed5d9f77e41ffdafe4b15
+  metadata.gz: 763fdc0eeba122974a101d60e8b17045d3d55dcf63f7144fe4db181a6808b858
+  data.tar.gz: 2afff645fa2b925e9317100043a2c574574c7245a99edbc29c8da5c508cb766a
 SHA512:
-  metadata.gz: 7ba436dc94c82c4b6642e8084e76455d3f32a786bc718364c89f2e3769b3786ddcfb65f0e1526d62b56c5d11a6e6e67cab501f3063b8e5e0c59fa2e46a219292
-  data.tar.gz: b6ef5c5bd6e318348fb4bb9fb794c37ff57be2e13bdcaaeef707246a524b3ea6b5d65ba62e0a2d3d7dbfab696ac6fbbfbfad33f943c473d1c2abc8747849658d
+  metadata.gz: a0e3cf85dfd919a884f90de3bdb1a6896bedc12ed327ae237858ae4182d2565e7d4c19453b2d68f77717ed8763c65ba6565ba654ef34c49aed2273e51d43c5cb
+  data.tar.gz: 387787c6e6f2ab20ff30714374591fd7211fbb9b3181e35df0a0e27fe800444af1d7428c00c7648c1ba2e3b7f1a1e6672574f0e3a38cfa1124d1bcaadd83adcd

data/CHANGELOG.md

@@ -1,3 +1,25 @@
+## 0.3.4 / 2021-03-02
+
+* Added configurable polling period (thanks [jesseproudman](https://github.com/jesseproudman))
+
+## 0.3.3 / 2021-01-12
+
+* Added support for Ruby 3.0 (thanks [blackjiro](https://github.com/blackjiro))
+
+## 0.3.2 / 2020-11-24
+
+* Added optional `request_token` and `work_group` parameters to the query execute method (thanks [mediafinger](https://github.com/mediafinger))
+
+## 0.3.1 / 2020-07-20
+
+* Bumped development rake version from 0.10 to 0.13 for security fixes
+* Fixed warning about string defaulting to string (#2)
+
+## 0.3.0 / 2019-07-02
+
+* Added enumerator-based result access methods: `#rows` and `#records`
+* Fixed bug dropping headers from memoized rows across `#to_h` and `#to_a(header_row: true)`
+
 ## 0.2.0 / 2019-03-20
 
 * Added support for NULL values.  Columns with a nullable status of "NULLABLE" or "UNKNOWN" will return nil for NULL values

data/Gemfile.lock

@@ -1,33 +1,36 @@
 PATH
   remote: .
   specs:
-    athens (0.2.0)
+    athens (0.3.4)
       aws-sdk-athena (~> 1)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    aws-eventstream (1.0.1)
-    aws-partitions (1.136.0)
-    aws-sdk-athena (1.7.0)
-      aws-sdk-core (~> 3, >= 3.39.0)
-      aws-sigv4 (~> 1.0)
-    aws-sdk-core (3.46.0)
-      aws-eventstream (~> 1.0)
-      aws-partitions (~> 1.0)
-      aws-sigv4 (~> 1.0)
+    aws-eventstream (1.1.0)
+    aws-partitions (1.430.0)
+    aws-sdk-athena (1.35.0)
+      aws-sdk-core (~> 3, >= 3.112.0)
+      aws-sigv4 (~> 1.1)
+    aws-sdk-core (3.112.0)
+      aws-eventstream (~> 1, >= 1.0.2)
+      aws-partitions (~> 1, >= 1.239.0)
+      aws-sigv4 (~> 1.1)
       jmespath (~> 1.0)
-    aws-sigv4 (1.0.3)
+    aws-sigv4 (1.2.2)
+      aws-eventstream (~> 1, >= 1.0.2)
     jmespath (1.4.0)
-    rake (10.5.0)
+    rake (13.0.1)
+    rexml (3.2.4)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
   athens!
-  bundler (~> 1.17)
-  rake (~> 10.0)
+  bundler (>= 1.17)
+  rake (~> 13.0)
+  rexml (~> 3.2)
 
 BUNDLED WITH
-   1.17.2
+   2.2.5

data/README.md

@@ -44,8 +44,8 @@ When your query is done, grab the results as an array:
 ```ruby
 results = query.to_a
 # [
-#   ['column_1', 'column_2', 'column_3'], 
-#   [15, 'data', true], 
+#   ['column_1', 'column_2', 'column_3'],
+#   [15, 'data', true],
 #   [20, 'foo', false],
 #   ...
 # ]
@@ -55,12 +55,29 @@ Or as a hash (which is really an array where each row is a hash):
 ```ruby
 results = query.to_h
 # [
-#   {'column_1': 15, 'column_2': 'data', 'column_3': true}, 
+#   {'column_1': 15, 'column_2': 'data', 'column_3': true},
 #   {'column_1': 20, 'column_2': 'foo', 'column_3': false},
 #   ...
 # ]
 ```
 
+Results are also available as unbuffered enumerators of row arrays:
+```ruby
+query.rows.each {|row| ...}
+# ['column_1', 'column_2', 'column_3']
+# [15, 'data', true]
+# [20, 'foo', false],
+# ...
+```
+
+Or hashes:
+```ruby
+query.records.each {|record| ...}
+# {'column_1': 15, 'column_2': 'data', 'column_3': true}
+# {'column_1': 20, 'column_2': 'foo', 'column_3': false}
+# ...
+```
+
 Athens attempts to parse the sql data types into their ruby equivalents, although there's currently no support for the more complex Array/Map types.
 
 ### Configuration
@@ -70,9 +87,10 @@ Configure your AWS settings in an `Athens.configure` block (in rails put this in
 ```ruby
 Athens.configure do |config|
   config.output_location = "s3://my-bucket/my-folder/athena/results/"  # Required
-  config.aws_access_key = 'access'     # Optional
-  config.aws_secret_key = 'secret'     # Optional
-  config.aws_region     = 'us-east-1'  # Optional
+  config.aws_access_key      = 'access'     # Optional
+  config.aws_secret_key      = 'secret'     # Optional
+  config.aws_region          = 'us-east-1'  # Optional
+  config.wait_polling_period = 0.25         # Optional - What period should we poll for the complete query?
 end
 ```
 
@@ -129,6 +147,13 @@ query.cancel   # Attempts to cancel an in-progress query, returns true or false
 query.to_a(header_row: false)  # If you want your query results returned without a header row of column names
 ```
 
+The execute method also optionally supports the `request_token` and `work_group` [parameters](https://docs.aws.amazon.com/athena/latest/APIReference/API_StartQueryExecution.html#API_StartQueryExecution_RequestSyntax):
+
+```ruby
+conn = Athens::Connection.new(database: 'sample')
+query = conn.execute("SELECT * FROM mytable", request_token: single_use_token, work_group: my_work_group)
+```
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
@@ -148,3 +173,4 @@ The gem is available as open source under the terms of the [WTFPL License](http:
 ## Code of Conduct
 
 Everyone interacting in the Athens project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/getletterpress/athens/blob/master/CODE_OF_CONDUCT.md).
+

data/Rakefile

@@ -1,2 +1 @@
 require "bundler/gem_tasks"
-task :default => :spec

data/athens.gemspec

@@ -32,10 +32,11 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.required_ruby_version = '~> 2.4'
+  spec.required_ruby_version = '>= 2.4'
 
   spec.add_dependency "aws-sdk-athena", "~> 1"
 
-  spec.add_development_dependency "bundler", "~> 1.17"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "bundler", ">= 1.17"
+  spec.add_development_dependency "rake", "~> 13.0"
+  spec.add_development_dependency "rexml", "~> 3.2"
 end

data/lib/athens/configuration.rb

@@ -1,15 +1,17 @@
 module Athens
   class Configuration
-    attr_accessor :aws_access_key, 
+    attr_accessor :aws_access_key,
                   :aws_secret_key,
                   :aws_region,
-                  :output_location
+                  :output_location,
+                  :wait_polling_period
 
     def initialize
       @aws_access_key = nil
       @aws_secret_key = nil
       @aws_region = nil
       @output_location = nil
+      @wait_polling_period = 0.25
     end
   end
-end
+end

data/lib/athens/connection.rb

@@ -8,7 +8,7 @@ module Athens
     def initialize(database: nil, aws_client_override: {})
       @database_name = database
 
-      client_config = { 
+      client_config = {
         access_key_id: Athens.configuration.aws_access_key,
         secret_access_key: Athens.configuration.aws_secret_key,
         region: Athens.configuration.aws_region
@@ -19,26 +19,28 @@ module Athens
 
     # Runs a query against Athena, returning an Athens::Query object
     # that you can use to wait for it to finish or get the results
-    def execute(query)
+    def execute(query, request_token: nil, work_group: nil)
       if @database_name
         resp = @client.start_query_execution(
           query_string: query,
           query_execution_context: context,
-          result_configuration: result_config
+          result_configuration: result_config,
+          client_request_token: request_token,
+          work_group: work_group
         )
       else
         resp = @client.start_query_execution(
           query_string: query,
           result_configuration: result_config
         )
-      end        
+      end
 
       return Athens::Query.new(self, resp.query_execution_id)
     end
 
     private
 
-      def context 
+      def context
         Aws::Athena::Types::QueryExecutionContext.new(database: @database_name)
       end
 

data/lib/athens/query.rb

@@ -49,7 +49,7 @@ module Athens
         end
 
         # Wait a bit and check again
-        sleep(0.25)
+        sleep(Athens.configuration.wait_polling_period.to_f)
       end
     end
 
@@ -69,12 +69,10 @@ module Athens
       end
     end
 
-    def to_a(header_row: true)
+    def rows
       raise InvalidRequestError.new("Query must be in SUCCEEDED state to return results") unless @state == 'SUCCEEDED'
 
-      if @results.nil?
-        # Need to load and map all of the rows from the original result
-        @results = []
+      Enumerator.new do |y|
         result = @connection.client.get_query_results({query_execution_id: @query_execution_id})
 
         metadata = result.result_set.result_set_metadata
@@ -85,53 +83,43 @@ module Athens
           break if rows.empty?
 
           if first
-            @results << rows.shift.data.map {|col| col.var_char_value}
+            y << rows.shift.data.map {|col| col.var_char_value}
             first = false
           end
 
-          rows.each do |row|
-            @results << map_types(metadata, row)
-          end
+          rows.each {|row| y << map_types(metadata, row)}
 
-          if result.next_token
-            result = @connection.client.get_query_results({
-              query_execution_id: @query_execution_id,
-              next_token: result.next_token
-            })
-          else
-            # No more rows, break out and return our mapped data
-            break
-          end
-        end
-      end
+          break unless result.next_token
 
-      if header_row
-        return @results
-      else
-        return @results[1, @results.size]
+          result = @connection.client.get_query_results({
+            query_execution_id: @query_execution_id,
+            next_token: result.next_token
+          })
+        end
       end
     end
 
-    def to_h
-      if @hash_results.nil?
-        all_rows = self.to_a(header_row: true)
-
-        headers = all_rows.shift
+    def records
+      Enumerator.new do |y|
+        headers = nil
 
-        @hash_results = []
-
-        unless headers.nil?
-          all_rows.each do |row|
-            map = {}
-            headers.each_with_index do |header, index|
-              map[header] = row[index]
-            end
-            @hash_results << map
+        rows.each_with_index do |row|
+          if headers.nil?
+            headers = row
+            next
           end
+
+          y << Hash[headers.zip(row)]
         end
       end
+    end
+
+    def to_a(header_row: true)
+      (@results ||= rows.to_a).drop(header_row ? 0 : 1)
+    end
 
-      return @hash_results
+    def to_h
+      @hash_results ||= records.to_a
     end
 
     private
@@ -163,7 +151,7 @@ module Athens
               mapped << data.to_i
             when 'timestamp'
               mapped << Time.parse(data)
-            when 'varchar'
+            when 'varchar', 'string'
               mapped << data
             when 'float', 'double'
               mapped << data.to_f
@@ -185,9 +173,8 @@ module Athens
         end
 
         return mapped
-      end    
+      end
 
 
   end
 end
-

data/lib/athens/version.rb

@@ -1,3 +1,3 @@
 module Athens
-  VERSION = "0.2.0"
+  VERSION = "0.3.4"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: athens
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.4
 platform: ruby
 authors:
 - Chris Schulte
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2019-03-20 00:00:00.000000000 Z
+date: 2021-03-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-athena
@@ -28,14 +28,14 @@ dependencies:
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.17'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.17'
 - !ruby/object:Gem::Dependency
@@ -44,14 +44,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: rexml
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '3.2'
 description: Allows you to easily access AWS Athena databases and run queries
 email:
 - chris@oceanbreezesoftware.com
@@ -84,13 +98,13 @@ metadata:
   homepage_uri: https://github.com/getletterpress/athens
   source_code_uri: https://github.com/getletterpress/athens
   changelog_uri: https://github.com/getletterpress/athens/CHANGELOG.md
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - "~>"
+  - - ">="
     - !ruby/object:Gem::Version
       version: '2.4'
 required_rubygems_version: !ruby/object:Gem::Requirement
@@ -99,9 +113,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.8
-signing_key: 
+rubygems_version: 3.1.4
+signing_key:
 specification_version: 4
 summary: Run simple SQL queries in AWS Athena
 test_files: []