sneaql-aws 0.0.2-java

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: a02e256c853198e83c025a8eff04f34d95632953
+  data.tar.gz: 2bf1e880ac1c31fb6bfc54ac55785f8754ad01b2
+SHA512:
+  metadata.gz: ac2df46e763416b6c3baf0db10ba3cb454b4f195a839a9e4e16e2696fc27fb344127d4f2369964368e574856e053b8fe9b06d422c58b876fb5dfe4dc8c126f55
+  data.tar.gz: b3d4a472e468dfa050d241c5008a77bb713a658da3c6bb38483d13456bc9af3e3dc9a110095187153164240b58b75beec24150d0acd73d5c58414350a4af38d5

data/lib/sneaql-aws.rb

@@ -0,0 +1,103 @@
+require 'sneaql'
+require 'aws-sdk'
+
+# AWS extensions for SneaQL
+module SneaqlAWS
+  # Command tags providing AWS functionalities to SneaQL
+  module Commands
+    # runs the query then stores the array of hashes into the recordset hash
+    class SneaqlAWSS3ObjectList < Sneaql::Core::SneaqlCommand
+      Sneaql::Core::RegisterMappedClass.new(
+        :command,
+        'aws_s3_object_list',
+        SneaqlAWS::Commands::SneaqlAWSS3ObjectList
+      )
+
+      # @param [String] recordset_name name of the recordset in which to store the results
+      def action(recordset_name, bucket_name, bucket_region, object_prefix)
+        r = s3_object_search(
+          bucket_name,
+          bucket_region,
+          object_prefix
+        )
+        
+        @logger.debug("adding #{r.length} recs as #{recordset_name}")
+        @recordset_manager.store_recordset(recordset_name, r)
+      end
+
+      # argument types
+      def arg_definition
+        [:recordset, :expression, :expression, :expression]
+      end
+      
+      # @return [Array] returns array of hashes from SQL results
+      def s3_object_search(bucket, region, prefix)
+        bucket = @expression_handler.evaluate_expression(bucket)
+        region = @expression_handler.evaluate_expression(region)
+        prefix = @expression_handler.evaluate_expression(prefix)
+        
+        @logger.debug("creating s3 connection...")
+        # create connection to s3
+        # aws credentials will be resolved through normal SDK precedence rules
+        s3 = Aws::S3::Resource.new(region: region)
+        
+        @logger.info("searching for objects in bucket #{bucket} in region #{region} matching prefix #{prefix}")
+        # perform the initial object search
+        resp = s3.client.list_objects_v2(
+          {
+            bucket: bucket,
+            prefix: prefix,
+            fetch_owner: true
+          }
+        )
+        
+        # array to hold results
+        res = []
+        
+        # response contents are converted to native ruby hashes
+        # then appended to the res array for use as a recordset 
+        resp['contents'].each do |obj|
+          res << {
+            key: obj.key,
+            last_modified: obj.last_modified.to_s,
+            etag: obj.etag,
+            size: obj.size,
+            storage_class: obj.storage_class,
+            owner_name: obj.owner['display_name'],
+            owner_id: obj.owner['id']
+          }
+        end
+        
+        # check to see if there is a continuation token
+        next_token = resp.next_continuation_token
+        
+        # repeat until all results have been gathered
+        while next_token do
+          resp = s3.client.list_objects_v2(
+            {
+              bucket: bucket,
+              continuation_token: next_token
+            }
+          )
+          
+          resp['contents'].each do |obj|
+            res << {
+              key: obj.key,
+              last_modified: obj.last_modified.to_s,
+              etag: obj.etag,
+              size: obj.size,
+              storage_class: obj.storage_class,
+              owner_name: obj.owner['display_name'],
+              owner_id: obj.owner['id']
+            }
+          end
+          
+          next_token = resp.next_continuation_token
+        end
+        
+        @logger.info("#{res.length} objects found")
+        return res
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,100 @@
+--- !ruby/object:Gem::Specification
+name: sneaql-aws
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+platform: java
+authors:
+- jeremy winters
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2017-07-28 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  name: logger
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.9'
+  name: minitest
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.9'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  name: git
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.6'
+  name: aws-sdk
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.6'
+description: provides extensions to sneaql allowing interaction with AWS
+email: jeremy.winters@full360.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/sneaql-aws.rb
+homepage: https://www.full360.com
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.6.6
+signing_key:
+specification_version: 4
+summary: sneaql extensions to interact with AWS
+test_files: []