sneaql-aws 0.0.2-java → 0.0.3-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: a02e256c853198e83c025a8eff04f34d95632953
-  data.tar.gz: 2bf1e880ac1c31fb6bfc54ac55785f8754ad01b2
+SHA256:
+  metadata.gz: 446cdd3ebe709888e9ef72e80a2aa77bfa873528a6be895faa1065a076039663
+  data.tar.gz: 733dc9f2f106313bb58de5f06500813214b5145215fca9d81b14118a6b70f846
 SHA512:
-  metadata.gz: ac2df46e763416b6c3baf0db10ba3cb454b4f195a839a9e4e16e2696fc27fb344127d4f2369964368e574856e053b8fe9b06d422c58b876fb5dfe4dc8c126f55
-  data.tar.gz: b3d4a472e468dfa050d241c5008a77bb713a658da3c6bb38483d13456bc9af3e3dc9a110095187153164240b58b75beec24150d0acd73d5c58414350a4af38d5
+  metadata.gz: 3c844063a7ab258e9974529d5bc3ea93fd816ca4655ca7a8a9e005b20bc63e3b349e8537bd63f31e0bff1ef8b4041d88e09d744defd11407bee0ff254ba59049
+  data.tar.gz: 8c4e445c0c357a3333e9ab3f10e20b4cee651d030776b2d657be91b30857f8a1272df13bbadcd5925c8d680e384e75bd6dee8092767cebcbd2deb92e78993e72

data/lib/sneaql-aws.rb

@@ -1,10 +1,59 @@
 require 'sneaql'
 require 'aws-sdk'
+require 'json'
+require 'zlib'
 
 # AWS extensions for SneaQL
 module SneaqlAWS
+  module Exceptions
+    # Exception used to to gracefully exit test
+    class AWSResourceCreationException < Sneaql::Exceptions::BaseError
+      def initialize(msg = 'Unable to create AWS resource')
+        super
+      end
+    end
+    
+    # Exception used to to gracefully exit test
+    class AWSS3Exception < Sneaql::Exceptions::BaseError
+      def initialize(msg = 'Error interacting with AWS S3 service')
+        super
+      end
+    end
+  end
+  
   # Command tags providing AWS functionalities to SneaQL
   module Commands
+    
+    class AWSInteractions
+      def initialize(logger)
+        @logger = logger ? logger : Logger.new(STDOUT)  
+      end
+      
+      def s3_resource(region)
+        @logger.debug("creating S3 resource...")
+        Aws::S3::Resource.new(region: region)
+      rescue => e
+        @logger.error(e.message)
+        e.backtrace.each { |b| @logger.error(b) }
+        raise SneaqlAWS::Exceptions::AWSResourceCreationException
+      end
+
+      def upload_to_s3(bucket_name, bucket_region, object_key, local_file_path)
+        interactions = AWSInteractions.new(@logger)
+        s3 = s3_resource(bucket_region)
+        @logger.info("uploading local file #{local_file_path} to s3://#{bucket_name}/#{object_key}")
+        s3 = Aws::S3::Resource.new(region: bucket_region)
+        File.open(local_file_path, 'rb') do |file|
+          s3.client.put_object(bucket: bucket_name, key: object_key, body: file)
+        end
+      rescue => e
+        @logger.error(e.message)
+        e.backtrace.each { |b| @logger.error(b) }
+        raise SneaqlAWS::Exceptions::AWSS3Exception
+      end
+      
+    end
+    
     # runs the query then stores the array of hashes into the recordset hash
     class SneaqlAWSS3ObjectList < Sneaql::Core::SneaqlCommand
       Sneaql::Core::RegisterMappedClass.new(
@@ -21,8 +70,8 @@ module SneaqlAWS
           object_prefix
         )
         
-        @logger.debug("adding #{r.length} recs as #{recordset_name}")
         @recordset_manager.store_recordset(recordset_name, r)
+        @logger.info("added #{r.length} records to recordset #{recordset_name}")
       end
 
       # argument types
@@ -35,11 +84,11 @@ module SneaqlAWS
         bucket = @expression_handler.evaluate_expression(bucket)
         region = @expression_handler.evaluate_expression(region)
         prefix = @expression_handler.evaluate_expression(prefix)
-        
-        @logger.debug("creating s3 connection...")
+
         # create connection to s3
         # aws credentials will be resolved through normal SDK precedence rules
-        s3 = Aws::S3::Resource.new(region: region)
+        interactions = AWSInteractions.new(@logger)
+        s3 = interactions.s3_resource(region)
         
         @logger.info("searching for objects in bucket #{bucket} in region #{region} matching prefix #{prefix}")
         # perform the initial object search
@@ -58,6 +107,8 @@ module SneaqlAWS
         # then appended to the res array for use as a recordset 
         resp['contents'].each do |obj|
           res << {
+            region: region,
+            bucket: bucket,
             key: obj.key,
             last_modified: obj.last_modified.to_s,
             etag: obj.etag,
@@ -82,6 +133,8 @@ module SneaqlAWS
           
           resp['contents'].each do |obj|
             res << {
+              region: region,
+              bucket: bucket,
               key: obj.key,
               last_modified: obj.last_modified.to_s,
               etag: obj.etag,
@@ -99,5 +152,89 @@ module SneaqlAWS
         return res
       end
     end
+    
+    # writes a recordset to S3 in a gzip json format
+    class SneaqlAWSRecordsetToS3 < Sneaql::Core::SneaqlCommand
+      Sneaql::Core::RegisterMappedClass.new(
+        :command,
+        'aws_recordset_to_s3',
+        SneaqlAWS::Commands::SneaqlAWSRecordsetToS3
+      )
+
+      def action(recordset_name, bucket, region, key, object_type)
+        bucket_name = @expression_handler.evaluate_expression(bucket)
+        bucket_region = @expression_handler.evaluate_expression(region)
+        object_key = @expression_handler.evaluate_expression(key)
+        
+        recordset_to_file(
+          recordset_name,
+          local_json_path,
+          object_type
+        )
+        
+        interactions = AWSInteractions.new(@logger)
+        interactions.upload_to_s3(
+          bucket_name,
+          bucket_region,
+          object_key,
+          local_json_path
+        )
+      end
+
+      # argument types
+      def arg_definition
+        [:recordset, :expression, :expression, :expression, :symbol]
+      end
+
+      def local_json_path
+        "/tmp/json_export_#{self.object_id}.gz"
+      end
+
+      def recordset_to_file(recordset, local_file_path, object_type)
+        @logger.debug("writing recordset #{recordset} to file of type #{object_type}")
+        
+        if object_type == 'gzipjson'
+          json_file = File.open(local_file_path, 'w')
+          zd = Zlib::GzipWriter.new(json_file)
+
+          json_gzip_streamer = Proc.new do |z, h|
+            h.keys.each do |k|
+              if h[k].class == Float
+                h[k] = nil if (h[k].nan? or h[k].infinite?)
+              end
+            end
+            z << (h.to_json + "\n") && z.write(z.flush)
+          end
+          
+          @recordset_manager.recordset[recordset].each do |r|
+            json_gzip_streamer.call(zd, r)
+          end
+          
+          zd.close
+          json_file.close
+          
+        elsif object_type == 'json'
+          json_file = File.open(local_file_path, 'w')
+          
+          json_formatter = Proc.new do |f, h|
+            h.keys.each do |k|
+              if h[k].class == Float
+                h[k] = nil if (h[k].nan? or h[k].infinite?)
+              end
+            end
+            f.puts h.to_json
+          end
+          
+          @recordset_manager.recordset[recordset].each do |r|
+            json_formatter.call(json_file, r)
+          end
+          
+          json_file.close
+        end
+        
+        @logger.info("#{@recordset_manager.recordset[recordset].length} written for recordset #{recordset} ")
+      end
+      
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sneaql-aws
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: java
 authors:
 - jeremy winters
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-07-28 00:00:00.000000000 Z
+date: 2017-08-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -93,7 +93,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.6
+rubygems_version: 2.6.11
 signing_key:
 specification_version: 4
 summary: sneaql extensions to interact with AWS