sneaql-aws 0.0.2-java → 0.0.3-java

Sign up to get free protection for your applications and to get access to all the features.
Files changed (3) hide show
  1. checksums.yaml +5 -5
  2. data/lib/sneaql-aws.rb +141 -4
  3. metadata +3 -3
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
- SHA1:
3
- metadata.gz: a02e256c853198e83c025a8eff04f34d95632953
4
- data.tar.gz: 2bf1e880ac1c31fb6bfc54ac55785f8754ad01b2
2
+ SHA256:
3
+ metadata.gz: 446cdd3ebe709888e9ef72e80a2aa77bfa873528a6be895faa1065a076039663
4
+ data.tar.gz: 733dc9f2f106313bb58de5f06500813214b5145215fca9d81b14118a6b70f846
5
5
  SHA512:
6
- metadata.gz: ac2df46e763416b6c3baf0db10ba3cb454b4f195a839a9e4e16e2696fc27fb344127d4f2369964368e574856e053b8fe9b06d422c58b876fb5dfe4dc8c126f55
7
- data.tar.gz: b3d4a472e468dfa050d241c5008a77bb713a658da3c6bb38483d13456bc9af3e3dc9a110095187153164240b58b75beec24150d0acd73d5c58414350a4af38d5
6
+ metadata.gz: 3c844063a7ab258e9974529d5bc3ea93fd816ca4655ca7a8a9e005b20bc63e3b349e8537bd63f31e0bff1ef8b4041d88e09d744defd11407bee0ff254ba59049
7
+ data.tar.gz: 8c4e445c0c357a3333e9ab3f10e20b4cee651d030776b2d657be91b30857f8a1272df13bbadcd5925c8d680e384e75bd6dee8092767cebcbd2deb92e78993e72
data/lib/sneaql-aws.rb CHANGED
@@ -1,10 +1,59 @@
1
1
  require 'sneaql'
2
2
  require 'aws-sdk'
3
+ require 'json'
4
+ require 'zlib'
3
5
 
4
6
  # AWS extensions for SneaQL
5
7
  module SneaqlAWS
8
+ module Exceptions
9
+ # Exception used to to gracefully exit test
10
+ class AWSResourceCreationException < Sneaql::Exceptions::BaseError
11
+ def initialize(msg = 'Unable to create AWS resource')
12
+ super
13
+ end
14
+ end
15
+
16
+ # Exception used to to gracefully exit test
17
+ class AWSS3Exception < Sneaql::Exceptions::BaseError
18
+ def initialize(msg = 'Error interacting with AWS S3 service')
19
+ super
20
+ end
21
+ end
22
+ end
23
+
6
24
  # Command tags providing AWS functionalities to SneaQL
7
25
  module Commands
26
+
27
+ class AWSInteractions
28
+ def initialize(logger)
29
+ @logger = logger ? logger : Logger.new(STDOUT)
30
+ end
31
+
32
+ def s3_resource(region)
33
+ @logger.debug("creating S3 resource...")
34
+ Aws::S3::Resource.new(region: region)
35
+ rescue => e
36
+ @logger.error(e.message)
37
+ e.backtrace.each { |b| @logger.error(b) }
38
+ raise SneaqlAWS::Exceptions::AWSResourceCreationException
39
+ end
40
+
41
+ def upload_to_s3(bucket_name, bucket_region, object_key, local_file_path)
42
+ interactions = AWSInteractions.new(@logger)
43
+ s3 = s3_resource(bucket_region)
44
+ @logger.info("uploading local file #{local_file_path} to s3://#{bucket_name}/#{object_key}")
45
+ s3 = Aws::S3::Resource.new(region: bucket_region)
46
+ File.open(local_file_path, 'rb') do |file|
47
+ s3.client.put_object(bucket: bucket_name, key: object_key, body: file)
48
+ end
49
+ rescue => e
50
+ @logger.error(e.message)
51
+ e.backtrace.each { |b| @logger.error(b) }
52
+ raise SneaqlAWS::Exceptions::AWSS3Exception
53
+ end
54
+
55
+ end
56
+
8
57
  # runs the query then stores the array of hashes into the recordset hash
9
58
  class SneaqlAWSS3ObjectList < Sneaql::Core::SneaqlCommand
10
59
  Sneaql::Core::RegisterMappedClass.new(
@@ -21,8 +70,8 @@ module SneaqlAWS
21
70
  object_prefix
22
71
  )
23
72
 
24
- @logger.debug("adding #{r.length} recs as #{recordset_name}")
25
73
  @recordset_manager.store_recordset(recordset_name, r)
74
+ @logger.info("added #{r.length} records to recordset #{recordset_name}")
26
75
  end
27
76
 
28
77
  # argument types
@@ -35,11 +84,11 @@ module SneaqlAWS
35
84
  bucket = @expression_handler.evaluate_expression(bucket)
36
85
  region = @expression_handler.evaluate_expression(region)
37
86
  prefix = @expression_handler.evaluate_expression(prefix)
38
-
39
- @logger.debug("creating s3 connection...")
87
+
40
88
  # create connection to s3
41
89
  # aws credentials will be resolved through normal SDK precedence rules
42
- s3 = Aws::S3::Resource.new(region: region)
90
+ interactions = AWSInteractions.new(@logger)
91
+ s3 = interactions.s3_resource(region)
43
92
 
44
93
  @logger.info("searching for objects in bucket #{bucket} in region #{region} matching prefix #{prefix}")
45
94
  # perform the initial object search
@@ -58,6 +107,8 @@ module SneaqlAWS
58
107
  # then appended to the res array for use as a recordset
59
108
  resp['contents'].each do |obj|
60
109
  res << {
110
+ region: region,
111
+ bucket: bucket,
61
112
  key: obj.key,
62
113
  last_modified: obj.last_modified.to_s,
63
114
  etag: obj.etag,
@@ -82,6 +133,8 @@ module SneaqlAWS
82
133
 
83
134
  resp['contents'].each do |obj|
84
135
  res << {
136
+ region: region,
137
+ bucket: bucket,
85
138
  key: obj.key,
86
139
  last_modified: obj.last_modified.to_s,
87
140
  etag: obj.etag,
@@ -99,5 +152,89 @@ module SneaqlAWS
99
152
  return res
100
153
  end
101
154
  end
155
+
156
+ # writes a recordset to S3 in a gzip json format
157
+ class SneaqlAWSRecordsetToS3 < Sneaql::Core::SneaqlCommand
158
+ Sneaql::Core::RegisterMappedClass.new(
159
+ :command,
160
+ 'aws_recordset_to_s3',
161
+ SneaqlAWS::Commands::SneaqlAWSRecordsetToS3
162
+ )
163
+
164
+ def action(recordset_name, bucket, region, key, object_type)
165
+ bucket_name = @expression_handler.evaluate_expression(bucket)
166
+ bucket_region = @expression_handler.evaluate_expression(region)
167
+ object_key = @expression_handler.evaluate_expression(key)
168
+
169
+ recordset_to_file(
170
+ recordset_name,
171
+ local_json_path,
172
+ object_type
173
+ )
174
+
175
+ interactions = AWSInteractions.new(@logger)
176
+ interactions.upload_to_s3(
177
+ bucket_name,
178
+ bucket_region,
179
+ object_key,
180
+ local_json_path
181
+ )
182
+ end
183
+
184
+ # argument types
185
+ def arg_definition
186
+ [:recordset, :expression, :expression, :expression, :symbol]
187
+ end
188
+
189
+ def local_json_path
190
+ "/tmp/json_export_#{self.object_id}.gz"
191
+ end
192
+
193
+ def recordset_to_file(recordset, local_file_path, object_type)
194
+ @logger.debug("writing recordset #{recordset} to file of type #{object_type}")
195
+
196
+ if object_type == 'gzipjson'
197
+ json_file = File.open(local_file_path, 'w')
198
+ zd = Zlib::GzipWriter.new(json_file)
199
+
200
+ json_gzip_streamer = Proc.new do |z, h|
201
+ h.keys.each do |k|
202
+ if h[k].class == Float
203
+ h[k] = nil if (h[k].nan? or h[k].infinite?)
204
+ end
205
+ end
206
+ z << (h.to_json + "\n") && z.write(z.flush)
207
+ end
208
+
209
+ @recordset_manager.recordset[recordset].each do |r|
210
+ json_gzip_streamer.call(zd, r)
211
+ end
212
+
213
+ zd.close
214
+ json_file.close
215
+
216
+ elsif object_type == 'json'
217
+ json_file = File.open(local_file_path, 'w')
218
+
219
+ json_formatter = Proc.new do |f, h|
220
+ h.keys.each do |k|
221
+ if h[k].class == Float
222
+ h[k] = nil if (h[k].nan? or h[k].infinite?)
223
+ end
224
+ end
225
+ f.puts h.to_json
226
+ end
227
+
228
+ @recordset_manager.recordset[recordset].each do |r|
229
+ json_formatter.call(json_file, r)
230
+ end
231
+
232
+ json_file.close
233
+ end
234
+
235
+ @logger.info("#{@recordset_manager.recordset[recordset].length} written for recordset #{recordset} ")
236
+ end
237
+
238
+ end
102
239
  end
103
240
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: sneaql-aws
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.2
4
+ version: 0.0.3
5
5
  platform: java
6
6
  authors:
7
7
  - jeremy winters
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2017-07-28 00:00:00.000000000 Z
11
+ date: 2017-08-15 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -93,7 +93,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
93
93
  version: '0'
94
94
  requirements: []
95
95
  rubyforge_project:
96
- rubygems_version: 2.6.6
96
+ rubygems_version: 2.6.11
97
97
  signing_key:
98
98
  specification_version: 4
99
99
  summary: sneaql extensions to interact with AWS