sneaql-aws 0.0.2-java → 0.0.3-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/lib/sneaql-aws.rb +141 -4
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 446cdd3ebe709888e9ef72e80a2aa77bfa873528a6be895faa1065a076039663
|
4
|
+
data.tar.gz: 733dc9f2f106313bb58de5f06500813214b5145215fca9d81b14118a6b70f846
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3c844063a7ab258e9974529d5bc3ea93fd816ca4655ca7a8a9e005b20bc63e3b349e8537bd63f31e0bff1ef8b4041d88e09d744defd11407bee0ff254ba59049
|
7
|
+
data.tar.gz: 8c4e445c0c357a3333e9ab3f10e20b4cee651d030776b2d657be91b30857f8a1272df13bbadcd5925c8d680e384e75bd6dee8092767cebcbd2deb92e78993e72
|
data/lib/sneaql-aws.rb
CHANGED
@@ -1,10 +1,59 @@
|
|
1
1
|
require 'sneaql'
|
2
2
|
require 'aws-sdk'
|
3
|
+
require 'json'
|
4
|
+
require 'zlib'
|
3
5
|
|
4
6
|
# AWS extensions for SneaQL
|
5
7
|
module SneaqlAWS
|
8
|
+
module Exceptions
|
9
|
+
# Exception used to to gracefully exit test
|
10
|
+
class AWSResourceCreationException < Sneaql::Exceptions::BaseError
|
11
|
+
def initialize(msg = 'Unable to create AWS resource')
|
12
|
+
super
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
# Exception used to to gracefully exit test
|
17
|
+
class AWSS3Exception < Sneaql::Exceptions::BaseError
|
18
|
+
def initialize(msg = 'Error interacting with AWS S3 service')
|
19
|
+
super
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
6
24
|
# Command tags providing AWS functionalities to SneaQL
|
7
25
|
module Commands
|
26
|
+
|
27
|
+
class AWSInteractions
|
28
|
+
def initialize(logger)
|
29
|
+
@logger = logger ? logger : Logger.new(STDOUT)
|
30
|
+
end
|
31
|
+
|
32
|
+
def s3_resource(region)
|
33
|
+
@logger.debug("creating S3 resource...")
|
34
|
+
Aws::S3::Resource.new(region: region)
|
35
|
+
rescue => e
|
36
|
+
@logger.error(e.message)
|
37
|
+
e.backtrace.each { |b| @logger.error(b) }
|
38
|
+
raise SneaqlAWS::Exceptions::AWSResourceCreationException
|
39
|
+
end
|
40
|
+
|
41
|
+
def upload_to_s3(bucket_name, bucket_region, object_key, local_file_path)
|
42
|
+
interactions = AWSInteractions.new(@logger)
|
43
|
+
s3 = s3_resource(bucket_region)
|
44
|
+
@logger.info("uploading local file #{local_file_path} to s3://#{bucket_name}/#{object_key}")
|
45
|
+
s3 = Aws::S3::Resource.new(region: bucket_region)
|
46
|
+
File.open(local_file_path, 'rb') do |file|
|
47
|
+
s3.client.put_object(bucket: bucket_name, key: object_key, body: file)
|
48
|
+
end
|
49
|
+
rescue => e
|
50
|
+
@logger.error(e.message)
|
51
|
+
e.backtrace.each { |b| @logger.error(b) }
|
52
|
+
raise SneaqlAWS::Exceptions::AWSS3Exception
|
53
|
+
end
|
54
|
+
|
55
|
+
end
|
56
|
+
|
8
57
|
# runs the query then stores the array of hashes into the recordset hash
|
9
58
|
class SneaqlAWSS3ObjectList < Sneaql::Core::SneaqlCommand
|
10
59
|
Sneaql::Core::RegisterMappedClass.new(
|
@@ -21,8 +70,8 @@ module SneaqlAWS
|
|
21
70
|
object_prefix
|
22
71
|
)
|
23
72
|
|
24
|
-
@logger.debug("adding #{r.length} recs as #{recordset_name}")
|
25
73
|
@recordset_manager.store_recordset(recordset_name, r)
|
74
|
+
@logger.info("added #{r.length} records to recordset #{recordset_name}")
|
26
75
|
end
|
27
76
|
|
28
77
|
# argument types
|
@@ -35,11 +84,11 @@ module SneaqlAWS
|
|
35
84
|
bucket = @expression_handler.evaluate_expression(bucket)
|
36
85
|
region = @expression_handler.evaluate_expression(region)
|
37
86
|
prefix = @expression_handler.evaluate_expression(prefix)
|
38
|
-
|
39
|
-
@logger.debug("creating s3 connection...")
|
87
|
+
|
40
88
|
# create connection to s3
|
41
89
|
# aws credentials will be resolved through normal SDK precedence rules
|
42
|
-
|
90
|
+
interactions = AWSInteractions.new(@logger)
|
91
|
+
s3 = interactions.s3_resource(region)
|
43
92
|
|
44
93
|
@logger.info("searching for objects in bucket #{bucket} in region #{region} matching prefix #{prefix}")
|
45
94
|
# perform the initial object search
|
@@ -58,6 +107,8 @@ module SneaqlAWS
|
|
58
107
|
# then appended to the res array for use as a recordset
|
59
108
|
resp['contents'].each do |obj|
|
60
109
|
res << {
|
110
|
+
region: region,
|
111
|
+
bucket: bucket,
|
61
112
|
key: obj.key,
|
62
113
|
last_modified: obj.last_modified.to_s,
|
63
114
|
etag: obj.etag,
|
@@ -82,6 +133,8 @@ module SneaqlAWS
|
|
82
133
|
|
83
134
|
resp['contents'].each do |obj|
|
84
135
|
res << {
|
136
|
+
region: region,
|
137
|
+
bucket: bucket,
|
85
138
|
key: obj.key,
|
86
139
|
last_modified: obj.last_modified.to_s,
|
87
140
|
etag: obj.etag,
|
@@ -99,5 +152,89 @@ module SneaqlAWS
|
|
99
152
|
return res
|
100
153
|
end
|
101
154
|
end
|
155
|
+
|
156
|
+
# writes a recordset to S3 in a gzip json format
|
157
|
+
class SneaqlAWSRecordsetToS3 < Sneaql::Core::SneaqlCommand
|
158
|
+
Sneaql::Core::RegisterMappedClass.new(
|
159
|
+
:command,
|
160
|
+
'aws_recordset_to_s3',
|
161
|
+
SneaqlAWS::Commands::SneaqlAWSRecordsetToS3
|
162
|
+
)
|
163
|
+
|
164
|
+
def action(recordset_name, bucket, region, key, object_type)
|
165
|
+
bucket_name = @expression_handler.evaluate_expression(bucket)
|
166
|
+
bucket_region = @expression_handler.evaluate_expression(region)
|
167
|
+
object_key = @expression_handler.evaluate_expression(key)
|
168
|
+
|
169
|
+
recordset_to_file(
|
170
|
+
recordset_name,
|
171
|
+
local_json_path,
|
172
|
+
object_type
|
173
|
+
)
|
174
|
+
|
175
|
+
interactions = AWSInteractions.new(@logger)
|
176
|
+
interactions.upload_to_s3(
|
177
|
+
bucket_name,
|
178
|
+
bucket_region,
|
179
|
+
object_key,
|
180
|
+
local_json_path
|
181
|
+
)
|
182
|
+
end
|
183
|
+
|
184
|
+
# argument types
|
185
|
+
def arg_definition
|
186
|
+
[:recordset, :expression, :expression, :expression, :symbol]
|
187
|
+
end
|
188
|
+
|
189
|
+
def local_json_path
|
190
|
+
"/tmp/json_export_#{self.object_id}.gz"
|
191
|
+
end
|
192
|
+
|
193
|
+
def recordset_to_file(recordset, local_file_path, object_type)
|
194
|
+
@logger.debug("writing recordset #{recordset} to file of type #{object_type}")
|
195
|
+
|
196
|
+
if object_type == 'gzipjson'
|
197
|
+
json_file = File.open(local_file_path, 'w')
|
198
|
+
zd = Zlib::GzipWriter.new(json_file)
|
199
|
+
|
200
|
+
json_gzip_streamer = Proc.new do |z, h|
|
201
|
+
h.keys.each do |k|
|
202
|
+
if h[k].class == Float
|
203
|
+
h[k] = nil if (h[k].nan? or h[k].infinite?)
|
204
|
+
end
|
205
|
+
end
|
206
|
+
z << (h.to_json + "\n") && z.write(z.flush)
|
207
|
+
end
|
208
|
+
|
209
|
+
@recordset_manager.recordset[recordset].each do |r|
|
210
|
+
json_gzip_streamer.call(zd, r)
|
211
|
+
end
|
212
|
+
|
213
|
+
zd.close
|
214
|
+
json_file.close
|
215
|
+
|
216
|
+
elsif object_type == 'json'
|
217
|
+
json_file = File.open(local_file_path, 'w')
|
218
|
+
|
219
|
+
json_formatter = Proc.new do |f, h|
|
220
|
+
h.keys.each do |k|
|
221
|
+
if h[k].class == Float
|
222
|
+
h[k] = nil if (h[k].nan? or h[k].infinite?)
|
223
|
+
end
|
224
|
+
end
|
225
|
+
f.puts h.to_json
|
226
|
+
end
|
227
|
+
|
228
|
+
@recordset_manager.recordset[recordset].each do |r|
|
229
|
+
json_formatter.call(json_file, r)
|
230
|
+
end
|
231
|
+
|
232
|
+
json_file.close
|
233
|
+
end
|
234
|
+
|
235
|
+
@logger.info("#{@recordset_manager.recordset[recordset].length} written for recordset #{recordset} ")
|
236
|
+
end
|
237
|
+
|
238
|
+
end
|
102
239
|
end
|
103
240
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sneaql-aws
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.3
|
5
5
|
platform: java
|
6
6
|
authors:
|
7
7
|
- jeremy winters
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-08-15 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -93,7 +93,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
93
93
|
version: '0'
|
94
94
|
requirements: []
|
95
95
|
rubyforge_project:
|
96
|
-
rubygems_version: 2.6.
|
96
|
+
rubygems_version: 2.6.11
|
97
97
|
signing_key:
|
98
98
|
specification_version: 4
|
99
99
|
summary: sneaql extensions to interact with AWS
|