sneaql-aws 0.0.2-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/lib/sneaql-aws.rb +103 -0
- metadata +100 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: a02e256c853198e83c025a8eff04f34d95632953
|
4
|
+
data.tar.gz: 2bf1e880ac1c31fb6bfc54ac55785f8754ad01b2
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: ac2df46e763416b6c3baf0db10ba3cb454b4f195a839a9e4e16e2696fc27fb344127d4f2369964368e574856e053b8fe9b06d422c58b876fb5dfe4dc8c126f55
|
7
|
+
data.tar.gz: b3d4a472e468dfa050d241c5008a77bb713a658da3c6bb38483d13456bc9af3e3dc9a110095187153164240b58b75beec24150d0acd73d5c58414350a4af38d5
|
data/lib/sneaql-aws.rb
ADDED
@@ -0,0 +1,103 @@
|
|
1
|
+
require 'sneaql'
|
2
|
+
require 'aws-sdk'
|
3
|
+
|
4
|
+
# AWS extensions for SneaQL
|
5
|
+
module SneaqlAWS
|
6
|
+
# Command tags providing AWS functionalities to SneaQL
|
7
|
+
module Commands
|
8
|
+
# runs the query then stores the array of hashes into the recordset hash
|
9
|
+
class SneaqlAWSS3ObjectList < Sneaql::Core::SneaqlCommand
|
10
|
+
Sneaql::Core::RegisterMappedClass.new(
|
11
|
+
:command,
|
12
|
+
'aws_s3_object_list',
|
13
|
+
SneaqlAWS::Commands::SneaqlAWSS3ObjectList
|
14
|
+
)
|
15
|
+
|
16
|
+
# @param [String] recordset_name name of the recordset in which to store the results
|
17
|
+
def action(recordset_name, bucket_name, bucket_region, object_prefix)
|
18
|
+
r = s3_object_search(
|
19
|
+
bucket_name,
|
20
|
+
bucket_region,
|
21
|
+
object_prefix
|
22
|
+
)
|
23
|
+
|
24
|
+
@logger.debug("adding #{r.length} recs as #{recordset_name}")
|
25
|
+
@recordset_manager.store_recordset(recordset_name, r)
|
26
|
+
end
|
27
|
+
|
28
|
+
# argument types
|
29
|
+
def arg_definition
|
30
|
+
[:recordset, :expression, :expression, :expression]
|
31
|
+
end
|
32
|
+
|
33
|
+
# @return [Array] returns array of hashes from SQL results
|
34
|
+
def s3_object_search(bucket, region, prefix)
|
35
|
+
bucket = @expression_handler.evaluate_expression(bucket)
|
36
|
+
region = @expression_handler.evaluate_expression(region)
|
37
|
+
prefix = @expression_handler.evaluate_expression(prefix)
|
38
|
+
|
39
|
+
@logger.debug("creating s3 connection...")
|
40
|
+
# create connection to s3
|
41
|
+
# aws credentials will be resolved through normal SDK precedence rules
|
42
|
+
s3 = Aws::S3::Resource.new(region: region)
|
43
|
+
|
44
|
+
@logger.info("searching for objects in bucket #{bucket} in region #{region} matching prefix #{prefix}")
|
45
|
+
# perform the initial object search
|
46
|
+
resp = s3.client.list_objects_v2(
|
47
|
+
{
|
48
|
+
bucket: bucket,
|
49
|
+
prefix: prefix,
|
50
|
+
fetch_owner: true
|
51
|
+
}
|
52
|
+
)
|
53
|
+
|
54
|
+
# array to hold results
|
55
|
+
res = []
|
56
|
+
|
57
|
+
# response contents are converted to native ruby hashes
|
58
|
+
# then appended to the res array for use as a recordset
|
59
|
+
resp['contents'].each do |obj|
|
60
|
+
res << {
|
61
|
+
key: obj.key,
|
62
|
+
last_modified: obj.last_modified.to_s,
|
63
|
+
etag: obj.etag,
|
64
|
+
size: obj.size,
|
65
|
+
storage_class: obj.storage_class,
|
66
|
+
owner_name: obj.owner['display_name'],
|
67
|
+
owner_id: obj.owner['id']
|
68
|
+
}
|
69
|
+
end
|
70
|
+
|
71
|
+
# check to see if there is a continuation token
|
72
|
+
next_token = resp.next_continuation_token
|
73
|
+
|
74
|
+
# repeat until all results have been gathered
|
75
|
+
while next_token do
|
76
|
+
resp = s3.client.list_objects_v2(
|
77
|
+
{
|
78
|
+
bucket: bucket,
|
79
|
+
continuation_token: next_token
|
80
|
+
}
|
81
|
+
)
|
82
|
+
|
83
|
+
resp['contents'].each do |obj|
|
84
|
+
res << {
|
85
|
+
key: obj.key,
|
86
|
+
last_modified: obj.last_modified.to_s,
|
87
|
+
etag: obj.etag,
|
88
|
+
size: obj.size,
|
89
|
+
storage_class: obj.storage_class,
|
90
|
+
owner_name: obj.owner['display_name'],
|
91
|
+
owner_id: obj.owner['id']
|
92
|
+
}
|
93
|
+
end
|
94
|
+
|
95
|
+
next_token = resp.next_continuation_token
|
96
|
+
end
|
97
|
+
|
98
|
+
@logger.info("#{res.length} objects found")
|
99
|
+
return res
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|
103
|
+
end
|
metadata
ADDED
@@ -0,0 +1,100 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: sneaql-aws
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.2
|
5
|
+
platform: java
|
6
|
+
authors:
|
7
|
+
- jeremy winters
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2017-07-28 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
requirement: !ruby/object:Gem::Requirement
|
15
|
+
requirements:
|
16
|
+
- - "~>"
|
17
|
+
- !ruby/object:Gem::Version
|
18
|
+
version: '1.2'
|
19
|
+
name: logger
|
20
|
+
prerelease: false
|
21
|
+
type: :runtime
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '1.2'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
requirement: !ruby/object:Gem::Requirement
|
29
|
+
requirements:
|
30
|
+
- - "~>"
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: '5.9'
|
33
|
+
name: minitest
|
34
|
+
prerelease: false
|
35
|
+
type: :development
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '5.9'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
requirement: !ruby/object:Gem::Requirement
|
43
|
+
requirements:
|
44
|
+
- - "~>"
|
45
|
+
- !ruby/object:Gem::Version
|
46
|
+
version: '1.3'
|
47
|
+
name: git
|
48
|
+
prerelease: false
|
49
|
+
type: :runtime
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '1.3'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
requirement: !ruby/object:Gem::Requirement
|
57
|
+
requirements:
|
58
|
+
- - "~>"
|
59
|
+
- !ruby/object:Gem::Version
|
60
|
+
version: '2.6'
|
61
|
+
name: aws-sdk
|
62
|
+
prerelease: false
|
63
|
+
type: :runtime
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '2.6'
|
69
|
+
description: provides extensions to sneaql allowing interaction with AWS
|
70
|
+
email: jeremy.winters@full360.com
|
71
|
+
executables: []
|
72
|
+
extensions: []
|
73
|
+
extra_rdoc_files: []
|
74
|
+
files:
|
75
|
+
- lib/sneaql-aws.rb
|
76
|
+
homepage: https://www.full360.com
|
77
|
+
licenses:
|
78
|
+
- MIT
|
79
|
+
metadata: {}
|
80
|
+
post_install_message:
|
81
|
+
rdoc_options: []
|
82
|
+
require_paths:
|
83
|
+
- lib
|
84
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
85
|
+
requirements:
|
86
|
+
- - ">="
|
87
|
+
- !ruby/object:Gem::Version
|
88
|
+
version: '2.0'
|
89
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
90
|
+
requirements:
|
91
|
+
- - ">="
|
92
|
+
- !ruby/object:Gem::Version
|
93
|
+
version: '0'
|
94
|
+
requirements: []
|
95
|
+
rubyforge_project:
|
96
|
+
rubygems_version: 2.6.6
|
97
|
+
signing_key:
|
98
|
+
specification_version: 4
|
99
|
+
summary: sneaql extensions to interact with AWS
|
100
|
+
test_files: []
|