sneaql-aws 0.0.2-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. checksums.yaml +7 -0
  2. data/lib/sneaql-aws.rb +103 -0
  3. metadata +100 -0
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: a02e256c853198e83c025a8eff04f34d95632953
4
+ data.tar.gz: 2bf1e880ac1c31fb6bfc54ac55785f8754ad01b2
5
+ SHA512:
6
+ metadata.gz: ac2df46e763416b6c3baf0db10ba3cb454b4f195a839a9e4e16e2696fc27fb344127d4f2369964368e574856e053b8fe9b06d422c58b876fb5dfe4dc8c126f55
7
+ data.tar.gz: b3d4a472e468dfa050d241c5008a77bb713a658da3c6bb38483d13456bc9af3e3dc9a110095187153164240b58b75beec24150d0acd73d5c58414350a4af38d5
data/lib/sneaql-aws.rb ADDED
@@ -0,0 +1,103 @@
1
+ require 'sneaql'
2
+ require 'aws-sdk'
3
+
4
+ # AWS extensions for SneaQL
5
+ module SneaqlAWS
6
+ # Command tags providing AWS functionalities to SneaQL
7
+ module Commands
8
+ # runs the query then stores the array of hashes into the recordset hash
9
+ class SneaqlAWSS3ObjectList < Sneaql::Core::SneaqlCommand
10
+ Sneaql::Core::RegisterMappedClass.new(
11
+ :command,
12
+ 'aws_s3_object_list',
13
+ SneaqlAWS::Commands::SneaqlAWSS3ObjectList
14
+ )
15
+
16
+ # @param [String] recordset_name name of the recordset in which to store the results
17
+ def action(recordset_name, bucket_name, bucket_region, object_prefix)
18
+ r = s3_object_search(
19
+ bucket_name,
20
+ bucket_region,
21
+ object_prefix
22
+ )
23
+
24
+ @logger.debug("adding #{r.length} recs as #{recordset_name}")
25
+ @recordset_manager.store_recordset(recordset_name, r)
26
+ end
27
+
28
+ # argument types
29
+ def arg_definition
30
+ [:recordset, :expression, :expression, :expression]
31
+ end
32
+
33
+ # @return [Array] returns array of hashes from SQL results
34
+ def s3_object_search(bucket, region, prefix)
35
+ bucket = @expression_handler.evaluate_expression(bucket)
36
+ region = @expression_handler.evaluate_expression(region)
37
+ prefix = @expression_handler.evaluate_expression(prefix)
38
+
39
+ @logger.debug("creating s3 connection...")
40
+ # create connection to s3
41
+ # aws credentials will be resolved through normal SDK precedence rules
42
+ s3 = Aws::S3::Resource.new(region: region)
43
+
44
+ @logger.info("searching for objects in bucket #{bucket} in region #{region} matching prefix #{prefix}")
45
+ # perform the initial object search
46
+ resp = s3.client.list_objects_v2(
47
+ {
48
+ bucket: bucket,
49
+ prefix: prefix,
50
+ fetch_owner: true
51
+ }
52
+ )
53
+
54
+ # array to hold results
55
+ res = []
56
+
57
+ # response contents are converted to native ruby hashes
58
+ # then appended to the res array for use as a recordset
59
+ resp['contents'].each do |obj|
60
+ res << {
61
+ key: obj.key,
62
+ last_modified: obj.last_modified.to_s,
63
+ etag: obj.etag,
64
+ size: obj.size,
65
+ storage_class: obj.storage_class,
66
+ owner_name: obj.owner['display_name'],
67
+ owner_id: obj.owner['id']
68
+ }
69
+ end
70
+
71
+ # check to see if there is a continuation token
72
+ next_token = resp.next_continuation_token
73
+
74
+ # repeat until all results have been gathered
75
+ while next_token do
76
+ resp = s3.client.list_objects_v2(
77
+ {
78
+ bucket: bucket,
79
+ continuation_token: next_token
80
+ }
81
+ )
82
+
83
+ resp['contents'].each do |obj|
84
+ res << {
85
+ key: obj.key,
86
+ last_modified: obj.last_modified.to_s,
87
+ etag: obj.etag,
88
+ size: obj.size,
89
+ storage_class: obj.storage_class,
90
+ owner_name: obj.owner['display_name'],
91
+ owner_id: obj.owner['id']
92
+ }
93
+ end
94
+
95
+ next_token = resp.next_continuation_token
96
+ end
97
+
98
+ @logger.info("#{res.length} objects found")
99
+ return res
100
+ end
101
+ end
102
+ end
103
+ end
metadata ADDED
@@ -0,0 +1,100 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: sneaql-aws
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.2
5
+ platform: java
6
+ authors:
7
+ - jeremy winters
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2017-07-28 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ requirement: !ruby/object:Gem::Requirement
15
+ requirements:
16
+ - - "~>"
17
+ - !ruby/object:Gem::Version
18
+ version: '1.2'
19
+ name: logger
20
+ prerelease: false
21
+ type: :runtime
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '1.2'
27
+ - !ruby/object:Gem::Dependency
28
+ requirement: !ruby/object:Gem::Requirement
29
+ requirements:
30
+ - - "~>"
31
+ - !ruby/object:Gem::Version
32
+ version: '5.9'
33
+ name: minitest
34
+ prerelease: false
35
+ type: :development
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '5.9'
41
+ - !ruby/object:Gem::Dependency
42
+ requirement: !ruby/object:Gem::Requirement
43
+ requirements:
44
+ - - "~>"
45
+ - !ruby/object:Gem::Version
46
+ version: '1.3'
47
+ name: git
48
+ prerelease: false
49
+ type: :runtime
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '1.3'
55
+ - !ruby/object:Gem::Dependency
56
+ requirement: !ruby/object:Gem::Requirement
57
+ requirements:
58
+ - - "~>"
59
+ - !ruby/object:Gem::Version
60
+ version: '2.6'
61
+ name: aws-sdk
62
+ prerelease: false
63
+ type: :runtime
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - "~>"
67
+ - !ruby/object:Gem::Version
68
+ version: '2.6'
69
+ description: provides extensions to sneaql allowing interaction with AWS
70
+ email: jeremy.winters@full360.com
71
+ executables: []
72
+ extensions: []
73
+ extra_rdoc_files: []
74
+ files:
75
+ - lib/sneaql-aws.rb
76
+ homepage: https://www.full360.com
77
+ licenses:
78
+ - MIT
79
+ metadata: {}
80
+ post_install_message:
81
+ rdoc_options: []
82
+ require_paths:
83
+ - lib
84
+ required_ruby_version: !ruby/object:Gem::Requirement
85
+ requirements:
86
+ - - ">="
87
+ - !ruby/object:Gem::Version
88
+ version: '2.0'
89
+ required_rubygems_version: !ruby/object:Gem::Requirement
90
+ requirements:
91
+ - - ">="
92
+ - !ruby/object:Gem::Version
93
+ version: '0'
94
+ requirements: []
95
+ rubyforge_project:
96
+ rubygems_version: 2.6.6
97
+ signing_key:
98
+ specification_version: 4
99
+ summary: sneaql extensions to interact with AWS
100
+ test_files: []