dataduck 0.5.4 → 0.5.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/dataduck/etl.rb +1 -1
- data/lib/dataduck/redshift_destination.rb +1 -0
- data/lib/dataduck/s3_object.rb +18 -0
- data/lib/dataduck/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 878a28b576d47ad2c2ba6a1df7d984e937bad5dd
|
4
|
+
data.tar.gz: 73ef27e0862f684e079a040631fd001a8bade409
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 39e8be0c1acbbd2819a9dad50107ad77d5142ed2fdb7f2b9e946a0ed4fb9d0061db630d7be2004872381c270390fb169033dbeb7a8b10c2db574f5cbe37e6d34
|
7
|
+
data.tar.gz: 3b9fcbf4268df2af1d3f6131d4e986c7995b633c56d001578f6e332c95131e3cfea956a46ac9c997fde61c77bd092985d3e5120bfeb6fd585dda8fcd9969752b
|
data/lib/dataduck/etl.rb
CHANGED
@@ -196,6 +196,7 @@ module DataDuck
|
|
196
196
|
s3_object = self.upload_table_to_s3!(table)
|
197
197
|
self.create_output_tables!(table)
|
198
198
|
self.query(self.copy_query(table, s3_object.s3_path))
|
199
|
+
s3_object.delete!
|
199
200
|
|
200
201
|
if table.staging_name != table.building_name
|
201
202
|
self.merge_from_staging!(table)
|
data/lib/dataduck/s3_object.rb
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
require 'aws-sdk'
|
2
|
+
require_relative 'logs'
|
2
3
|
|
3
4
|
module DataDuck
|
4
5
|
class S3Object
|
@@ -12,7 +13,24 @@ module DataDuck
|
|
12
13
|
@region = region
|
13
14
|
end
|
14
15
|
|
16
|
+
def delete!
|
17
|
+
Logs.debug("Deleting S3 file #{ self.s3_path }...")
|
18
|
+
|
19
|
+
s3 = Aws::S3::Client.new(
|
20
|
+
region: @region,
|
21
|
+
access_key_id: @aws_key,
|
22
|
+
secret_access_key: @aws_secret,
|
23
|
+
)
|
24
|
+
|
25
|
+
s3.delete_object({
|
26
|
+
bucket: @bucket,
|
27
|
+
key: self.full_path,
|
28
|
+
})
|
29
|
+
end
|
30
|
+
|
15
31
|
def upload!
|
32
|
+
Logs.debug("Uploading S3 file #{ self.s3_path }...")
|
33
|
+
|
16
34
|
s3 = Aws::S3::Client.new(
|
17
35
|
region: @region,
|
18
36
|
access_key_id: @aws_key,
|
data/lib/dataduck/version.rb
CHANGED