datapimp 1.2.9 → 1.2.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/datapimp.gemspec +2 -1
- data/lib/datapimp/cli/create.rb +95 -0
- data/lib/datapimp/clients/amazon.rb +6 -0
- data/lib/datapimp/templates/cloudfront/403.html.erb +10 -0
- data/lib/datapimp/templates/cloudfront/aws_cloudfront_distribution_template.json +110 -0
- data/lib/datapimp/templates/cloudfront/aws_cloudfront_distribution_with_new_buckets_template.json +153 -0
- metadata +19 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 3792340fb939bb27f1adb1c3cc84f000931f3862
|
4
|
+
data.tar.gz: 2e35324289e4b19730dbf02fa004a2942d199107
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b9b636585d5ae0e17bcfe46d1eae30c9ccde4fcfabf07e12de66536d12a3e30d52007ff3aaabcf209680dacd37e78a17ead6f24f8d21318a11ed8037fbc70c02
|
7
|
+
data.tar.gz: af0ecf140e8caf0659f4db26a516412f4dbbc6fa5fb24d702b745c7470f052afc052a7eb5f12873d6cc393aab92f9aff891ad842d6dd162fea1b5bae7d1985eb
|
data/datapimp.gemspec
CHANGED
@@ -23,6 +23,7 @@ Gem::Specification.new do |spec|
|
|
23
23
|
spec.add_dependency 'commander', '>= 4.3'
|
24
24
|
spec.add_dependency 'terminal-table'
|
25
25
|
spec.add_dependency 'fog-aws', '>= 0.1'
|
26
|
+
spec.add_dependency 'aws-sdk', '~> 2'
|
26
27
|
spec.add_dependency 'dropbox-api', '>= 0.4.7'
|
27
28
|
spec.add_dependency 'google_drive', '>= 1.0'
|
28
29
|
spec.add_dependency 'google-api-client', '>= 0.8'
|
@@ -38,7 +39,7 @@ Gem::Specification.new do |spec|
|
|
38
39
|
spec.add_dependency 'github-fs', '~> 0'
|
39
40
|
spec.add_dependency 'colored', '> 0.0'
|
40
41
|
spec.add_dependency 'multi_json', '~> 1.10'
|
41
|
-
|
42
|
+
|
42
43
|
# one of these will go
|
43
44
|
spec.add_dependency 'pivotal-tracker', '~> 0.5.13'
|
44
45
|
spec.add_dependency 'tracker_api', '>= 0.2.10'
|
data/lib/datapimp/cli/create.rb
CHANGED
@@ -131,3 +131,98 @@ command 'create cloudfront distribution' do |c|
|
|
131
131
|
|
132
132
|
end
|
133
133
|
end
|
134
|
+
|
135
|
+
# bin/datapimp create cf protected distribution --name z-test --bucket 'warbler.architects.io' --error-bucket z-test-error-bucket --domains hola.com,hello.com --app-url https://blueprints.architects.io --origin-access-identity E2RCKW2LSUD589 --trace
|
136
|
+
command 'create cf protected distribution' do |c|
|
137
|
+
c.syntax = "datapimp create cf protected distribution"
|
138
|
+
c.description = "create a cloudfront PROTECTED distribution using signed cookies"
|
139
|
+
|
140
|
+
Datapimp::Cli.accepts_keys_for(c, :amazon)
|
141
|
+
|
142
|
+
c.option '--name NAME', String, 'The name for this distribution'
|
143
|
+
c.option '--bucket NAME', String, 'The name of the *existing* bucket that will provide the content'
|
144
|
+
c.option '--error-bucket NAME', String, 'The name of the *existing* bucket that will hold the errors folder and 403.html file'
|
145
|
+
c.option '--domains DOMAINS', Array, 'What domains will be pointing to this bucket?'
|
146
|
+
c.option '--app-url NAME', String, 'The url of the AUTH Applitacion'
|
147
|
+
c.option '--origin-access-identity NAME', String, 'The Origin Access Identity to be used to create the distribution'
|
148
|
+
|
149
|
+
c.action do |args, options|
|
150
|
+
cf = Datapimp::Sync.amazon.cloud_formation
|
151
|
+
|
152
|
+
template_body = File.read(File.join(File.dirname(__FILE__), '..', 'templates/cloudfront', 'aws_cloudfront_distribution_template.json'))
|
153
|
+
|
154
|
+
res = cf.create_stack(
|
155
|
+
stack_name: options.name,
|
156
|
+
template_body: template_body,
|
157
|
+
# disable_rollback: true,
|
158
|
+
parameters: [
|
159
|
+
{
|
160
|
+
parameter_key: "AppLocation",
|
161
|
+
parameter_value: URI.parse(options.app_url).host,
|
162
|
+
use_previous_value: true
|
163
|
+
},
|
164
|
+
{
|
165
|
+
parameter_key: "BucketName",
|
166
|
+
parameter_value: options.bucket,
|
167
|
+
use_previous_value: true
|
168
|
+
},
|
169
|
+
{
|
170
|
+
parameter_key: "ErrorBucketName",
|
171
|
+
parameter_value: options.error_bucket,
|
172
|
+
use_previous_value: true
|
173
|
+
},
|
174
|
+
{
|
175
|
+
parameter_key: "Aliases",
|
176
|
+
parameter_value: options.domains.join(','),
|
177
|
+
use_previous_value: true
|
178
|
+
},
|
179
|
+
{
|
180
|
+
parameter_key: "DistributionComment",
|
181
|
+
parameter_value: "#{options.name} distribution",
|
182
|
+
use_previous_value: true
|
183
|
+
},
|
184
|
+
{
|
185
|
+
parameter_key: "OriginAccessIdentity",
|
186
|
+
parameter_value: options.origin_access_identity,
|
187
|
+
use_previous_value: true
|
188
|
+
}
|
189
|
+
]
|
190
|
+
)
|
191
|
+
|
192
|
+
begin
|
193
|
+
puts "Waiting for stack creation process to finish ..."
|
194
|
+
sleep 30
|
195
|
+
stack = cf.describe_stacks(stack_name: options.name).stacks.first
|
196
|
+
end while stack.stack_status == "CREATE_IN_PROGRESS"
|
197
|
+
|
198
|
+
if stack.stack_status != "CREATE_COMPLETE"
|
199
|
+
puts "stack failed to create"
|
200
|
+
exit 1
|
201
|
+
end
|
202
|
+
|
203
|
+
s3 = Aws::S3::Client.new(region: cf.config.region)
|
204
|
+
template_body_403 = ERB.new(File.read(File.join(File.dirname(__FILE__), '../templates/cloudfront', '403.html.erb'))).result(binding)
|
205
|
+
|
206
|
+
# S3 403.html error file
|
207
|
+
begin
|
208
|
+
s3.put_object(
|
209
|
+
bucket: options.error_bucket,
|
210
|
+
key: 'errors/403.html',
|
211
|
+
content_type: 'text/html',
|
212
|
+
cache_control: 'max-age=300',
|
213
|
+
acl: 'public-read',
|
214
|
+
body: template_body_403
|
215
|
+
)
|
216
|
+
rescue Aws::S3::Errors::NoSuchBucket
|
217
|
+
error_bucket = "#{options.error_bucket}.s3.amazonaws.com"
|
218
|
+
s3.put_object(
|
219
|
+
bucket: error_bucket,
|
220
|
+
key: 'errors/403.html',
|
221
|
+
content_type: 'text/html',
|
222
|
+
cache_control: 'max-age=300',
|
223
|
+
acl: 'public-read',
|
224
|
+
body: template_body_403
|
225
|
+
)
|
226
|
+
end
|
227
|
+
end
|
228
|
+
end
|
@@ -54,6 +54,12 @@ module Datapimp
|
|
54
54
|
})
|
55
55
|
end
|
56
56
|
|
57
|
+
# This needs ENV['AWS_ACCESS_KEY_ID'] and ENV['AWS_SECRET_ACCESS_KEY']
|
58
|
+
def cloud_formation
|
59
|
+
require 'aws-sdk'
|
60
|
+
@cloud_formation ||= Aws::CloudFormation::Client.new(region: aws_region)
|
61
|
+
end
|
62
|
+
|
57
63
|
def s3_bucket_website_url
|
58
64
|
if s3_bucket.is_a?(Fog::Storage::AWS::Directory)
|
59
65
|
website_url_for(s3_bucket)
|
@@ -0,0 +1,10 @@
|
|
1
|
+
<html>
|
2
|
+
<head>
|
3
|
+
<title>Redirecting...</title>
|
4
|
+
</head>
|
5
|
+
<body>
|
6
|
+
<script>
|
7
|
+
window.location="https://<%= options.app_url %>/authorization/get_ticket?service=https://<%= options.app_url %>&asset_url=" + document.location.href;
|
8
|
+
</script>
|
9
|
+
</body>
|
10
|
+
</html>
|
@@ -0,0 +1,110 @@
|
|
1
|
+
{
|
2
|
+
"AWSTemplateFormatVersion": "2010-09-09",
|
3
|
+
"Description": "Cloudfront Demo Setup",
|
4
|
+
"Parameters": {
|
5
|
+
"BucketName": {
|
6
|
+
"Type": "String",
|
7
|
+
"Description": "Name of the bucket to create"
|
8
|
+
},
|
9
|
+
"Aliases": {
|
10
|
+
"Type": "CommaDelimitedList",
|
11
|
+
"Description": "CNAMES for this distribution"
|
12
|
+
},
|
13
|
+
"ErrorBucketName": {
|
14
|
+
"Type": "String",
|
15
|
+
"Description": "Name of the bucket to hold the 403.html error page"
|
16
|
+
},
|
17
|
+
"AppLocation": {
|
18
|
+
"Type": "String",
|
19
|
+
"Description": "The DNS name you have deployed the app to (for example example.com)"
|
20
|
+
},
|
21
|
+
"DistributionComment": {
|
22
|
+
"Type": "String",
|
23
|
+
"Description": "Commeht section for the distribution"
|
24
|
+
},
|
25
|
+
"OriginAccessIdentity": {
|
26
|
+
"Type": "String",
|
27
|
+
"Description": "The value that CloudFront returned in the Id element when the origin access identity was created."
|
28
|
+
}
|
29
|
+
},
|
30
|
+
"Mappings": {},
|
31
|
+
"Conditions": {},
|
32
|
+
"Resources": {
|
33
|
+
"Distribution": {
|
34
|
+
"Type": "AWS::CloudFront::Distribution",
|
35
|
+
"Properties": {
|
36
|
+
"DistributionConfig": {
|
37
|
+
"Enabled": true,
|
38
|
+
"Comment": {"Ref": "DistributionComment"},
|
39
|
+
"Aliases": {"Ref": "Aliases"},
|
40
|
+
"Origins": [
|
41
|
+
{
|
42
|
+
"DomainName": { "Ref": "BucketName" },
|
43
|
+
"Id": "S3",
|
44
|
+
"S3OriginConfig": {
|
45
|
+
"OriginAccessIdentity": {"Fn::Join": ["", ["origin-access-identity/cloudfront/", {"Ref": "OriginAccessIdentity"}]]}
|
46
|
+
}
|
47
|
+
},
|
48
|
+
{
|
49
|
+
"DomainName": { "Ref": "ErrorBucketName" },
|
50
|
+
"Id": "ErrorS3",
|
51
|
+
"S3OriginConfig": {
|
52
|
+
"OriginAccessIdentity": {"Fn::Join": ["", ["origin-access-identity/cloudfront/", {"Ref": "OriginAccessIdentity"}]]}
|
53
|
+
}
|
54
|
+
},
|
55
|
+
{
|
56
|
+
"DomainName": { "Ref": "AppLocation" },
|
57
|
+
"Id": "Application",
|
58
|
+
"CustomOriginConfig": {
|
59
|
+
"OriginProtocolPolicy": "match-viewer"
|
60
|
+
}
|
61
|
+
}
|
62
|
+
],
|
63
|
+
"CacheBehaviors": [
|
64
|
+
{
|
65
|
+
"TargetOriginId": "Application",
|
66
|
+
"PathPattern": "/authorization/*",
|
67
|
+
"ForwardedValues": {
|
68
|
+
"QueryString": true,
|
69
|
+
"Cookies": {
|
70
|
+
"Forward": "whitelist",
|
71
|
+
"WhitelistedNames": ["DUMMY"]
|
72
|
+
}
|
73
|
+
},
|
74
|
+
"ViewerProtocolPolicy": "allow-all"
|
75
|
+
},
|
76
|
+
{
|
77
|
+
"TargetOriginId": "ErrorS3",
|
78
|
+
"PathPattern": "/errors/*",
|
79
|
+
"ForwardedValues": {
|
80
|
+
"QueryString": false
|
81
|
+
},
|
82
|
+
"ViewerProtocolPolicy": "allow-all"
|
83
|
+
}
|
84
|
+
],
|
85
|
+
"DefaultCacheBehavior": {
|
86
|
+
"TargetOriginId": "S3",
|
87
|
+
"ForwardedValues": {
|
88
|
+
"QueryString": false
|
89
|
+
},
|
90
|
+
"ViewerProtocolPolicy": "allow-all",
|
91
|
+
"TrustedSigners": ["self"]
|
92
|
+
},
|
93
|
+
"DefaultRootObject": "index.html",
|
94
|
+
"CustomErrorResponses": [
|
95
|
+
{
|
96
|
+
"ErrorCode": 403,
|
97
|
+
"ResponsePagePath": "/errors/403.html",
|
98
|
+
"ResponseCode": 403
|
99
|
+
}
|
100
|
+
]
|
101
|
+
}
|
102
|
+
}
|
103
|
+
}
|
104
|
+
},
|
105
|
+
"Outputs": {
|
106
|
+
"Distribution": {
|
107
|
+
"Value": {"Ref": "Distribution"}
|
108
|
+
}
|
109
|
+
}
|
110
|
+
}
|
data/lib/datapimp/templates/cloudfront/aws_cloudfront_distribution_with_new_buckets_template.json
ADDED
@@ -0,0 +1,153 @@
|
|
1
|
+
{
|
2
|
+
"AWSTemplateFormatVersion": "2010-09-09",
|
3
|
+
"Description": "Cloudfront Demo Setup",
|
4
|
+
"Parameters": {
|
5
|
+
"BucketName": {
|
6
|
+
"Type": "String",
|
7
|
+
"Description": "Name of the bucket to create"
|
8
|
+
},
|
9
|
+
"Aliases": {
|
10
|
+
"Type": "CommaDelimitedList",
|
11
|
+
"Description": "CNAMES for this distribution"
|
12
|
+
},
|
13
|
+
"ErrorBucketName": {
|
14
|
+
"Type": "String",
|
15
|
+
"Description": "Name of the bucket to hold the 403.html error page"
|
16
|
+
},
|
17
|
+
"AppLocation": {
|
18
|
+
"Type": "String",
|
19
|
+
"Description": "The DNS name you have deployed the app to (for example example.com)"
|
20
|
+
},
|
21
|
+
"DistributionComment": {
|
22
|
+
"Type": "String",
|
23
|
+
"Description": "Commeht section for the distribution"
|
24
|
+
},
|
25
|
+
"OriginAccessIdentity": {
|
26
|
+
"Type": "String",
|
27
|
+
"Description": "The value that CloudFront returned in the Id element when the origin access identity was created."
|
28
|
+
}
|
29
|
+
},
|
30
|
+
"Mappings": {},
|
31
|
+
"Conditions": {},
|
32
|
+
"Resources": {
|
33
|
+
"Bucket": {
|
34
|
+
"Type": "AWS::S3::Bucket",
|
35
|
+
"Properties": {
|
36
|
+
"AccessControl": "Private",
|
37
|
+
"BucketName": { "Ref": "BucketName" }
|
38
|
+
}
|
39
|
+
},
|
40
|
+
"ErrorBucket": {
|
41
|
+
"Type": "AWS::S3::Bucket",
|
42
|
+
"Properties": {
|
43
|
+
"AccessControl": "Private",
|
44
|
+
"BucketName": { "Ref": "ErrorBucketName" }
|
45
|
+
}
|
46
|
+
},
|
47
|
+
|
48
|
+
"BucketPolicy": {
|
49
|
+
"Type": "AWS::S3::BucketPolicy",
|
50
|
+
"Properties": {
|
51
|
+
"Bucket": { "Ref": "Bucket" },
|
52
|
+
"PolicyDocument": {
|
53
|
+
"Version": "2008-10-17",
|
54
|
+
"Id": "PolicyForCloudFrontPrivateContent",
|
55
|
+
"Statement": [
|
56
|
+
{
|
57
|
+
"Sid": "1",
|
58
|
+
"Effect": "Allow",
|
59
|
+
"Principal": {
|
60
|
+
"AWS": {"Fn::Join": [" ", ["arn:aws:iam::cloudfront:user/CloudFront Origin Access Identity", { "Ref": "OriginAccessIdentity"}]]}
|
61
|
+
},
|
62
|
+
"Action": "s3:GetObject",
|
63
|
+
"Resource": {"Fn::Join": ["", ["arn:aws:s3:::", {"Ref": "Bucket"}, "/*"]]}
|
64
|
+
}
|
65
|
+
]
|
66
|
+
}
|
67
|
+
}
|
68
|
+
},
|
69
|
+
|
70
|
+
"Distribution": {
|
71
|
+
"Type": "AWS::CloudFront::Distribution",
|
72
|
+
"Properties": {
|
73
|
+
"DistributionConfig": {
|
74
|
+
"Enabled": true,
|
75
|
+
"Comment": {"Ref": "DistributionComment"},
|
76
|
+
"Aliases": {"Ref": "Aliases"},
|
77
|
+
"Origins": [
|
78
|
+
{
|
79
|
+
"DomainName": {"Fn::GetAtt":[ "Bucket", "DomainName"]},
|
80
|
+
"Id": "S3",
|
81
|
+
"S3OriginConfig": {
|
82
|
+
"OriginAccessIdentity": {"Fn::Join": ["", ["origin-access-identity/cloudfront/", {"Ref": "OriginAccessIdentity"}]]}
|
83
|
+
}
|
84
|
+
},
|
85
|
+
{
|
86
|
+
"DomainName": {"Fn::GetAtt":[ "ErrorBucket", "DomainName"]},
|
87
|
+
"Id": "ErrorS3",
|
88
|
+
"S3OriginConfig": {
|
89
|
+
"OriginAccessIdentity": {"Fn::Join": ["", ["origin-access-identity/cloudfront/", {"Ref": "OriginAccessIdentity"}]]}
|
90
|
+
}
|
91
|
+
},
|
92
|
+
{
|
93
|
+
"DomainName": {"Ref": "AppLocation"},
|
94
|
+
"Id": "Application",
|
95
|
+
"CustomOriginConfig": {
|
96
|
+
"OriginProtocolPolicy": "match-viewer"
|
97
|
+
}
|
98
|
+
}
|
99
|
+
],
|
100
|
+
"CacheBehaviors": [
|
101
|
+
{
|
102
|
+
"TargetOriginId": "Application",
|
103
|
+
"PathPattern": "/authorization/*",
|
104
|
+
"ForwardedValues": {
|
105
|
+
"QueryString": true,
|
106
|
+
"Cookies": {
|
107
|
+
"Forward": "whitelist",
|
108
|
+
"WhitelistedNames": ["DUMMY"]
|
109
|
+
}
|
110
|
+
},
|
111
|
+
"ViewerProtocolPolicy": "allow-all"
|
112
|
+
},
|
113
|
+
{
|
114
|
+
"TargetOriginId": "ErrorS3",
|
115
|
+
"PathPattern": "/errors/*",
|
116
|
+
"ForwardedValues": {
|
117
|
+
"QueryString": false
|
118
|
+
},
|
119
|
+
"ViewerProtocolPolicy": "allow-all"
|
120
|
+
}
|
121
|
+
],
|
122
|
+
"DefaultCacheBehavior": {
|
123
|
+
"TargetOriginId": "S3",
|
124
|
+
"ForwardedValues": {
|
125
|
+
"QueryString": false
|
126
|
+
},
|
127
|
+
"ViewerProtocolPolicy": "allow-all",
|
128
|
+
"TrustedSigners": ["self"]
|
129
|
+
},
|
130
|
+
"DefaultRootObject": "index.html",
|
131
|
+
"CustomErrorResponses": [
|
132
|
+
{
|
133
|
+
"ErrorCode": 403,
|
134
|
+
"ResponsePagePath": "/errors/403.html",
|
135
|
+
"ResponseCode": 403
|
136
|
+
}
|
137
|
+
]
|
138
|
+
}
|
139
|
+
}
|
140
|
+
}
|
141
|
+
},
|
142
|
+
"Outputs": {
|
143
|
+
"Distribution": {
|
144
|
+
"Value": {"Ref": "Distribution"}
|
145
|
+
},
|
146
|
+
"ErrorBucket": {
|
147
|
+
"Value": {"Ref": "ErrorBucket"}
|
148
|
+
},
|
149
|
+
"Bucket": {
|
150
|
+
"Value": {"Ref": "Bucket"}
|
151
|
+
}
|
152
|
+
}
|
153
|
+
}
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: datapimp
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.2.
|
4
|
+
version: 1.2.10
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jonathan Soeder
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-08-
|
11
|
+
date: 2015-08-20 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: pry
|
@@ -80,6 +80,20 @@ dependencies:
|
|
80
80
|
- - ">="
|
81
81
|
- !ruby/object:Gem::Version
|
82
82
|
version: '0.1'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: aws-sdk
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - "~>"
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '2'
|
90
|
+
type: :runtime
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - "~>"
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '2'
|
83
97
|
- !ruby/object:Gem::Dependency
|
84
98
|
name: dropbox-api
|
85
99
|
requirement: !ruby/object:Gem::Requirement
|
@@ -511,6 +525,9 @@ files:
|
|
511
525
|
- lib/datapimp/sync/dropbox_folder.rb
|
512
526
|
- lib/datapimp/sync/google_drive_folder.rb
|
513
527
|
- lib/datapimp/sync/s3_bucket.rb
|
528
|
+
- lib/datapimp/templates/cloudfront/403.html.erb
|
529
|
+
- lib/datapimp/templates/cloudfront/aws_cloudfront_distribution_template.json
|
530
|
+
- lib/datapimp/templates/cloudfront/aws_cloudfront_distribution_with_new_buckets_template.json
|
514
531
|
- lib/datapimp/util.rb
|
515
532
|
- lib/datapimp/version.rb
|
516
533
|
- spec/datapimp/sync/github_spec.rb
|