s3_website_revived 4.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (48) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +15 -0
  3. data/.travis.yml +5 -0
  4. data/Gemfile +3 -0
  5. data/LICENSE +42 -0
  6. data/README.md +591 -0
  7. data/Rakefile +2 -0
  8. data/additional-docs/debugging.md +21 -0
  9. data/additional-docs/development.md +29 -0
  10. data/additional-docs/example-configurations.md +113 -0
  11. data/additional-docs/running-from-ec2-with-dropbox.md +6 -0
  12. data/additional-docs/setting-up-aws-credentials.md +52 -0
  13. data/assembly.sbt +3 -0
  14. data/bin/s3_website +285 -0
  15. data/build.sbt +48 -0
  16. data/changelog.md +596 -0
  17. data/lib/s3_website/version.rb +3 -0
  18. data/lib/s3_website.rb +7 -0
  19. data/project/assembly.sbt +1 -0
  20. data/project/build.properties +1 -0
  21. data/project/plugins.sbt +1 -0
  22. data/release +41 -0
  23. data/resources/configuration_file_template.yml +67 -0
  24. data/resources/s3_website.jar.md5 +1 -0
  25. data/s3_website-4.0.0.jar +0 -0
  26. data/s3_website.gemspec +34 -0
  27. data/sbt +3 -0
  28. data/src/main/resources/log4j.properties +6 -0
  29. data/src/main/scala/s3/website/ByteHelper.scala +18 -0
  30. data/src/main/scala/s3/website/CloudFront.scala +144 -0
  31. data/src/main/scala/s3/website/Logger.scala +67 -0
  32. data/src/main/scala/s3/website/Push.scala +246 -0
  33. data/src/main/scala/s3/website/Ruby.scala +14 -0
  34. data/src/main/scala/s3/website/S3.scala +239 -0
  35. data/src/main/scala/s3/website/UploadHelper.scala +76 -0
  36. data/src/main/scala/s3/website/model/Config.scala +249 -0
  37. data/src/main/scala/s3/website/model/S3Endpoint.scala +35 -0
  38. data/src/main/scala/s3/website/model/Site.scala +159 -0
  39. data/src/main/scala/s3/website/model/push.scala +225 -0
  40. data/src/main/scala/s3/website/model/ssg.scala +30 -0
  41. data/src/main/scala/s3/website/package.scala +182 -0
  42. data/src/test/scala/s3/website/AwsSdkSpec.scala +15 -0
  43. data/src/test/scala/s3/website/ConfigSpec.scala +150 -0
  44. data/src/test/scala/s3/website/S3EndpointSpec.scala +15 -0
  45. data/src/test/scala/s3/website/S3WebsiteSpec.scala +1480 -0
  46. data/src/test/scala/s3/website/UnitTest.scala +11 -0
  47. data/vagrant/Vagrantfile +25 -0
  48. metadata +195 -0
data/changelog.md ADDED
@@ -0,0 +1,596 @@
1
+ # Changelog
2
+
3
+ This project uses [Semantic Versioning](http://semver.org).
4
+
5
+ ## 3.4.0
6
+
7
+ * Add support for default credentials providers in the `s3_website cfg create`
8
+ `s3_website cfg apply` commands
9
+
10
+ See <https://github.com/laurilehmijoki/configure-s3-website/pull/22>
11
+
12
+ ## 3.3.0
13
+
14
+ * Support `http_error_code_returned_equals` in redirect rules
15
+
16
+ See <https://github.com/laurilehmijoki/configure-s3-website/pull/21> for
17
+ discussion
18
+
19
+ ## 3.2.0
20
+
21
+ * Fall back to [the default credentials sources](http://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html) if `s3_id` is not provided in `s3_website.yml`
22
+
23
+ ## 3.1.0
24
+
25
+ Support for [session tokens](http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp.html)
26
+
27
+ ## 3.0.0
28
+
29
+ The CloudFront client now uses the official AWS SDK. If your `s3_website.yml`
30
+ file contains the `cloudfront_distribution_config` setting, you might need to modify
31
+ the it. The modifications should be rather straightforward, just follow the
32
+ [instructions](https://github.com/laurilehmijoki/configure-s3-website/blob/master/changelog.md#200).
33
+
34
+ In other words, if you don't use CloudFront, upgrading to 3.0.0 should not
35
+ result in any problems.
36
+
37
+ ## 2.16.0
38
+
39
+ * Add command `s3_website install`
40
+
41
+ ## 2.15.1
42
+
43
+ * Support all AWS regions
44
+
45
+ ## 2.15.0
46
+
47
+ * Add new setting `content_type`
48
+
49
+ See <https://github.com/laurilehmijoki/s3_website/issues/232> for discussion
50
+
51
+ ## 2.14.3
52
+
53
+ * Fix mime type of an already-gzipped .json file
54
+
55
+ See <https://github.com/laurilehmijoki/s3_website/pull/231>
56
+
57
+ ## 2.14.2
58
+
59
+ * Apply correct mime type on already-gzipped files
60
+
61
+ See <https://github.com/laurilehmijoki/s3_website/issues/229#issuecomment-237229421> for discussion
62
+
63
+ ## 2.14.1
64
+
65
+ * Do not gzip a file that is already gzipped
66
+
67
+ See <https://github.com/laurilehmijoki/s3_website/issues/229> for discussion
68
+
69
+ ## 2.14.0
70
+
71
+ * Add support for CloudFront wildcard invalidations
72
+
73
+ Introduced a new setting, `cloudfront_wildcard_invalidation: (true|false)`
74
+
75
+ ## 2.13.0
76
+
77
+ * Add support for `dnf`, a Linux package manager
78
+
79
+ ## 2.12.3
80
+
81
+ * Fix <https://github.com/laurilehmijoki/s3_website/issues/208>
82
+
83
+ ## 2.12.2
84
+
85
+ * Merge <https://github.com/laurilehmijoki/s3_website/pull/190>
86
+
87
+ ## 2.12.1
88
+
89
+ * Fix broken build
90
+
91
+ ## 2.12.0
92
+
93
+ * Automatically detect Middleman generated websites
94
+
95
+ ## 2.11.2
96
+
97
+ * A new fix based on
98
+ <https://github.com/laurilehmijoki/s3_website/issues/181#issuecomment-141397992>
99
+
100
+ ## 2.11.1
101
+
102
+ * Prevent runaway recursion in file listing
103
+
104
+ See <https://github.com/laurilehmijoki/s3_website/issues/181> for discussion
105
+
106
+ ## 2.11.0
107
+
108
+ * Add the `s3_key_prefix` setting
109
+
110
+ ## 2.10.0
111
+
112
+ * Support glob hashes in `cache_control`
113
+
114
+ ## 2.9.0
115
+
116
+ * Add setting `cache_control`
117
+
118
+ ## 2.8.6
119
+
120
+ * Detect changed file even though the file has the same contents with another file on the S3 bucket
121
+
122
+ See <https://github.com/laurilehmijoki/s3_website/issues/156> for discussion.
123
+
124
+ ## 2.8.5
125
+
126
+ * URL encode (ä|ö|ü) in invalidation path
127
+
128
+ ## 2.8.4
129
+
130
+ * URL encode ' in invalidation path
131
+
132
+ See <https://github.com/laurilehmijoki/s3_website/issues/63> for discussion.
133
+
134
+ * Accept ' in `exclude_from_upload` and `ignore_on_server`
135
+
136
+ ## 2.8.3
137
+
138
+ * Fix bug where the setting `cloudfront_invalidate_root: true` resulted in a
139
+ CloudFront invalidation even if there were no changes to push.
140
+
141
+ See <https://github.com/laurilehmijoki/s3_website/issues/149> for discussion.
142
+
143
+ ## 2.8.1
144
+
145
+ * Change CloudFront `OriginProtocolPolicy` to `http-only`
146
+
147
+ See <https://github.com/laurilehmijoki/s3_website/issues/152> for discussion.
148
+
149
+ ## 2.8.0
150
+
151
+ * Automatically gzip `.ico` files, if `gzip: true`
152
+
153
+ ## 2.7.6
154
+
155
+ * Null-check the result of File.listFiles
156
+
157
+ See <https://github.com/laurilehmijoki/s3_website/issues/145> for discussion.
158
+
159
+ ## 2.7.5
160
+
161
+ * Remove superfluous dot from error message
162
+
163
+ ## 2.7.4
164
+
165
+ * Show a helpful error message if the configured site is missing
166
+
167
+ See <https://github.com/laurilehmijoki/s3_website/issues/144> for discussion.
168
+
169
+ ## 2.7.3
170
+
171
+ * Support valid URI characters in max_age glob
172
+
173
+ See <https://github.com/laurilehmijoki/s3_website/issues/140> for discussion.
174
+
175
+ ## 2.7.2
176
+
177
+ * Fix Windows issue
178
+
179
+ See <https://github.com/laurilehmijoki/s3_website/issues/105> for discussion.
180
+
181
+ ## 2.7.1
182
+
183
+ * Loosen dependency requirements
184
+
185
+ See <https://github.com/laurilehmijoki/s3_website/pull/135> for discussion.
186
+
187
+ ## 2.7.0
188
+
189
+ * Add setting `treat_zero_length_objects_as_redirects`
190
+
191
+ Before, `s3_website push` always uploaded one zero-length object for each
192
+ configured redirect. From now on, you can instruct s3\_website to treat
193
+ zero-length S3 objects as existing redirects. If you have a large amount of
194
+ redirects on your site, you may find that this feature decreases the duration
195
+ of `s3_website push`.
196
+
197
+ See <https://github.com/laurilehmijoki/s3_website/issues/132> for discussion.
198
+
199
+ ## 2.6.1
200
+
201
+ * Always invalidate the object */index.html* if the setting `cloudfront_invalidate_root` is on
202
+
203
+ See https://github.com/laurilehmijoki/s3_website/pull/130 for discussion
204
+
205
+ ## 2.6.0
206
+
207
+ * Support `--config-dir` in `cfg apply`
208
+
209
+ ## 2.5.1
210
+
211
+ * Print **Would have updated|redirected|created** when running with `--dry-run`
212
+
213
+ ## 2.5.0
214
+
215
+ * Add `push --force` option
216
+
217
+ When the user pushes with force, s3_website skips the diff. This is helpful for the
218
+ users who wish to update the S3 object metadata.
219
+
220
+ ## 2.4.0
221
+
222
+ * Add `ignore_on_server: _DELETE_NOTHING_ON_THE_S3_BUCKET_` for the sake of convenience
223
+
224
+ See https://github.com/laurilehmijoki/s3_website/issues/121 for discussion.
225
+
226
+ ## 2.3.1
227
+
228
+ * Add Windows support
229
+
230
+ ## 2.3.0
231
+
232
+ * The command `s3_website cfg apply` now supports `--headless` and
233
+ `--autocreate-cloudfront-dist`
234
+
235
+ ## 2.2.0
236
+
237
+ * Specify the location of the website in the *s3_website.yml* file
238
+
239
+ Just add the setting `site: path-to-your-website` into the file.
240
+
241
+ * Fix Nanoc auto detection
242
+
243
+ Previously, a website in *public/output* was not automatically detected.
244
+
245
+ ## 2.1.16
246
+
247
+ * Support non-US-ASCII files when using `ignore_on_server`
248
+
249
+ Fixes https://github.com/laurilehmijoki/s3_website/issues/102
250
+
251
+ ## 2.1.15
252
+
253
+ * Support non-US-ASCII files when using `max_age`
254
+
255
+ Fixes https://github.com/laurilehmijoki/s3_website/issues/102
256
+
257
+ ## 2.1.14
258
+
259
+ * Hide false AWS SDK alarms
260
+
261
+ Fixes https://github.com/laurilehmijoki/s3_website/issues/104
262
+
263
+ ## 2.1.13
264
+
265
+ * Print JVM stack trace on error and `--verbose`
266
+
267
+ This eases debugging
268
+
269
+ ## 2.1.12
270
+
271
+ * Exit with status 1 when encountering an unrecognised CLI option
272
+
273
+ Fixes https://github.com/laurilehmijoki/s3_website/issues/103
274
+
275
+ ## 2.1.11
276
+
277
+ * Fix documentation for the `--config-dir` option
278
+
279
+ The `--config_dir` option has changed to `--config-dir` in version 2 of this
280
+ gem.
281
+
282
+ ## 2.1.10
283
+
284
+ * Remove warning on Ruby 1.8
285
+
286
+ ## 2.1.9
287
+
288
+ * Separate development and production code more clearly in the s3_website
289
+ executable
290
+
291
+ ## 2.1.8
292
+
293
+ * Remove unused code in the s3_website executable
294
+
295
+ ## 2.1.7
296
+
297
+ * Remove local db
298
+
299
+ It turned out to be too complex to maintain
300
+
301
+ ## 2.1.6
302
+
303
+ * Automatically add slash to redirects if needed
304
+
305
+ ## 2.1.5
306
+
307
+ * Target JVM 1.6 in build.sbt
308
+
309
+ ## 2.1.4
310
+
311
+ * Fix reason-for-upload message
312
+
313
+ ## 2.1.3
314
+
315
+ * Fix boolean logic in reason-for-upload
316
+
317
+ ## 2.1.2
318
+
319
+ * Show a more informative message if the jar file is corrupted.
320
+
321
+ ## 2.1.1
322
+
323
+ * Verify that the s3_website.jar is not corrupted. Download it again, if it is.
324
+
325
+ ## 2.1.0
326
+
327
+ * Show the upload reason when calling `push --verbose`
328
+
329
+ ## 2.0.1
330
+
331
+ * Rename binary s3_website_monadic to s3_website
332
+
333
+ ## 2.0.0
334
+
335
+ ### New features
336
+
337
+ * Faster uploads for extra large sites
338
+
339
+ Use proper multithreading with JVM threads.
340
+
341
+ * Simulate deployments with `push --dry-run`
342
+
343
+ * Support CloudFront invalidations when the site contains over 3000 files
344
+
345
+ * Display transferred bytes
346
+
347
+ * Display upload speed
348
+
349
+ * `push --verbose` switch
350
+
351
+ ### Bug fixes
352
+
353
+ * Fault tolerance – do not crash if one of the uploads fails
354
+
355
+ Before, the push command crashed if something unexpected happened. From now
356
+ on, s3_website will run all the operations it can, and report errors in the
357
+ end.
358
+
359
+ ### Java is now required
360
+
361
+ * The `push` command is now written in Scala
362
+
363
+ This means that you need Java 1.6 or above to run the command `s3_website
364
+ push`.
365
+
366
+ ### Removed features
367
+
368
+ * `extensionless_mime_type`
369
+
370
+ s3_website now relies on Apache Tika to infer the mime type.
371
+
372
+ * the `--headless` switch on the command-line
373
+
374
+ s3_website always deletes the files that are on the S3 bucket but not on the local file system.
375
+ Use the settings `ignore_on_server` and `exclude_from_upload` to control the retained files.
376
+
377
+ * You can no longer use this gem as a Ruby library. You can migrate by calling
378
+ the `s3_website push --site=x --config-dir=y` system command from your Ruby code.
379
+
380
+ * `gzip_zopfli: true`
381
+
382
+ At the time of writing, there does not exist a stable zopfli implementation
383
+ for Java.
384
+
385
+ ### Other backward incompatible changes
386
+
387
+ * The option `--config_dir` has changed to `--config-dir`
388
+
389
+ ## 1.8.1
390
+
391
+ * Do not push the *.env* file
392
+
393
+ ## 1.8.0
394
+
395
+ * Add support for (dotenv)[https://github.com/bkeepers/dotenv]
396
+
397
+ ## 1.7.6
398
+
399
+ * Remove a test setting from *Gemfile*
400
+
401
+ ## 1.7.5
402
+
403
+ * Improve significantly the performance of the push command.
404
+
405
+ See <https://github.com/laurilehmijoki/s3_website/pull/88> for more info.
406
+
407
+ ## 1.7.4
408
+
409
+ * Fix issue https://github.com/laurilehmijoki/s3_website/issues/83
410
+
411
+ ## 1.7.3
412
+
413
+ * Fix Digest::Digest deprecation warn on Ruby 2.1.0
414
+
415
+ This warning did appear then one used the `cfg create` or `cfg apply`
416
+ commands.
417
+
418
+ ## 1.7.2
419
+
420
+ * This release contains no code changes (the indended change is in the 1.7.3
421
+ release)
422
+
423
+ ## 1.7.1
424
+
425
+ * Do not override ERB when adding CloudFront dist
426
+
427
+ ## 1.7.0
428
+
429
+ * Add zopfli compression support
430
+
431
+ ## 1.6.13
432
+
433
+ * Depend on any 1-series version of the aws-sdk gem
434
+
435
+ ## 1.6.12
436
+
437
+ * Fix bug <https://github.com/laurilehmijoki/s3_website/issues/63>
438
+
439
+ ## 1.6.11
440
+
441
+ * Loosen the dependency spec of mime-types (#70)
442
+
443
+ ## 1.6.10
444
+
445
+ * Fix bug <https://github.com/laurilehmijoki/s3_website/issues/38>
446
+
447
+ ## 1.6.9
448
+
449
+ * Fix Digest::Digest deprecation warn on Ruby 2.1.0
450
+
451
+ ## 1.6.8
452
+
453
+ * Fix content-type problem
454
+ (<https://github.com/laurilehmijoki/s3_website/pull/66>)
455
+
456
+ ## 1.6.7
457
+
458
+ * Support the eu-west-1 location constraint for the commands `cfg apply` and
459
+ `cfg create`
460
+
461
+ ## 1.6.6
462
+
463
+ * Mark all text documents as UTF-8 encoded
464
+
465
+ ## 1.6.5
466
+
467
+ * In case of error, exit with status 1
468
+
469
+ ## 1.6.4
470
+
471
+ * Add systematic error handling
472
+
473
+ Fixes issue https://github.com/laurilehmijoki/s3_website/issues/52.
474
+
475
+ ## 1.6.3
476
+
477
+ * Invalidate a deleted file on CloudFront
478
+
479
+ ## 1.6.2
480
+
481
+ * Fix issue <https://github.com/laurilehmijoki/s3_website/pull/54>
482
+
483
+ ## 1.6.1
484
+
485
+ * Fix issue <https://github.com/laurilehmijoki/s3_website/issues/30>
486
+
487
+ ## 1.6.0
488
+
489
+ * Add support for excluding files from upload
490
+ * s3_website.yml now supports `exclude_from_upload`
491
+ * Support multiple values on the `ignore_on_server` setting
492
+
493
+ ## 1.5.0
494
+
495
+ * Add support for specifying the MIME type for extensionless files
496
+
497
+ ## 1.4.5
498
+
499
+ * If max-age=0, set `Cache-Control: no-cache, max-age=0`
500
+
501
+ ## 1.4.4
502
+
503
+ * Add support for eu-west-1 as a location constraint
504
+
505
+ ## 1.4.3
506
+
507
+ * Decrease the default concurrency level to 3
508
+
509
+ See https://github.com/laurilehmijoki/s3_website/issues/8#issuecomment-24855991
510
+ for discussion.
511
+
512
+ ## 1.4.2
513
+
514
+ * Fix `s3_website cfg apply` for CloudFront setup (#33)
515
+
516
+ ## 1.4.1
517
+
518
+ * Fix diff for Windows users
519
+
520
+ See
521
+ <https://github.com/laurilehmijoki/s3_website/issues/8#issuecomment-23683568>
522
+ for discussion.
523
+
524
+ ## 1.4.0
525
+
526
+ * Add setting `cloudfront_invalidate_root`
527
+
528
+ ## 1.3.2
529
+
530
+ * Move blacklist filtering into a better place
531
+
532
+ ## 1.3.1
533
+
534
+ * Print to stdout the initial state of the diff progress indicator
535
+
536
+ ## 1.3.0
537
+
538
+ * Show a progress indicator when calculating diff
539
+
540
+ ## 1.2.1
541
+
542
+ * Use `print` instead of `puts` when printing to stdout in a concurrent context
543
+
544
+ ## 1.2.0
545
+
546
+ * Use the `--config_dir` CLI option to specify the directory from where to read
547
+ the `s3_website.yml` config file
548
+
549
+ ## 1.1.2
550
+
551
+ * Mention the MIT license in the gemspec file
552
+
553
+ ## 1.1.1
554
+
555
+ * Mention the new `concurrency_level` setting in the sample config file
556
+
557
+ ## 1.1.0
558
+
559
+ * Add possibility to define the concurrency level in *s3_website.yml*
560
+
561
+ ## 1.0.3
562
+
563
+ * Reject blacklisted files in a more appropriate place
564
+
565
+ ## 1.0.2
566
+
567
+ * Never upload the file *s3_website.yml*
568
+
569
+ ## 1.0.1
570
+
571
+ * Set default concurrency level to 100. Related to issue [#6](https://github.com/laurilehmijoki/s3_website/issues/6).
572
+
573
+ ## 1.0.0
574
+
575
+ * Make 0.4.0 the version 1.0.0
576
+
577
+ ## 0.4.0
578
+
579
+ * Include the available configs in the sample s3_website.yml file
580
+
581
+ ## 0.3.0
582
+
583
+ * Add Nanoc support
584
+
585
+ ## 0.2.1
586
+
587
+ * Remove Gemfile.lock
588
+ * Rename gemspec file
589
+
590
+ ## 0.2.0
591
+
592
+ * Add command `s3_website cfg apply`
593
+
594
+ ## 0.1.0
595
+
596
+ * First version
@@ -0,0 +1,3 @@
1
+ module S3Website
2
+ VERSION = '4.0.0'
3
+ end
data/lib/s3_website.rb ADDED
@@ -0,0 +1,7 @@
1
+ require 'rubygems'
2
+ require 'thor'
3
+ require 'dotenv'
4
+
5
+ Dotenv.load
6
+
7
+ require File.dirname(__FILE__) + "/s3_website/version"
@@ -0,0 +1 @@
1
+ addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.11.2")
@@ -0,0 +1 @@
1
+ sbt.version=0.13.18
@@ -0,0 +1 @@
1
+ addSbtPlugin("com.github.mpeltonen" % "sbt-idea" % "1.6.0")
data/release ADDED
@@ -0,0 +1,41 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'octokit'
4
+ require_relative 'lib/s3_website/version'
5
+
6
+ version = S3Website::VERSION
7
+ unless ENV['S3_WEBSITE_RELEASE_KEY']
8
+ puts "Set the S3_WEBSITE_RELEASE_KEY environmental variable. It should be your personal GitHub access token."
9
+ puts "Go to https://github.com/settings/applications#personal-access-tokens."
10
+ exit 1
11
+ end
12
+
13
+ def run(cmd)
14
+ unless system(cmd)
15
+ exit 1
16
+ end
17
+ end
18
+
19
+ puts "Building s3_website.jar..."
20
+ jar_location = 'target/scala-2.11/s3_website.jar'
21
+ run "./sbt clean assembly"
22
+
23
+ # The bin s3_website uses the MD5 checksum to verify that the jar is not corrupted
24
+ jar_md5 = Digest::MD5.file(jar_location).hexdigest
25
+ File.write('resources/s3_website.jar.md5', jar_md5)
26
+
27
+ tag_name = "v#{version}"
28
+ run "git tag #{tag_name} && git push origin #{tag_name}"
29
+ run "git push"
30
+
31
+ run "rake build"
32
+
33
+ client = Octokit::Client.new(:login => 'laurilehmijoki', :password => ENV['S3_WEBSITE_RELEASE_KEY'])
34
+ release = client.create_release(
35
+ 'laurilehmijoki/s3_website',
36
+ tag_name,
37
+ :body => "See https://github.com/laurilehmijoki/s3_website/blob/master/changelog.md##{version.gsub('.', '')}"
38
+ )
39
+ puts "Uploading s3_website.jar..."
40
+ client.upload_asset(release[:url], jar_location)
41
+ run "gem push pkg/s3_website-#{version}.gem"