s3_website_revived 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +15 -0
  3. data/.travis.yml +5 -0
  4. data/Gemfile +3 -0
  5. data/LICENSE +42 -0
  6. data/README.md +591 -0
  7. data/Rakefile +2 -0
  8. data/additional-docs/debugging.md +21 -0
  9. data/additional-docs/development.md +29 -0
  10. data/additional-docs/example-configurations.md +113 -0
  11. data/additional-docs/running-from-ec2-with-dropbox.md +6 -0
  12. data/additional-docs/setting-up-aws-credentials.md +52 -0
  13. data/assembly.sbt +3 -0
  14. data/bin/s3_website +285 -0
  15. data/build.sbt +48 -0
  16. data/changelog.md +596 -0
  17. data/lib/s3_website/version.rb +3 -0
  18. data/lib/s3_website.rb +7 -0
  19. data/project/assembly.sbt +1 -0
  20. data/project/build.properties +1 -0
  21. data/project/plugins.sbt +1 -0
  22. data/release +41 -0
  23. data/resources/configuration_file_template.yml +67 -0
  24. data/resources/s3_website.jar.md5 +1 -0
  25. data/s3_website-4.0.0.jar +0 -0
  26. data/s3_website.gemspec +34 -0
  27. data/sbt +3 -0
  28. data/src/main/resources/log4j.properties +6 -0
  29. data/src/main/scala/s3/website/ByteHelper.scala +18 -0
  30. data/src/main/scala/s3/website/CloudFront.scala +144 -0
  31. data/src/main/scala/s3/website/Logger.scala +67 -0
  32. data/src/main/scala/s3/website/Push.scala +246 -0
  33. data/src/main/scala/s3/website/Ruby.scala +14 -0
  34. data/src/main/scala/s3/website/S3.scala +239 -0
  35. data/src/main/scala/s3/website/UploadHelper.scala +76 -0
  36. data/src/main/scala/s3/website/model/Config.scala +249 -0
  37. data/src/main/scala/s3/website/model/S3Endpoint.scala +35 -0
  38. data/src/main/scala/s3/website/model/Site.scala +159 -0
  39. data/src/main/scala/s3/website/model/push.scala +225 -0
  40. data/src/main/scala/s3/website/model/ssg.scala +30 -0
  41. data/src/main/scala/s3/website/package.scala +182 -0
  42. data/src/test/scala/s3/website/AwsSdkSpec.scala +15 -0
  43. data/src/test/scala/s3/website/ConfigSpec.scala +150 -0
  44. data/src/test/scala/s3/website/S3EndpointSpec.scala +15 -0
  45. data/src/test/scala/s3/website/S3WebsiteSpec.scala +1480 -0
  46. data/src/test/scala/s3/website/UnitTest.scala +11 -0
  47. data/vagrant/Vagrantfile +25 -0
  48. metadata +195 -0
data/changelog.md ADDED
@@ -0,0 +1,596 @@
1
+ # Changelog
2
+
3
+ This project uses [Semantic Versioning](http://semver.org).
4
+
5
+ ## 3.4.0
6
+
7
+ * Add support for default credentials providers in the `s3_website cfg create`
8
+ `s3_website cfg apply` commands
9
+
10
+ See <https://github.com/laurilehmijoki/configure-s3-website/pull/22>
11
+
12
+ ## 3.3.0
13
+
14
+ * Support `http_error_code_returned_equals` in redirect rules
15
+
16
+ See <https://github.com/laurilehmijoki/configure-s3-website/pull/21> for
17
+ discussion
18
+
19
+ ## 3.2.0
20
+
21
+ * Fall back to [the default credentials sources](http://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html) if `s3_id` is not provided in `s3_website.yml`
22
+
23
+ ## 3.1.0
24
+
25
+ Support for [session tokens](http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp.html)
26
+
27
+ ## 3.0.0
28
+
29
+ The CloudFront client now uses the official AWS SDK. If your `s3_website.yml`
30
+ file contains the `cloudfront_distribution_config` setting, you might need to modify
31
+ the it. The modifications should be rather straightforward, just follow the
32
+ [instructions](https://github.com/laurilehmijoki/configure-s3-website/blob/master/changelog.md#200).
33
+
34
+ In other words, if you don't use CloudFront, upgrading to 3.0.0 should not
35
+ result in any problems.
36
+
37
+ ## 2.16.0
38
+
39
+ * Add command `s3_website install`
40
+
41
+ ## 2.15.1
42
+
43
+ * Support all AWS regions
44
+
45
+ ## 2.15.0
46
+
47
+ * Add new setting `content_type`
48
+
49
+ See <https://github.com/laurilehmijoki/s3_website/issues/232> for discussion
50
+
51
+ ## 2.14.3
52
+
53
+ * Fix mime type of an already-gzipped .json file
54
+
55
+ See <https://github.com/laurilehmijoki/s3_website/pull/231>
56
+
57
+ ## 2.14.2
58
+
59
+ * Apply correct mime type on already-gzipped files
60
+
61
+ See <https://github.com/laurilehmijoki/s3_website/issues/229#issuecomment-237229421> for discussion
62
+
63
+ ## 2.14.1
64
+
65
+ * Do not gzip a file that is already gzipped
66
+
67
+ See <https://github.com/laurilehmijoki/s3_website/issues/229> for discussion
68
+
69
+ ## 2.14.0
70
+
71
+ * Add support for CloudFront wildcard invalidations
72
+
73
+ Introduced a new setting, `cloudfront_wildcard_invalidation: (true|false)`
74
+
75
+ ## 2.13.0
76
+
77
+ * Add support for `dnf`, a Linux package manager
78
+
79
+ ## 2.12.3
80
+
81
+ * Fix <https://github.com/laurilehmijoki/s3_website/issues/208>
82
+
83
+ ## 2.12.2
84
+
85
+ * Merge <https://github.com/laurilehmijoki/s3_website/pull/190>
86
+
87
+ ## 2.12.1
88
+
89
+ * Fix broken build
90
+
91
+ ## 2.12.0
92
+
93
+ * Automatically detect Middleman generated websites
94
+
95
+ ## 2.11.2
96
+
97
+ * A new fix based on
98
+ <https://github.com/laurilehmijoki/s3_website/issues/181#issuecomment-141397992>
99
+
100
+ ## 2.11.1
101
+
102
+ * Prevent runaway recursion in file listing
103
+
104
+ See <https://github.com/laurilehmijoki/s3_website/issues/181> for discussion
105
+
106
+ ## 2.11.0
107
+
108
+ * Add the `s3_key_prefix` setting
109
+
110
+ ## 2.10.0
111
+
112
+ * Support glob hashes in `cache_control`
113
+
114
+ ## 2.9.0
115
+
116
+ * Add setting `cache_control`
117
+
118
+ ## 2.8.6
119
+
120
+ * Detect changed file even though the file has the same contents with another file on the S3 bucket
121
+
122
+ See <https://github.com/laurilehmijoki/s3_website/issues/156> for discussion.
123
+
124
+ ## 2.8.5
125
+
126
+ * URL encode (ä|ö|ü) in invalidation path
127
+
128
+ ## 2.8.4
129
+
130
+ * URL encode ' in invalidation path
131
+
132
+ See <https://github.com/laurilehmijoki/s3_website/issues/63> for discussion.
133
+
134
+ * Accept ' in `exclude_from_upload` and `ignore_on_server`
135
+
136
+ ## 2.8.3
137
+
138
+ * Fix bug where the setting `cloudfront_invalidate_root: true` resulted in a
139
+ CloudFront invalidation even if there were no changes to push.
140
+
141
+ See <https://github.com/laurilehmijoki/s3_website/issues/149> for discussion.
142
+
143
+ ## 2.8.1
144
+
145
+ * Change CloudFront `OriginProtocolPolicy` to `http-only`
146
+
147
+ See <https://github.com/laurilehmijoki/s3_website/issues/152> for discussion.
148
+
149
+ ## 2.8.0
150
+
151
+ * Automatically gzip `.ico` files, if `gzip: true`
152
+
153
+ ## 2.7.6
154
+
155
+ * Null-check the result of File.listFiles
156
+
157
+ See <https://github.com/laurilehmijoki/s3_website/issues/145> for discussion.
158
+
159
+ ## 2.7.5
160
+
161
+ * Remove superfluous dot from error message
162
+
163
+ ## 2.7.4
164
+
165
+ * Show a helpful error message if the configured site is missing
166
+
167
+ See <https://github.com/laurilehmijoki/s3_website/issues/144> for discussion.
168
+
169
+ ## 2.7.3
170
+
171
+ * Support valid URI characters in max_age glob
172
+
173
+ See <https://github.com/laurilehmijoki/s3_website/issues/140> for discussion.
174
+
175
+ ## 2.7.2
176
+
177
+ * Fix Windows issue
178
+
179
+ See <https://github.com/laurilehmijoki/s3_website/issues/105> for discussion.
180
+
181
+ ## 2.7.1
182
+
183
+ * Loosen dependency requirements
184
+
185
+ See <https://github.com/laurilehmijoki/s3_website/pull/135> for discussion.
186
+
187
+ ## 2.7.0
188
+
189
+ * Add setting `treat_zero_length_objects_as_redirects`
190
+
191
+ Before, `s3_website push` always uploaded one zero-length object for each
192
+ configured redirect. From now on, you can instruct s3\_website to treat
193
+ zero-length S3 objects as existing redirects. If you have a large amount of
194
+ redirects on your site, you may find that this feature decreases the duration
195
+ of `s3_website push`.
196
+
197
+ See <https://github.com/laurilehmijoki/s3_website/issues/132> for discussion.
198
+
199
+ ## 2.6.1
200
+
201
+ * Always invalidate the object */index.html* if the setting `cloudfront_invalidate_root` is on
202
+
203
+ See https://github.com/laurilehmijoki/s3_website/pull/130 for discussion
204
+
205
+ ## 2.6.0
206
+
207
+ * Support `--config-dir` in `cfg apply`
208
+
209
+ ## 2.5.1
210
+
211
+ * Print **Would have updated|redirected|created** when running with `--dry-run`
212
+
213
+ ## 2.5.0
214
+
215
+ * Add `push --force` option
216
+
217
+ When the user pushes with force, s3_website skips the diff. This is helpful for the
218
+ users who wish to update the S3 object metadata.
219
+
220
+ ## 2.4.0
221
+
222
+ * Add `ignore_on_server: _DELETE_NOTHING_ON_THE_S3_BUCKET_` for the sake of convenience
223
+
224
+ See https://github.com/laurilehmijoki/s3_website/issues/121 for discussion.
225
+
226
+ ## 2.3.1
227
+
228
+ * Add Windows support
229
+
230
+ ## 2.3.0
231
+
232
+ * The command `s3_website cfg apply` now supports `--headless` and
233
+ `--autocreate-cloudfront-dist`
234
+
235
+ ## 2.2.0
236
+
237
+ * Specify the location of the website in the *s3_website.yml* file
238
+
239
+ Just add the setting `site: path-to-your-website` into the file.
240
+
241
+ * Fix Nanoc auto detection
242
+
243
+ Previously, a website in *public/output* was not automatically detected.
244
+
245
+ ## 2.1.16
246
+
247
+ * Support non-US-ASCII files when using `ignore_on_server`
248
+
249
+ Fixes https://github.com/laurilehmijoki/s3_website/issues/102
250
+
251
+ ## 2.1.15
252
+
253
+ * Support non-US-ASCII files when using `max_age`
254
+
255
+ Fixes https://github.com/laurilehmijoki/s3_website/issues/102
256
+
257
+ ## 2.1.14
258
+
259
+ * Hide false AWS SDK alarms
260
+
261
+ Fixes https://github.com/laurilehmijoki/s3_website/issues/104
262
+
263
+ ## 2.1.13
264
+
265
+ * Print JVM stack trace on error and `--verbose`
266
+
267
+ This eases debugging
268
+
269
+ ## 2.1.12
270
+
271
+ * Exit with status 1 when encountering an unrecognised CLI option
272
+
273
+ Fixes https://github.com/laurilehmijoki/s3_website/issues/103
274
+
275
+ ## 2.1.11
276
+
277
+ * Fix documentation for the `--config-dir` option
278
+
279
+ The `--config_dir` option has changed to `--config-dir` in version 2 of this
280
+ gem.
281
+
282
+ ## 2.1.10
283
+
284
+ * Remove warning on Ruby 1.8
285
+
286
+ ## 2.1.9
287
+
288
+ * Separate development and production code more clearly in the s3_website
289
+ executable
290
+
291
+ ## 2.1.8
292
+
293
+ * Remove unused code in the s3_website executable
294
+
295
+ ## 2.1.7
296
+
297
+ * Remove local db
298
+
299
+ It turned out to be too complex to maintain
300
+
301
+ ## 2.1.6
302
+
303
+ * Automatically add slash to redirects if needed
304
+
305
+ ## 2.1.5
306
+
307
+ * Target JVM 1.6 in build.sbt
308
+
309
+ ## 2.1.4
310
+
311
+ * Fix reason-for-upload message
312
+
313
+ ## 2.1.3
314
+
315
+ * Fix boolean logic in reason-for-upload
316
+
317
+ ## 2.1.2
318
+
319
+ * Show a more informative message if the jar file is corrupted.
320
+
321
+ ## 2.1.1
322
+
323
+ * Verify that the s3_website.jar is not corrupted. Download it again, if it is.
324
+
325
+ ## 2.1.0
326
+
327
+ * Show the upload reason when calling `push --verbose`
328
+
329
+ ## 2.0.1
330
+
331
+ * Rename binary s3_website_monadic to s3_website
332
+
333
+ ## 2.0.0
334
+
335
+ ### New features
336
+
337
+ * Faster uploads for extra large sites
338
+
339
+ Use proper multithreading with JVM threads.
340
+
341
+ * Simulate deployments with `push --dry-run`
342
+
343
+ * Support CloudFront invalidations when the site contains over 3000 files
344
+
345
+ * Display transferred bytes
346
+
347
+ * Display upload speed
348
+
349
+ * `push --verbose` switch
350
+
351
+ ### Bug fixes
352
+
353
+ * Fault tolerance – do not crash if one of the uploads fails
354
+
355
+ Before, the push command crashed if something unexpected happened. From now
356
+ on, s3_website will run all the operations it can, and report errors in the
357
+ end.
358
+
359
+ ### Java is now required
360
+
361
+ * The `push` command is now written in Scala
362
+
363
+ This means that you need Java 1.6 or above to run the command `s3_website
364
+ push`.
365
+
366
+ ### Removed features
367
+
368
+ * `extensionless_mime_type`
369
+
370
+ s3_website now relies on Apache Tika to infer the mime type.
371
+
372
+ * the `--headless` switch on the command-line
373
+
374
+ s3_website always deletes the files that are on the S3 bucket but not on the local file system.
375
+ Use the settings `ignore_on_server` and `exclude_from_upload` to control the retained files.
376
+
377
+ * You can no longer use this gem as a Ruby library. You can migrate by calling
378
+ the `s3_website push --site=x --config-dir=y` system command from your Ruby code.
379
+
380
+ * `gzip_zopfli: true`
381
+
382
+ At the time of writing, there does not exist a stable zopfli implementation
383
+ for Java.
384
+
385
+ ### Other backward incompatible changes
386
+
387
+ * The option `--config_dir` has changed to `--config-dir`
388
+
389
+ ## 1.8.1
390
+
391
+ * Do not push the *.env* file
392
+
393
+ ## 1.8.0
394
+
395
+ * Add support for (dotenv)[https://github.com/bkeepers/dotenv]
396
+
397
+ ## 1.7.6
398
+
399
+ * Remove a test setting from *Gemfile*
400
+
401
+ ## 1.7.5
402
+
403
+ * Improve significantly the performance of the push command.
404
+
405
+ See <https://github.com/laurilehmijoki/s3_website/pull/88> for more info.
406
+
407
+ ## 1.7.4
408
+
409
+ * Fix issue https://github.com/laurilehmijoki/s3_website/issues/83
410
+
411
+ ## 1.7.3
412
+
413
+ * Fix Digest::Digest deprecation warn on Ruby 2.1.0
414
+
415
+ This warning did appear then one used the `cfg create` or `cfg apply`
416
+ commands.
417
+
418
+ ## 1.7.2
419
+
420
+ * This release contains no code changes (the indended change is in the 1.7.3
421
+ release)
422
+
423
+ ## 1.7.1
424
+
425
+ * Do not override ERB when adding CloudFront dist
426
+
427
+ ## 1.7.0
428
+
429
+ * Add zopfli compression support
430
+
431
+ ## 1.6.13
432
+
433
+ * Depend on any 1-series version of the aws-sdk gem
434
+
435
+ ## 1.6.12
436
+
437
+ * Fix bug <https://github.com/laurilehmijoki/s3_website/issues/63>
438
+
439
+ ## 1.6.11
440
+
441
+ * Loosen the dependency spec of mime-types (#70)
442
+
443
+ ## 1.6.10
444
+
445
+ * Fix bug <https://github.com/laurilehmijoki/s3_website/issues/38>
446
+
447
+ ## 1.6.9
448
+
449
+ * Fix Digest::Digest deprecation warn on Ruby 2.1.0
450
+
451
+ ## 1.6.8
452
+
453
+ * Fix content-type problem
454
+ (<https://github.com/laurilehmijoki/s3_website/pull/66>)
455
+
456
+ ## 1.6.7
457
+
458
+ * Support the eu-west-1 location constraint for the commands `cfg apply` and
459
+ `cfg create`
460
+
461
+ ## 1.6.6
462
+
463
+ * Mark all text documents as UTF-8 encoded
464
+
465
+ ## 1.6.5
466
+
467
+ * In case of error, exit with status 1
468
+
469
+ ## 1.6.4
470
+
471
+ * Add systematic error handling
472
+
473
+ Fixes issue https://github.com/laurilehmijoki/s3_website/issues/52.
474
+
475
+ ## 1.6.3
476
+
477
+ * Invalidate a deleted file on CloudFront
478
+
479
+ ## 1.6.2
480
+
481
+ * Fix issue <https://github.com/laurilehmijoki/s3_website/pull/54>
482
+
483
+ ## 1.6.1
484
+
485
+ * Fix issue <https://github.com/laurilehmijoki/s3_website/issues/30>
486
+
487
+ ## 1.6.0
488
+
489
+ * Add support for excluding files from upload
490
+ * s3_website.yml now supports `exclude_from_upload`
491
+ * Support multiple values on the `ignore_on_server` setting
492
+
493
+ ## 1.5.0
494
+
495
+ * Add support for specifying the MIME type for extensionless files
496
+
497
+ ## 1.4.5
498
+
499
+ * If max-age=0, set `Cache-Control: no-cache, max-age=0`
500
+
501
+ ## 1.4.4
502
+
503
+ * Add support for eu-west-1 as a location constraint
504
+
505
+ ## 1.4.3
506
+
507
+ * Decrease the default concurrency level to 3
508
+
509
+ See https://github.com/laurilehmijoki/s3_website/issues/8#issuecomment-24855991
510
+ for discussion.
511
+
512
+ ## 1.4.2
513
+
514
+ * Fix `s3_website cfg apply` for CloudFront setup (#33)
515
+
516
+ ## 1.4.1
517
+
518
+ * Fix diff for Windows users
519
+
520
+ See
521
+ <https://github.com/laurilehmijoki/s3_website/issues/8#issuecomment-23683568>
522
+ for discussion.
523
+
524
+ ## 1.4.0
525
+
526
+ * Add setting `cloudfront_invalidate_root`
527
+
528
+ ## 1.3.2
529
+
530
+ * Move blacklist filtering into a better place
531
+
532
+ ## 1.3.1
533
+
534
+ * Print to stdout the initial state of the diff progress indicator
535
+
536
+ ## 1.3.0
537
+
538
+ * Show a progress indicator when calculating diff
539
+
540
+ ## 1.2.1
541
+
542
+ * Use `print` instead of `puts` when printing to stdout in a concurrent context
543
+
544
+ ## 1.2.0
545
+
546
+ * Use the `--config_dir` CLI option to specify the directory from where to read
547
+ the `s3_website.yml` config file
548
+
549
+ ## 1.1.2
550
+
551
+ * Mention the MIT license in the gemspec file
552
+
553
+ ## 1.1.1
554
+
555
+ * Mention the new `concurrency_level` setting in the sample config file
556
+
557
+ ## 1.1.0
558
+
559
+ * Add possibility to define the concurrency level in *s3_website.yml*
560
+
561
+ ## 1.0.3
562
+
563
+ * Reject blacklisted files in a more appropriate place
564
+
565
+ ## 1.0.2
566
+
567
+ * Never upload the file *s3_website.yml*
568
+
569
+ ## 1.0.1
570
+
571
+ * Set default concurrency level to 100. Related to issue [#6](https://github.com/laurilehmijoki/s3_website/issues/6).
572
+
573
+ ## 1.0.0
574
+
575
+ * Make 0.4.0 the version 1.0.0
576
+
577
+ ## 0.4.0
578
+
579
+ * Include the available configs in the sample s3_website.yml file
580
+
581
+ ## 0.3.0
582
+
583
+ * Add Nanoc support
584
+
585
+ ## 0.2.1
586
+
587
+ * Remove Gemfile.lock
588
+ * Rename gemspec file
589
+
590
+ ## 0.2.0
591
+
592
+ * Add command `s3_website cfg apply`
593
+
594
+ ## 0.1.0
595
+
596
+ * First version
@@ -0,0 +1,3 @@
1
+ module S3Website
2
+ VERSION = '4.0.0'
3
+ end
data/lib/s3_website.rb ADDED
@@ -0,0 +1,7 @@
1
+ require 'rubygems'
2
+ require 'thor'
3
+ require 'dotenv'
4
+
5
+ Dotenv.load
6
+
7
+ require File.dirname(__FILE__) + "/s3_website/version"
@@ -0,0 +1 @@
1
+ addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.11.2")
@@ -0,0 +1 @@
1
+ sbt.version=0.13.18
@@ -0,0 +1 @@
1
+ addSbtPlugin("com.github.mpeltonen" % "sbt-idea" % "1.6.0")
data/release ADDED
@@ -0,0 +1,41 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'octokit'
4
+ require_relative 'lib/s3_website/version'
5
+
6
+ version = S3Website::VERSION
7
+ unless ENV['S3_WEBSITE_RELEASE_KEY']
8
+ puts "Set the S3_WEBSITE_RELEASE_KEY environmental variable. It should be your personal GitHub access token."
9
+ puts "Go to https://github.com/settings/applications#personal-access-tokens."
10
+ exit 1
11
+ end
12
+
13
+ def run(cmd)
14
+ unless system(cmd)
15
+ exit 1
16
+ end
17
+ end
18
+
19
+ puts "Building s3_website.jar..."
20
+ jar_location = 'target/scala-2.11/s3_website.jar'
21
+ run "./sbt clean assembly"
22
+
23
+ # The bin s3_website uses the MD5 checksum to verify that the jar is not corrupted
24
+ jar_md5 = Digest::MD5.file(jar_location).hexdigest
25
+ File.write('resources/s3_website.jar.md5', jar_md5)
26
+
27
+ tag_name = "v#{version}"
28
+ run "git tag #{tag_name} && git push origin #{tag_name}"
29
+ run "git push"
30
+
31
+ run "rake build"
32
+
33
+ client = Octokit::Client.new(:login => 'laurilehmijoki', :password => ENV['S3_WEBSITE_RELEASE_KEY'])
34
+ release = client.create_release(
35
+ 'laurilehmijoki/s3_website',
36
+ tag_name,
37
+ :body => "See https://github.com/laurilehmijoki/s3_website/blob/master/changelog.md##{version.gsub('.', '')}"
38
+ )
39
+ puts "Uploading s3_website.jar..."
40
+ client.upload_asset(release[:url], jar_location)
41
+ run "gem push pkg/s3_website-#{version}.gem"