rstreet 1.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.env.example +3 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +22 -0
- data/README.md +163 -0
- data/Rakefile +2 -0
- data/bin/rstreet +9 -0
- data/lib/bucket_communicator.rb +42 -0
- data/lib/files_reader.rb +20 -0
- data/lib/manifest_builder.rb +48 -0
- data/lib/opts_parser.rb +47 -0
- data/lib/rstreet.rb +58 -0
- data/lib/rstreet/version.rb +3 -0
- data/lib/s3.rb +19 -0
- data/lib/uploadable.rb +24 -0
- data/lib/uploadable_collector.rb +72 -0
- data/rstreet.gemspec +26 -0
- metadata +134 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 5c9c064930f22deb7fb0a3fab8b8323509bcc533
|
4
|
+
data.tar.gz: e429f6c7e6ac79f70bed6253eb741cf9308007ef
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 8960137d9358d4e36fe35cc9020c48027be18c870be28585689dbed2ebe9d589d762a859b37bfa01814f5acc92540d2e7737c21000df306d08b38bb4b122530e
|
7
|
+
data.tar.gz: 1209e3e3483d148cc166287b5fba2bd42cb92c11bc1593c952e0082d0629e7ebd866d320cb61a3c362c72e543ce287517fc17e77600ea4e320b5ef58acad87a2
|
data/.env.example
ADDED
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2015 jaketrent
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,163 @@
|
|
1
|
+
# Rstreet
|
2
|
+
|
3
|
+
Smart Uploader for AWS S3 buckets.
|
4
|
+
|
5
|
+
Generates a manifest file upon first upload;
|
6
|
+
subsequent uploads only send changed files. Especially useful as a deploy tool
|
7
|
+
for uploading static websites to S3. Benefits of using Street:
|
8
|
+
|
9
|
+
* Upload only changed files; reduces **PUT** requests.
|
10
|
+
* GZip Compressed **manifest** file stored on S3 is very small.
|
11
|
+
* Mime-type lookup on upload to facilitate proper browser handling.
|
12
|
+
* Non-destructive *(mostly)*. Works with existing buckets and data. See *Manifest
|
13
|
+
File* section below for more details.
|
14
|
+
|
15
|
+
Ruby port of [tgroshon's](https://github.com/tgroshon) [street.js](https://github.com/tgroshon/street.js)
|
16
|
+
|
17
|
+
## tl;dr
|
18
|
+
|
19
|
+
* Install with `gem install rstreet` to get the `rstreet` cli tool.
|
20
|
+
* Export **AWS_ACCESS_KEY_ID** and **AWS_SECRET_ACCESS_KEY** to your environment.
|
21
|
+
* Run `rstreet -e -b <Target S3 Bucket> path/to/upload/dir`
|
22
|
+
|
23
|
+
## Installation
|
24
|
+
|
25
|
+
Add this line to your application's Gemfile:
|
26
|
+
|
27
|
+
```ruby
|
28
|
+
gem 'rstreet'
|
29
|
+
```
|
30
|
+
|
31
|
+
And then execute:
|
32
|
+
|
33
|
+
$ bundle
|
34
|
+
|
35
|
+
Or install it yourself as:
|
36
|
+
|
37
|
+
$ gem install rstreet
|
38
|
+
|
39
|
+
## Command Line Options
|
40
|
+
|
41
|
+
A helpful interface into Street is using the command line tool `rstreet`. The
|
42
|
+
command line tool requires a path to a directory that you want to upload to S3 as
|
43
|
+
the last argument.
|
44
|
+
Run `rstreet --help` to see a list of options.
|
45
|
+
|
46
|
+
#### -e, --load-env, Load Environment Variables ####
|
47
|
+
|
48
|
+
Rstreet can load environment variables declared in a `.env` file of the current
|
49
|
+
working directory. With this option, Rstreet searches for 3 AWS variables:
|
50
|
+
|
51
|
+
1. **AWS_ACCESS_KEY_ID**
|
52
|
+
2. **AWS_SECRET_ACCESS_KEY**
|
53
|
+
3. **S3_BUCKET**
|
54
|
+
|
55
|
+
Instead of using Environment Variables, you can some or all of these through
|
56
|
+
the command line options `--aws-key`, `--aws-secret`, and `--bucket`
|
57
|
+
respectively.
|
58
|
+
|
59
|
+
*Example*: `rstreet -e path/to/upload/dir`
|
60
|
+
|
61
|
+
#### -b, --bucket [bucket], S3 Destination Bucket ####
|
62
|
+
|
63
|
+
The Amazon S3 bucket to be the destination of your uploaded files.
|
64
|
+
|
65
|
+
*Example*: `rstreet -b <S3 bucket name> path/to/upload/dir`
|
66
|
+
|
67
|
+
#### -k, --aws-key [key], AWS Access Key Id ####
|
68
|
+
|
69
|
+
The AWS Access Key Id to be used for authenticating the S3 session. User must
|
70
|
+
have `PUT` and `GET` permissions on the bucket.
|
71
|
+
|
72
|
+
*Example*: `street -k <AWS Access Key Id> path/to/upload/dir`
|
73
|
+
|
74
|
+
#### -s, --aws-secret [secret], AWS Secret Access Key ####
|
75
|
+
|
76
|
+
The AWS Secret Access Key associated with the AWS Access Key Id you are using.
|
77
|
+
|
78
|
+
*Example*: `rstreet -s <AWS Secret Key> path/to/upload/dir`
|
79
|
+
|
80
|
+
#### -v, --verbose, Run with expanded messages ####
|
81
|
+
|
82
|
+
Shows the number of files uploaded, but not much else. This option will do more
|
83
|
+
in the future.
|
84
|
+
|
85
|
+
*Example*: `rstreet -v /path/to/upload/dir`
|
86
|
+
|
87
|
+
#### -n, --dry-run, Run but do not upload ####
|
88
|
+
|
89
|
+
Best if used in conjuction with the `--verbose` option.
|
90
|
+
|
91
|
+
*Example*:
|
92
|
+
|
93
|
+
`rstreet -n path/to/upload/dir`
|
94
|
+
|
95
|
+
`rstreet -nv path/to/upload/dir`
|
96
|
+
|
97
|
+
`rstreet -n path/to/upload/dir`
|
98
|
+
|
99
|
+
## Run Programmatically
|
100
|
+
|
101
|
+
If you want to incorporate the Rstreet uploader into another program, that's
|
102
|
+
also possible. Perhaps you'd like to use it as a part of an API that can upload
|
103
|
+
to S3 or your own build script. Usage is simple. It takes a hash of options that
|
104
|
+
match the commandline interface. However, note that the names of the options are slightly
|
105
|
+
different.
|
106
|
+
|
107
|
+
```ruby
|
108
|
+
require "rstreet"
|
109
|
+
|
110
|
+
options = {
|
111
|
+
src: 'path/to/upload/dir', # Path to directory to upload.
|
112
|
+
dry_run: false, # Disable upload mechanism.
|
113
|
+
s3_bucket: 'bucketname.com', # Name of S3 Bucket to upload to.
|
114
|
+
aws_key: 'AWS Access Key Id', # AWS Access Key Id for authentication w/ S3.
|
115
|
+
aws_secret: 'AWS Secret Key', # AWS Secret Key for authentication w/ S3.
|
116
|
+
load_env: true, # Load Environment Variables with 'dotenv'.
|
117
|
+
verbose: true, # Trigger extra messages.
|
118
|
+
}
|
119
|
+
|
120
|
+
Rstreet::Uploader.new(options).run
|
121
|
+
```
|
122
|
+
|
123
|
+
You can make this part of a [Grunt](http://gruntjs.com/) task or your own,
|
124
|
+
standalone deploy script. Dealer's choice!
|
125
|
+
|
126
|
+
## Manifest File
|
127
|
+
|
128
|
+
The `manifest.json.gz` file is a GZipped JSON file that maps S3 Object Keys (file
|
129
|
+
names) to MD5 Hashes.
|
130
|
+
|
131
|
+
The `manifest.json.gz` file is generated each time Street is run. On the first run
|
132
|
+
of Street on a specific directory, the `manifest.json.gz` file is written to that
|
133
|
+
directory (called the *upload directory*), and all files it found are uploaded to
|
134
|
+
S3. The next time Street is run, a **new manifest** is generated locally, the
|
135
|
+
**old manifest** is pulled down from S3, and the two files are compared to
|
136
|
+
search for differences. The **new manifest** is only written if files have
|
137
|
+
been changed. If files have been changed, the new `manifest.json.gz` and changed
|
138
|
+
files are then uploaded to S3 where they replace the objects of the same name.
|
139
|
+
|
140
|
+
This approach has two important effects that you should be aware of:
|
141
|
+
|
142
|
+
1. **\*IMPORTANT\*** Upon first run of Street, all files in the directory
|
143
|
+
and a new `manifest.json.gz` file will be uploaded and replace S3 objects with
|
144
|
+
the same name.
|
145
|
+
|
146
|
+
2. A `manifest.json.gz` file on S3 does *not* hold information on all the objects in
|
147
|
+
that bucket; only on objects that Street has specifically uploaded.
|
148
|
+
|
149
|
+
Keep these in mind as you work.
|
150
|
+
|
151
|
+
## Setup AWS Access Policies
|
152
|
+
|
153
|
+
Make sure that the credentials that you're providing to the tool have sufficient access to read and write to the bucket you're attempting to access.
|
154
|
+
|
155
|
+
For an easy default, attach the managed access control policy "AmazonS3FullAccess" to your user/group.
|
156
|
+
|
157
|
+
## Contributing
|
158
|
+
|
159
|
+
1. Fork it ( https://github.com/[jaketrent]/rstreet/fork )
|
160
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
161
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
162
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
163
|
+
5. Create a new Pull Request
|
data/Rakefile
ADDED
data/bin/rstreet
ADDED
@@ -0,0 +1,42 @@
|
|
1
|
+
require "s3"
|
2
|
+
|
3
|
+
require "manifest_builder"
|
4
|
+
|
5
|
+
module Rstreet
|
6
|
+
class BucketCommunicator
|
7
|
+
def initialize(s3_bucket, dry_run)
|
8
|
+
@s3, @dry_run = S3.new(s3_bucket), dry_run
|
9
|
+
end
|
10
|
+
|
11
|
+
def pull_manifest(manifest_uploadable)
|
12
|
+
manifest_file = @s3.get_file(manifest_uploadable)
|
13
|
+
if manifest_file
|
14
|
+
ManifestBuilder.new(manifest_file).read
|
15
|
+
else
|
16
|
+
ManifestBuilder.empty_manifest
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
def upload(uploadables)
|
21
|
+
# TODO: refactor into sep procs
|
22
|
+
uploadables.map do |u|
|
23
|
+
if dry_run?
|
24
|
+
puts "dry run upload: #{u.name}"
|
25
|
+
else
|
26
|
+
begin
|
27
|
+
@s3.put_file(u)
|
28
|
+
puts "uploaded: #{u.name}"
|
29
|
+
rescue StandardError => e
|
30
|
+
puts "error uploading: #{u.name}", e
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
private
|
37
|
+
|
38
|
+
def dry_run?
|
39
|
+
@dry_run
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
data/lib/files_reader.rb
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
require 'set'
|
2
|
+
|
3
|
+
module Rstreet
|
4
|
+
class FilesReader
|
5
|
+
def self.read_all(dir)
|
6
|
+
dir = File.realpath dir
|
7
|
+
# TODO: print if verbose
|
8
|
+
# puts "Found files in #{@dir}:"
|
9
|
+
|
10
|
+
files = Dir.glob("#{dir}/**/*").to_set
|
11
|
+
dirs = Dir.glob("#{dir}/**/*/").map { |d| d[0..-2] }.to_set
|
12
|
+
|
13
|
+
# puts files.inspect
|
14
|
+
# puts ""
|
15
|
+
# puts dirs.inspect
|
16
|
+
|
17
|
+
files.subtract(dirs).to_a
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,48 @@
|
|
1
|
+
require "json"
|
2
|
+
|
3
|
+
module Rstreet
|
4
|
+
# TODO: name something more general to manifests (does more than builds)
|
5
|
+
class ManifestBuilder
|
6
|
+
attr_reader :manifest
|
7
|
+
|
8
|
+
def initialize(manifest_file)
|
9
|
+
@manifest = {}
|
10
|
+
@manifest_file = manifest_file
|
11
|
+
end
|
12
|
+
|
13
|
+
def self.empty_manifest
|
14
|
+
{}
|
15
|
+
end
|
16
|
+
|
17
|
+
def read
|
18
|
+
gz = Zlib::GzipReader.new(StringIO.new(@manifest_file))
|
19
|
+
@manifest = JSON.parse(gz.read).to_h
|
20
|
+
gz.close
|
21
|
+
@manifest
|
22
|
+
end
|
23
|
+
|
24
|
+
def write()
|
25
|
+
File.open(@manifest_file, "w") do |f|
|
26
|
+
gz = Zlib::GzipWriter.new(f)
|
27
|
+
gz.write @manifest.to_json.to_s
|
28
|
+
gz.close
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def add(uploadable)
|
33
|
+
@manifest[uploadable.name] = uploadable.hash
|
34
|
+
end
|
35
|
+
|
36
|
+
def diff(other_manifest)
|
37
|
+
# TODO: determine if this is the diff algorithm we want
|
38
|
+
# TODO: in the future, account for deletions and delete objects from bucket
|
39
|
+
# 8 [ruby-2.1.5]::rstreet]>a.to_set
|
40
|
+
# => #<Set: {[:a, 1], [:b, 2], [:c, 3]}>
|
41
|
+
# 9 [ruby-2.1.5]::rstreet]>a.to_set.difference b.to_set
|
42
|
+
# => #<Set: {[:b, 2], [:c, 3]}>
|
43
|
+
# 10 [ruby-2.1.5]::rstreet]>b.to_set.difference a.to_set
|
44
|
+
# => #<Set: {[:b, 4]}>
|
45
|
+
@manifest.to_set.difference(other_manifest.to_set).to_h.keys
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
data/lib/opts_parser.rb
ADDED
@@ -0,0 +1,47 @@
|
|
1
|
+
require "optparse"
|
2
|
+
require "ostruct"
|
3
|
+
|
4
|
+
module Rstreet
|
5
|
+
class OptsParser
|
6
|
+
def self.parse(args)
|
7
|
+
options = OpenStruct.new
|
8
|
+
options.src = args[-1]
|
9
|
+
|
10
|
+
opt_parser = OptionParser.new do |opts|
|
11
|
+
opts.banner = "Usage: rstreet [options] <src_dir>"
|
12
|
+
opts.separator ""
|
13
|
+
opts.separator "Options:"
|
14
|
+
|
15
|
+
# TODO: DRY up
|
16
|
+
opts.on("-e", "--load-env",
|
17
|
+
"Load environment from .env file") do |load_env|
|
18
|
+
options.load_env = true
|
19
|
+
end
|
20
|
+
opts.on("-b b", "--bucket b",
|
21
|
+
"S3 destination bucket") do |s3_bucket|
|
22
|
+
options.s3_bucket = s3_bucket
|
23
|
+
end
|
24
|
+
opts.on("-k k", "--aws-key k",
|
25
|
+
"AWS Access Key ID") do |aws_key|
|
26
|
+
options.aws_key = aws_key
|
27
|
+
end
|
28
|
+
opts.on("-s s", "--aws-secret s",
|
29
|
+
"AWS Secret Access Key") do |aws_secret|
|
30
|
+
options.aws_secret = aws_secret
|
31
|
+
end
|
32
|
+
opts.on("-v", "--verbose",
|
33
|
+
"Run with expanded messages") do |verbose|
|
34
|
+
options.verbose = true
|
35
|
+
end
|
36
|
+
opts.on("-n", "--dry-run",
|
37
|
+
"Run but do not upload") do |dry_run|
|
38
|
+
options.dry_run = true
|
39
|
+
end
|
40
|
+
|
41
|
+
end
|
42
|
+
|
43
|
+
opt_parser.parse!(args)
|
44
|
+
options
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
data/lib/rstreet.rb
ADDED
@@ -0,0 +1,58 @@
|
|
1
|
+
require "aws-sdk"
|
2
|
+
require "dotenv"
|
3
|
+
|
4
|
+
require "bucket_communicator"
|
5
|
+
require "rstreet/version"
|
6
|
+
require "uploadable_collector"
|
7
|
+
|
8
|
+
module Rstreet
|
9
|
+
class Uploader
|
10
|
+
def initialize(options)
|
11
|
+
@options = options
|
12
|
+
load_env if should_load_env?
|
13
|
+
config_aws
|
14
|
+
validate_options
|
15
|
+
end
|
16
|
+
|
17
|
+
def run
|
18
|
+
collector = UploadableCollector.new(@options.src)
|
19
|
+
uploadables = collector.collect
|
20
|
+
|
21
|
+
bucket_communicator = BucketCommunicator.new(@options.s3_bucket, @options.dry_run)
|
22
|
+
old_manifest = bucket_communicator.pull_manifest(collector.manifest_uploadable)
|
23
|
+
current_manifest_builder = collector.manifest_builder
|
24
|
+
diff = current_manifest_builder.diff(old_manifest)
|
25
|
+
|
26
|
+
to_upload = collector.find_uploadables(diff)
|
27
|
+
bucket_communicator.upload(to_upload)
|
28
|
+
end
|
29
|
+
|
30
|
+
private
|
31
|
+
|
32
|
+
def config_aws
|
33
|
+
AWS.config(access_key_id: @options.aws_key, secret_access_key: @options.aws_secret)
|
34
|
+
end
|
35
|
+
|
36
|
+
def load_env
|
37
|
+
Dotenv.load
|
38
|
+
@options.s3_bucket ||= ENV["S3_BUCKET"]
|
39
|
+
@options.aws_key ||= ENV["AWS_ACCESS_KEY_ID"]
|
40
|
+
@options.aws_secret ||= ENV["AWS_SECRET_ACCESS_KEY"]
|
41
|
+
end
|
42
|
+
|
43
|
+
def should_load_env?
|
44
|
+
@options.load_env
|
45
|
+
end
|
46
|
+
|
47
|
+
def validate_options
|
48
|
+
# TODO: DRY'ify
|
49
|
+
raise ArgumentError, "Specify a source directory in options.src" if @options.src.nil?
|
50
|
+
|
51
|
+
raise ArgumentError, "Specify an S3 Bucket in an env var called S3_BUCKET or options.s3_bucket" if @options.s3_bucket.nil?
|
52
|
+
|
53
|
+
raise ArgumentError, "Specify an AWS Access Key Id in an env var called AWS_ACCESS_KEY_ID or options.aws_key" if @options.aws_key.nil?
|
54
|
+
|
55
|
+
raise ArgumentError, "Specify an AWS Secret Access Key in env var called AWS_SECRET_ACCESS_KEY or options.aws_secret" if @options.aws_secret.nil?
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
data/lib/s3.rb
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
require "aws-sdk"
|
2
|
+
|
3
|
+
module Rstreet
|
4
|
+
class S3
|
5
|
+
def initialize(s3_bucket)
|
6
|
+
@s3 = AWS::S3.new
|
7
|
+
@s3_bucket = s3_bucket
|
8
|
+
end
|
9
|
+
|
10
|
+
def get_file(uploadable)
|
11
|
+
@s3.buckets[@s3_bucket].objects[uploadable.name].read
|
12
|
+
rescue AWS::S3::Errors::NoSuchKey
|
13
|
+
end
|
14
|
+
|
15
|
+
def put_file(uploadable)
|
16
|
+
@s3.buckets[@s3_bucket].objects[uploadable.name].write(file: uploadable.path, acl: :authenticated_read)
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
data/lib/uploadable.rb
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
require "mime/types"
|
2
|
+
|
3
|
+
module Rstreet
|
4
|
+
class Uploadable
|
5
|
+
attr_reader :name, :path, :hash
|
6
|
+
|
7
|
+
def initialize(name, path)
|
8
|
+
@name, @path = name, path
|
9
|
+
@hash = gen_hash || ""
|
10
|
+
@content_type = gen_content_type || "application/octet-stream"
|
11
|
+
end
|
12
|
+
|
13
|
+
private
|
14
|
+
|
15
|
+
def gen_hash
|
16
|
+
Digest::MD5.file(@path).hexdigest if File.file?(@path)
|
17
|
+
end
|
18
|
+
|
19
|
+
def gen_content_type
|
20
|
+
MIME::Types.type_for(@path).first.content_type
|
21
|
+
rescue
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,72 @@
|
|
1
|
+
require "files_reader"
|
2
|
+
require "manifest_builder"
|
3
|
+
require "uploadable"
|
4
|
+
|
5
|
+
module Rstreet
|
6
|
+
class UploadableCollector
|
7
|
+
|
8
|
+
# TODO: Shouldn't need to get this for the pull_manifest in rstreet.rb
|
9
|
+
# Instead, consider some class method with the default file path info
|
10
|
+
attr_reader :manifest_builder, :manifest_uploadable
|
11
|
+
|
12
|
+
def initialize(src)
|
13
|
+
@src = File.realpath src
|
14
|
+
# TODO: get rid of cache and consolidate inside of manifest builder
|
15
|
+
@uploadables_cache = {}
|
16
|
+
init_manifest_builder
|
17
|
+
end
|
18
|
+
|
19
|
+
def collect
|
20
|
+
files = FilesReader.read_all(@src)
|
21
|
+
uploadables = convert_files_to_uploadables files
|
22
|
+
uploadables = add_manifest_to_uploadables uploadables
|
23
|
+
save_uploadables_in_manifest uploadables
|
24
|
+
uploadables
|
25
|
+
end
|
26
|
+
|
27
|
+
# TODO: be more consistent with name, file, path var names
|
28
|
+
def find_uploadables(file_names)
|
29
|
+
@uploadables_cache.select { |k, v| file_names.include? k }.values
|
30
|
+
end
|
31
|
+
|
32
|
+
private
|
33
|
+
|
34
|
+
def save_uploadables_in_manifest(uploadables)
|
35
|
+
uploadables.each do |u|
|
36
|
+
@manifest_builder.add u
|
37
|
+
@uploadables_cache[u.name] = u
|
38
|
+
end
|
39
|
+
@manifest_builder.write
|
40
|
+
end
|
41
|
+
|
42
|
+
def init_manifest_builder
|
43
|
+
manifest_file = File.join(@src, ".manifest.json.gz")
|
44
|
+
@manifest_uploadable = convert_to_uploadable(manifest_file)
|
45
|
+
@manifest_builder = ManifestBuilder.new manifest_file
|
46
|
+
end
|
47
|
+
|
48
|
+
def convert_files_to_uploadables(files)
|
49
|
+
files.map do |file|
|
50
|
+
convert_to_uploadable(file)
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
def add_manifest_to_uploadables(uploadables)
|
55
|
+
uploadables << @manifest_uploadable if uploadables.any?
|
56
|
+
uploadables
|
57
|
+
end
|
58
|
+
|
59
|
+
def convert_to_uploadable(file)
|
60
|
+
Uploadable.new(format_uploadable_name(file), file)
|
61
|
+
end
|
62
|
+
|
63
|
+
def format_uploadable_name(file)
|
64
|
+
file.sub(ensure_ends_with_separator(@src), "")
|
65
|
+
end
|
66
|
+
|
67
|
+
def ensure_ends_with_separator(path)
|
68
|
+
path.end_with?(File::SEPARATOR) ? path : "#{path}#{File::SEPARATOR}"
|
69
|
+
end
|
70
|
+
|
71
|
+
end
|
72
|
+
end
|
data/rstreet.gemspec
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'rstreet/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "rstreet"
|
8
|
+
spec.version = Rstreet::VERSION
|
9
|
+
spec.authors = ["jaketrent"]
|
10
|
+
spec.email = ["trent.jake@gmail.com"]
|
11
|
+
spec.summary = %q{Smart uploader for S3}
|
12
|
+
spec.description = %q{Smart s3 uploader. Generates a manifest file on first upload. Minimizes future uploads by only sending changed files.}
|
13
|
+
spec.homepage = "https://github.com/jaketrent/rstreet"
|
14
|
+
spec.license = "MIT"
|
15
|
+
|
16
|
+
spec.files = `git ls-files -z`.split("\x0")
|
17
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
18
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
19
|
+
spec.require_paths = ["lib"]
|
20
|
+
|
21
|
+
spec.add_development_dependency "bundler", "~> 1.7"
|
22
|
+
spec.add_development_dependency "rake", "~> 10.0"
|
23
|
+
spec.add_development_dependency "dotenv", "~> 1.0.2"
|
24
|
+
spec.add_development_dependency "mime-types", "~> 2.4.3"
|
25
|
+
spec.add_development_dependency "aws-sdk", "~> 2"
|
26
|
+
end
|
metadata
ADDED
@@ -0,0 +1,134 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: rstreet
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 1.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- jaketrent
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2015-02-22 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: bundler
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1.7'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '1.7'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rake
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '10.0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '10.0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: dotenv
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: 1.0.2
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: 1.0.2
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: mime-types
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: 2.4.3
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: 2.4.3
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: aws-sdk
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - "~>"
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '2'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - "~>"
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '2'
|
83
|
+
description: Smart s3 uploader. Generates a manifest file on first upload. Minimizes
|
84
|
+
future uploads by only sending changed files.
|
85
|
+
email:
|
86
|
+
- trent.jake@gmail.com
|
87
|
+
executables:
|
88
|
+
- rstreet
|
89
|
+
extensions: []
|
90
|
+
extra_rdoc_files: []
|
91
|
+
files:
|
92
|
+
- ".env.example"
|
93
|
+
- ".gitignore"
|
94
|
+
- Gemfile
|
95
|
+
- LICENSE.txt
|
96
|
+
- README.md
|
97
|
+
- Rakefile
|
98
|
+
- bin/rstreet
|
99
|
+
- lib/bucket_communicator.rb
|
100
|
+
- lib/files_reader.rb
|
101
|
+
- lib/manifest_builder.rb
|
102
|
+
- lib/opts_parser.rb
|
103
|
+
- lib/rstreet.rb
|
104
|
+
- lib/rstreet/version.rb
|
105
|
+
- lib/s3.rb
|
106
|
+
- lib/uploadable.rb
|
107
|
+
- lib/uploadable_collector.rb
|
108
|
+
- rstreet.gemspec
|
109
|
+
homepage: https://github.com/jaketrent/rstreet
|
110
|
+
licenses:
|
111
|
+
- MIT
|
112
|
+
metadata: {}
|
113
|
+
post_install_message:
|
114
|
+
rdoc_options: []
|
115
|
+
require_paths:
|
116
|
+
- lib
|
117
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
118
|
+
requirements:
|
119
|
+
- - ">="
|
120
|
+
- !ruby/object:Gem::Version
|
121
|
+
version: '0'
|
122
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
123
|
+
requirements:
|
124
|
+
- - ">="
|
125
|
+
- !ruby/object:Gem::Version
|
126
|
+
version: '0'
|
127
|
+
requirements: []
|
128
|
+
rubyforge_project:
|
129
|
+
rubygems_version: 2.4.3
|
130
|
+
signing_key:
|
131
|
+
specification_version: 4
|
132
|
+
summary: Smart uploader for S3
|
133
|
+
test_files: []
|
134
|
+
has_rdoc:
|