bubbles 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +14 -0
- data/.rspec +2 -0
- data/.travis.yml +10 -0
- data/Gemfile +4 -0
- data/LICENSE +20 -0
- data/README.md +135 -0
- data/Rakefile +6 -0
- data/bin/console +14 -0
- data/bin/setup +8 -0
- data/bubbles.gemspec +28 -0
- data/exe/bubbles +5 -0
- data/lib/bubbles.rb +37 -0
- data/lib/bubbles/bubblicious_file.rb +40 -0
- data/lib/bubbles/command_queue.rb +43 -0
- data/lib/bubbles/common_uploader_interface.rb +13 -0
- data/lib/bubbles/config.rb +136 -0
- data/lib/bubbles/dir_watcher.rb +53 -0
- data/lib/bubbles/uploaders/local_dir.rb +24 -0
- data/lib/bubbles/uploaders/s3.rb +36 -0
- data/lib/bubbles/version.rb +3 -0
- metadata +124 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: f81c23e3e967680cb5a8607cf97afea1a28f5da9
|
4
|
+
data.tar.gz: edf10a7ed78c0583082d88e4d0846f459ee6780d
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 879504d35b51e5e986cfbcce49f2a7b6335b0c0dc153178cf29ad2aa1146826a5381e29c6216ee286e4505b51c5e7327bf08d0fa6abe42b0b3b52f70bfb3f36e
|
7
|
+
data.tar.gz: cba515653ced3cb7bcc1fdd54abef80240e1e7d8ebe346b053a0b4457de4106274f3140caa40b7246d7a1ba011117f9f056ea099419808cdac374a1c29b6b31d
|
data/.gitignore
ADDED
data/.rspec
ADDED
data/.travis.yml
ADDED
data/Gemfile
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2017 Equivalent (Tomas Valent)
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy of
|
6
|
+
this software and associated documentation files (the "Software"), to deal in
|
7
|
+
the Software without restriction, including without limitation the rights to
|
8
|
+
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
|
9
|
+
the Software, and to permit persons to whom the Software is furnished to do so,
|
10
|
+
subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
13
|
+
copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
17
|
+
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
18
|
+
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
19
|
+
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
20
|
+
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,135 @@
|
|
1
|
+
# Bubbles
|
2
|
+
|
3
|
+
[](https://travis-ci.org/equivalent/bubbles)
|
4
|
+
|
5
|
+
Daemonized file uploader that watch a folder and uploads any files files
|
6
|
+
to AWS S3 or Local directory (e.g. mounted NAS volume). Designed for Raspberry pi zero
|
7
|
+
|
8
|
+
|
9
|
+
Notes:
|
10
|
+
|
11
|
+
* for AWS S3 upload we use AWS-SDK [s3 put_object](http://docs.aws.amazon.com/sdkforruby/api/Aws/S3/Client.html#put_object-instance_method) the single file upload should not exceed 5GB. I'm not planing to introduce S3 multipart uploads but Pull Requests are welcome.
|
12
|
+
|
13
|
+
|
14
|
+
## Installation
|
15
|
+
|
16
|
+
Add this line to your application's Gemfile:
|
17
|
+
|
18
|
+
```ruby
|
19
|
+
gem 'bubbles'
|
20
|
+
```
|
21
|
+
|
22
|
+
And then execute:
|
23
|
+
|
24
|
+
$ bundle
|
25
|
+
|
26
|
+
Or install it yourself as:
|
27
|
+
|
28
|
+
$ gem install bubbles
|
29
|
+
|
30
|
+
## Usage
|
31
|
+
|
32
|
+
##### The easy way
|
33
|
+
|
34
|
+
Gem by default looks into to two locations for configuration file. So
|
35
|
+
either create `~/.bubbles/config` or `/var/lib/bubbles/config.yml`
|
36
|
+
|
37
|
+
> note: make sure you have correct read access permission so that ruby
|
38
|
+
> can read that file
|
39
|
+
|
40
|
+
...with content similar to this:
|
41
|
+
|
42
|
+
```yml
|
43
|
+
s3_access_key_id: xxxxxxxxxxxxxxxxxxxx
|
44
|
+
s3_secret_access_key: yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy
|
45
|
+
s3_bucket: bucket_name
|
46
|
+
s3_region: eu-west-1
|
47
|
+
|
48
|
+
source_dir: /home/myuser/source_folder
|
49
|
+
processing_dir: /home/myuser/processing_folder
|
50
|
+
```
|
51
|
+
|
52
|
+
Now all is left is to lunch bubbles with `bubbles` or `bundle exec bubbles`
|
53
|
+
|
54
|
+
> note: files in `source_folder` will get removed after successful upload
|
55
|
+
|
56
|
+
##### More advanced config
|
57
|
+
|
58
|
+
|
59
|
+
```yml
|
60
|
+
source_dir: /var/myuser/source_folder # source from where to pick up files
|
61
|
+
processing_dir: /home/myuser/processing_folder
|
62
|
+
|
63
|
+
log_level: 0 # debug log level
|
64
|
+
log_path: /var/log/bubbles.log # default is STDOOT
|
65
|
+
|
66
|
+
sleep_interval: 1 # sleep between next command
|
67
|
+
num_of_files_to_schedule: 1 # how many files schedule for processing at the same time
|
68
|
+
|
69
|
+
uploaders:
|
70
|
+
- 'Bubbles::Uploaders::S3' # by default only S3 uploader is used
|
71
|
+
- 'Bubbles::Uploaders::LocalDir'
|
72
|
+
|
73
|
+
s3_access_key_id: xxxxxxxxxxxxxxxxxxxx
|
74
|
+
s3_secret_access_key: yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy
|
75
|
+
s3_bucket: bucket_name
|
76
|
+
s3_region: eu-west-1
|
77
|
+
s3_path: foo_folder/bar_folder/car_folder # will upload to s3://bucket/foo_folder/bar_folder/car_folder
|
78
|
+
|
79
|
+
local_dir_uploader_path: /mnt/network_smb_storage
|
80
|
+
```
|
81
|
+
|
82
|
+
> Look into [lib/bubbles/config.rb](https://github.com/equivalent/bubbles/blob/master/lib/bubbles/config.rb) for more details.
|
83
|
+
|
84
|
+
##### Full Ruby way
|
85
|
+
|
86
|
+
You can create custom ruby file and you initialize custom
|
87
|
+
`Bubbles::Config.new` instance and set all the options you want there:
|
88
|
+
|
89
|
+
```ruby
|
90
|
+
# ~/my_bubbles_runner.rb
|
91
|
+
require 'bubbles'
|
92
|
+
|
93
|
+
c = Bubbles::Config.new
|
94
|
+
|
95
|
+
c.log_level = 0
|
96
|
+
c.source_dir = /var/myuser/source_folder
|
97
|
+
c.processing_dir = /var/myuser/processing_folder
|
98
|
+
c.uploader_classes = [Bubbles::Uploaders::S3, Bubbles::Uploaders::LocalDir]
|
99
|
+
c.local_dir_uploader_path = /mnt/network_smb_storage
|
100
|
+
c.s3_region = 'eu-west-1'
|
101
|
+
c.s3_bucket = 'mybckt'
|
102
|
+
c.s3_access_key_id = 'xxxxxxxxxxx'
|
103
|
+
c.s3_secret_access_key = 'yyyyyyyyyyy'
|
104
|
+
|
105
|
+
# ...
|
106
|
+
|
107
|
+
command_queue = Bubbles::CommandQueue.new(config: c)
|
108
|
+
command_queue << Bubbles::DirWatcher.new(config: c, command_queue: command_queue)
|
109
|
+
|
110
|
+
loop do
|
111
|
+
command_queue.call_next
|
112
|
+
sleep c.sleep_interval
|
113
|
+
end
|
114
|
+
```
|
115
|
+
|
116
|
+
..and execute:
|
117
|
+
|
118
|
+
```sh
|
119
|
+
ruby my_bubbles_runner.rb
|
120
|
+
# or
|
121
|
+
bundle exec ruby my_bubbles_runner.rb
|
122
|
+
```
|
123
|
+
|
124
|
+
> Look into [lib/bubbles/config.rb](https://github.com/equivalent/bubbles/blob/master/lib/bubbles/config.rb) for more details.
|
125
|
+
|
126
|
+
## Development
|
127
|
+
|
128
|
+
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
129
|
+
To install this gem onto your local machine, run `bundle exec rake install`.
|
130
|
+
|
131
|
+
## Contributing
|
132
|
+
|
133
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/equivalent/bubbles.
|
134
|
+
|
135
|
+
|
data/Rakefile
ADDED
data/bin/console
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require "bundler/setup"
|
4
|
+
require "bubbles"
|
5
|
+
|
6
|
+
# You can add fixtures and/or initialization code here to make experimenting
|
7
|
+
# with your gem easier. You can also use a different console, if you like.
|
8
|
+
|
9
|
+
# (If you use this, don't forget to add pry to your Gemfile!)
|
10
|
+
# require "pry"
|
11
|
+
# Pry.start
|
12
|
+
|
13
|
+
require "irb"
|
14
|
+
IRB.start(__FILE__)
|
data/bin/setup
ADDED
data/bubbles.gemspec
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'bubbles/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "bubbles"
|
8
|
+
spec.version = Bubbles::VERSION
|
9
|
+
spec.authors = ["Tomas Valent"]
|
10
|
+
spec.licenses = ['MIT']
|
11
|
+
spec.email = ["equivalent@eq8.eu"]
|
12
|
+
|
13
|
+
spec.summary = %q{Lightweight daemon file uploader to cloud}
|
14
|
+
spec.description = %q{Daemonized file uploader that watch a folder and uploads any files files to AWS S3. Designed for Raspberry pi zero}
|
15
|
+
spec.homepage = "https://github.com/equivalent/bubbles"
|
16
|
+
|
17
|
+
spec.files = `git ls-files -z`.split("\x0").reject do |f|
|
18
|
+
f.match(%r{^(test|spec|features)/})
|
19
|
+
end
|
20
|
+
spec.bindir = "exe"
|
21
|
+
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
22
|
+
spec.require_paths = ["lib"]
|
23
|
+
|
24
|
+
spec.add_dependency "aws-sdk", "~> 2"
|
25
|
+
spec.add_development_dependency "bundler", "~> 1.14"
|
26
|
+
spec.add_development_dependency "rake", "~> 10.0"
|
27
|
+
spec.add_development_dependency "rspec", "~> 3.0"
|
28
|
+
end
|
data/exe/bubbles
ADDED
data/lib/bubbles.rb
ADDED
@@ -0,0 +1,37 @@
|
|
1
|
+
require 'fileutils'
|
2
|
+
require 'pathname'
|
3
|
+
require 'logger'
|
4
|
+
require 'securerandom'
|
5
|
+
require 'yaml'
|
6
|
+
require 'forwardable'
|
7
|
+
require 'aws-sdk'
|
8
|
+
require "bubbles/version"
|
9
|
+
require "bubbles/config"
|
10
|
+
require "bubbles/command_queue"
|
11
|
+
require "bubbles/bubblicious_file"
|
12
|
+
require "bubbles/dir_watcher"
|
13
|
+
require 'bubbles/common_uploader_interface'
|
14
|
+
require "bubbles/uploaders/s3"
|
15
|
+
require "bubbles/uploaders/local_dir"
|
16
|
+
|
17
|
+
module Bubbles
|
18
|
+
extend self
|
19
|
+
|
20
|
+
def config
|
21
|
+
@config ||= Config.new
|
22
|
+
end
|
23
|
+
|
24
|
+
def run
|
25
|
+
command_queue = Bubbles::CommandQueue.new(config: config)
|
26
|
+
|
27
|
+
command_queue << Bubbles::DirWatcher.new({
|
28
|
+
config: config,
|
29
|
+
command_queue: command_queue
|
30
|
+
})
|
31
|
+
|
32
|
+
loop do
|
33
|
+
command_queue.call_next
|
34
|
+
sleep config.sleep_interval
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
module Bubbles
|
2
|
+
class BubbliciousFile
|
3
|
+
extend Forwardable
|
4
|
+
|
5
|
+
def initialize(file:, config:)
|
6
|
+
@file = file
|
7
|
+
@config = config
|
8
|
+
end
|
9
|
+
|
10
|
+
def move_to_processing_dir
|
11
|
+
config.logger.debug("BubbliciousFile: moving file #{file} to #{uid_file}")
|
12
|
+
FileUtils.mv(file, uid_file)
|
13
|
+
end
|
14
|
+
|
15
|
+
def remove_file
|
16
|
+
config.logger.debug("BubbliciousFile: removing file #{uid_file}")
|
17
|
+
FileUtils.rm(uid_file)
|
18
|
+
end
|
19
|
+
|
20
|
+
def metadata
|
21
|
+
{ original_name: file.basename.to_s }
|
22
|
+
end
|
23
|
+
|
24
|
+
def uid_file
|
25
|
+
Pathname.new(processing_dir).join(uid_file_name)
|
26
|
+
end
|
27
|
+
|
28
|
+
def uid_file_name
|
29
|
+
@uid_file_name ||= "#{uniq_filename_randomizer.call}#{file.extname}"
|
30
|
+
end
|
31
|
+
|
32
|
+
private
|
33
|
+
attr_reader :config
|
34
|
+
def_delegators :config, :uniq_filename_randomizer, :processing_dir
|
35
|
+
|
36
|
+
def file
|
37
|
+
Pathname.new(@file)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
module Bubbles
|
2
|
+
class CommandQueue
|
3
|
+
extend Forwardable
|
4
|
+
|
5
|
+
def_delegators :queue, :size
|
6
|
+
|
7
|
+
def initialize(config:)
|
8
|
+
@config = config
|
9
|
+
end
|
10
|
+
|
11
|
+
def queue
|
12
|
+
@queue ||= []
|
13
|
+
end
|
14
|
+
|
15
|
+
def <<(command_object)
|
16
|
+
queue << command_object
|
17
|
+
end
|
18
|
+
|
19
|
+
def call_next
|
20
|
+
if command = queue.shift
|
21
|
+
log command
|
22
|
+
command.call
|
23
|
+
else
|
24
|
+
log "Nothing in the command queue"
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def reschedule(command_object)
|
29
|
+
queue.unshift(command_object)
|
30
|
+
end
|
31
|
+
|
32
|
+
def inspect
|
33
|
+
"<##{self.class.name} queue:#{queue.inspect} >"
|
34
|
+
end
|
35
|
+
|
36
|
+
private
|
37
|
+
attr_reader :config
|
38
|
+
|
39
|
+
def log(command)
|
40
|
+
config.logger.debug("Processing: #{command.inspect}")
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
module Bubbles
|
2
|
+
module CommonUploaderInterface
|
3
|
+
def self.included(base)
|
4
|
+
base.send(:attr_reader, :config, :command_queue, :bfile)
|
5
|
+
end
|
6
|
+
|
7
|
+
def initialize(bfile:, command_queue:, config:)
|
8
|
+
@bfile = bfile
|
9
|
+
@config = config
|
10
|
+
@command_queue = command_queue
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
@@ -0,0 +1,136 @@
|
|
1
|
+
module Bubbles
|
2
|
+
class Config
|
3
|
+
def self.home_config
|
4
|
+
Pathname.new(Dir.home).join('.bubbles/config.yml').to_s
|
5
|
+
end
|
6
|
+
|
7
|
+
def self.var_config
|
8
|
+
'/var/lib/bubbles/config.yml'
|
9
|
+
end
|
10
|
+
|
11
|
+
attr_writer :config_path, :logger, :source_dir, :processing_dir, :log_path,
|
12
|
+
:log_level, :sleep_interval, :uploader_classes, :num_of_files_to_schedule,
|
13
|
+
:uniq_filename_randomizer, :local_dir_uploader_path, :s3_access_key_id, :s3_secret_access_key,
|
14
|
+
:s3_region, :s3_path, :s3_bucket, :use_default_config_locations
|
15
|
+
|
16
|
+
def log_path
|
17
|
+
@log_path || config_yml['log_path'] || STDOUT
|
18
|
+
end
|
19
|
+
|
20
|
+
def log_level
|
21
|
+
@log_level || config_yml['log_level'] || :debug
|
22
|
+
end
|
23
|
+
|
24
|
+
def logger
|
25
|
+
@logger ||= Logger.new(log_path).tap { |l| l.level = log_level }
|
26
|
+
end
|
27
|
+
|
28
|
+
def source_dir
|
29
|
+
@source_dir ||= config_yml.fetch('source_dir') { raise_config_required }
|
30
|
+
pathnamed(@source_dir)
|
31
|
+
end
|
32
|
+
|
33
|
+
def processing_dir
|
34
|
+
@processing_dir ||= config_yml.fetch('processing_dir') { raise_config_required }
|
35
|
+
pathnamed(@processing_dir)
|
36
|
+
end
|
37
|
+
|
38
|
+
def uploader_classes
|
39
|
+
return @uploader_classes if @uploader_classes
|
40
|
+
if uploaders = config_yml['uploaders']
|
41
|
+
uploaders.map { |u| Object.const_get(u) }
|
42
|
+
else
|
43
|
+
[Bubbles::Uploaders::S3]
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
def local_dir_uploader_path
|
48
|
+
err_msg = 'As you are using LocalDir uploader,' +
|
49
|
+
' you need to specify `local_dir_uploader_path` in your config' +
|
50
|
+
' so the uploader knows where to upload'
|
51
|
+
@local_dir_uploader_path ||= config_yml.fetch('local_dir_uploader_path') { raise err_msg }
|
52
|
+
pathnamed(@local_dir_uploader_path)
|
53
|
+
end
|
54
|
+
|
55
|
+
# number of seconds between every command execution in queue seconds, defaults to 1
|
56
|
+
def sleep_interval
|
57
|
+
@sleep_interval || 1
|
58
|
+
end
|
59
|
+
|
60
|
+
# how many files should DirWatcher schedule for upload, defaults to 1
|
61
|
+
def num_of_files_to_schedule
|
62
|
+
@num_of_files_to_schedule || 1
|
63
|
+
end
|
64
|
+
|
65
|
+
def config_path
|
66
|
+
if @config_path
|
67
|
+
raise "Config file #{@config_path} does not exist" unless File.exist?(@config_path)
|
68
|
+
@config_path
|
69
|
+
elsif File.exist?(self.class.var_config) && use_default_config_locations
|
70
|
+
self.class.var_config
|
71
|
+
elsif File.exist?(self.class.home_config) && use_default_config_locations
|
72
|
+
self.class.home_config
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
def uniq_filename_randomizer
|
77
|
+
@uniq_filename_randomizer ||= ->() { SecureRandom.uuid }
|
78
|
+
end
|
79
|
+
|
80
|
+
def s3_credentials
|
81
|
+
@aws_credentials ||= Aws::Credentials.new(s3_access_key_id, s3_secret_access_key)
|
82
|
+
end
|
83
|
+
|
84
|
+
def s3_region
|
85
|
+
@s3_region \
|
86
|
+
|| config_yml['s3_region'] \
|
87
|
+
|| raise('Please provide s3_region in your config file')
|
88
|
+
end
|
89
|
+
|
90
|
+
def s3_path
|
91
|
+
@s3_path \
|
92
|
+
|| config_yml['s3_path'] \
|
93
|
+
|| ''
|
94
|
+
end
|
95
|
+
|
96
|
+
def s3_bucket
|
97
|
+
@s3_bucket \
|
98
|
+
|| config_yml['s3_bucket'] \
|
99
|
+
|| raise('Please provide s3_bucket in your config file')
|
100
|
+
end
|
101
|
+
|
102
|
+
private
|
103
|
+
def use_default_config_locations
|
104
|
+
return @use_default_config_locations unless @use_default_config_locations.nil?
|
105
|
+
true
|
106
|
+
end
|
107
|
+
|
108
|
+
def s3_access_key_id
|
109
|
+
@s3_access_key_id \
|
110
|
+
|| config_yml['s3_access_key_id'] \
|
111
|
+
|| raise('Please provide s3_access_key_id in your config file')
|
112
|
+
end
|
113
|
+
|
114
|
+
def s3_secret_access_key
|
115
|
+
@s3_secret_access_key \
|
116
|
+
|| config_yml['s3_secret_access_key'] \
|
117
|
+
|| raise('Please provide s3_secret_access_key in your config file')
|
118
|
+
end
|
119
|
+
|
120
|
+
def config_yml
|
121
|
+
if config_path
|
122
|
+
@config_yml ||= YAML.load_file(config_path)
|
123
|
+
else
|
124
|
+
{}
|
125
|
+
end
|
126
|
+
end
|
127
|
+
|
128
|
+
def raise_config_required
|
129
|
+
raise "Please provide configuration file. You can do this by creating #{self.class.home_config} or check project github README.md"
|
130
|
+
end
|
131
|
+
|
132
|
+
def pathnamed(location_obj)
|
133
|
+
location_obj.respond_to?(:basename) ? location_obj : Pathname.new(location_obj)
|
134
|
+
end
|
135
|
+
end
|
136
|
+
end
|
@@ -0,0 +1,53 @@
|
|
1
|
+
module Bubbles
|
2
|
+
class DirWatcher
|
3
|
+
extend Forwardable
|
4
|
+
DestinationIsNotDirectory = Class.new(StandardError)
|
5
|
+
|
6
|
+
def initialize(config:, command_queue:)
|
7
|
+
@config = config
|
8
|
+
@command_queue = command_queue
|
9
|
+
end
|
10
|
+
|
11
|
+
def call
|
12
|
+
check_source_dir_existence
|
13
|
+
check_processing_dir_existence
|
14
|
+
|
15
|
+
source_dir_files
|
16
|
+
.last(num_of_files_to_schedule)
|
17
|
+
.each do |file|
|
18
|
+
bfile = BubbliciousFile.new(file: file, config: config)
|
19
|
+
bfile.move_to_processing_dir
|
20
|
+
|
21
|
+
uploader_classes.each do |uploader_class|
|
22
|
+
command_queue << uploader_class.new(bfile: bfile, command_queue: command_queue, config: config)
|
23
|
+
end
|
24
|
+
|
25
|
+
command_queue << bfile.public_method(:remove_file)
|
26
|
+
end
|
27
|
+
|
28
|
+
command_queue << self
|
29
|
+
end
|
30
|
+
|
31
|
+
def inspect
|
32
|
+
"#<#{self.class.name} source_dir: #{source_dir}, processing_dir: #{processing_dir}>"
|
33
|
+
end
|
34
|
+
|
35
|
+
def source_dir_files
|
36
|
+
Dir
|
37
|
+
.glob(source_dir.join('**/*').to_s)
|
38
|
+
.select { |x| Pathname.new(x).file? }
|
39
|
+
end
|
40
|
+
|
41
|
+
private
|
42
|
+
attr_reader :command_queue, :config
|
43
|
+
def_delegators :config, :source_dir, :processing_dir, :num_of_files_to_schedule, :uploader_classes
|
44
|
+
|
45
|
+
def check_source_dir_existence
|
46
|
+
raise DestinationIsNotDirectory unless source_dir.directory?
|
47
|
+
end
|
48
|
+
|
49
|
+
def check_processing_dir_existence
|
50
|
+
raise DestinationIsNotDirectory unless processing_dir.directory?
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
module Bubbles
|
2
|
+
module Uploaders
|
3
|
+
class LocalDir
|
4
|
+
extend Forwardable
|
5
|
+
include Bubbles::CommonUploaderInterface
|
6
|
+
|
7
|
+
def call
|
8
|
+
config.logger.debug("#{self.class.name}: transfering #{uid_file} to #{local_dir_uploader_path}")
|
9
|
+
FileUtils.cp(uid_file, local_dir_uploader_path)
|
10
|
+
rescue Errno::ENOENT => e
|
11
|
+
config.logger.error("#{e.message}")
|
12
|
+
command_queue.reschedule(self)
|
13
|
+
end
|
14
|
+
|
15
|
+
def inspect
|
16
|
+
"<##{self.class.name} uid_file: #{uid_file} to: #{local_dir_uploader_path}>"
|
17
|
+
end
|
18
|
+
|
19
|
+
private
|
20
|
+
def_delegators :config, :local_dir_uploader_path
|
21
|
+
def_delegators :bfile, :uid_file
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
module Bubbles
|
2
|
+
module Uploaders
|
3
|
+
class S3
|
4
|
+
extend Forwardable
|
5
|
+
include Bubbles::CommonUploaderInterface
|
6
|
+
|
7
|
+
def call
|
8
|
+
File.open(uid_file, 'rb') do |file|
|
9
|
+
s3.put_object(bucket: s3_bucket, key: s3_full_path, body: file)
|
10
|
+
end
|
11
|
+
rescue => e
|
12
|
+
config.logger.error("#{e.message}")
|
13
|
+
command_queue.reschedule(self)
|
14
|
+
end
|
15
|
+
|
16
|
+
def inspect
|
17
|
+
"<##{self.class.name} uid_file: #{uid_file} to: s3://#{s3_bucket}#{s3_full_path}>"
|
18
|
+
end
|
19
|
+
|
20
|
+
private
|
21
|
+
def_delegators :config, :s3_bucket, :s3_path, :s3_credentials, :s3_region
|
22
|
+
def_delegators :bfile, :uid_file, :uid_file_name
|
23
|
+
|
24
|
+
def s3
|
25
|
+
@s3 ||= Aws::S3::Client.new({
|
26
|
+
region: s3_region,
|
27
|
+
credentials: s3_credentials
|
28
|
+
})
|
29
|
+
end
|
30
|
+
|
31
|
+
def s3_full_path
|
32
|
+
Pathname.new(s3_path).join(uid_file_name).to_s
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
metadata
ADDED
@@ -0,0 +1,124 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: bubbles
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Tomas Valent
|
8
|
+
autorequire:
|
9
|
+
bindir: exe
|
10
|
+
cert_chain: []
|
11
|
+
date: 2017-04-18 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: aws-sdk
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '2'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '2'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: bundler
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '1.14'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '1.14'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rake
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '10.0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '10.0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: rspec
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '3.0'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '3.0'
|
69
|
+
description: Daemonized file uploader that watch a folder and uploads any files files
|
70
|
+
to AWS S3. Designed for Raspberry pi zero
|
71
|
+
email:
|
72
|
+
- equivalent@eq8.eu
|
73
|
+
executables:
|
74
|
+
- bubbles
|
75
|
+
extensions: []
|
76
|
+
extra_rdoc_files: []
|
77
|
+
files:
|
78
|
+
- ".gitignore"
|
79
|
+
- ".rspec"
|
80
|
+
- ".travis.yml"
|
81
|
+
- Gemfile
|
82
|
+
- LICENSE
|
83
|
+
- README.md
|
84
|
+
- Rakefile
|
85
|
+
- bin/console
|
86
|
+
- bin/setup
|
87
|
+
- bubbles.gemspec
|
88
|
+
- exe/bubbles
|
89
|
+
- lib/bubbles.rb
|
90
|
+
- lib/bubbles/bubblicious_file.rb
|
91
|
+
- lib/bubbles/command_queue.rb
|
92
|
+
- lib/bubbles/common_uploader_interface.rb
|
93
|
+
- lib/bubbles/config.rb
|
94
|
+
- lib/bubbles/dir_watcher.rb
|
95
|
+
- lib/bubbles/uploaders/local_dir.rb
|
96
|
+
- lib/bubbles/uploaders/s3.rb
|
97
|
+
- lib/bubbles/version.rb
|
98
|
+
- tmp/dummy_local_dir_uploader_dir/.gitkeep
|
99
|
+
- tmp/dummy_processing_dir/.gitkeep
|
100
|
+
homepage: https://github.com/equivalent/bubbles
|
101
|
+
licenses:
|
102
|
+
- MIT
|
103
|
+
metadata: {}
|
104
|
+
post_install_message:
|
105
|
+
rdoc_options: []
|
106
|
+
require_paths:
|
107
|
+
- lib
|
108
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
109
|
+
requirements:
|
110
|
+
- - ">="
|
111
|
+
- !ruby/object:Gem::Version
|
112
|
+
version: '0'
|
113
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - ">="
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0'
|
118
|
+
requirements: []
|
119
|
+
rubyforge_project:
|
120
|
+
rubygems_version: 2.5.1
|
121
|
+
signing_key:
|
122
|
+
specification_version: 4
|
123
|
+
summary: Lightweight daemon file uploader to cloud
|
124
|
+
test_files: []
|