multi_sync 0.0.2 → 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +6 -8
- data/Gemfile +0 -1
- data/README.md +79 -51
- data/Rakefile +0 -22
- data/lib/multi_sync.rb +27 -5
- data/lib/multi_sync/attributes/pathname.rb +1 -1
- data/lib/multi_sync/client.rb +124 -112
- data/lib/multi_sync/configuration.rb +4 -1
- data/lib/multi_sync/extensions/jekyll.rb +24 -0
- data/lib/multi_sync/extensions/middleman.rb +1 -7
- data/lib/multi_sync/extensions/rails.rb +0 -2
- data/lib/multi_sync/{mixins/pluralize_helper.rb → helpers/pluralize.rb} +2 -2
- data/lib/multi_sync/logging.rb +7 -0
- data/lib/multi_sync/resource.rb +0 -2
- data/lib/multi_sync/resources/local_resource.rb +0 -1
- data/lib/multi_sync/source.rb +0 -2
- data/lib/multi_sync/sources/local_source.rb +3 -1
- data/lib/multi_sync/sources/manifest_source.rb +14 -16
- data/lib/multi_sync/target.rb +3 -3
- data/lib/multi_sync/targets/aws_target.rb +6 -10
- data/lib/multi_sync/targets/local_target.rb +7 -15
- data/lib/multi_sync/version.rb +1 -1
- data/lib/tasks/multi_sync_rails.rake +4 -4
- data/multi_sync.gemspec +3 -2
- data/spec/support/fog.rb +0 -1
- data/spec/support/timecop.rb +7 -1
- data/spec/support/tmpdir.rb +18 -0
- data/spec/unit/multi_sync/client_spec.rb +51 -74
- data/spec/unit/multi_sync/configuration_spec.rb +54 -83
- data/spec/unit/multi_sync/resources/local_resource_spec.rb +9 -7
- data/spec/unit/multi_sync/sources/local_source_spec.rb +14 -14
- data/spec/unit/multi_sync/sources/manifest_source_spec.rb +16 -18
- data/spec/unit/multi_sync/targets/aws_target_spec.rb +8 -8
- data/spec/unit/multi_sync/targets/local_target_spec.rb +9 -9
- data/spec/unit/multi_sync_spec.rb +44 -31
- metadata +102 -91
- data/gemfiles/middleman-3.1.x.gemfile +0 -5
- data/gemfiles/rails-3.2.x.gemfile +0 -5
- data/gemfiles/rails-4.0.x.gemfile +0 -5
- data/lib/multi_sync/mixins/log_helper.rb +0 -9
- data/spec/support/fakefs.rb +0 -7
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e9f1a53316b76079921573131ddede125112c3c4
|
4
|
+
data.tar.gz: 36193eacd42d2c64268688cb0a2068c5071f306c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: aadc84bd39bd8fdd4dcddd2c2ee70d3493f49db344cd29df962bb088b5759f76f5d5dd97039f49c858e3cfe02db3078080cc984819146643c6797d17e8087e86
|
7
|
+
data.tar.gz: b6ab4df2946516aea524b84a8d506e272646faa4180a74f1f8eabcd5701c238ad234d964c0df02c95a9ceb90e00ddbe949feeebc88d2df93387a9a40a09f869f
|
data/.travis.yml
CHANGED
@@ -3,22 +3,20 @@ cache: bundler
|
|
3
3
|
bundler_args: --without development
|
4
4
|
rvm:
|
5
5
|
- ruby-head
|
6
|
+
- ruby
|
6
7
|
- jruby-head
|
8
|
+
- jruby
|
7
9
|
- 2.1.0
|
8
10
|
- 2.0.0
|
9
11
|
- 1.9.3
|
10
|
-
- rbx-
|
11
|
-
- jruby-19mode
|
12
|
-
# gemfile:
|
13
|
-
# - gemfiles/Gemfile.middleman-3.1.x
|
14
|
-
# - gemfiles/Gemfile.rails-3.2.x
|
15
|
-
# - Gemfile
|
12
|
+
- rbx-2
|
16
13
|
matrix:
|
17
14
|
fast_finish: true
|
18
15
|
allow_failures:
|
19
16
|
- rvm: ruby-head
|
17
|
+
- rvm: ruby
|
20
18
|
- rvm: jruby-head
|
21
|
-
- rvm:
|
22
|
-
- rvm:
|
19
|
+
- rvm: jruby
|
20
|
+
- rvm: rbx-2
|
23
21
|
notifications:
|
24
22
|
email: false
|
data/Gemfile
CHANGED
data/README.md
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
|
3
3
|
:heavy_exclamation_mark: **currently a functioning WIP thats not quite finished yet but its close!** :heavy_exclamation_mark:
|
4
4
|
|
5
|
-
|
5
|
+
Flexible synchronisation for your assets.
|
6
6
|
|
7
7
|
`MultiSync` stands on the shoulders of giants. On one side is [Celluloid](http://celluloid.io) allowing for the synchronisation of assets to be highly parallel. On the other is [Fog::Storage](https://github.com/fog/fog) allowing `MulitSync` to support [various well known storage services](#storage-services).
|
8
8
|
|
@@ -15,7 +15,7 @@ Listed below are examples of how to get setup and started.
|
|
15
15
|
## Installation
|
16
16
|
|
17
17
|
```ruby
|
18
|
-
gem 'multi_sync', '~> 0.0.
|
18
|
+
gem 'multi_sync', '~> 0.0.2'
|
19
19
|
```
|
20
20
|
|
21
21
|
```ruby
|
@@ -43,104 +43,132 @@ end
|
|
43
43
|
|
44
44
|
### Source
|
45
45
|
|
46
|
-
|
46
|
+
All `source`s takes one argument which is a `Hash` of configuration detailed below. There are currently two type's of `source`s which are
|
47
|
+
|
48
|
+
#### Source Types
|
49
|
+
|
50
|
+
- `local_source` - Uses all files within the `source_dir`
|
51
|
+
- `manifest_source` - Tries to find a `Sprocket`s `manifest.{yml,json}` file within the `source_dir`
|
47
52
|
|
48
53
|
| Key | Type | Default | Description |
|
49
54
|
| :-- | :--- | :------ | :---------- |
|
50
|
-
| `
|
51
|
-
| `
|
52
|
-
| `
|
53
|
-
| `targets` | `Symbol`, `Array` | All targets | The target(s) this source should sync against |
|
55
|
+
| `source_dir` | `Pathname`, `String` | `nil` | The location this `source` should use |
|
56
|
+
| `resource_options` | `Hash` | `{}` | A hash of options for this `source`'s resources |
|
57
|
+
| `targets` | `Symbol`, `Array` | All targets | The `target`(s) this `source` should sync against |
|
54
58
|
| `include` | `String` ([shell glob](http://www.ruby-doc.org/core-2.1.1/Dir.html#method-c-glob)) | `**/*` | A shell globe to use for inclusion |
|
55
59
|
| `exclude` | `String` ([shell glob](http://www.ruby-doc.org/core-2.1.1/Dir.html#method-c-glob)) | `nil` | A shell globe to use for exclusion |
|
56
60
|
___
|
57
61
|
|
58
62
|
```ruby
|
59
|
-
# A source
|
60
|
-
|
61
|
-
type: :local,
|
63
|
+
# A `local` `source` which will use all files within '../build'
|
64
|
+
local_source({
|
62
65
|
source_dir: '../build'
|
63
|
-
}
|
66
|
+
})
|
64
67
|
```
|
65
68
|
___
|
66
69
|
|
67
70
|
```ruby
|
68
|
-
# A source
|
69
|
-
|
70
|
-
type: :manifest,
|
71
|
+
# A `manifest` `source` which will use a Sprockets ':manifest' within '../public/assets'
|
72
|
+
manifest_source({
|
71
73
|
source_dir: '../public/assets'
|
72
|
-
}
|
74
|
+
})
|
73
75
|
```
|
74
76
|
___
|
75
77
|
|
76
78
|
```ruby
|
77
|
-
# A source
|
78
|
-
# within '../build' including only 'mp4, mpg, mov'
|
79
|
-
|
80
|
-
type: :local,
|
79
|
+
# A `local` `source` which will use all files
|
80
|
+
# within '../build' including only 'mp4, mpg, mov' files
|
81
|
+
local_source({
|
81
82
|
source_dir: '../build',
|
82
83
|
include: '*.{mp4,mpg,mov}'
|
83
|
-
}
|
84
|
+
})
|
84
85
|
```
|
85
86
|
___
|
86
87
|
|
87
88
|
```ruby
|
88
|
-
# A source
|
89
|
-
# within '../build' excluding any 'jpg, gif, png'
|
90
|
-
|
91
|
-
type: :local,
|
89
|
+
# A `local` `source` which will use all files
|
90
|
+
# within '../build' excluding any 'jpg, gif, png' files
|
91
|
+
local_source({
|
92
92
|
source_dir: '../build',
|
93
93
|
exclude: '*.{jpg,gif,png}'
|
94
|
-
}
|
94
|
+
})
|
95
95
|
```
|
96
96
|
___
|
97
97
|
|
98
98
|
```ruby
|
99
|
-
# A source
|
100
|
-
# within '../public/assets' excluding any 'jpg, gif, png' files
|
101
|
-
# and only synchronising with a target named `:www`
|
102
|
-
source :www, {
|
103
|
-
type: :manifest,
|
104
|
-
source_dir: '../public/assets',
|
105
|
-
exclude: '*.{jpg,gif,png}',
|
106
|
-
targets: :www
|
107
|
-
}
|
108
|
-
```
|
109
|
-
___
|
110
|
-
|
111
|
-
```ruby
|
112
|
-
# A source named ':image_assets' which will use a Sprockets ':manifest'
|
99
|
+
# A `manifest` `source` which will use use a Sprockets `manifest`
|
113
100
|
# within '../public/assets' including only 'jpg, gif, png' files
|
114
|
-
# which sets `cache_control` and `expires` headers
|
115
|
-
|
116
|
-
source :image_assets, {
|
117
|
-
type: :manifest,
|
101
|
+
# which sets `cache_control` and `expires` headers
|
102
|
+
manifest_source({
|
118
103
|
source_dir: '../public/assets',
|
119
104
|
include: '*.{jpg,gif,png}',
|
120
105
|
resource_options: {
|
121
106
|
cache_control: 'public, max-age=31557600',
|
122
107
|
expires: CGI.rfc1123_date(Time.now + 31557600)
|
123
|
-
}
|
124
|
-
|
125
|
-
}
|
108
|
+
}
|
109
|
+
})
|
126
110
|
```
|
127
111
|
|
128
112
|
### Target
|
129
113
|
|
114
|
+
All `target`s takes one argument which is a `Hash` of configuration detailed below. There is currently only one `target` type which is:
|
115
|
+
|
116
|
+
#### Target Types
|
117
|
+
|
118
|
+
- `aws_target` - Synchronises to `aws` (`S3`)
|
119
|
+
|
120
|
+
| Key | Type | Default | Description |
|
121
|
+
| :-- | :--- | :------ | :---------- |
|
122
|
+
| `target_dir` | `Pathname`, `String` | `nil` | the name of the `target`'s directory (eg s3 bucket name) |
|
123
|
+
| `destination_dir` | `Pathname`, `String` | `nil` | the name of the `target` destination's directory (eg folder within target) |
|
124
|
+
| `credentials` | `Hash` | inherits [Fog Credentials](https://github.com/karlfreeman/multi_sync#fog-credentials-support) | credentionals needed by [Fog](http://fog.io) |
|
125
|
+
___
|
126
|
+
|
127
|
+
```ruby
|
128
|
+
# An `aws` `target` which will sync to the root of a bucket named 's3-bucket-name'
|
129
|
+
# in region 'us-east-1', with access_key_id 'xxx', and secret_access_key 'xxx'
|
130
|
+
aws_target({
|
131
|
+
target_dir: 's3-bucket-name'
|
132
|
+
credentials: {
|
133
|
+
region: 'us-east-1',
|
134
|
+
aws_access_key_id: 'xxx',
|
135
|
+
aws_secret_access_key: 'xxx'
|
136
|
+
}
|
137
|
+
})
|
138
|
+
```
|
139
|
+
___
|
140
|
+
|
130
141
|
```ruby
|
131
|
-
|
142
|
+
# An `aws` `target` which will sync to a bucket named 's3-bucket-name'
|
143
|
+
# using credentials sourced from Fog's credentials (.fog or FOG_RC)
|
144
|
+
aws_target({
|
145
|
+
target_dir: 's3-bucket-name'
|
146
|
+
})
|
132
147
|
```
|
148
|
+
___
|
133
149
|
|
150
|
+
```ruby
|
151
|
+
# An `aws` `target` which will sync to a bucket named 's3-bucket-name'
|
152
|
+
# within a directory named 'directory-within-s3'
|
153
|
+
# in region 'us-east-1', with access_key_id 'xxx', and secret_access_key 'xxx'
|
154
|
+
aws_target({
|
155
|
+
target_dir: 's3-bucket-name'
|
156
|
+
destination_dir: 'directory-within-s3'
|
157
|
+
credentials: {
|
158
|
+
region: 'us-east-1',
|
159
|
+
aws_access_key_id: 'xxx',
|
160
|
+
aws_secret_access_key: 'xxx'
|
161
|
+
}
|
162
|
+
})
|
163
|
+
```
|
134
164
|
|
135
165
|
## Supported Libraries
|
136
166
|
|
137
|
-
- [Rails](https://github.com/karlfreeman/multi_sync
|
167
|
+
- [Rails](https://github.com/karlfreeman/multi_sync-rails)
|
138
168
|
- Sinatra (WIP)
|
139
|
-
- Middleman
|
169
|
+
- [Middleman](https://github.com/karlfreeman/multi_sync-middleman)
|
140
170
|
- Jekyll (WIP)
|
141
171
|
- Nanoc (WIP)
|
142
|
-
- [POR](https://github.com/karlfreeman/multi_sync/wiki/por)
|
143
|
-
- Rake (WIP)
|
144
172
|
|
145
173
|
## Badges
|
146
174
|
|
data/Rakefile
CHANGED
@@ -19,27 +19,5 @@ begin
|
|
19
19
|
rescue LoadError
|
20
20
|
end
|
21
21
|
|
22
|
-
namespace :spec do
|
23
|
-
|
24
|
-
desc 'Run specs with middleman'
|
25
|
-
task :middleman do
|
26
|
-
ENV['BUNDLE_GEMFILE'] = 'gemfiles/middleman-3.1.x.gemfile'
|
27
|
-
Rake::Task['spec'].execute
|
28
|
-
end
|
29
|
-
|
30
|
-
desc 'Run specs with rails 3.2'
|
31
|
-
task :rails_3_2 do
|
32
|
-
ENV['BUNDLE_GEMFILE'] = 'gemfiles/rails-3.2.x.gemfile'
|
33
|
-
Rake::Task['spec'].execute
|
34
|
-
end
|
35
|
-
|
36
|
-
desc 'Run specs with rails 4.0'
|
37
|
-
task :rails_4_0 do
|
38
|
-
ENV['BUNDLE_GEMFILE'] = 'gemfiles/rails-4.0.x.gemfile'
|
39
|
-
Rake::Task['spec'].execute
|
40
|
-
end
|
41
|
-
|
42
|
-
end
|
43
|
-
|
44
22
|
task default: :spec
|
45
23
|
task test: :spec
|
data/lib/multi_sync.rb
CHANGED
@@ -12,8 +12,9 @@ module MultiSync
|
|
12
12
|
|
13
13
|
# a list of libraries, extension file and class name
|
14
14
|
REQUIREMENT_MAP = [
|
15
|
-
['rails', 'multi_sync/extensions/rails']
|
16
|
-
['middleman-core', 'multi_sync/extensions/middleman']
|
15
|
+
['rails', 'multi_sync/extensions/rails'],
|
16
|
+
['middleman-core', 'multi_sync/extensions/middleman'],
|
17
|
+
['jekyll', 'multi_sync/extensions/jekyll']
|
17
18
|
]
|
18
19
|
|
19
20
|
# delegate all MultiSync::Configuration's attribute accessors to the configuration
|
@@ -25,8 +26,22 @@ module MultiSync
|
|
25
26
|
# delegate all MultiSync::Client's attribute accessors to the configuration
|
26
27
|
def_delegators :client, *MultiSync::Client.attribute_set.map(&:name)
|
27
28
|
|
28
|
-
|
29
|
-
|
29
|
+
# include sync method
|
30
|
+
def_delegator :client, :sync
|
31
|
+
|
32
|
+
# create methods for each source (local_source(options), manifest_source(options))
|
33
|
+
MultiSync::Client::SUPPORTED_SOURCE_TYPES.each do |type, clazz|
|
34
|
+
define_singleton_method "#{type}_source" do |options = {}|
|
35
|
+
client.add_source(clazz, options)
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
# create methods for each target (aws_target(options))
|
40
|
+
MultiSync::Client::SUPPORTED_TARGET_TYPES.each do |type, clazz|
|
41
|
+
define_singleton_method "#{type}_target" do |options = {}|
|
42
|
+
client.add_target(clazz, options)
|
43
|
+
end
|
44
|
+
end
|
30
45
|
|
31
46
|
# Configure
|
32
47
|
#
|
@@ -40,7 +55,7 @@ module MultiSync
|
|
40
55
|
#
|
41
56
|
# @return [MultiSync]
|
42
57
|
def self.run(&block)
|
43
|
-
configure(&block).
|
58
|
+
configure(&block).sync
|
44
59
|
end
|
45
60
|
|
46
61
|
# Prepare
|
@@ -64,6 +79,13 @@ module MultiSync
|
|
64
79
|
@configuration ||= MultiSync::Configuration.new(options)
|
65
80
|
end
|
66
81
|
|
82
|
+
# Return the MultiSync::VERSION
|
83
|
+
#
|
84
|
+
# @return [String]
|
85
|
+
def self.version
|
86
|
+
MultiSync::VERSION
|
87
|
+
end
|
88
|
+
|
67
89
|
# Reset the MultiSync::Client
|
68
90
|
def self.reset_client!
|
69
91
|
remove_instance_variable :@client if defined?(@client)
|
data/lib/multi_sync/client.rb
CHANGED
@@ -1,27 +1,31 @@
|
|
1
|
-
require 'set'
|
2
1
|
require 'virtus'
|
3
2
|
require 'lazily'
|
4
3
|
require 'celluloid'
|
5
|
-
%w(sources targets).each do |dir|
|
4
|
+
%w(sources targets helpers).each do |dir|
|
6
5
|
Dir.glob(File.expand_path("../#{dir}/**/*.rb", __FILE__), &method(:require))
|
7
6
|
end
|
8
|
-
require 'multi_sync/mixins/pluralize_helper'
|
9
7
|
|
10
8
|
module MultiSync
|
11
9
|
class Client
|
12
10
|
include Virtus.model
|
13
|
-
include MultiSync::
|
11
|
+
include MultiSync::Helpers::Pluralize
|
14
12
|
|
15
13
|
attribute :supervisor, Celluloid::SupervisionGroup
|
16
|
-
attribute :
|
17
|
-
attribute :
|
18
|
-
attribute :
|
14
|
+
attribute :running_upload_jobs, Array, default: []
|
15
|
+
attribute :running_delete_jobs, Array, default: []
|
16
|
+
attribute :complete_upload_jobs, Array, default: []
|
17
|
+
attribute :complete_delete_jobs, Array, default: []
|
18
|
+
attribute :incomplete_upload_jobs, Array, default: []
|
19
|
+
attribute :incomplete_delete_jobs, Array, default: []
|
19
20
|
attribute :sources, Array, default: []
|
20
21
|
attribute :sync_attempts, Integer, default: 0
|
21
22
|
attribute :file_sync_attempts, Integer, default: 0
|
22
23
|
attribute :started_at, Time, required: false
|
23
24
|
attribute :finished_at, Time, required: false
|
24
25
|
|
26
|
+
SUPPORTED_SOURCE_TYPES = [[:local, MultiSync::LocalSource], [:manifest, MultiSync::ManifestSource]]
|
27
|
+
SUPPORTED_TARGET_TYPES = [[:local, MultiSync::LocalTarget], [:aws, MultiSync::AwsTarget]]
|
28
|
+
|
25
29
|
# Initialize a new Client object
|
26
30
|
#
|
27
31
|
# @param options [Hash]
|
@@ -30,94 +34,63 @@ module MultiSync
|
|
30
34
|
super
|
31
35
|
end
|
32
36
|
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
37
|
+
#
|
38
|
+
#
|
39
|
+
#
|
40
|
+
def add_target(clazz, options = {})
|
41
|
+
# TODO: friendly pool names?
|
42
|
+
pool_name = Celluloid.uuid
|
43
|
+
supervisor.pool(clazz, as: pool_name, args: [options], size: MultiSync.target_pool_size)
|
44
|
+
pool_name
|
40
45
|
end
|
41
|
-
alias_method :target, :add_target
|
42
46
|
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
47
|
+
#
|
48
|
+
#
|
49
|
+
#
|
50
|
+
def add_source(clazz, options = {})
|
51
|
+
source = clazz.new(options)
|
52
|
+
sources << source
|
53
|
+
source
|
49
54
|
end
|
50
|
-
alias_method :source, :add_source
|
51
|
-
|
52
|
-
def synchronize
|
53
|
-
MultiSync.debug 'Preventing synchronization as there are no sources found.' && return if sync_pointless?
|
54
|
-
MultiSync.debug 'Starting synchronization...'
|
55
55
|
|
56
|
-
|
57
|
-
|
56
|
+
#
|
57
|
+
#
|
58
|
+
#
|
59
|
+
def sync
|
60
|
+
MultiSync.warn 'Preventing synchronization as there are no sources found.' && return if sync_pointless?
|
58
61
|
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
+
if first_run?
|
63
|
+
MultiSync.debug 'Starting synchronization...'
|
64
|
+
determine_sync
|
65
|
+
else
|
66
|
+
MultiSync.debug 'Restarting synchronization...'
|
62
67
|
end
|
63
68
|
|
64
|
-
|
65
|
-
|
69
|
+
sync_attempted
|
70
|
+
|
71
|
+
MultiSync.debug 'Fetching upload jobs from the future...'
|
72
|
+
(running_upload_jobs | incomplete_upload_jobs).lazily.each do | job |
|
66
73
|
begin
|
67
|
-
|
74
|
+
complete_upload_jobs << job.value
|
68
75
|
rescue => error
|
69
76
|
self.file_sync_attempts = file_sync_attempts + 1
|
70
77
|
MultiSync.warn error.inspect
|
71
|
-
|
72
|
-
else
|
73
|
-
complete_jobs << completed_job
|
74
|
-
true
|
78
|
+
incomplete_upload_jobs << job
|
75
79
|
end
|
76
80
|
end
|
77
81
|
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
minutes, seconds = elapsed.divmod 60.0
|
87
|
-
bytes = get_total_file_size_from_complete_jobs
|
88
|
-
MultiSync.debug "Sync completed in #{pluralize(minutes.round, 'minute')} and #{pluralize(seconds.round, 'second')}"
|
89
|
-
MultiSync.debug "#{pluralize(complete_jobs.length, 'file')} were synchronised (#{pluralize(get_complete_deleted_jobs.length, 'deleted file')} and #{pluralize(get_complete_upload_jobs.length, 'uploaded file')}) from #{pluralize(sources.length, 'source')} to #{pluralize(supervisor.actors.length, 'target')}"
|
90
|
-
if bytes > 1024.0
|
91
|
-
kilobytes = bytes / 1024.0
|
92
|
-
MultiSync.debug "The upload weight totalled %.#{0}f #{pluralize(kilobytes, 'kilobyte', nil, false)}" % kilobytes
|
93
|
-
else
|
94
|
-
MultiSync.debug "The upload weight totalled %.#{0}f #{pluralize(bytes, 'byte', nil, false)}" % bytes
|
82
|
+
MultiSync.debug 'Fetching delete jobs from the future...'
|
83
|
+
(running_delete_jobs | incomplete_delete_jobs).lazily.each do | job |
|
84
|
+
begin
|
85
|
+
complete_delete_jobs << job.value
|
86
|
+
rescue => error
|
87
|
+
self.file_sync_attempts = file_sync_attempts + 1
|
88
|
+
MultiSync.warn error.inspect
|
89
|
+
incomplete_delete_jobs << job
|
95
90
|
end
|
96
|
-
MultiSync.debug "#{pluralize(file_sync_attempts, 'failed request')} were detected and re-tried"
|
97
|
-
else
|
98
|
-
MultiSync.debug "Sync failed to complete with #{pluralize(incomplete_jobs.length, 'outstanding file')} to be synchronised"
|
99
|
-
MultiSync.debug "#{pluralize(complete_jobs.length, 'file')} were synchronised (#{pluralize(get_complete_deleted_jobs.length, 'deleted file')} and #{pluralize(get_complete_upload_jobs.length, 'uploaded file')}) from #{pluralize(sources.length, 'source')} to #{pluralize(supervisor.actors.length, 'target')}"
|
100
91
|
end
|
101
92
|
|
102
|
-
|
103
|
-
end
|
104
|
-
alias_method :fin, :finalize
|
105
|
-
|
106
|
-
def get_complete_deleted_jobs
|
107
|
-
complete_jobs.select { |job| job[:method] == :delete }
|
108
|
-
end
|
109
|
-
|
110
|
-
def get_complete_upload_jobs
|
111
|
-
complete_jobs.select { |job| job[:method] == :upload }
|
112
|
-
end
|
113
|
-
|
114
|
-
def get_total_file_size_from_complete_jobs
|
115
|
-
total_file_size = 0
|
116
|
-
get_complete_upload_jobs.each do | job |
|
117
|
-
job_content_length = job[:response].content_length || job[:response].determine_content_length || 0
|
118
|
-
total_file_size += job_content_length
|
119
|
-
end
|
120
|
-
total_file_size
|
93
|
+
finish_sync
|
121
94
|
end
|
122
95
|
|
123
96
|
private
|
@@ -129,10 +102,9 @@ module MultiSync
|
|
129
102
|
|
130
103
|
starting_synchronizing_msg = "ynchronizing: '#{source.source_dir}'"
|
131
104
|
starting_synchronizing_msg.prepend MultiSync.force ? 'Forcefully s' : 'S'
|
132
|
-
MultiSync.
|
105
|
+
MultiSync.debug starting_synchronizing_msg
|
133
106
|
|
134
107
|
source_files = source.files
|
135
|
-
source_files.sort! # sort to make sure the source's indexes match the targets
|
136
108
|
|
137
109
|
# when no targets are specified, assume all targets
|
138
110
|
source.targets = supervisor_actor_names if source.targets.empty?
|
@@ -147,7 +119,7 @@ module MultiSync
|
|
147
119
|
|
148
120
|
MultiSync.debug 'Fetching files from the target...'
|
149
121
|
|
150
|
-
target_files =
|
122
|
+
target_files = supervisor[target_id].files
|
151
123
|
target_files.sort! # sort to make sure the target's indexs match the sources
|
152
124
|
|
153
125
|
MultiSync.debug "#{pluralize(target_files.length, 'file')} found from the target"
|
@@ -162,28 +134,30 @@ module MultiSync
|
|
162
134
|
abandoned_files_msg += ", however we're skipping them as :delete_abandoned_files is false" unless MultiSync.delete_abandoned_files
|
163
135
|
MultiSync.debug abandoned_files_msg
|
164
136
|
|
165
|
-
# remove missing_files from source_files (as we know they
|
166
|
-
# remove abandoned_files from target_files (as we know they
|
137
|
+
# remove missing_files from source_files (as we know they're missing so don't need to check for them)
|
138
|
+
# remove abandoned_files from target_files (as we know they're abandoned so don't need to check for them)
|
167
139
|
outdated_files.concat determine_outdated_files(source_files - missing_files, target_files - abandoned_files)
|
168
140
|
MultiSync.debug "#{outdated_files.length} of the files are outdated"
|
169
141
|
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
142
|
+
MultiSync.debug 'Scheduling jobs in the future...'
|
143
|
+
|
144
|
+
# outdated files
|
145
|
+
outdated_files.lazily.each do | resource |
|
146
|
+
running_upload_jobs << supervisor[target_id].future.upload(resource)
|
175
147
|
end
|
176
148
|
|
177
149
|
# missing files
|
178
150
|
if MultiSync.upload_missing_files
|
179
|
-
missing_files.lazily.each do |
|
180
|
-
|
151
|
+
missing_files.lazily.each do | resource |
|
152
|
+
running_upload_jobs << supervisor[target_id].future.upload(resource)
|
181
153
|
end
|
182
154
|
end
|
183
155
|
|
184
|
-
#
|
185
|
-
|
186
|
-
|
156
|
+
# abandoned files
|
157
|
+
if MultiSync.delete_abandoned_files
|
158
|
+
abandoned_files.lazily.each do | resource |
|
159
|
+
running_delete_jobs << supervisor[target_id].future.delete(resource)
|
160
|
+
end
|
187
161
|
end
|
188
162
|
|
189
163
|
end
|
@@ -191,39 +165,77 @@ module MultiSync
|
|
191
165
|
end
|
192
166
|
end
|
193
167
|
|
168
|
+
def sync_attempted
|
169
|
+
self.started_at = Time.now if first_run?
|
170
|
+
self.sync_attempts = sync_attempts.next
|
171
|
+
if sync_attempts > MultiSync.max_sync_attempts
|
172
|
+
MultiSync.warn "Sync was attempted more then #{MultiSync.max_sync_attempts} times"
|
173
|
+
fail ArgumentError, "Sync was attempted more then #{MultiSync.max_sync_attempts} times"
|
174
|
+
end
|
175
|
+
end
|
176
|
+
|
177
|
+
def finish_sync
|
178
|
+
# recurse when there are incomplete_jobs still
|
179
|
+
incomplete_jobs.length != 0 ? sync : self.finished_at = Time.now
|
180
|
+
|
181
|
+
if finished_at
|
182
|
+
elapsed = finished_at.to_f - started_at.to_f
|
183
|
+
minutes, seconds = elapsed.divmod 60.0
|
184
|
+
bytes = complete_upload_jobs_bytes
|
185
|
+
kilobytes = bytes / 1024.0
|
186
|
+
MultiSync.debug "Sync completed in #{pluralize(minutes.round, 'minute')} and #{pluralize(seconds.round, 'second')}"
|
187
|
+
MultiSync.debug 'The combined upload weight was ' + ((bytes > 1024.0) ? pluralize(kilobytes.round, 'kilobyte') : pluralize(bytes.round, 'byte'))
|
188
|
+
MultiSync.debug "#{pluralize(file_sync_attempts, 'failed request')} were detected and re-tried"
|
189
|
+
else
|
190
|
+
MultiSync.debug "Sync failed to complete with #{pluralize(incomplete_jobs.length, 'outstanding file')} to be synchronised"
|
191
|
+
end
|
192
|
+
MultiSync.debug "#{pluralize(complete_jobs.length, 'file')} were synchronised (#{pluralize(complete_delete_jobs.length, 'deleted file')} and #{pluralize(complete_upload_jobs.length, 'uploaded file')}) from #{pluralize(sources.length, 'source')} to #{pluralize(supervisor_actor_names.length, 'target')}"
|
193
|
+
|
194
|
+
supervisor.terminate
|
195
|
+
end
|
196
|
+
|
197
|
+
def complete_jobs
|
198
|
+
complete_upload_jobs | complete_delete_jobs
|
199
|
+
end
|
200
|
+
|
201
|
+
def incomplete_jobs
|
202
|
+
incomplete_upload_jobs | incomplete_delete_jobs
|
203
|
+
end
|
204
|
+
|
205
|
+
def complete_upload_jobs_bytes
|
206
|
+
total_bytes = 0
|
207
|
+
complete_upload_jobs.each do | job |
|
208
|
+
total_bytes += job.content_length || job.determine_content_length || 0
|
209
|
+
end
|
210
|
+
total_bytes
|
211
|
+
end
|
212
|
+
|
194
213
|
def determine_missing_files(source_files, target_files)
|
195
|
-
|
196
|
-
missing_files
|
214
|
+
source_files - target_files
|
197
215
|
end
|
198
216
|
|
199
217
|
def determine_abandoned_files(source_files, target_files)
|
200
|
-
|
201
|
-
abandoned_files
|
218
|
+
target_files - source_files
|
202
219
|
end
|
203
220
|
|
204
221
|
def determine_outdated_files(source_files, target_files)
|
205
222
|
outdated_files = []
|
223
|
+
equivalent_files = []
|
206
224
|
|
207
225
|
# TODO: replace with celluloid pool of futures
|
208
226
|
# check each source file against the matching target_file's etag
|
209
227
|
source_files.lazily.each_with_index do |file, i|
|
210
|
-
|
228
|
+
if !file.matching_etag?(target_files[i]) || MultiSync.force
|
229
|
+
outdated_files << file
|
230
|
+
else
|
231
|
+
equivalent_files << file
|
232
|
+
end
|
211
233
|
end
|
212
234
|
|
213
|
-
|
214
|
-
|
235
|
+
# TODO: move to a better place
|
236
|
+
MultiSync.debug "#{equivalent_files.length} of the files are identical"
|
215
237
|
|
216
|
-
|
217
|
-
self.started_at = Time.now if first_run?
|
218
|
-
self.sync_attempts = sync_attempts.next
|
219
|
-
if sync_attempts > MultiSync.max_sync_attempts
|
220
|
-
MultiSync.warn "Sync was attempted more then #{MultiSync.max_sync_attempts} times"
|
221
|
-
fail ArgumentError, "Sync was attempted more then #{MultiSync.max_sync_attempts} times"
|
222
|
-
end
|
223
|
-
end
|
224
|
-
|
225
|
-
def finish_sync
|
226
|
-
incomplete_jobs.length != 0 ? synchronize : self.finished_at = Time.now
|
238
|
+
outdated_files
|
227
239
|
end
|
228
240
|
|
229
241
|
def first_run?
|
@@ -235,7 +247,7 @@ module MultiSync
|
|
235
247
|
end
|
236
248
|
|
237
249
|
def supervisor_actor_names
|
238
|
-
supervisor.actors.map
|
250
|
+
supervisor.actors.map(&:registered_name)
|
239
251
|
end
|
240
252
|
end
|
241
253
|
end
|