datapimp 1.2.7 → 1.2.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/datapimp/cli/create.rb +2 -1
- data/lib/datapimp/sync/dropbox_folder.rb +48 -12
- data/lib/datapimp/sync/s3_bucket.rb +16 -3
- metadata +2 -3
- data/lib/datapimp/sync/dropbox_delta.rb +0 -67
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b303a51b6bbca4a93af57cfaa463562e17fa8758
|
4
|
+
data.tar.gz: 640333b946ff814751d2077fbaea43e5b1af0c10
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8d830340d599872cfd2f0efd11bd9f8a8fcb9d45a6d5918a5f45cb27959b2e45ca90366f67dabb758e5fe94477cdca87f868ba1736517127efb7cf6bbe7de7b4
|
7
|
+
data.tar.gz: b480ce5379e3474a7c2f6175762b10c7e065f879b05e5ce6353b635a9a3684da908adc4edde70939f9bdbaadd5fe36f489a0545760ddbcd4951228ffd09cc91a
|
data/lib/datapimp/cli/create.rb
CHANGED
@@ -82,10 +82,11 @@ command 'create s3 bucket' do |c|
|
|
82
82
|
|
83
83
|
c.option '--setup-website', 'Setup the bucket for website hosting'
|
84
84
|
c.option '--create-redirect-bucket', 'Setup a redirect bucket'
|
85
|
+
c.option '--private', nil, 'Make this bucket private.'
|
85
86
|
|
86
87
|
c.action do |args, options|
|
87
88
|
raise 'Must specify bucket name' unless args.first
|
88
|
-
Datapimp::Sync::S3Bucket.new(remote: args.first, redirect: !!(options.create_redirect_bucket), setup_website: !!(options.setup_website)).run_create_action()
|
89
|
+
Datapimp::Sync::S3Bucket.new(remote: args.first, redirect: !!(options.create_redirect_bucket), setup_website: !!(options.setup_website)).run_create_action(make_private: !!options.private)
|
89
90
|
end
|
90
91
|
end
|
91
92
|
|
@@ -8,21 +8,58 @@ module Datapimp
|
|
8
8
|
log "DropboxFolder run:#{action}"
|
9
9
|
|
10
10
|
if action == :push
|
11
|
-
run_push_action
|
11
|
+
run_push_action(options)
|
12
12
|
elsif action == :pull
|
13
|
-
run_pull_action
|
13
|
+
run_pull_action(options)
|
14
14
|
elsif action == :create
|
15
|
-
run_create_action
|
15
|
+
run_create_action(options)
|
16
|
+
elsif action == :delta
|
17
|
+
run_delta_action(options)
|
16
18
|
end
|
17
19
|
end
|
18
20
|
|
19
|
-
def
|
20
|
-
|
21
|
+
def run_delta_action(options={})
|
22
|
+
delta.entries.each do |entry|
|
23
|
+
remote_dropbox_path = entry.path
|
24
|
+
|
25
|
+
begin
|
26
|
+
reg = Regexp.new("#{remote}\/?", "i")
|
27
|
+
relative_local_path = remote_dropbox_path.gsub(reg, '')
|
28
|
+
target = local_path.join(relative_local_path)
|
29
|
+
|
30
|
+
if entry.is_dir
|
31
|
+
log "Creating directory: #{ target }" unless target.exist?
|
32
|
+
FileUtils.mkdir_p(target)
|
33
|
+
elsif entry.is_deleted && target.exist?
|
34
|
+
log "Deleting #{ target }"
|
35
|
+
target.unlink
|
36
|
+
elsif entry.is_deleted && !target.exist?
|
37
|
+
log "Skipping #{ entry.path }"
|
38
|
+
elsif entry.bytes == target.size
|
39
|
+
log "Skipping #{ entry.path }"
|
40
|
+
elsif !entry.is_deleted && target.exist? && entry.bytes != target.size
|
41
|
+
log "Downloading #{ target }"
|
42
|
+
target.open("wb") {|fh| fh.write(entry.download) }
|
43
|
+
else
|
44
|
+
log "Found something we can't handle"
|
45
|
+
binding.pry
|
46
|
+
end
|
47
|
+
rescue
|
48
|
+
nil
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
log "Processed #{ delta.entries.length } in cursor: #{ delta.cursor }"
|
53
|
+
cursor_path.open("w+") {|fh| fh.write(delta.cursor) }
|
54
|
+
|
55
|
+
delta.entries.length
|
21
56
|
end
|
22
57
|
|
23
|
-
def
|
24
|
-
|
58
|
+
def run_create_action(options={})
|
59
|
+
dropbox.mkdir(remote)
|
60
|
+
end
|
25
61
|
|
62
|
+
def run_pull_action(options={})
|
26
63
|
remote_path_entries.each do |entry|
|
27
64
|
remote_dropbox_path = entry.path
|
28
65
|
|
@@ -31,7 +68,7 @@ module Datapimp
|
|
31
68
|
relative_local_path = remote_dropbox_path.gsub("#{remote}/",'')
|
32
69
|
target = local_path.join(relative_local_path)
|
33
70
|
|
34
|
-
next if target.exist? && target.size == entry.bytes
|
71
|
+
next if !entry.is_dir && target.exist? && target.size == entry.bytes
|
35
72
|
|
36
73
|
if entry.is_dir
|
37
74
|
log "Creating folder #{ relative_local_path }"
|
@@ -48,7 +85,7 @@ module Datapimp
|
|
48
85
|
end
|
49
86
|
end
|
50
87
|
|
51
|
-
def run_push_action
|
88
|
+
def run_push_action(options={})
|
52
89
|
if remote_path_missing?
|
53
90
|
run_create_action()
|
54
91
|
end
|
@@ -73,7 +110,7 @@ module Datapimp
|
|
73
110
|
# how to modify the state of `local_path` to match what exists
|
74
111
|
# on Dropbox.
|
75
112
|
def delta
|
76
|
-
@delta ||= dropbox.delta(cursor,
|
113
|
+
@delta ||= dropbox.delta(cursor, path_prefix: remote.with_leading_slash)
|
77
114
|
end
|
78
115
|
|
79
116
|
# A Pointer to the local path we will be syncing with the Dropbox remote
|
@@ -89,7 +126,7 @@ module Datapimp
|
|
89
126
|
end
|
90
127
|
|
91
128
|
def cursor_path
|
92
|
-
local_path.join('.
|
129
|
+
local_path.join('.dropbox_cursor')
|
93
130
|
end
|
94
131
|
|
95
132
|
def remote_path
|
@@ -123,4 +160,3 @@ module Datapimp
|
|
123
160
|
|
124
161
|
end
|
125
162
|
end
|
126
|
-
|
@@ -121,6 +121,11 @@ module Datapimp
|
|
121
121
|
deploy_manifest_path.open("w+") {|fh| fh.write(deploy_manifest.to_json) }
|
122
122
|
end
|
123
123
|
|
124
|
+
def run_reset_action(options={})
|
125
|
+
bucket = directories.get(remote)
|
126
|
+
bucket.files.each {|f| key = f.key; f.delete rescue nil; f.destroy rescue nil; log "Deleting #{ key }"}
|
127
|
+
end
|
128
|
+
|
124
129
|
def run_pull_action(options={})
|
125
130
|
directories = Datapimp::Sync.amazon.storage.directories
|
126
131
|
bucket = directories.get(remote)
|
@@ -133,21 +138,27 @@ module Datapimp
|
|
133
138
|
|
134
139
|
bucket.files.each do |file|
|
135
140
|
local_file = local_path.join(file.key)
|
136
|
-
|
141
|
+
|
142
|
+
if local_file.exist? && file.etag == Digest::MD5.hexdigest(local_file.read)
|
143
|
+
log "Skipping #{ file.key }"
|
144
|
+
next
|
145
|
+
end
|
137
146
|
|
138
147
|
FileUtils.mkdir_p(local_file.dirname)
|
139
148
|
|
140
|
-
local_file.open("w+") {|fh| log("Updated #{ file.key }");
|
149
|
+
local_file.open("w+") {|fh| fh.write(file.body); log("Updated #{ file.key }"); }
|
141
150
|
end
|
142
151
|
end
|
143
152
|
|
144
153
|
def run_create_action(options={})
|
145
154
|
directories = Datapimp::Sync.amazon.storage.directories
|
146
155
|
|
156
|
+
make_private = !!options[:make_private]
|
157
|
+
|
147
158
|
bucket = if existing = directories.get(remote)
|
148
159
|
existing
|
149
160
|
else
|
150
|
-
directories.create(key:remote, public:
|
161
|
+
directories.create(key:remote, public: !make_private)
|
151
162
|
end
|
152
163
|
|
153
164
|
storage.put_bucket_website(remote, :IndexDocument => 'index.html', :ErrorDocument => 'error.html')
|
@@ -166,6 +177,8 @@ module Datapimp
|
|
166
177
|
run_update_acl_action(options)
|
167
178
|
elsif action == :pull
|
168
179
|
run_pull_action(options)
|
180
|
+
elsif action == :reset
|
181
|
+
run_reset_options(options)
|
169
182
|
end
|
170
183
|
end
|
171
184
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: datapimp
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.2.
|
4
|
+
version: 1.2.8
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jonathan Soeder
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-07-
|
11
|
+
date: 2015-07-30 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: pry
|
@@ -508,7 +508,6 @@ files:
|
|
508
508
|
- lib/datapimp/sources/pivotal.rb
|
509
509
|
- lib/datapimp/sync.rb
|
510
510
|
- lib/datapimp/sync/cloudfront_distribution.rb
|
511
|
-
- lib/datapimp/sync/dropbox_delta.rb
|
512
511
|
- lib/datapimp/sync/dropbox_folder.rb
|
513
512
|
- lib/datapimp/sync/google_drive_folder.rb
|
514
513
|
- lib/datapimp/sync/s3_bucket.rb
|
@@ -1,67 +0,0 @@
|
|
1
|
-
module Datapimp
|
2
|
-
module Sync
|
3
|
-
class DropboxDelta
|
4
|
-
|
5
|
-
attr_accessor :client,
|
6
|
-
:data,
|
7
|
-
:cursor,
|
8
|
-
:entries,
|
9
|
-
:path_prefix
|
10
|
-
|
11
|
-
def initialize(client, cursor, path_prefix=nil)
|
12
|
-
@client = client
|
13
|
-
@cursor = cursor
|
14
|
-
@path_prefix = path_prefix
|
15
|
-
end
|
16
|
-
|
17
|
-
def processed!
|
18
|
-
# TODO
|
19
|
-
# Should update cursor
|
20
|
-
end
|
21
|
-
|
22
|
-
def entries
|
23
|
-
return @entries if @entries
|
24
|
-
fetch
|
25
|
-
@entries
|
26
|
-
end
|
27
|
-
|
28
|
-
def _dropbox_delta at=nil
|
29
|
-
at ||= cursor
|
30
|
-
response = client.delta(at, path_prefix)
|
31
|
-
self.cursor = response["cursor"]
|
32
|
-
response
|
33
|
-
end
|
34
|
-
|
35
|
-
def data
|
36
|
-
@data ||= fetch
|
37
|
-
end
|
38
|
-
|
39
|
-
def on_reset path_prefix, cursor
|
40
|
-
end
|
41
|
-
|
42
|
-
def fetch
|
43
|
-
return @response if @response
|
44
|
-
|
45
|
-
response = _dropbox_delta
|
46
|
-
|
47
|
-
if response["reset"] == true
|
48
|
-
on_reset(path_prefix, cursor)
|
49
|
-
end
|
50
|
-
|
51
|
-
self.entries = {}.to_mash
|
52
|
-
|
53
|
-
response["entries"].each do |entry|
|
54
|
-
path, meta = entry
|
55
|
-
self.entries[path] = meta
|
56
|
-
end
|
57
|
-
|
58
|
-
if response["has_more"] == true
|
59
|
-
# TODO Implement
|
60
|
-
end
|
61
|
-
|
62
|
-
@response = response
|
63
|
-
end
|
64
|
-
|
65
|
-
end
|
66
|
-
end
|
67
|
-
end
|