datapimp 1.2.7 → 1.2.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: ee1c9e42c2e0221ca734f17b15f0265597e74f90
4
- data.tar.gz: 03bc06e371e3ec0fee16682b39e51a4d48bf4623
3
+ metadata.gz: b303a51b6bbca4a93af57cfaa463562e17fa8758
4
+ data.tar.gz: 640333b946ff814751d2077fbaea43e5b1af0c10
5
5
  SHA512:
6
- metadata.gz: 15b3917ea6d396ac439c514f29dd1040b3942afe19cb33cb6fb32935f146b13844132bd24f8b17816ad730327d8e1e3a23f1b5f06535790f16ed3920ffbc898a
7
- data.tar.gz: 3447b34fe02b2c3750295e604bee6a9b71060abd106e9e0417c7ea2ee90f8bf08ed52dedcb2b8a10a409bb7b524bc91d27f05f20d021b400b121dd9b312bffdf
6
+ metadata.gz: 8d830340d599872cfd2f0efd11bd9f8a8fcb9d45a6d5918a5f45cb27959b2e45ca90366f67dabb758e5fe94477cdca87f868ba1736517127efb7cf6bbe7de7b4
7
+ data.tar.gz: b480ce5379e3474a7c2f6175762b10c7e065f879b05e5ce6353b635a9a3684da908adc4edde70939f9bdbaadd5fe36f489a0545760ddbcd4951228ffd09cc91a
@@ -82,10 +82,11 @@ command 'create s3 bucket' do |c|
82
82
 
83
83
  c.option '--setup-website', 'Setup the bucket for website hosting'
84
84
  c.option '--create-redirect-bucket', 'Setup a redirect bucket'
85
+ c.option '--private', nil, 'Make this bucket private.'
85
86
 
86
87
  c.action do |args, options|
87
88
  raise 'Must specify bucket name' unless args.first
88
- Datapimp::Sync::S3Bucket.new(remote: args.first, redirect: !!(options.create_redirect_bucket), setup_website: !!(options.setup_website)).run_create_action()
89
+ Datapimp::Sync::S3Bucket.new(remote: args.first, redirect: !!(options.create_redirect_bucket), setup_website: !!(options.setup_website)).run_create_action(make_private: !!options.private)
89
90
  end
90
91
  end
91
92
 
@@ -8,21 +8,58 @@ module Datapimp
8
8
  log "DropboxFolder run:#{action}"
9
9
 
10
10
  if action == :push
11
- run_push_action
11
+ run_push_action(options)
12
12
  elsif action == :pull
13
- run_pull_action
13
+ run_pull_action(options)
14
14
  elsif action == :create
15
- run_create_action
15
+ run_create_action(options)
16
+ elsif action == :delta
17
+ run_delta_action(options)
16
18
  end
17
19
  end
18
20
 
19
- def run_create_action
20
- dropbox.mkdir(remote)
21
+ def run_delta_action(options={})
22
+ delta.entries.each do |entry|
23
+ remote_dropbox_path = entry.path
24
+
25
+ begin
26
+ reg = Regexp.new("#{remote}\/?", "i")
27
+ relative_local_path = remote_dropbox_path.gsub(reg, '')
28
+ target = local_path.join(relative_local_path)
29
+
30
+ if entry.is_dir
31
+ log "Creating directory: #{ target }" unless target.exist?
32
+ FileUtils.mkdir_p(target)
33
+ elsif entry.is_deleted && target.exist?
34
+ log "Deleting #{ target }"
35
+ target.unlink
36
+ elsif entry.is_deleted && !target.exist?
37
+ log "Skipping #{ entry.path }"
38
+ elsif entry.bytes == target.size
39
+ log "Skipping #{ entry.path }"
40
+ elsif !entry.is_deleted && target.exist? && entry.bytes != target.size
41
+ log "Downloading #{ target }"
42
+ target.open("wb") {|fh| fh.write(entry.download) }
43
+ else
44
+ log "Found something we can't handle"
45
+ binding.pry
46
+ end
47
+ rescue
48
+ nil
49
+ end
50
+ end
51
+
52
+ log "Processed #{ delta.entries.length } in cursor: #{ delta.cursor }"
53
+ cursor_path.open("w+") {|fh| fh.write(delta.cursor) }
54
+
55
+ delta.entries.length
21
56
  end
22
57
 
23
- def run_pull_action
24
- binding.pry
58
+ def run_create_action(options={})
59
+ dropbox.mkdir(remote)
60
+ end
25
61
 
62
+ def run_pull_action(options={})
26
63
  remote_path_entries.each do |entry|
27
64
  remote_dropbox_path = entry.path
28
65
 
@@ -31,7 +68,7 @@ module Datapimp
31
68
  relative_local_path = remote_dropbox_path.gsub("#{remote}/",'')
32
69
  target = local_path.join(relative_local_path)
33
70
 
34
- next if target.exist? && target.size == entry.bytes
71
+ next if !entry.is_dir && target.exist? && target.size == entry.bytes
35
72
 
36
73
  if entry.is_dir
37
74
  log "Creating folder #{ relative_local_path }"
@@ -48,7 +85,7 @@ module Datapimp
48
85
  end
49
86
  end
50
87
 
51
- def run_push_action
88
+ def run_push_action(options={})
52
89
  if remote_path_missing?
53
90
  run_create_action()
54
91
  end
@@ -73,7 +110,7 @@ module Datapimp
73
110
  # how to modify the state of `local_path` to match what exists
74
111
  # on Dropbox.
75
112
  def delta
76
- @delta ||= dropbox.delta(cursor, remote_path)
113
+ @delta ||= dropbox.delta(cursor, path_prefix: remote.with_leading_slash)
77
114
  end
78
115
 
79
116
  # A Pointer to the local path we will be syncing with the Dropbox remote
@@ -89,7 +126,7 @@ module Datapimp
89
126
  end
90
127
 
91
128
  def cursor_path
92
- local_path.join('.dropbox-cursor')
129
+ local_path.join('.dropbox_cursor')
93
130
  end
94
131
 
95
132
  def remote_path
@@ -123,4 +160,3 @@ module Datapimp
123
160
 
124
161
  end
125
162
  end
126
-
@@ -121,6 +121,11 @@ module Datapimp
121
121
  deploy_manifest_path.open("w+") {|fh| fh.write(deploy_manifest.to_json) }
122
122
  end
123
123
 
124
+ def run_reset_action(options={})
125
+ bucket = directories.get(remote)
126
+ bucket.files.each {|f| key = f.key; f.delete rescue nil; f.destroy rescue nil; log "Deleting #{ key }"}
127
+ end
128
+
124
129
  def run_pull_action(options={})
125
130
  directories = Datapimp::Sync.amazon.storage.directories
126
131
  bucket = directories.get(remote)
@@ -133,21 +138,27 @@ module Datapimp
133
138
 
134
139
  bucket.files.each do |file|
135
140
  local_file = local_path.join(file.key)
136
- next if local_file.exist? && file.etag == Digest::MD5.hexdigest(local_file.read)
141
+
142
+ if local_file.exist? && file.etag == Digest::MD5.hexdigest(local_file.read)
143
+ log "Skipping #{ file.key }"
144
+ next
145
+ end
137
146
 
138
147
  FileUtils.mkdir_p(local_file.dirname)
139
148
 
140
- local_file.open("w+") {|fh| log("Updated #{ file.key }"); fh.write(file.body) }
149
+ local_file.open("w+") {|fh| fh.write(file.body); log("Updated #{ file.key }"); }
141
150
  end
142
151
  end
143
152
 
144
153
  def run_create_action(options={})
145
154
  directories = Datapimp::Sync.amazon.storage.directories
146
155
 
156
+ make_private = !!options[:make_private]
157
+
147
158
  bucket = if existing = directories.get(remote)
148
159
  existing
149
160
  else
150
- directories.create(key:remote, public: true)
161
+ directories.create(key:remote, public: !make_private)
151
162
  end
152
163
 
153
164
  storage.put_bucket_website(remote, :IndexDocument => 'index.html', :ErrorDocument => 'error.html')
@@ -166,6 +177,8 @@ module Datapimp
166
177
  run_update_acl_action(options)
167
178
  elsif action == :pull
168
179
  run_pull_action(options)
180
+ elsif action == :reset
181
+ run_reset_options(options)
169
182
  end
170
183
  end
171
184
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: datapimp
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.2.7
4
+ version: 1.2.8
5
5
  platform: ruby
6
6
  authors:
7
7
  - Jonathan Soeder
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-07-20 00:00:00.000000000 Z
11
+ date: 2015-07-30 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: pry
@@ -508,7 +508,6 @@ files:
508
508
  - lib/datapimp/sources/pivotal.rb
509
509
  - lib/datapimp/sync.rb
510
510
  - lib/datapimp/sync/cloudfront_distribution.rb
511
- - lib/datapimp/sync/dropbox_delta.rb
512
511
  - lib/datapimp/sync/dropbox_folder.rb
513
512
  - lib/datapimp/sync/google_drive_folder.rb
514
513
  - lib/datapimp/sync/s3_bucket.rb
@@ -1,67 +0,0 @@
1
- module Datapimp
2
- module Sync
3
- class DropboxDelta
4
-
5
- attr_accessor :client,
6
- :data,
7
- :cursor,
8
- :entries,
9
- :path_prefix
10
-
11
- def initialize(client, cursor, path_prefix=nil)
12
- @client = client
13
- @cursor = cursor
14
- @path_prefix = path_prefix
15
- end
16
-
17
- def processed!
18
- # TODO
19
- # Should update cursor
20
- end
21
-
22
- def entries
23
- return @entries if @entries
24
- fetch
25
- @entries
26
- end
27
-
28
- def _dropbox_delta at=nil
29
- at ||= cursor
30
- response = client.delta(at, path_prefix)
31
- self.cursor = response["cursor"]
32
- response
33
- end
34
-
35
- def data
36
- @data ||= fetch
37
- end
38
-
39
- def on_reset path_prefix, cursor
40
- end
41
-
42
- def fetch
43
- return @response if @response
44
-
45
- response = _dropbox_delta
46
-
47
- if response["reset"] == true
48
- on_reset(path_prefix, cursor)
49
- end
50
-
51
- self.entries = {}.to_mash
52
-
53
- response["entries"].each do |entry|
54
- path, meta = entry
55
- self.entries[path] = meta
56
- end
57
-
58
- if response["has_more"] == true
59
- # TODO Implement
60
- end
61
-
62
- @response = response
63
- end
64
-
65
- end
66
- end
67
- end