qcloud_cos 0.4.0 → 0.4.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop.yml +8 -0
- data/README.md +2 -2
- data/bin/qcloud-cos +7 -8
- data/lib/qcloud_cos.rb +1 -1
- data/lib/qcloud_cos/api.rb +14 -47
- data/lib/qcloud_cos/cli.rb +87 -89
- data/lib/qcloud_cos/convenient_api.rb +21 -2
- data/lib/qcloud_cos/model/folder_object.rb +12 -7
- data/lib/qcloud_cos/multipart.rb +82 -0
- data/lib/qcloud_cos/version.rb +1 -1
- data/wiki/get_started.md +1 -1
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 82a6d54335364ee45eb1d60e4fe33c02181d13f8
|
4
|
+
data.tar.gz: f3ee2074c3a196a7f46ce1ba2077fbb85d16a354
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b08e8867cd08cd0fba2eefec5754dcacc30633a68d8cbdca71af7b745db0006994e81379bbe25d94680b293cfb45c651d1a7afc0efc1a960d42feb20caa10934
|
7
|
+
data.tar.gz: 76f4f556cf7fe094c5db2d0515be9cc7d463902b115fc439fe8c79905ebe486510f4b306811a5c790da272d43012ae5c587f6d208930d0df8391a713055ad801
|
data/.rubocop.yml
CHANGED
data/README.md
CHANGED
@@ -54,7 +54,7 @@ Here is original Restful API, It has the most detailed and authoritative explana
|
|
54
54
|
|
55
55
|
Here is our RDoc Document, It's well format to help you find more detail about methods.
|
56
56
|
|
57
|
-
+ [RDoc Document](http://www.rubydoc.info/gems/qcloud_cos/0.
|
57
|
+
+ [RDoc Document](http://www.rubydoc.info/gems/qcloud_cos/0.4.1)
|
58
58
|
|
59
59
|
|
60
60
|
Here are some more guides for help you. Welcome to advice.
|
@@ -76,4 +76,4 @@ We use minitest for test and rubocop for Syntax checker, If you want to make con
|
|
76
76
|
|
77
77
|
## License
|
78
78
|
|
79
|
-
licensed under the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0.html)
|
79
|
+
licensed under the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0.html)
|
data/bin/qcloud-cos
CHANGED
@@ -10,10 +10,12 @@ program :description, 'command-line tool for Qcloud COS'
|
|
10
10
|
program :help, 'Author', 'Newell Zhu<zlx.star@gmail.com>'
|
11
11
|
default_command :help
|
12
12
|
|
13
|
+
global_option '-b', '--bucket Bucket Name', 'specify bucket name, it will override default bucket'
|
14
|
+
|
13
15
|
command :config do |c|
|
14
16
|
c.syntax = 'qcloud-cos config'
|
15
17
|
c.description = 'Init config, eg: qcloud-cos config'
|
16
|
-
c.action do |
|
18
|
+
c.action do |_args, _options|
|
17
19
|
QcloudCos::Cli.config
|
18
20
|
end
|
19
21
|
end
|
@@ -25,7 +27,6 @@ command :info do |c|
|
|
25
27
|
c.example 'Obtain information for /test/', '$ qcloud-cos info /test/'
|
26
28
|
c.example 'Obtain information for /production.log', '$ qcloud-cos info /production.log'
|
27
29
|
c.example 'Obtain information for /production.log from bucket2', '$ qcloud-cos info --bucket bucket2 /production.log'
|
28
|
-
c.option '--bucket Bucket Name', String, 'specify bucket name, it will override default bucket'
|
29
30
|
c.action do |args, options|
|
30
31
|
abort unless QcloudCos::Cli.environment_configed?
|
31
32
|
begin
|
@@ -44,13 +45,13 @@ command :list do |c|
|
|
44
45
|
c.example 'List all objects under /test/', '$ qcloud-cos list /test/'
|
45
46
|
c.example 'List first 10 objects under /test/', '$ qcloud-cos list --num 10 /test/'
|
46
47
|
c.example 'List all objects under / for bucket: bucket2', '$ qcloud-cos list --num 10 --bucket bucket2 /test/'
|
47
|
-
c.option '--bucket Bucket Name', String, 'specify bucket name, it will override default bucket'
|
48
48
|
c.option '--num Num', Integer, 'specify max objects, default is 100'
|
49
49
|
c.action do |args, options|
|
50
50
|
abort unless QcloudCos::Cli.environment_configed?
|
51
|
+
options.default num: 100
|
51
52
|
begin
|
52
53
|
cli = QcloudCos::Cli.init
|
53
|
-
cli.list(args, options).map{|path| puts path }
|
54
|
+
cli.list(args, options).map { |path| puts path }
|
54
55
|
rescue => e
|
55
56
|
say_error e.message
|
56
57
|
end
|
@@ -65,11 +66,11 @@ command :upload do |c|
|
|
65
66
|
c.example 'Upload all files under ./test/ to /data/', '$ qcloud-cos upload test/ /data/'
|
66
67
|
c.example 'Upload all files under ./test/ to /data/ with other bucket: bucket2', '$ qcloud-cos upload --bucket bucket2 test/ /data/'
|
67
68
|
c.example 'Upload production.log to /data/ with slice_size and min', '$ qcloud-cos upload --size 10 --min 100 production.log /data/'
|
68
|
-
c.option '--
|
69
|
-
c.option '--size Slice Size', Integer, "specify slice size for slice upload in Bytes, default: #{QcloudCos::DEFAULT_SLICE_SIZE}(#{QcloudCos::DEFAULT_SLICE_SIZE/1024/1024}M)"
|
69
|
+
c.option '--size Slice Size', Integer, "specify slice size for slice upload in Bytes, default: #{QcloudCos::DEFAULT_SLICE_SIZE}(#{QcloudCos::DEFAULT_SLICE_SIZE / 1024 / 1024}M)"
|
70
70
|
c.option '--min Min Slice File Size', Integer, "specify min slice file size in Bytes, default: default: #{QcloudCos::MIN_SLICE_FILE_SIZE * 1024 * 1024}(#{QcloudCos::MIN_SLICE_FILE_SIZE}M)"
|
71
71
|
c.action do |args, options|
|
72
72
|
abort unless QcloudCos::Cli.environment_configed?
|
73
|
+
options.default size: QcloudCos::DEFAULT_SLICE_SIZE, min: QcloudCos::MIN_SLICE_FILE_SIZE * 1024 * 1024
|
73
74
|
begin
|
74
75
|
cli = QcloudCos::Cli.init
|
75
76
|
cli.upload(args, options)
|
@@ -86,7 +87,6 @@ command :download do |c|
|
|
86
87
|
c.example 'Download file from /data/production.log and save under ./data/', '$ qcloud-cos download /data/production.log ./data'
|
87
88
|
c.example 'Download whole folder /data/test/ and save under ./data/', '$ qcloud-cos download /data/test/ ./data'
|
88
89
|
c.example 'Download whole folder /data/test/ from bucket2 and save under ./data/', '$ qcloud-cos download --bucket bucket2 /data/test/ ./data'
|
89
|
-
c.option '--bucket Bucket Name', String, 'specify bucket name, it will override default bucket'
|
90
90
|
c.action do |args, options|
|
91
91
|
abort unless QcloudCos::Cli.environment_configed?
|
92
92
|
begin
|
@@ -105,7 +105,6 @@ command :remove do |c|
|
|
105
105
|
c.example 'Remove folder /data/test/', '$ qcloud-cos remove /data/test/'
|
106
106
|
c.example 'Remove folder /data/test/ in recursive', '$ qcloud-cos remove --recursive /data/test/'
|
107
107
|
c.example 'Remove folder /data/test/ from bucket2', '$ qcloud-cos download --bucket bucket2 /data/test/'
|
108
|
-
c.option '--bucket Bucket Name', String, 'specify bucket name, it will override default bucket'
|
109
108
|
c.option '--[no-]recursive', 'specify recursive or not when remove folder'
|
110
109
|
c.action do |args, options|
|
111
110
|
abort unless QcloudCos::Cli.environment_configed?
|
data/lib/qcloud_cos.rb
CHANGED
data/lib/qcloud_cos/api.rb
CHANGED
@@ -1,10 +1,11 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
require 'qcloud_cos/utils'
|
3
|
+
require 'qcloud_cos/multipart'
|
3
4
|
require 'qcloud_cos/model/list'
|
4
5
|
|
5
6
|
module QcloudCos
|
6
7
|
module Api
|
7
|
-
#
|
8
|
+
# 列出文件或者目录
|
8
9
|
#
|
9
10
|
# @param path [String] 指定目标路径, 以 / 结尾, 则列出该目录下文件或者文件夹,不以 / 结尾,就搜索该前缀的文件或者文件夹
|
10
11
|
# @param options [Hash] 额外参数
|
@@ -135,42 +136,13 @@ module QcloudCos
|
|
135
136
|
fail FileNotExistError unless File.exist?(src_path)
|
136
137
|
bucket = validates(dst_path, options)
|
137
138
|
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
return data if data.key?('url') # 妙传命中
|
146
|
-
|
147
|
-
slice_size = data['slice_size'] || DEFAULT_SLICE_SIZE
|
148
|
-
session = data['session'] || options['session']
|
149
|
-
offset = data['offset'] || 0
|
150
|
-
|
151
|
-
fail MissingSessionIdError unless session
|
152
|
-
|
153
|
-
while offset < filesize
|
154
|
-
filecontent = IO.read(src_path, slice_size, offset)
|
155
|
-
|
156
|
-
retry_times = 0
|
157
|
-
begin
|
158
|
-
result = upload_part(dst_path, session, offset, filecontent, options)
|
159
|
-
progress = [offset + slice_size, filesize].min
|
160
|
-
yield((progress.to_f/filesize).round(2)) if block_given?
|
161
|
-
|
162
|
-
if result.key?('data') && result['data'].key?('session')
|
163
|
-
session = result['data']['session']
|
164
|
-
elsif result.key?('data') && result['data'].key?('url')
|
165
|
-
return result['data']
|
166
|
-
end
|
167
|
-
rescue => e
|
168
|
-
retry_times += 1
|
169
|
-
retry if retry_times <= config.max_retry_times
|
170
|
-
raise e
|
171
|
-
end
|
172
|
-
offset += slice_size
|
173
|
-
end
|
139
|
+
multipart = QcloudCos::Multipart.new(
|
140
|
+
dst_path,
|
141
|
+
src_path,
|
142
|
+
options.merge(bucket: bucket, authorization: authorization)
|
143
|
+
)
|
144
|
+
multipart.upload(&block)
|
145
|
+
multipart.result
|
174
146
|
end
|
175
147
|
|
176
148
|
# 初始化分片上传
|
@@ -279,17 +251,12 @@ module QcloudCos
|
|
279
251
|
|
280
252
|
return delete(path, options) if options['recursive'] != true
|
281
253
|
|
282
|
-
|
283
|
-
|
284
|
-
|
285
|
-
|
286
|
-
|
287
|
-
elsif object.is_a?(QcloudCos::FileObject)
|
288
|
-
delete_file("#{path}#{object.name}", options)
|
289
|
-
end
|
254
|
+
all(path, options).each do |object|
|
255
|
+
if object.is_a?(QcloudCos::FolderObject)
|
256
|
+
delete_folder("#{path}#{object.name}/", options)
|
257
|
+
elsif object.is_a?(QcloudCos::FileObject)
|
258
|
+
delete_file("#{path}#{object.name}", options)
|
290
259
|
end
|
291
|
-
break unless objects.has_more
|
292
|
-
options['context'] = objects.context
|
293
260
|
end
|
294
261
|
delete(path)
|
295
262
|
end
|
data/lib/qcloud_cos/cli.rb
CHANGED
@@ -21,30 +21,23 @@ module QcloudCos
|
|
21
21
|
# 交互模式配置环境
|
22
22
|
# 命令: $ qcloud-cos config
|
23
23
|
def self.config(config_path = nil)
|
24
|
-
config_path
|
24
|
+
config_path ||= QcloudCos::QCLOUD_COS_CONFIG
|
25
25
|
return Commander::UI.say_error("#{config_path} already exist, remove it first or direct edit it!") if File.exist?(config_path)
|
26
26
|
|
27
27
|
app_id = Commander::UI.ask 'Qcloud COS APP ID: '
|
28
|
-
return Commander::UI.say_error(
|
28
|
+
return Commander::UI.say_error('Missing Qcloud COS APP ID') if app_id.empty?
|
29
29
|
|
30
30
|
secret_id = Commander::UI.ask 'Qcloud COS Secret ID: '
|
31
|
-
return Commander::UI.say_error(
|
31
|
+
return Commander::UI.say_error('Missing Qcloud COS Secret ID') if secret_id.empty?
|
32
32
|
|
33
33
|
secret_key = Commander::UI.ask 'Qcloud COS Secret Key: '
|
34
|
-
return Commander::UI.say_error(
|
34
|
+
return Commander::UI.say_error('Missing Qcloud COS Secret Key') if secret_key.empty?
|
35
35
|
|
36
36
|
endpoint = Commander::UI.ask "Default Qcloud COS Endpoint [#{QcloudCos::DEFAULT_ENDPOINT}]: "
|
37
37
|
endpoint = QcloudCos::DEFAULT_ENDPOINT if endpoint.empty?
|
38
38
|
bucket = Commander::UI.ask 'Default Qcloud COS Bucket: '
|
39
39
|
|
40
|
-
write_config(
|
41
|
-
config_path,
|
42
|
-
app_id: app_id,
|
43
|
-
secret_id: secret_id,
|
44
|
-
secret_key: secret_key,
|
45
|
-
endpoint: endpoint,
|
46
|
-
bucket: bucket
|
47
|
-
)
|
40
|
+
write_config(config_path, app_id: app_id, secret_id: secret_id, secret_key: secret_key, endpoint: endpoint, bucket: bucket)
|
48
41
|
end
|
49
42
|
|
50
43
|
# 检查环境是否配置
|
@@ -52,7 +45,7 @@ module QcloudCos
|
|
52
45
|
# @return [Boolean]
|
53
46
|
def self.environment_configed?
|
54
47
|
configed = File.exist?(QcloudCos::QCLOUD_COS_CONFIG) || !ENV['QCLOUD_COS_APP_ID'].to_s.empty?
|
55
|
-
Commander::UI.say_error(
|
48
|
+
Commander::UI.say_error('Use `qcloud-cos config` first or export your environments') unless configed
|
56
49
|
configed
|
57
50
|
end
|
58
51
|
|
@@ -62,7 +55,7 @@ module QcloudCos
|
|
62
55
|
load_config.each { |k, v| config.send("#{k}=", v) }
|
63
56
|
end
|
64
57
|
|
65
|
-
|
58
|
+
new
|
66
59
|
end
|
67
60
|
|
68
61
|
# 查看信息
|
@@ -82,7 +75,7 @@ module QcloudCos
|
|
82
75
|
# qcloud-cos info --bucket bucket2 /production.log
|
83
76
|
def info(args, options)
|
84
77
|
path = args.shift || '/'
|
85
|
-
opts =
|
78
|
+
opts = parse_options(options)
|
86
79
|
|
87
80
|
QcloudCos.stat(path, opts)['data']
|
88
81
|
end
|
@@ -106,8 +99,7 @@ module QcloudCos
|
|
106
99
|
#
|
107
100
|
def list(args, options)
|
108
101
|
path = args.shift || '/'
|
109
|
-
opts =
|
110
|
-
opts[:num] = options.num || 100
|
102
|
+
opts = parse_options(options)
|
111
103
|
|
112
104
|
objects = QcloudCos.list(path, opts)
|
113
105
|
objects.map do |object|
|
@@ -133,20 +125,16 @@ module QcloudCos
|
|
133
125
|
# qcloud-cos upload --bucket bucket2 test/ /data/
|
134
126
|
def upload(args, options)
|
135
127
|
path = args.shift
|
136
|
-
return Commander::UI.say_error(
|
128
|
+
return Commander::UI.say_error('file missing, see example: $ qcloud-cos upload -h') unless path
|
137
129
|
return Commander::UI.say_error("file #{path} not exist") unless File.exist?(path)
|
138
130
|
|
139
131
|
dest_path = args.shift || '/'
|
140
|
-
return Commander::UI.say_error(
|
141
|
-
|
142
|
-
opts = parse(options)
|
143
|
-
opts[:slice_size] = options.size || QcloudCos::DEFAULT_SLICE_SIZE
|
144
|
-
opts[:min] = options.min || QcloudCos::MIN_SLICE_FILE_SIZE * 1024 * 1024
|
132
|
+
return Commander::UI.say_error('dest_path must end with /, see example: $ qcloud-cos upload -h') unless dest_path.end_with?('/')
|
145
133
|
|
146
134
|
if path.end_with?('/')
|
147
|
-
upload_folder(path, dest_path,
|
135
|
+
upload_folder(path, dest_path, parse_options(options))
|
148
136
|
else
|
149
|
-
upload_file(path, dest_path,
|
137
|
+
upload_file(path, dest_path, parse_options(options))
|
150
138
|
end
|
151
139
|
end
|
152
140
|
|
@@ -169,14 +157,14 @@ module QcloudCos
|
|
169
157
|
#
|
170
158
|
def download(args, options)
|
171
159
|
path = args.shift
|
172
|
-
return Commander::UI.say_error(
|
173
|
-
opts =
|
174
|
-
|
160
|
+
return Commander::UI.say_error('missing path, see example: $ qcloud-cos download -h') unless path
|
161
|
+
opts = parse_options(options)
|
162
|
+
save_path = args.shift || '.'
|
175
163
|
|
176
164
|
if path.end_with?('/')
|
177
|
-
download_folder(path, opts)
|
165
|
+
download_folder(path, save_path, opts)
|
178
166
|
else
|
179
|
-
download_file(path, opts)
|
167
|
+
download_file(path, save_path, opts)
|
180
168
|
end
|
181
169
|
end
|
182
170
|
|
@@ -195,12 +183,11 @@ module QcloudCos
|
|
195
183
|
# qcloud-cos remove --recursive /data/test/
|
196
184
|
#
|
197
185
|
# # 删除 bucket2 下面的目录 /data/test/
|
198
|
-
# qcloud-cos
|
186
|
+
# qcloud-cos remove --bucket bucket2 /data/test/
|
199
187
|
def remove(args, options)
|
200
188
|
path = args.shift
|
201
|
-
return Commander::UI.say_error(
|
202
|
-
|
203
|
-
opts = parse(options)
|
189
|
+
return Commander::UI.say_error('missing dest_path, see example: $ qcloud-cos remove -h') unless path
|
190
|
+
opts = parse_options(options)
|
204
191
|
|
205
192
|
if path.end_with?('/')
|
206
193
|
QcloudCos.delete_folder(path, opts.merge(recursive: !!options.recursive))
|
@@ -213,47 +200,61 @@ module QcloudCos
|
|
213
200
|
|
214
201
|
def upload_file(path, dest_path, opts)
|
215
202
|
dest_path = File.join(dest_path, path.split('/').last)
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
220
|
-
|
203
|
+
|
204
|
+
create_file_with_lint(path, dest_path) do
|
205
|
+
if File.size(path) > opts[:min]
|
206
|
+
QcloudCos.upload_slice(dest_path, path, Utils.hash_slice(opts, :bucket).merge(slice_size: opts[:size]))
|
207
|
+
else
|
208
|
+
QcloudCos.upload(dest_path, File.new(path), Utils.hash_slice(opts, :bucket))
|
209
|
+
end
|
221
210
|
end
|
211
|
+
end
|
212
|
+
|
213
|
+
def create_file_with_lint(path, dest_path, &block)
|
214
|
+
yield if block_given?
|
222
215
|
Commander::UI.say_ok "#{path} uploaded to #{dest_path}..."
|
223
216
|
rescue => e
|
224
217
|
Commander::UI.say_error "Failed when uploaded #{path} ===> #{e.message}"
|
225
|
-
ensure
|
226
|
-
file && file.close
|
227
218
|
end
|
228
219
|
|
229
220
|
def upload_folder(path, dest_path, opts)
|
230
|
-
|
231
|
-
Dir.glob("#{path}**/*") do |file_path|
|
232
|
-
split_path = file_path.sub(path, '').split('/')
|
233
|
-
new_path =
|
234
|
-
if File.file?(file_path)
|
235
|
-
split_path.size > 1 ? File.join(dest_path, *split_path[0..-2]) : dest_path
|
236
|
-
else
|
237
|
-
file_path = File.join(file_path, '')
|
238
|
-
File.join(dest_path, *split_path, '')
|
239
|
-
end
|
240
|
-
|
241
|
-
file_path_map[file_path] = new_path
|
242
|
-
end
|
243
|
-
remove_subdirectories(file_path_map)
|
221
|
+
path_map = find_upload_path_map(path, dest_path)
|
244
222
|
|
245
|
-
|
223
|
+
path_map.each do |file_path, dest|
|
246
224
|
if file_path.end_with?('/')
|
247
|
-
|
248
|
-
Commander::UI.say_ok "Create folder #{dest_path}"
|
225
|
+
create_folder_with_lint(dest, opts)
|
249
226
|
else
|
250
|
-
upload_file(file_path,
|
227
|
+
upload_file(file_path, dest, opts)
|
251
228
|
end
|
252
229
|
end
|
253
230
|
end
|
254
231
|
|
255
|
-
def
|
256
|
-
|
232
|
+
def create_folder_with_lint(dest, opts)
|
233
|
+
QcloudCos.create_folder(dest, opts)
|
234
|
+
Commander::UI.say_ok "Create folder #{dest}"
|
235
|
+
rescue => e
|
236
|
+
Commander::UI.say_ok "Failed when create folder #{dest} ====> #{e.message}"
|
237
|
+
end
|
238
|
+
|
239
|
+
def find_upload_path_map(path, dest_path)
|
240
|
+
path_map = Hash[Dir.glob("#{path}**/*").map do |file_path|
|
241
|
+
file_path = File.join(file_path, '') unless File.file?(file_path)
|
242
|
+
[file_path, find_upload_dest_path(file_path, path, dest_path)]
|
243
|
+
end]
|
244
|
+
remove_subdirectories!(path_map)
|
245
|
+
path_map
|
246
|
+
end
|
247
|
+
|
248
|
+
def find_upload_dest_path(file_path, parent_path, dest_path)
|
249
|
+
split_path = file_path.sub(parent_path, '').split('/')
|
250
|
+
if File.file?(file_path)
|
251
|
+
split_path.size > 1 ? File.join(dest_path, *split_path[0..-2]) : dest_path
|
252
|
+
else
|
253
|
+
File.join(dest_path, *split_path, '')
|
254
|
+
end
|
255
|
+
end
|
256
|
+
|
257
|
+
def download_file(path, save_path, opts)
|
257
258
|
FileUtils.mkdir_p(save_path) unless File.exist?(save_path)
|
258
259
|
|
259
260
|
file_path = File.join(save_path, path.split('/').last)
|
@@ -264,31 +265,26 @@ module QcloudCos
|
|
264
265
|
Commander::UI.say_error("Failed when Download #{path} ===> #{e.message}")
|
265
266
|
end
|
266
267
|
|
267
|
-
def download_folder(path, opts)
|
268
|
-
|
269
|
-
|
270
|
-
|
271
|
-
|
272
|
-
|
273
|
-
|
274
|
-
|
275
|
-
|
276
|
-
|
277
|
-
end
|
278
|
-
download_folder("#{path}#{object.name}/", opts.merge(save_path: new_path))
|
279
|
-
elsif object.is_a?(QcloudCos::FileObject)
|
280
|
-
if object.access_url
|
281
|
-
download_file("#{path}#{object.name}", opts.merge(save_path: save_path, access_url: object.access_url))
|
282
|
-
else
|
283
|
-
download_file("#{path}#{object.name}", opts.merge(save_path: save_path))
|
284
|
-
end
|
285
|
-
end
|
268
|
+
def download_folder(path, save_path, opts)
|
269
|
+
mkdir_with_lint(path, save_path)
|
270
|
+
|
271
|
+
QcloudCos.all(path, opts).each do |object|
|
272
|
+
new_path = "#{path}#{object.name}"
|
273
|
+
|
274
|
+
if object.is_a?(QcloudCos::FolderObject)
|
275
|
+
download_folder(File.join(new_path, ''), File.join(save_path, object.name), opts)
|
276
|
+
elsif object.is_a?(QcloudCos::FileObject)
|
277
|
+
download_file(new_path, save_path, opts.merge(access_url: object.access_url))
|
286
278
|
end
|
287
|
-
break unless objects.has_more
|
288
|
-
opts['context'] = objects.context
|
289
279
|
end
|
290
280
|
end
|
291
281
|
|
282
|
+
def mkdir_with_lint(path, save_path)
|
283
|
+
return if File.exist?(save_path)
|
284
|
+
FileUtils.mkdir_p(save_path)
|
285
|
+
Commander::UI.say_ok "Save #{path} to #{save_path}/"
|
286
|
+
end
|
287
|
+
|
292
288
|
def self.load_config
|
293
289
|
file_config = load_file_config
|
294
290
|
|
@@ -308,13 +304,16 @@ module QcloudCos
|
|
308
304
|
end.compact]
|
309
305
|
end
|
310
306
|
|
311
|
-
def
|
307
|
+
def parse_options(options)
|
312
308
|
opts = {}
|
313
309
|
opts[:bucket] = options.bucket if options.bucket
|
310
|
+
opts[:min] = options.min if options.min
|
311
|
+
opts[:size] = options.size if options.size
|
312
|
+
opts[:num] = options.num if options.num
|
314
313
|
opts
|
315
314
|
end
|
316
315
|
|
317
|
-
def remove_subdirectories(path_map)
|
316
|
+
def remove_subdirectories!(path_map)
|
318
317
|
new_map = path_map.dup
|
319
318
|
new_map.each do |_, dest_path|
|
320
319
|
path_map.reject! do |file_path, dest|
|
@@ -324,22 +323,21 @@ module QcloudCos
|
|
324
323
|
end
|
325
324
|
|
326
325
|
def save_to_file(access_url, file_path)
|
327
|
-
File.open(file_path,
|
326
|
+
File.open(file_path, 'wb') do |f|
|
328
327
|
f.write HTTParty.get(access_url)
|
329
328
|
end
|
330
329
|
end
|
331
330
|
|
332
331
|
def self.write_config(config_path, options)
|
333
|
-
File.open(config_path,
|
332
|
+
File.open(config_path, 'w') do |file|
|
334
333
|
file.puts "app_id=#{options[:app_id]}"
|
335
334
|
file.puts "secret_id=#{options[:secret_id]}"
|
336
335
|
file.puts "secret_key=#{options[:secret_key]}"
|
337
336
|
file.puts "endpoint=#{options[:endpoint]}"
|
338
337
|
file.puts "bucket=#{options[:bucket]}"
|
339
|
-
file.puts
|
340
|
-
file.puts
|
338
|
+
file.puts 'ssl_ca_file='
|
339
|
+
file.puts 'max_retry_times='
|
341
340
|
end
|
342
341
|
end
|
343
|
-
|
344
342
|
end
|
345
343
|
end
|
@@ -1,13 +1,12 @@
|
|
1
1
|
module QcloudCos
|
2
2
|
module ConvenientApi
|
3
|
-
|
4
3
|
# 获取 Bucket 信息
|
5
4
|
#
|
6
5
|
# @param bucket_name [String] :bucket (config.bucket) 指定当前 bucket, 默认是配置里面的 bucket
|
7
6
|
#
|
8
7
|
# @return [Hash] 返回 Bucket 信息
|
9
8
|
def bucket_info(bucket_name = nil)
|
10
|
-
bucket_name
|
9
|
+
bucket_name ||= config.bucket
|
11
10
|
stat('/', bucket: bucket_name)['data']
|
12
11
|
rescue
|
13
12
|
{}
|
@@ -104,5 +103,25 @@ module QcloudCos
|
|
104
103
|
fail FileNotExistError
|
105
104
|
end
|
106
105
|
end
|
106
|
+
|
107
|
+
# 列出所有文件或者目录
|
108
|
+
#
|
109
|
+
# @param path [String] 指定目标路径, 以 / 结尾, 则列出该目录下文件或者文件夹,不以 / 结尾,就搜索该前缀的文件或者文件夹
|
110
|
+
# @param options [Hash] 额外参数
|
111
|
+
# @option options [String] :bucket (config.bucket_name) 指定当前 bucket, 默认是配置里面的 bucket
|
112
|
+
# @option options [String] :pattern (eListBoth) 指定拉取的内容,可选值: eListBoth, eListDirOnly, eListFileOnly
|
113
|
+
# @option options [Integer] :order (0) 指定拉取文件的顺序, 默认为正序(=0), 可选值: 0, 1
|
114
|
+
#
|
115
|
+
# @return [Hash]
|
116
|
+
def all(path, options = {})
|
117
|
+
results = []
|
118
|
+
loop do
|
119
|
+
objects = QcloudCos.list(path, options)
|
120
|
+
results += objects.to_a
|
121
|
+
break unless objects.has_more
|
122
|
+
options['context'] = objects.context
|
123
|
+
end
|
124
|
+
results
|
125
|
+
end
|
107
126
|
end
|
108
127
|
end
|
@@ -22,13 +22,18 @@ module QcloudCos
|
|
22
22
|
if !path.end_with?('/')
|
23
23
|
fail InvalidFolderPathError, '文件夹路径必须以 / 结尾'
|
24
24
|
elsif !(names = path.split('/')).empty?
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
25
|
+
validate_name(names)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
# 校验文件夹名字
|
30
|
+
def self.validate_name(names)
|
31
|
+
if names.detect { |name| RETAINED_FIELDS.include?(name.downcase) }
|
32
|
+
fail InvalidFolderPathError, %(文件夹名字不能是保留字段: '#{RETAINED_FIELDS.join("', '")}')
|
33
|
+
elsif names.detect { |name| name.match(/[\/?*:|\\<>"]/) }
|
34
|
+
fail InvalidFolderPathError, %(文件夹名字不能包含保留字符: '#{RETAINED_SYMBOLS.join("', '")}')
|
35
|
+
elsif names.detect { |name| name.length > MAXLENGTH }
|
36
|
+
fail InvalidFolderPathError, %(文件夹名字不能超过#{MAXLENGTH}个字符)
|
32
37
|
end
|
33
38
|
end
|
34
39
|
end
|
@@ -0,0 +1,82 @@
|
|
1
|
+
module QcloudCos
|
2
|
+
class Multipart
|
3
|
+
attr_reader :dst_path, :src_path, :bucket, :authorization, :result, :options
|
4
|
+
|
5
|
+
def initialize(dst_path, src_path, options = {})
|
6
|
+
@dst_path = dst_path
|
7
|
+
@src_path = src_path
|
8
|
+
@options = options
|
9
|
+
@bucket = options.delete(:bucket)
|
10
|
+
@authorization = options.delete(:authorization)
|
11
|
+
end
|
12
|
+
|
13
|
+
def upload(&block)
|
14
|
+
init_multipart
|
15
|
+
return if complete?
|
16
|
+
fail QcloudCos::MissingSessionIdError unless session
|
17
|
+
|
18
|
+
offset = @result['offset'] || 0
|
19
|
+
|
20
|
+
while offset < filesize
|
21
|
+
filecontent = IO.read(src_path, slice_size, offset)
|
22
|
+
break if upload_part(offset, filecontent, &block)
|
23
|
+
offset += slice_size
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
private
|
28
|
+
|
29
|
+
def init_multipart
|
30
|
+
@result ||= QcloudCos.init_slice_upload(dst_path, filesize, sha, options.merge('sign' => sign))['data']
|
31
|
+
end
|
32
|
+
|
33
|
+
def upload_part(offset, content, &block)
|
34
|
+
retry_for(QcloudCos.config.max_retry_times) do
|
35
|
+
@result = QcloudCos.upload_part(dst_path, session, offset, content, options)['data']
|
36
|
+
notify_progress(offset + slice_size, &block)
|
37
|
+
|
38
|
+
return true if complete?
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def complete?
|
43
|
+
@result.key?('url')
|
44
|
+
end
|
45
|
+
|
46
|
+
def notify_progress(progress, &block)
|
47
|
+
progress = [progress, filesize].min
|
48
|
+
yield((progress.to_f / filesize).round(2)) if block_given?
|
49
|
+
end
|
50
|
+
|
51
|
+
def filesize
|
52
|
+
@filesize ||= File.size(src_path)
|
53
|
+
end
|
54
|
+
|
55
|
+
def sha
|
56
|
+
@sha ||= Utils.generate_file_sha(src_path)
|
57
|
+
end
|
58
|
+
|
59
|
+
def sign
|
60
|
+
@sign ||= authorization.sign(bucket)
|
61
|
+
end
|
62
|
+
|
63
|
+
def slice_size
|
64
|
+
@slice_size ||= @result['slice_size'] || QcloudCos::DEFAULT_SLICE_SIZE
|
65
|
+
end
|
66
|
+
|
67
|
+
def session
|
68
|
+
@session ||= @result['session'] || options['session']
|
69
|
+
end
|
70
|
+
|
71
|
+
def retry_for(max_times, &block)
|
72
|
+
retry_times = 0
|
73
|
+
begin
|
74
|
+
yield if block_given?
|
75
|
+
rescue => e
|
76
|
+
retry_times += 1
|
77
|
+
retry if retry_times <= max_times
|
78
|
+
raise e
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
data/lib/qcloud_cos/version.rb
CHANGED
data/wiki/get_started.md
CHANGED
@@ -1082,5 +1082,5 @@ $ qcloud-cos download --bucket bucket2 /data/test/
|
|
1082
1082
|
|
1083
1083
|
## 其它资源
|
1084
1084
|
|
1085
|
-
+ [RDoc 文档](http://www.rubydoc.info/gems/qcloud_cos/0.
|
1085
|
+
+ [RDoc 文档](http://www.rubydoc.info/gems/qcloud_cos/0.4.1)
|
1086
1086
|
+ [腾讯 COS 详细文档](http://www.qcloud.com/doc/product/227/%E4%BA%A7%E5%93%81%E4%BB%8B%E7%BB%8D)
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: qcloud_cos
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.4.
|
4
|
+
version: 0.4.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Newell Zhu
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-01
|
11
|
+
date: 2016-02-01 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: httparty
|
@@ -208,6 +208,7 @@ files:
|
|
208
208
|
- lib/qcloud_cos/model/folder_object.rb
|
209
209
|
- lib/qcloud_cos/model/list.rb
|
210
210
|
- lib/qcloud_cos/model/objectable.rb
|
211
|
+
- lib/qcloud_cos/multipart.rb
|
211
212
|
- lib/qcloud_cos/utils.rb
|
212
213
|
- lib/qcloud_cos/version.rb
|
213
214
|
- qcloud_cos.gemspec
|