qcloud_cos 0.4.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +8 -0
- data/README.md +2 -2
- data/bin/qcloud-cos +7 -8
- data/lib/qcloud_cos.rb +1 -1
- data/lib/qcloud_cos/api.rb +14 -47
- data/lib/qcloud_cos/cli.rb +87 -89
- data/lib/qcloud_cos/convenient_api.rb +21 -2
- data/lib/qcloud_cos/model/folder_object.rb +12 -7
- data/lib/qcloud_cos/multipart.rb +82 -0
- data/lib/qcloud_cos/version.rb +1 -1
- data/wiki/get_started.md +1 -1
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 82a6d54335364ee45eb1d60e4fe33c02181d13f8
|
4
|
+
data.tar.gz: f3ee2074c3a196a7f46ce1ba2077fbb85d16a354
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b08e8867cd08cd0fba2eefec5754dcacc30633a68d8cbdca71af7b745db0006994e81379bbe25d94680b293cfb45c651d1a7afc0efc1a960d42feb20caa10934
|
7
|
+
data.tar.gz: 76f4f556cf7fe094c5db2d0515be9cc7d463902b115fc439fe8c79905ebe486510f4b306811a5c790da272d43012ae5c587f6d208930d0df8391a713055ad801
|
data/.rubocop.yml
CHANGED
data/README.md
CHANGED
@@ -54,7 +54,7 @@ Here is original Restful API, It has the most detailed and authoritative explana
|
|
54
54
|
|
55
55
|
Here is our RDoc Document, It's well format to help you find more detail about methods.
|
56
56
|
|
57
|
-
+ [RDoc Document](http://www.rubydoc.info/gems/qcloud_cos/0.
|
57
|
+
+ [RDoc Document](http://www.rubydoc.info/gems/qcloud_cos/0.4.1)
|
58
58
|
|
59
59
|
|
60
60
|
Here are some more guides for help you. Welcome to advice.
|
@@ -76,4 +76,4 @@ We use minitest for test and rubocop for Syntax checker, If you want to make con
|
|
76
76
|
|
77
77
|
## License
|
78
78
|
|
79
|
-
licensed under the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0.html)
|
79
|
+
licensed under the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0.html)
|
data/bin/qcloud-cos
CHANGED
@@ -10,10 +10,12 @@ program :description, 'command-line tool for Qcloud COS'
|
|
10
10
|
program :help, 'Author', 'Newell Zhu<zlx.star@gmail.com>'
|
11
11
|
default_command :help
|
12
12
|
|
13
|
+
global_option '-b', '--bucket Bucket Name', 'specify bucket name, it will override default bucket'
|
14
|
+
|
13
15
|
command :config do |c|
|
14
16
|
c.syntax = 'qcloud-cos config'
|
15
17
|
c.description = 'Init config, eg: qcloud-cos config'
|
16
|
-
c.action do |
|
18
|
+
c.action do |_args, _options|
|
17
19
|
QcloudCos::Cli.config
|
18
20
|
end
|
19
21
|
end
|
@@ -25,7 +27,6 @@ command :info do |c|
|
|
25
27
|
c.example 'Obtain information for /test/', '$ qcloud-cos info /test/'
|
26
28
|
c.example 'Obtain information for /production.log', '$ qcloud-cos info /production.log'
|
27
29
|
c.example 'Obtain information for /production.log from bucket2', '$ qcloud-cos info --bucket bucket2 /production.log'
|
28
|
-
c.option '--bucket Bucket Name', String, 'specify bucket name, it will override default bucket'
|
29
30
|
c.action do |args, options|
|
30
31
|
abort unless QcloudCos::Cli.environment_configed?
|
31
32
|
begin
|
@@ -44,13 +45,13 @@ command :list do |c|
|
|
44
45
|
c.example 'List all objects under /test/', '$ qcloud-cos list /test/'
|
45
46
|
c.example 'List first 10 objects under /test/', '$ qcloud-cos list --num 10 /test/'
|
46
47
|
c.example 'List all objects under / for bucket: bucket2', '$ qcloud-cos list --num 10 --bucket bucket2 /test/'
|
47
|
-
c.option '--bucket Bucket Name', String, 'specify bucket name, it will override default bucket'
|
48
48
|
c.option '--num Num', Integer, 'specify max objects, default is 100'
|
49
49
|
c.action do |args, options|
|
50
50
|
abort unless QcloudCos::Cli.environment_configed?
|
51
|
+
options.default num: 100
|
51
52
|
begin
|
52
53
|
cli = QcloudCos::Cli.init
|
53
|
-
cli.list(args, options).map{|path| puts path }
|
54
|
+
cli.list(args, options).map { |path| puts path }
|
54
55
|
rescue => e
|
55
56
|
say_error e.message
|
56
57
|
end
|
@@ -65,11 +66,11 @@ command :upload do |c|
|
|
65
66
|
c.example 'Upload all files under ./test/ to /data/', '$ qcloud-cos upload test/ /data/'
|
66
67
|
c.example 'Upload all files under ./test/ to /data/ with other bucket: bucket2', '$ qcloud-cos upload --bucket bucket2 test/ /data/'
|
67
68
|
c.example 'Upload production.log to /data/ with slice_size and min', '$ qcloud-cos upload --size 10 --min 100 production.log /data/'
|
68
|
-
c.option '--
|
69
|
-
c.option '--size Slice Size', Integer, "specify slice size for slice upload in Bytes, default: #{QcloudCos::DEFAULT_SLICE_SIZE}(#{QcloudCos::DEFAULT_SLICE_SIZE/1024/1024}M)"
|
69
|
+
c.option '--size Slice Size', Integer, "specify slice size for slice upload in Bytes, default: #{QcloudCos::DEFAULT_SLICE_SIZE}(#{QcloudCos::DEFAULT_SLICE_SIZE / 1024 / 1024}M)"
|
70
70
|
c.option '--min Min Slice File Size', Integer, "specify min slice file size in Bytes, default: default: #{QcloudCos::MIN_SLICE_FILE_SIZE * 1024 * 1024}(#{QcloudCos::MIN_SLICE_FILE_SIZE}M)"
|
71
71
|
c.action do |args, options|
|
72
72
|
abort unless QcloudCos::Cli.environment_configed?
|
73
|
+
options.default size: QcloudCos::DEFAULT_SLICE_SIZE, min: QcloudCos::MIN_SLICE_FILE_SIZE * 1024 * 1024
|
73
74
|
begin
|
74
75
|
cli = QcloudCos::Cli.init
|
75
76
|
cli.upload(args, options)
|
@@ -86,7 +87,6 @@ command :download do |c|
|
|
86
87
|
c.example 'Download file from /data/production.log and save under ./data/', '$ qcloud-cos download /data/production.log ./data'
|
87
88
|
c.example 'Download whole folder /data/test/ and save under ./data/', '$ qcloud-cos download /data/test/ ./data'
|
88
89
|
c.example 'Download whole folder /data/test/ from bucket2 and save under ./data/', '$ qcloud-cos download --bucket bucket2 /data/test/ ./data'
|
89
|
-
c.option '--bucket Bucket Name', String, 'specify bucket name, it will override default bucket'
|
90
90
|
c.action do |args, options|
|
91
91
|
abort unless QcloudCos::Cli.environment_configed?
|
92
92
|
begin
|
@@ -105,7 +105,6 @@ command :remove do |c|
|
|
105
105
|
c.example 'Remove folder /data/test/', '$ qcloud-cos remove /data/test/'
|
106
106
|
c.example 'Remove folder /data/test/ in recursive', '$ qcloud-cos remove --recursive /data/test/'
|
107
107
|
c.example 'Remove folder /data/test/ from bucket2', '$ qcloud-cos download --bucket bucket2 /data/test/'
|
108
|
-
c.option '--bucket Bucket Name', String, 'specify bucket name, it will override default bucket'
|
109
108
|
c.option '--[no-]recursive', 'specify recursive or not when remove folder'
|
110
109
|
c.action do |args, options|
|
111
110
|
abort unless QcloudCos::Cli.environment_configed?
|
data/lib/qcloud_cos.rb
CHANGED
data/lib/qcloud_cos/api.rb
CHANGED
@@ -1,10 +1,11 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
require 'qcloud_cos/utils'
|
3
|
+
require 'qcloud_cos/multipart'
|
3
4
|
require 'qcloud_cos/model/list'
|
4
5
|
|
5
6
|
module QcloudCos
|
6
7
|
module Api
|
7
|
-
#
|
8
|
+
# 列出文件或者目录
|
8
9
|
#
|
9
10
|
# @param path [String] 指定目标路径, 以 / 结尾, 则列出该目录下文件或者文件夹,不以 / 结尾,就搜索该前缀的文件或者文件夹
|
10
11
|
# @param options [Hash] 额外参数
|
@@ -135,42 +136,13 @@ module QcloudCos
|
|
135
136
|
fail FileNotExistError unless File.exist?(src_path)
|
136
137
|
bucket = validates(dst_path, options)
|
137
138
|
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
return data if data.key?('url') # 妙传命中
|
146
|
-
|
147
|
-
slice_size = data['slice_size'] || DEFAULT_SLICE_SIZE
|
148
|
-
session = data['session'] || options['session']
|
149
|
-
offset = data['offset'] || 0
|
150
|
-
|
151
|
-
fail MissingSessionIdError unless session
|
152
|
-
|
153
|
-
while offset < filesize
|
154
|
-
filecontent = IO.read(src_path, slice_size, offset)
|
155
|
-
|
156
|
-
retry_times = 0
|
157
|
-
begin
|
158
|
-
result = upload_part(dst_path, session, offset, filecontent, options)
|
159
|
-
progress = [offset + slice_size, filesize].min
|
160
|
-
yield((progress.to_f/filesize).round(2)) if block_given?
|
161
|
-
|
162
|
-
if result.key?('data') && result['data'].key?('session')
|
163
|
-
session = result['data']['session']
|
164
|
-
elsif result.key?('data') && result['data'].key?('url')
|
165
|
-
return result['data']
|
166
|
-
end
|
167
|
-
rescue => e
|
168
|
-
retry_times += 1
|
169
|
-
retry if retry_times <= config.max_retry_times
|
170
|
-
raise e
|
171
|
-
end
|
172
|
-
offset += slice_size
|
173
|
-
end
|
139
|
+
multipart = QcloudCos::Multipart.new(
|
140
|
+
dst_path,
|
141
|
+
src_path,
|
142
|
+
options.merge(bucket: bucket, authorization: authorization)
|
143
|
+
)
|
144
|
+
multipart.upload(&block)
|
145
|
+
multipart.result
|
174
146
|
end
|
175
147
|
|
176
148
|
# 初始化分片上传
|
@@ -279,17 +251,12 @@ module QcloudCos
|
|
279
251
|
|
280
252
|
return delete(path, options) if options['recursive'] != true
|
281
253
|
|
282
|
-
|
283
|
-
|
284
|
-
|
285
|
-
|
286
|
-
|
287
|
-
elsif object.is_a?(QcloudCos::FileObject)
|
288
|
-
delete_file("#{path}#{object.name}", options)
|
289
|
-
end
|
254
|
+
all(path, options).each do |object|
|
255
|
+
if object.is_a?(QcloudCos::FolderObject)
|
256
|
+
delete_folder("#{path}#{object.name}/", options)
|
257
|
+
elsif object.is_a?(QcloudCos::FileObject)
|
258
|
+
delete_file("#{path}#{object.name}", options)
|
290
259
|
end
|
291
|
-
break unless objects.has_more
|
292
|
-
options['context'] = objects.context
|
293
260
|
end
|
294
261
|
delete(path)
|
295
262
|
end
|
data/lib/qcloud_cos/cli.rb
CHANGED
@@ -21,30 +21,23 @@ module QcloudCos
|
|
21
21
|
# 交互模式配置环境
|
22
22
|
# 命令: $ qcloud-cos config
|
23
23
|
def self.config(config_path = nil)
|
24
|
-
config_path
|
24
|
+
config_path ||= QcloudCos::QCLOUD_COS_CONFIG
|
25
25
|
return Commander::UI.say_error("#{config_path} already exist, remove it first or direct edit it!") if File.exist?(config_path)
|
26
26
|
|
27
27
|
app_id = Commander::UI.ask 'Qcloud COS APP ID: '
|
28
|
-
return Commander::UI.say_error(
|
28
|
+
return Commander::UI.say_error('Missing Qcloud COS APP ID') if app_id.empty?
|
29
29
|
|
30
30
|
secret_id = Commander::UI.ask 'Qcloud COS Secret ID: '
|
31
|
-
return Commander::UI.say_error(
|
31
|
+
return Commander::UI.say_error('Missing Qcloud COS Secret ID') if secret_id.empty?
|
32
32
|
|
33
33
|
secret_key = Commander::UI.ask 'Qcloud COS Secret Key: '
|
34
|
-
return Commander::UI.say_error(
|
34
|
+
return Commander::UI.say_error('Missing Qcloud COS Secret Key') if secret_key.empty?
|
35
35
|
|
36
36
|
endpoint = Commander::UI.ask "Default Qcloud COS Endpoint [#{QcloudCos::DEFAULT_ENDPOINT}]: "
|
37
37
|
endpoint = QcloudCos::DEFAULT_ENDPOINT if endpoint.empty?
|
38
38
|
bucket = Commander::UI.ask 'Default Qcloud COS Bucket: '
|
39
39
|
|
40
|
-
write_config(
|
41
|
-
config_path,
|
42
|
-
app_id: app_id,
|
43
|
-
secret_id: secret_id,
|
44
|
-
secret_key: secret_key,
|
45
|
-
endpoint: endpoint,
|
46
|
-
bucket: bucket
|
47
|
-
)
|
40
|
+
write_config(config_path, app_id: app_id, secret_id: secret_id, secret_key: secret_key, endpoint: endpoint, bucket: bucket)
|
48
41
|
end
|
49
42
|
|
50
43
|
# 检查环境是否配置
|
@@ -52,7 +45,7 @@ module QcloudCos
|
|
52
45
|
# @return [Boolean]
|
53
46
|
def self.environment_configed?
|
54
47
|
configed = File.exist?(QcloudCos::QCLOUD_COS_CONFIG) || !ENV['QCLOUD_COS_APP_ID'].to_s.empty?
|
55
|
-
Commander::UI.say_error(
|
48
|
+
Commander::UI.say_error('Use `qcloud-cos config` first or export your environments') unless configed
|
56
49
|
configed
|
57
50
|
end
|
58
51
|
|
@@ -62,7 +55,7 @@ module QcloudCos
|
|
62
55
|
load_config.each { |k, v| config.send("#{k}=", v) }
|
63
56
|
end
|
64
57
|
|
65
|
-
|
58
|
+
new
|
66
59
|
end
|
67
60
|
|
68
61
|
# 查看信息
|
@@ -82,7 +75,7 @@ module QcloudCos
|
|
82
75
|
# qcloud-cos info --bucket bucket2 /production.log
|
83
76
|
def info(args, options)
|
84
77
|
path = args.shift || '/'
|
85
|
-
opts =
|
78
|
+
opts = parse_options(options)
|
86
79
|
|
87
80
|
QcloudCos.stat(path, opts)['data']
|
88
81
|
end
|
@@ -106,8 +99,7 @@ module QcloudCos
|
|
106
99
|
#
|
107
100
|
def list(args, options)
|
108
101
|
path = args.shift || '/'
|
109
|
-
opts =
|
110
|
-
opts[:num] = options.num || 100
|
102
|
+
opts = parse_options(options)
|
111
103
|
|
112
104
|
objects = QcloudCos.list(path, opts)
|
113
105
|
objects.map do |object|
|
@@ -133,20 +125,16 @@ module QcloudCos
|
|
133
125
|
# qcloud-cos upload --bucket bucket2 test/ /data/
|
134
126
|
def upload(args, options)
|
135
127
|
path = args.shift
|
136
|
-
return Commander::UI.say_error(
|
128
|
+
return Commander::UI.say_error('file missing, see example: $ qcloud-cos upload -h') unless path
|
137
129
|
return Commander::UI.say_error("file #{path} not exist") unless File.exist?(path)
|
138
130
|
|
139
131
|
dest_path = args.shift || '/'
|
140
|
-
return Commander::UI.say_error(
|
141
|
-
|
142
|
-
opts = parse(options)
|
143
|
-
opts[:slice_size] = options.size || QcloudCos::DEFAULT_SLICE_SIZE
|
144
|
-
opts[:min] = options.min || QcloudCos::MIN_SLICE_FILE_SIZE * 1024 * 1024
|
132
|
+
return Commander::UI.say_error('dest_path must end with /, see example: $ qcloud-cos upload -h') unless dest_path.end_with?('/')
|
145
133
|
|
146
134
|
if path.end_with?('/')
|
147
|
-
upload_folder(path, dest_path,
|
135
|
+
upload_folder(path, dest_path, parse_options(options))
|
148
136
|
else
|
149
|
-
upload_file(path, dest_path,
|
137
|
+
upload_file(path, dest_path, parse_options(options))
|
150
138
|
end
|
151
139
|
end
|
152
140
|
|
@@ -169,14 +157,14 @@ module QcloudCos
|
|
169
157
|
#
|
170
158
|
def download(args, options)
|
171
159
|
path = args.shift
|
172
|
-
return Commander::UI.say_error(
|
173
|
-
opts =
|
174
|
-
|
160
|
+
return Commander::UI.say_error('missing path, see example: $ qcloud-cos download -h') unless path
|
161
|
+
opts = parse_options(options)
|
162
|
+
save_path = args.shift || '.'
|
175
163
|
|
176
164
|
if path.end_with?('/')
|
177
|
-
download_folder(path, opts)
|
165
|
+
download_folder(path, save_path, opts)
|
178
166
|
else
|
179
|
-
download_file(path, opts)
|
167
|
+
download_file(path, save_path, opts)
|
180
168
|
end
|
181
169
|
end
|
182
170
|
|
@@ -195,12 +183,11 @@ module QcloudCos
|
|
195
183
|
# qcloud-cos remove --recursive /data/test/
|
196
184
|
#
|
197
185
|
# # 删除 bucket2 下面的目录 /data/test/
|
198
|
-
# qcloud-cos
|
186
|
+
# qcloud-cos remove --bucket bucket2 /data/test/
|
199
187
|
def remove(args, options)
|
200
188
|
path = args.shift
|
201
|
-
return Commander::UI.say_error(
|
202
|
-
|
203
|
-
opts = parse(options)
|
189
|
+
return Commander::UI.say_error('missing dest_path, see example: $ qcloud-cos remove -h') unless path
|
190
|
+
opts = parse_options(options)
|
204
191
|
|
205
192
|
if path.end_with?('/')
|
206
193
|
QcloudCos.delete_folder(path, opts.merge(recursive: !!options.recursive))
|
@@ -213,47 +200,61 @@ module QcloudCos
|
|
213
200
|
|
214
201
|
def upload_file(path, dest_path, opts)
|
215
202
|
dest_path = File.join(dest_path, path.split('/').last)
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
220
|
-
|
203
|
+
|
204
|
+
create_file_with_lint(path, dest_path) do
|
205
|
+
if File.size(path) > opts[:min]
|
206
|
+
QcloudCos.upload_slice(dest_path, path, Utils.hash_slice(opts, :bucket).merge(slice_size: opts[:size]))
|
207
|
+
else
|
208
|
+
QcloudCos.upload(dest_path, File.new(path), Utils.hash_slice(opts, :bucket))
|
209
|
+
end
|
221
210
|
end
|
211
|
+
end
|
212
|
+
|
213
|
+
def create_file_with_lint(path, dest_path, &block)
|
214
|
+
yield if block_given?
|
222
215
|
Commander::UI.say_ok "#{path} uploaded to #{dest_path}..."
|
223
216
|
rescue => e
|
224
217
|
Commander::UI.say_error "Failed when uploaded #{path} ===> #{e.message}"
|
225
|
-
ensure
|
226
|
-
file && file.close
|
227
218
|
end
|
228
219
|
|
229
220
|
def upload_folder(path, dest_path, opts)
|
230
|
-
|
231
|
-
Dir.glob("#{path}**/*") do |file_path|
|
232
|
-
split_path = file_path.sub(path, '').split('/')
|
233
|
-
new_path =
|
234
|
-
if File.file?(file_path)
|
235
|
-
split_path.size > 1 ? File.join(dest_path, *split_path[0..-2]) : dest_path
|
236
|
-
else
|
237
|
-
file_path = File.join(file_path, '')
|
238
|
-
File.join(dest_path, *split_path, '')
|
239
|
-
end
|
240
|
-
|
241
|
-
file_path_map[file_path] = new_path
|
242
|
-
end
|
243
|
-
remove_subdirectories(file_path_map)
|
221
|
+
path_map = find_upload_path_map(path, dest_path)
|
244
222
|
|
245
|
-
|
223
|
+
path_map.each do |file_path, dest|
|
246
224
|
if file_path.end_with?('/')
|
247
|
-
|
248
|
-
Commander::UI.say_ok "Create folder #{dest_path}"
|
225
|
+
create_folder_with_lint(dest, opts)
|
249
226
|
else
|
250
|
-
upload_file(file_path,
|
227
|
+
upload_file(file_path, dest, opts)
|
251
228
|
end
|
252
229
|
end
|
253
230
|
end
|
254
231
|
|
255
|
-
def
|
256
|
-
|
232
|
+
def create_folder_with_lint(dest, opts)
|
233
|
+
QcloudCos.create_folder(dest, opts)
|
234
|
+
Commander::UI.say_ok "Create folder #{dest}"
|
235
|
+
rescue => e
|
236
|
+
Commander::UI.say_ok "Failed when create folder #{dest} ====> #{e.message}"
|
237
|
+
end
|
238
|
+
|
239
|
+
def find_upload_path_map(path, dest_path)
|
240
|
+
path_map = Hash[Dir.glob("#{path}**/*").map do |file_path|
|
241
|
+
file_path = File.join(file_path, '') unless File.file?(file_path)
|
242
|
+
[file_path, find_upload_dest_path(file_path, path, dest_path)]
|
243
|
+
end]
|
244
|
+
remove_subdirectories!(path_map)
|
245
|
+
path_map
|
246
|
+
end
|
247
|
+
|
248
|
+
def find_upload_dest_path(file_path, parent_path, dest_path)
|
249
|
+
split_path = file_path.sub(parent_path, '').split('/')
|
250
|
+
if File.file?(file_path)
|
251
|
+
split_path.size > 1 ? File.join(dest_path, *split_path[0..-2]) : dest_path
|
252
|
+
else
|
253
|
+
File.join(dest_path, *split_path, '')
|
254
|
+
end
|
255
|
+
end
|
256
|
+
|
257
|
+
def download_file(path, save_path, opts)
|
257
258
|
FileUtils.mkdir_p(save_path) unless File.exist?(save_path)
|
258
259
|
|
259
260
|
file_path = File.join(save_path, path.split('/').last)
|
@@ -264,31 +265,26 @@ module QcloudCos
|
|
264
265
|
Commander::UI.say_error("Failed when Download #{path} ===> #{e.message}")
|
265
266
|
end
|
266
267
|
|
267
|
-
def download_folder(path, opts)
|
268
|
-
|
269
|
-
|
270
|
-
|
271
|
-
|
272
|
-
|
273
|
-
|
274
|
-
|
275
|
-
|
276
|
-
|
277
|
-
end
|
278
|
-
download_folder("#{path}#{object.name}/", opts.merge(save_path: new_path))
|
279
|
-
elsif object.is_a?(QcloudCos::FileObject)
|
280
|
-
if object.access_url
|
281
|
-
download_file("#{path}#{object.name}", opts.merge(save_path: save_path, access_url: object.access_url))
|
282
|
-
else
|
283
|
-
download_file("#{path}#{object.name}", opts.merge(save_path: save_path))
|
284
|
-
end
|
285
|
-
end
|
268
|
+
def download_folder(path, save_path, opts)
|
269
|
+
mkdir_with_lint(path, save_path)
|
270
|
+
|
271
|
+
QcloudCos.all(path, opts).each do |object|
|
272
|
+
new_path = "#{path}#{object.name}"
|
273
|
+
|
274
|
+
if object.is_a?(QcloudCos::FolderObject)
|
275
|
+
download_folder(File.join(new_path, ''), File.join(save_path, object.name), opts)
|
276
|
+
elsif object.is_a?(QcloudCos::FileObject)
|
277
|
+
download_file(new_path, save_path, opts.merge(access_url: object.access_url))
|
286
278
|
end
|
287
|
-
break unless objects.has_more
|
288
|
-
opts['context'] = objects.context
|
289
279
|
end
|
290
280
|
end
|
291
281
|
|
282
|
+
def mkdir_with_lint(path, save_path)
|
283
|
+
return if File.exist?(save_path)
|
284
|
+
FileUtils.mkdir_p(save_path)
|
285
|
+
Commander::UI.say_ok "Save #{path} to #{save_path}/"
|
286
|
+
end
|
287
|
+
|
292
288
|
def self.load_config
|
293
289
|
file_config = load_file_config
|
294
290
|
|
@@ -308,13 +304,16 @@ module QcloudCos
|
|
308
304
|
end.compact]
|
309
305
|
end
|
310
306
|
|
311
|
-
def
|
307
|
+
def parse_options(options)
|
312
308
|
opts = {}
|
313
309
|
opts[:bucket] = options.bucket if options.bucket
|
310
|
+
opts[:min] = options.min if options.min
|
311
|
+
opts[:size] = options.size if options.size
|
312
|
+
opts[:num] = options.num if options.num
|
314
313
|
opts
|
315
314
|
end
|
316
315
|
|
317
|
-
def remove_subdirectories(path_map)
|
316
|
+
def remove_subdirectories!(path_map)
|
318
317
|
new_map = path_map.dup
|
319
318
|
new_map.each do |_, dest_path|
|
320
319
|
path_map.reject! do |file_path, dest|
|
@@ -324,22 +323,21 @@ module QcloudCos
|
|
324
323
|
end
|
325
324
|
|
326
325
|
def save_to_file(access_url, file_path)
|
327
|
-
File.open(file_path,
|
326
|
+
File.open(file_path, 'wb') do |f|
|
328
327
|
f.write HTTParty.get(access_url)
|
329
328
|
end
|
330
329
|
end
|
331
330
|
|
332
331
|
def self.write_config(config_path, options)
|
333
|
-
File.open(config_path,
|
332
|
+
File.open(config_path, 'w') do |file|
|
334
333
|
file.puts "app_id=#{options[:app_id]}"
|
335
334
|
file.puts "secret_id=#{options[:secret_id]}"
|
336
335
|
file.puts "secret_key=#{options[:secret_key]}"
|
337
336
|
file.puts "endpoint=#{options[:endpoint]}"
|
338
337
|
file.puts "bucket=#{options[:bucket]}"
|
339
|
-
file.puts
|
340
|
-
file.puts
|
338
|
+
file.puts 'ssl_ca_file='
|
339
|
+
file.puts 'max_retry_times='
|
341
340
|
end
|
342
341
|
end
|
343
|
-
|
344
342
|
end
|
345
343
|
end
|
@@ -1,13 +1,12 @@
|
|
1
1
|
module QcloudCos
|
2
2
|
module ConvenientApi
|
3
|
-
|
4
3
|
# 获取 Bucket 信息
|
5
4
|
#
|
6
5
|
# @param bucket_name [String] :bucket (config.bucket) 指定当前 bucket, 默认是配置里面的 bucket
|
7
6
|
#
|
8
7
|
# @return [Hash] 返回 Bucket 信息
|
9
8
|
def bucket_info(bucket_name = nil)
|
10
|
-
bucket_name
|
9
|
+
bucket_name ||= config.bucket
|
11
10
|
stat('/', bucket: bucket_name)['data']
|
12
11
|
rescue
|
13
12
|
{}
|
@@ -104,5 +103,25 @@ module QcloudCos
|
|
104
103
|
fail FileNotExistError
|
105
104
|
end
|
106
105
|
end
|
106
|
+
|
107
|
+
# 列出所有文件或者目录
|
108
|
+
#
|
109
|
+
# @param path [String] 指定目标路径, 以 / 结尾, 则列出该目录下文件或者文件夹,不以 / 结尾,就搜索该前缀的文件或者文件夹
|
110
|
+
# @param options [Hash] 额外参数
|
111
|
+
# @option options [String] :bucket (config.bucket_name) 指定当前 bucket, 默认是配置里面的 bucket
|
112
|
+
# @option options [String] :pattern (eListBoth) 指定拉取的内容,可选值: eListBoth, eListDirOnly, eListFileOnly
|
113
|
+
# @option options [Integer] :order (0) 指定拉取文件的顺序, 默认为正序(=0), 可选值: 0, 1
|
114
|
+
#
|
115
|
+
# @return [Hash]
|
116
|
+
def all(path, options = {})
|
117
|
+
results = []
|
118
|
+
loop do
|
119
|
+
objects = QcloudCos.list(path, options)
|
120
|
+
results += objects.to_a
|
121
|
+
break unless objects.has_more
|
122
|
+
options['context'] = objects.context
|
123
|
+
end
|
124
|
+
results
|
125
|
+
end
|
107
126
|
end
|
108
127
|
end
|
@@ -22,13 +22,18 @@ module QcloudCos
|
|
22
22
|
if !path.end_with?('/')
|
23
23
|
fail InvalidFolderPathError, '文件夹路径必须以 / 结尾'
|
24
24
|
elsif !(names = path.split('/')).empty?
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
25
|
+
validate_name(names)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
# 校验文件夹名字
|
30
|
+
def self.validate_name(names)
|
31
|
+
if names.detect { |name| RETAINED_FIELDS.include?(name.downcase) }
|
32
|
+
fail InvalidFolderPathError, %(文件夹名字不能是保留字段: '#{RETAINED_FIELDS.join("', '")}')
|
33
|
+
elsif names.detect { |name| name.match(/[\/?*:|\\<>"]/) }
|
34
|
+
fail InvalidFolderPathError, %(文件夹名字不能包含保留字符: '#{RETAINED_SYMBOLS.join("', '")}')
|
35
|
+
elsif names.detect { |name| name.length > MAXLENGTH }
|
36
|
+
fail InvalidFolderPathError, %(文件夹名字不能超过#{MAXLENGTH}个字符)
|
32
37
|
end
|
33
38
|
end
|
34
39
|
end
|
@@ -0,0 +1,82 @@
|
|
1
|
+
module QcloudCos
|
2
|
+
class Multipart
|
3
|
+
attr_reader :dst_path, :src_path, :bucket, :authorization, :result, :options
|
4
|
+
|
5
|
+
def initialize(dst_path, src_path, options = {})
|
6
|
+
@dst_path = dst_path
|
7
|
+
@src_path = src_path
|
8
|
+
@options = options
|
9
|
+
@bucket = options.delete(:bucket)
|
10
|
+
@authorization = options.delete(:authorization)
|
11
|
+
end
|
12
|
+
|
13
|
+
def upload(&block)
|
14
|
+
init_multipart
|
15
|
+
return if complete?
|
16
|
+
fail QcloudCos::MissingSessionIdError unless session
|
17
|
+
|
18
|
+
offset = @result['offset'] || 0
|
19
|
+
|
20
|
+
while offset < filesize
|
21
|
+
filecontent = IO.read(src_path, slice_size, offset)
|
22
|
+
break if upload_part(offset, filecontent, &block)
|
23
|
+
offset += slice_size
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
private
|
28
|
+
|
29
|
+
def init_multipart
|
30
|
+
@result ||= QcloudCos.init_slice_upload(dst_path, filesize, sha, options.merge('sign' => sign))['data']
|
31
|
+
end
|
32
|
+
|
33
|
+
def upload_part(offset, content, &block)
|
34
|
+
retry_for(QcloudCos.config.max_retry_times) do
|
35
|
+
@result = QcloudCos.upload_part(dst_path, session, offset, content, options)['data']
|
36
|
+
notify_progress(offset + slice_size, &block)
|
37
|
+
|
38
|
+
return true if complete?
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def complete?
|
43
|
+
@result.key?('url')
|
44
|
+
end
|
45
|
+
|
46
|
+
def notify_progress(progress, &block)
|
47
|
+
progress = [progress, filesize].min
|
48
|
+
yield((progress.to_f / filesize).round(2)) if block_given?
|
49
|
+
end
|
50
|
+
|
51
|
+
def filesize
|
52
|
+
@filesize ||= File.size(src_path)
|
53
|
+
end
|
54
|
+
|
55
|
+
def sha
|
56
|
+
@sha ||= Utils.generate_file_sha(src_path)
|
57
|
+
end
|
58
|
+
|
59
|
+
def sign
|
60
|
+
@sign ||= authorization.sign(bucket)
|
61
|
+
end
|
62
|
+
|
63
|
+
def slice_size
|
64
|
+
@slice_size ||= @result['slice_size'] || QcloudCos::DEFAULT_SLICE_SIZE
|
65
|
+
end
|
66
|
+
|
67
|
+
def session
|
68
|
+
@session ||= @result['session'] || options['session']
|
69
|
+
end
|
70
|
+
|
71
|
+
def retry_for(max_times, &block)
|
72
|
+
retry_times = 0
|
73
|
+
begin
|
74
|
+
yield if block_given?
|
75
|
+
rescue => e
|
76
|
+
retry_times += 1
|
77
|
+
retry if retry_times <= max_times
|
78
|
+
raise e
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
data/lib/qcloud_cos/version.rb
CHANGED
data/wiki/get_started.md
CHANGED
@@ -1082,5 +1082,5 @@ $ qcloud-cos download --bucket bucket2 /data/test/
|
|
1082
1082
|
|
1083
1083
|
## 其它资源
|
1084
1084
|
|
1085
|
-
+ [RDoc 文档](http://www.rubydoc.info/gems/qcloud_cos/0.
|
1085
|
+
+ [RDoc 文档](http://www.rubydoc.info/gems/qcloud_cos/0.4.1)
|
1086
1086
|
+ [腾讯 COS 详细文档](http://www.qcloud.com/doc/product/227/%E4%BA%A7%E5%93%81%E4%BB%8B%E7%BB%8D)
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: qcloud_cos
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.4.
|
4
|
+
version: 0.4.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Newell Zhu
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-01
|
11
|
+
date: 2016-02-01 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: httparty
|
@@ -208,6 +208,7 @@ files:
|
|
208
208
|
- lib/qcloud_cos/model/folder_object.rb
|
209
209
|
- lib/qcloud_cos/model/list.rb
|
210
210
|
- lib/qcloud_cos/model/objectable.rb
|
211
|
+
- lib/qcloud_cos/multipart.rb
|
211
212
|
- lib/qcloud_cos/utils.rb
|
212
213
|
- lib/qcloud_cos/version.rb
|
213
214
|
- qcloud_cos.gemspec
|