rabbit_swift 0.2.8 → 0.2.9
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile +0 -2
- data/RELEASE.md +11 -0
- data/bin/slo_client.rb +95 -0
- data/lib/rabbit_swift.rb +1 -0
- data/lib/rabbit_swift/client.rb +37 -4
- data/lib/rabbit_swift/large_object/static_large_object.rb +50 -0
- data/lib/rabbit_swift/version.rb +1 -1
- data/rabbit_swift.gemspec +3 -2
- metadata +24 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7570788a09d3209ca2aa282b96282eb050b43e13
|
4
|
+
data.tar.gz: 3510fd4456248d641eae7997db7ef7eea2620dd9
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 62c332ae6f2bc9f5da004522fdd0d9414ca6154a7775d894b042ba3e101e26cdad09e05516bf4566031a6b93a600e5bba0510d494634f8123cc5b0e7ac46baa6
|
7
|
+
data.tar.gz: ed758fe16687b2cee36750d4aa1862ae14ed09f6b07eb5232c8147a3e20554065d6877c530876bb3d0df5a9b893da7e7c70534cee7e226468de5f1b65f2f23b3
|
data/Gemfile
CHANGED
data/RELEASE.md
ADDED
data/bin/slo_client.rb
ADDED
@@ -0,0 +1,95 @@
|
|
1
|
+
require 'json'
|
2
|
+
require 'optparse'
|
3
|
+
require 'rabbit_swift'
|
4
|
+
|
5
|
+
#ruby -I./lib/ this.rb
|
6
|
+
#bundle exec ruby -I./lib bin/slo_client.rb -s ~/Downloads/test.zip -d /test -c ../chino/conf/conf.json -l 1048576
|
7
|
+
=begin
|
8
|
+
-c conf.json
|
9
|
+
{
|
10
|
+
"swift" : {
|
11
|
+
"auth_url" : "", //必須
|
12
|
+
"tenantName" : "", //必須
|
13
|
+
"username" : "", //必須
|
14
|
+
"password" : "", //必須
|
15
|
+
"endPoint" : "" //必須ではない、末尾に/はつけない
|
16
|
+
}
|
17
|
+
}
|
18
|
+
=end
|
19
|
+
|
20
|
+
opt = OptionParser.new
|
21
|
+
Version = "1.0.0"
|
22
|
+
|
23
|
+
src_path = nil
|
24
|
+
dest_path = nil
|
25
|
+
conf_path = nil
|
26
|
+
limit_file_size = nil
|
27
|
+
opt.on('-s SRC_PATH', 'src_file_path') {|v| src_path = v}
|
28
|
+
opt.on('-d DEST_PATH', 'dest_path') {|v| dest_path = v}
|
29
|
+
opt.on('-c CONF_PATH', 'conf_path') {|v| conf_path = v}
|
30
|
+
opt.on('-l LIMIT_BYTE', 'limit_file_size') {|v| limit_file_size = v}
|
31
|
+
opt.parse!(ARGV)
|
32
|
+
|
33
|
+
|
34
|
+
class Slo_client
|
35
|
+
|
36
|
+
attr_accessor :src_path, :dest_path, :conf_path, :swift, :token, :limit_file_size, :original_dest_path
|
37
|
+
|
38
|
+
def initialize(src_path, dest_path, conf_path, send_timeout: 7200, limit_file_size: RabbitSwift::LargeObject::StaticLargeObject::LIMIT_FILE_SIZE)
|
39
|
+
@src_path = src_path
|
40
|
+
@dest_path = dest_path
|
41
|
+
@original_dest_path = dest_path
|
42
|
+
@conf_path = conf_path
|
43
|
+
@limit_file_size = limit_file_size
|
44
|
+
File.open conf_path do |file|
|
45
|
+
conf = JSON.load(file.read)
|
46
|
+
@swift = conf['swift']
|
47
|
+
|
48
|
+
if @dest_path =~ /^\//
|
49
|
+
@dest_path = @swift['endPoint'] + dest_path
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
@swift['send_timeout'] = send_timeout
|
54
|
+
end
|
55
|
+
|
56
|
+
def upload
|
57
|
+
slo = RabbitSwift::LargeObject::StaticLargeObject.new(src_path, dest_path, limit_file_size: @limit_file_size)
|
58
|
+
|
59
|
+
# (指定されたファイルサイズ単位で)ファイルを分割する
|
60
|
+
rabbit_file_split = slo.split
|
61
|
+
# JSONマニフェストファイルをつくる
|
62
|
+
manifest_json = slo.create_manifest_list(rabbit_file_split.file_list)
|
63
|
+
puts manifest_json
|
64
|
+
|
65
|
+
rabbit_swift_client = RabbitSwift::Client.new(@swift)
|
66
|
+
@token = rabbit_swift_client.get_token
|
67
|
+
puts 'token -> ' + token
|
68
|
+
|
69
|
+
#ファイルを全てアップロード(with etag)
|
70
|
+
rabbit_file_split.file_list.each do |file_path|
|
71
|
+
status = rabbit_swift_client.upload(token, dest_path, file_path)
|
72
|
+
puts file_path
|
73
|
+
puts 'http_status -> ' + status.to_s
|
74
|
+
if (status == RabbitSwift::Client::UPLOAD_SUCCESS_HTTP_STATUS_CODE)
|
75
|
+
puts 'upload OK'
|
76
|
+
else
|
77
|
+
puts 'upload NG'
|
78
|
+
return
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
puts "dest_path->" + dest_path
|
83
|
+
#マニフェストをアップロード
|
84
|
+
rabbit_swift_client.upload_manifest(token, dest_path, @original_dest_path, @src_path, manifest_json)
|
85
|
+
|
86
|
+
#分割したファイルを削除
|
87
|
+
rabbit_file_split.delete_all
|
88
|
+
|
89
|
+
end
|
90
|
+
|
91
|
+
end
|
92
|
+
|
93
|
+
|
94
|
+
slo = Slo_client.new(src_path,dest_path,conf_path, limit_file_size: limit_file_size)
|
95
|
+
slo.upload
|
data/lib/rabbit_swift.rb
CHANGED
data/lib/rabbit_swift/client.rb
CHANGED
@@ -4,6 +4,7 @@ require 'json'
|
|
4
4
|
require 'httpclient'
|
5
5
|
require 'pathname'
|
6
6
|
require 'mime/types'
|
7
|
+
require 'digest/md5'
|
7
8
|
|
8
9
|
module RabbitSwift
|
9
10
|
|
@@ -82,10 +83,16 @@ module RabbitSwift
|
|
82
83
|
#相対パスがきた時のために絶対パスに変換
|
83
84
|
path_name_obj = Pathname.new(input_file_path);
|
84
85
|
file_path = path_name_obj.expand_path.to_s
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
86
|
+
mime = MIME::Types.type_for(file_path)[0]
|
87
|
+
auth_header =
|
88
|
+
if mime.nil?
|
89
|
+
{'X-Auth-Token' => token}
|
90
|
+
else
|
91
|
+
{'X-Auth-Token' => token,
|
92
|
+
'Content-Type' => MIME::Types.type_for(file_path)[0].to_s
|
93
|
+
}
|
94
|
+
end
|
95
|
+
|
89
96
|
auth_header['X-Web-Mode'] = 'true' if @web_mode
|
90
97
|
|
91
98
|
http_client = HTTPClient.new
|
@@ -143,6 +150,32 @@ module RabbitSwift
|
|
143
150
|
@res.status
|
144
151
|
end
|
145
152
|
|
153
|
+
def upload_manifest(token, end_point, dest_container,input_file_path, manifest_json)
|
154
|
+
#相対パスがきた時のために絶対パスに変換
|
155
|
+
path_name_obj = Pathname.new(input_file_path);
|
156
|
+
file_path = path_name_obj.expand_path.to_s
|
157
|
+
target_url = add_filename_to_url(end_point, file_path)
|
158
|
+
|
159
|
+
manifest_path = File.join(dest_container, File.basename(input_file_path))
|
160
|
+
p end_point
|
161
|
+
p File.basename(input_file_path)
|
162
|
+
p manifest_path
|
163
|
+
p manifest_path.sub!(/^./,'')
|
164
|
+
auth_header =
|
165
|
+
{'X-Auth-Token' => token,
|
166
|
+
'X-Object-Manifest' => manifest_path+'_',
|
167
|
+
'X-STATIC-LARGE-OBJECT' => true,
|
168
|
+
'Content-Type' => 'application/json'
|
169
|
+
}
|
170
|
+
http_client = HTTPClient.new
|
171
|
+
url = URI.parse(URI.encode(target_url + '?multipart-manifest=put'))
|
172
|
+
p url
|
173
|
+
p auth_header
|
174
|
+
response = http_client.put(url, manifest_json, auth_header)
|
175
|
+
p response
|
176
|
+
p response.status
|
177
|
+
end
|
178
|
+
|
146
179
|
private
|
147
180
|
|
148
181
|
def build_auth_json
|
@@ -0,0 +1,50 @@
|
|
1
|
+
require 'digest/md5'
|
2
|
+
require 'rabbit_file_split'
|
3
|
+
require 'json'
|
4
|
+
|
5
|
+
module RabbitSwift::LargeObject
|
6
|
+
class StaticLargeObject
|
7
|
+
|
8
|
+
LIMIT_FILE_SIZE = 5368709120;
|
9
|
+
|
10
|
+
# 参考 http://blog.bit-isle.jp/bird/2013/06/35
|
11
|
+
# Swift Server SLO https://github.com/openstack/swift/blob/7a9a0e14b1c6a8f51454379beac95cd594a4193b/swift/common/middleware/slo.py
|
12
|
+
# GNU Core Util split http://git.savannah.gnu.org/cgit/coreutils.git/tree/src/split.c
|
13
|
+
|
14
|
+
@file_path = nil
|
15
|
+
@split_file_path = nil
|
16
|
+
@dest_container_path = nil
|
17
|
+
@limit_file_size = nil
|
18
|
+
|
19
|
+
def initialize(file_path, dest_container_path, split_file_path: nil, limit_file_size: LIMIT_FILE_SIZE)
|
20
|
+
@file_path = file_path
|
21
|
+
@split_file_path = split_file_path
|
22
|
+
@dest_container_path = dest_container_path
|
23
|
+
@limit_file_size = limit_file_size.to_i
|
24
|
+
end
|
25
|
+
|
26
|
+
def split
|
27
|
+
rabbit_file_split = RabbitFileSplit::Bytes.new(@file_path)
|
28
|
+
#p @file_path
|
29
|
+
#p @limit_file_size
|
30
|
+
rabbit_file_split.split(@limit_file_size)
|
31
|
+
rabbit_file_split
|
32
|
+
end
|
33
|
+
|
34
|
+
def create_manifest_list(file_path_list)
|
35
|
+
manifest = []
|
36
|
+
|
37
|
+
file_path_list.each do |file_path|
|
38
|
+
md5 = Digest::MD5.file(file_path).to_s
|
39
|
+
manifest.push(
|
40
|
+
{path: File.join(@dest_container_path, File.basename(file_path)), etag: md5, size_bytes: File.size(file_path)})
|
41
|
+
end
|
42
|
+
JSON.generate(manifest)
|
43
|
+
end
|
44
|
+
|
45
|
+
def self.is_over_default_limit_object_size(file_size)
|
46
|
+
LIMIT_FILE_SIZE > file_size
|
47
|
+
end
|
48
|
+
|
49
|
+
end
|
50
|
+
end
|
data/lib/rabbit_swift/version.rb
CHANGED
data/rabbit_swift.gemspec
CHANGED
@@ -9,8 +9,8 @@ Gem::Specification.new do |spec|
|
|
9
9
|
spec.authors = ["AKB428"]
|
10
10
|
spec.email = ["otoraru@gmail.com"]
|
11
11
|
spec.summary = %q{OpenStack Swift Simple Client}
|
12
|
-
spec.description = %q{OpenStack
|
13
|
-
spec.homepage = ""
|
12
|
+
spec.description = %q{OpenStack Object Storage Simple Client}
|
13
|
+
spec.homepage = "http://akb428.hatenablog.com/"
|
14
14
|
spec.license = "MIT"
|
15
15
|
|
16
16
|
spec.files = `git ls-files -z`.split("\x0")
|
@@ -22,4 +22,5 @@ Gem::Specification.new do |spec|
|
|
22
22
|
spec.add_development_dependency "rake", "~> 10.0"
|
23
23
|
spec.add_dependency 'httpclient'
|
24
24
|
spec.add_dependency 'mime-types'
|
25
|
+
spec.add_dependency 'rabbit_file_split'
|
25
26
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rabbit_swift
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.9
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- AKB428
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-02-14 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -66,10 +66,25 @@ dependencies:
|
|
66
66
|
- - ">="
|
67
67
|
- !ruby/object:Gem::Version
|
68
68
|
version: '0'
|
69
|
-
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: rabbit_file_split
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ">="
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :runtime
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
83
|
+
description: OpenStack Object Storage Simple Client
|
70
84
|
email:
|
71
85
|
- otoraru@gmail.com
|
72
|
-
executables:
|
86
|
+
executables:
|
87
|
+
- slo_client.rb
|
73
88
|
extensions: []
|
74
89
|
extra_rdoc_files: []
|
75
90
|
files:
|
@@ -77,16 +92,19 @@ files:
|
|
77
92
|
- Gemfile
|
78
93
|
- LICENSE.txt
|
79
94
|
- README.md
|
95
|
+
- RELEASE.md
|
80
96
|
- Rakefile
|
97
|
+
- bin/slo_client.rb
|
81
98
|
- lib/rabbit_swift.rb
|
82
99
|
- lib/rabbit_swift/account/.gitkeep
|
83
100
|
- lib/rabbit_swift/client.rb
|
84
101
|
- lib/rabbit_swift/container/.gitkeep
|
102
|
+
- lib/rabbit_swift/large_object/static_large_object.rb
|
85
103
|
- lib/rabbit_swift/object/delete.rb
|
86
104
|
- lib/rabbit_swift/object/head.rb
|
87
105
|
- lib/rabbit_swift/version.rb
|
88
106
|
- rabbit_swift.gemspec
|
89
|
-
homepage:
|
107
|
+
homepage: http://akb428.hatenablog.com/
|
90
108
|
licenses:
|
91
109
|
- MIT
|
92
110
|
metadata: {}
|
@@ -111,3 +129,4 @@ signing_key:
|
|
111
129
|
specification_version: 4
|
112
130
|
summary: OpenStack Swift Simple Client
|
113
131
|
test_files: []
|
132
|
+
has_rdoc:
|