rabbit_swift 0.2.8 → 0.2.9

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 2a808886b3a129b3197ea093b486caa88db6f4d5
4
- data.tar.gz: 527381104142fa5138c6fd851a90f45110d5fb77
3
+ metadata.gz: 7570788a09d3209ca2aa282b96282eb050b43e13
4
+ data.tar.gz: 3510fd4456248d641eae7997db7ef7eea2620dd9
5
5
  SHA512:
6
- metadata.gz: 33a4e4e199f95f9791d123673006722e1315ced0b366cda613ddd255a42845f871b700364bacadc82692dd3e3b93ab7b2a78eb74a49bf035f763d65ca7b36e82
7
- data.tar.gz: ab7c2a7fd0d0207caf572ab07f9c4f57fc79352543afa51540765df226e230c34266d467b8f7eb3f64812d424e395df8a96eb7a0f94ea6023b391a1261f930bd
6
+ metadata.gz: 62c332ae6f2bc9f5da004522fdd0d9414ca6154a7775d894b042ba3e101e26cdad09e05516bf4566031a6b93a600e5bba0510d494634f8123cc5b0e7ac46baa6
7
+ data.tar.gz: ed758fe16687b2cee36750d4aa1862ae14ed09f6b07eb5232c8147a3e20554065d6877c530876bb3d0df5a9b893da7e7c70534cee7e226468de5f1b65f2f23b3
data/Gemfile CHANGED
@@ -2,5 +2,3 @@ source 'https://rubygems.org'
2
2
 
3
3
  # Specify your gem's dependencies in rabbit_swift.gemspec
4
4
  gemspec
5
- gem 'httpclient'
6
- gem 'mime-types'
data/RELEASE.md ADDED
@@ -0,0 +1,11 @@
1
+ # リリース履歴
2
+
3
+ ## 0.2.9
4
+
5
+ (2015/02/14)
6
+
7
+ Static Large Object (α1)
8
+
9
+ ## 0.2.8
10
+
11
+ not history
data/bin/slo_client.rb ADDED
@@ -0,0 +1,95 @@
1
+ require 'json'
2
+ require 'optparse'
3
+ require 'rabbit_swift'
4
+
5
+ #ruby -I./lib/ this.rb
6
+ #bundle exec ruby -I./lib bin/slo_client.rb -s ~/Downloads/test.zip -d /test -c ../chino/conf/conf.json -l 1048576
7
+ =begin
8
+ -c conf.json
9
+ {
10
+ "swift" : {
11
+ "auth_url" : "", //必須
12
+ "tenantName" : "", //必須
13
+ "username" : "", //必須
14
+ "password" : "", //必須
15
+ "endPoint" : "" //必須ではない、末尾に/はつけない
16
+ }
17
+ }
18
+ =end
19
+
20
+ opt = OptionParser.new
21
+ Version = "1.0.0"
22
+
23
+ src_path = nil
24
+ dest_path = nil
25
+ conf_path = nil
26
+ limit_file_size = nil
27
+ opt.on('-s SRC_PATH', 'src_file_path') {|v| src_path = v}
28
+ opt.on('-d DEST_PATH', 'dest_path') {|v| dest_path = v}
29
+ opt.on('-c CONF_PATH', 'conf_path') {|v| conf_path = v}
30
+ opt.on('-l LIMIT_BYTE', 'limit_file_size') {|v| limit_file_size = v}
31
+ opt.parse!(ARGV)
32
+
33
+
34
+ class Slo_client
35
+
36
+ attr_accessor :src_path, :dest_path, :conf_path, :swift, :token, :limit_file_size, :original_dest_path
37
+
38
+ def initialize(src_path, dest_path, conf_path, send_timeout: 7200, limit_file_size: RabbitSwift::LargeObject::StaticLargeObject::LIMIT_FILE_SIZE)
39
+ @src_path = src_path
40
+ @dest_path = dest_path
41
+ @original_dest_path = dest_path
42
+ @conf_path = conf_path
43
+ @limit_file_size = limit_file_size
44
+ File.open conf_path do |file|
45
+ conf = JSON.load(file.read)
46
+ @swift = conf['swift']
47
+
48
+ if @dest_path =~ /^\//
49
+ @dest_path = @swift['endPoint'] + dest_path
50
+ end
51
+ end
52
+
53
+ @swift['send_timeout'] = send_timeout
54
+ end
55
+
56
+ def upload
57
+ slo = RabbitSwift::LargeObject::StaticLargeObject.new(src_path, dest_path, limit_file_size: @limit_file_size)
58
+
59
+ # (指定されたファイルサイズ単位で)ファイルを分割する
60
+ rabbit_file_split = slo.split
61
+ # JSONマニフェストファイルをつくる
62
+ manifest_json = slo.create_manifest_list(rabbit_file_split.file_list)
63
+ puts manifest_json
64
+
65
+ rabbit_swift_client = RabbitSwift::Client.new(@swift)
66
+ @token = rabbit_swift_client.get_token
67
+ puts 'token -> ' + token
68
+
69
+ #ファイルを全てアップロード(with etag)
70
+ rabbit_file_split.file_list.each do |file_path|
71
+ status = rabbit_swift_client.upload(token, dest_path, file_path)
72
+ puts file_path
73
+ puts 'http_status -> ' + status.to_s
74
+ if (status == RabbitSwift::Client::UPLOAD_SUCCESS_HTTP_STATUS_CODE)
75
+ puts 'upload OK'
76
+ else
77
+ puts 'upload NG'
78
+ return
79
+ end
80
+ end
81
+
82
+ puts "dest_path->" + dest_path
83
+ #マニフェストをアップロード
84
+ rabbit_swift_client.upload_manifest(token, dest_path, @original_dest_path, @src_path, manifest_json)
85
+
86
+ #分割したファイルを削除
87
+ rabbit_file_split.delete_all
88
+
89
+ end
90
+
91
+ end
92
+
93
+
94
+ slo = Slo_client.new(src_path,dest_path,conf_path, limit_file_size: limit_file_size)
95
+ slo.upload
data/lib/rabbit_swift.rb CHANGED
@@ -1,5 +1,6 @@
1
1
  require "rabbit_swift/version"
2
2
  require "rabbit_swift/client"
3
+ require "rabbit_swift/large_object/static_large_object"
3
4
 
4
5
  module RabbitSwift
5
6
  # Your code goes here...
@@ -4,6 +4,7 @@ require 'json'
4
4
  require 'httpclient'
5
5
  require 'pathname'
6
6
  require 'mime/types'
7
+ require 'digest/md5'
7
8
 
8
9
  module RabbitSwift
9
10
 
@@ -82,10 +83,16 @@ module RabbitSwift
82
83
  #相対パスがきた時のために絶対パスに変換
83
84
  path_name_obj = Pathname.new(input_file_path);
84
85
  file_path = path_name_obj.expand_path.to_s
85
- auth_header = {
86
- 'X-Auth-Token' => token,
87
- 'Content-Type' => MIME::Types.type_for(file_path)[0].to_s
88
- }
86
+ mime = MIME::Types.type_for(file_path)[0]
87
+ auth_header =
88
+ if mime.nil?
89
+ {'X-Auth-Token' => token}
90
+ else
91
+ {'X-Auth-Token' => token,
92
+ 'Content-Type' => MIME::Types.type_for(file_path)[0].to_s
93
+ }
94
+ end
95
+
89
96
  auth_header['X-Web-Mode'] = 'true' if @web_mode
90
97
 
91
98
  http_client = HTTPClient.new
@@ -143,6 +150,32 @@ module RabbitSwift
143
150
  @res.status
144
151
  end
145
152
 
153
+ def upload_manifest(token, end_point, dest_container,input_file_path, manifest_json)
154
+ #相対パスがきた時のために絶対パスに変換
155
+ path_name_obj = Pathname.new(input_file_path);
156
+ file_path = path_name_obj.expand_path.to_s
157
+ target_url = add_filename_to_url(end_point, file_path)
158
+
159
+ manifest_path = File.join(dest_container, File.basename(input_file_path))
160
+ p end_point
161
+ p File.basename(input_file_path)
162
+ p manifest_path
163
+ p manifest_path.sub!(/^./,'')
164
+ auth_header =
165
+ {'X-Auth-Token' => token,
166
+ 'X-Object-Manifest' => manifest_path+'_',
167
+ 'X-STATIC-LARGE-OBJECT' => true,
168
+ 'Content-Type' => 'application/json'
169
+ }
170
+ http_client = HTTPClient.new
171
+ url = URI.parse(URI.encode(target_url + '?multipart-manifest=put'))
172
+ p url
173
+ p auth_header
174
+ response = http_client.put(url, manifest_json, auth_header)
175
+ p response
176
+ p response.status
177
+ end
178
+
146
179
  private
147
180
 
148
181
  def build_auth_json
@@ -0,0 +1,50 @@
1
+ require 'digest/md5'
2
+ require 'rabbit_file_split'
3
+ require 'json'
4
+
5
+ module RabbitSwift::LargeObject
6
+ class StaticLargeObject
7
+
8
+ LIMIT_FILE_SIZE = 5368709120;
9
+
10
+ # 参考 http://blog.bit-isle.jp/bird/2013/06/35
11
+ # Swift Server SLO https://github.com/openstack/swift/blob/7a9a0e14b1c6a8f51454379beac95cd594a4193b/swift/common/middleware/slo.py
12
+ # GNU Core Util split http://git.savannah.gnu.org/cgit/coreutils.git/tree/src/split.c
13
+
14
+ @file_path = nil
15
+ @split_file_path = nil
16
+ @dest_container_path = nil
17
+ @limit_file_size = nil
18
+
19
+ def initialize(file_path, dest_container_path, split_file_path: nil, limit_file_size: LIMIT_FILE_SIZE)
20
+ @file_path = file_path
21
+ @split_file_path = split_file_path
22
+ @dest_container_path = dest_container_path
23
+ @limit_file_size = limit_file_size.to_i
24
+ end
25
+
26
+ def split
27
+ rabbit_file_split = RabbitFileSplit::Bytes.new(@file_path)
28
+ #p @file_path
29
+ #p @limit_file_size
30
+ rabbit_file_split.split(@limit_file_size)
31
+ rabbit_file_split
32
+ end
33
+
34
+ def create_manifest_list(file_path_list)
35
+ manifest = []
36
+
37
+ file_path_list.each do |file_path|
38
+ md5 = Digest::MD5.file(file_path).to_s
39
+ manifest.push(
40
+ {path: File.join(@dest_container_path, File.basename(file_path)), etag: md5, size_bytes: File.size(file_path)})
41
+ end
42
+ JSON.generate(manifest)
43
+ end
44
+
45
+ def self.is_over_default_limit_object_size(file_size)
46
+ LIMIT_FILE_SIZE > file_size
47
+ end
48
+
49
+ end
50
+ end
@@ -1,3 +1,3 @@
1
1
  module RabbitSwift
2
- VERSION = "0.2.8"
2
+ VERSION = "0.2.9"
3
3
  end
data/rabbit_swift.gemspec CHANGED
@@ -9,8 +9,8 @@ Gem::Specification.new do |spec|
9
9
  spec.authors = ["AKB428"]
10
10
  spec.email = ["otoraru@gmail.com"]
11
11
  spec.summary = %q{OpenStack Swift Simple Client}
12
- spec.description = %q{OpenStack Swift Simple Client. main target for "ConoHa VPS"}
13
- spec.homepage = ""
12
+ spec.description = %q{OpenStack Object Storage Simple Client}
13
+ spec.homepage = "http://akb428.hatenablog.com/"
14
14
  spec.license = "MIT"
15
15
 
16
16
  spec.files = `git ls-files -z`.split("\x0")
@@ -22,4 +22,5 @@ Gem::Specification.new do |spec|
22
22
  spec.add_development_dependency "rake", "~> 10.0"
23
23
  spec.add_dependency 'httpclient'
24
24
  spec.add_dependency 'mime-types'
25
+ spec.add_dependency 'rabbit_file_split'
25
26
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: rabbit_swift
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.8
4
+ version: 0.2.9
5
5
  platform: ruby
6
6
  authors:
7
7
  - AKB428
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-01-21 00:00:00.000000000 Z
11
+ date: 2015-02-14 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -66,10 +66,25 @@ dependencies:
66
66
  - - ">="
67
67
  - !ruby/object:Gem::Version
68
68
  version: '0'
69
- description: OpenStack Swift Simple Client. main target for "ConoHa VPS"
69
+ - !ruby/object:Gem::Dependency
70
+ name: rabbit_file_split
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - ">="
74
+ - !ruby/object:Gem::Version
75
+ version: '0'
76
+ type: :runtime
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - ">="
81
+ - !ruby/object:Gem::Version
82
+ version: '0'
83
+ description: OpenStack Object Storage Simple Client
70
84
  email:
71
85
  - otoraru@gmail.com
72
- executables: []
86
+ executables:
87
+ - slo_client.rb
73
88
  extensions: []
74
89
  extra_rdoc_files: []
75
90
  files:
@@ -77,16 +92,19 @@ files:
77
92
  - Gemfile
78
93
  - LICENSE.txt
79
94
  - README.md
95
+ - RELEASE.md
80
96
  - Rakefile
97
+ - bin/slo_client.rb
81
98
  - lib/rabbit_swift.rb
82
99
  - lib/rabbit_swift/account/.gitkeep
83
100
  - lib/rabbit_swift/client.rb
84
101
  - lib/rabbit_swift/container/.gitkeep
102
+ - lib/rabbit_swift/large_object/static_large_object.rb
85
103
  - lib/rabbit_swift/object/delete.rb
86
104
  - lib/rabbit_swift/object/head.rb
87
105
  - lib/rabbit_swift/version.rb
88
106
  - rabbit_swift.gemspec
89
- homepage: ''
107
+ homepage: http://akb428.hatenablog.com/
90
108
  licenses:
91
109
  - MIT
92
110
  metadata: {}
@@ -111,3 +129,4 @@ signing_key:
111
129
  specification_version: 4
112
130
  summary: OpenStack Swift Simple Client
113
131
  test_files: []
132
+ has_rdoc: