rabbit_swift 0.2.8 → 0.2.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2a808886b3a129b3197ea093b486caa88db6f4d5
-  data.tar.gz: 527381104142fa5138c6fd851a90f45110d5fb77
+  metadata.gz: 7570788a09d3209ca2aa282b96282eb050b43e13
+  data.tar.gz: 3510fd4456248d641eae7997db7ef7eea2620dd9
 SHA512:
-  metadata.gz: 33a4e4e199f95f9791d123673006722e1315ced0b366cda613ddd255a42845f871b700364bacadc82692dd3e3b93ab7b2a78eb74a49bf035f763d65ca7b36e82
-  data.tar.gz: ab7c2a7fd0d0207caf572ab07f9c4f57fc79352543afa51540765df226e230c34266d467b8f7eb3f64812d424e395df8a96eb7a0f94ea6023b391a1261f930bd
+  metadata.gz: 62c332ae6f2bc9f5da004522fdd0d9414ca6154a7775d894b042ba3e101e26cdad09e05516bf4566031a6b93a600e5bba0510d494634f8123cc5b0e7ac46baa6
+  data.tar.gz: ed758fe16687b2cee36750d4aa1862ae14ed09f6b07eb5232c8147a3e20554065d6877c530876bb3d0df5a9b893da7e7c70534cee7e226468de5f1b65f2f23b3

data/Gemfile

@@ -2,5 +2,3 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in rabbit_swift.gemspec
 gemspec
-gem 'httpclient'
-gem 'mime-types'

data/RELEASE.md

@@ -0,0 +1,11 @@
+# リリース履歴
+
+## 0.2.9
+
+(2015/02/14)
+
+Static Large Object (α1)
+
+## 0.2.8
+
+not history

data/bin/slo_client.rb

@@ -0,0 +1,95 @@
+require 'json'
+require 'optparse'
+require 'rabbit_swift'
+
+#ruby -I./lib/ this.rb
+#bundle exec ruby -I./lib bin/slo_client.rb -s ~/Downloads/test.zip -d /test -c ../chino/conf/conf.json -l 1048576
+=begin
+-c conf.json
+{
+ "swift" : {
+        "auth_url" : "",　//必須
+        "tenantName" : "", //必須
+        "username" : "", //必須
+        "password" : "", //必須
+        "endPoint" : "" //必須ではない、末尾に/はつけない
+ }
+}
+=end
+
+opt = OptionParser.new
+Version = "1.0.0"
+
+src_path = nil
+dest_path = nil
+conf_path = nil
+limit_file_size = nil
+opt.on('-s SRC_PATH', 'src_file_path') {|v| src_path = v}
+opt.on('-d DEST_PATH', 'dest_path') {|v| dest_path = v}
+opt.on('-c CONF_PATH', 'conf_path') {|v| conf_path = v}
+opt.on('-l LIMIT_BYTE', 'limit_file_size') {|v| limit_file_size = v}
+opt.parse!(ARGV)
+
+
+class Slo_client
+
+  attr_accessor :src_path, :dest_path, :conf_path, :swift, :token, :limit_file_size, :original_dest_path
+
+  def initialize(src_path, dest_path, conf_path, send_timeout: 7200, limit_file_size:  RabbitSwift::LargeObject::StaticLargeObject::LIMIT_FILE_SIZE)
+    @src_path = src_path
+    @dest_path = dest_path
+    @original_dest_path = dest_path
+    @conf_path = conf_path
+    @limit_file_size = limit_file_size
+    File.open conf_path do |file|
+      conf = JSON.load(file.read)
+      @swift = conf['swift']
+
+      if @dest_path =~ /^\//
+        @dest_path = @swift['endPoint'] + dest_path
+      end
+    end
+
+    @swift['send_timeout'] = send_timeout
+  end
+
+  def upload
+    slo = RabbitSwift::LargeObject::StaticLargeObject.new(src_path, dest_path, limit_file_size: @limit_file_size)
+
+    # (指定されたファイルサイズ単位で)ファイルを分割する
+    rabbit_file_split = slo.split
+    # JSONマニフェストファイルをつくる
+    manifest_json = slo.create_manifest_list(rabbit_file_split.file_list)
+    puts manifest_json
+
+    rabbit_swift_client = RabbitSwift::Client.new(@swift)
+    @token = rabbit_swift_client.get_token
+    puts 'token -> ' + token
+
+    #ファイルを全てアップロード(with etag)
+    rabbit_file_split.file_list.each do |file_path|
+      status = rabbit_swift_client.upload(token, dest_path, file_path)
+      puts file_path
+      puts 'http_status -> ' + status.to_s
+      if (status == RabbitSwift::Client::UPLOAD_SUCCESS_HTTP_STATUS_CODE)
+        puts 'upload OK'
+      else
+        puts 'upload NG'
+        return
+      end
+    end
+
+    puts "dest_path->" + dest_path
+    #マニフェストをアップロード
+    rabbit_swift_client.upload_manifest(token, dest_path, @original_dest_path, @src_path, manifest_json)
+
+    #分割したファイルを削除
+    rabbit_file_split.delete_all
+
+  end
+
+end
+
+
+slo = Slo_client.new(src_path,dest_path,conf_path, limit_file_size: limit_file_size)
+slo.upload

data/lib/rabbit_swift.rb

@@ -1,5 +1,6 @@
 require "rabbit_swift/version"
 require "rabbit_swift/client"
+require "rabbit_swift/large_object/static_large_object"
 
 module RabbitSwift
   # Your code goes here...

data/lib/rabbit_swift/client.rb

@@ -4,6 +4,7 @@ require 'json'
 require 'httpclient'
 require 'pathname'
 require 'mime/types'
+require 'digest/md5'
 
 module RabbitSwift
 
@@ -82,10 +83,16 @@ module RabbitSwift
       #相対パスがきた時のために絶対パスに変換
       path_name_obj = Pathname.new(input_file_path);
       file_path = path_name_obj.expand_path.to_s
-      auth_header = {
-          'X-Auth-Token' => token,
-          'Content-Type' => MIME::Types.type_for(file_path)[0].to_s
-      }
+      mime = MIME::Types.type_for(file_path)[0]
+      auth_header =
+          if mime.nil?
+            {'X-Auth-Token' => token}
+          else
+            {'X-Auth-Token' => token,
+             'Content-Type' => MIME::Types.type_for(file_path)[0].to_s
+            }
+          end
+
       auth_header['X-Web-Mode'] = 'true' if @web_mode
 
       http_client = HTTPClient.new
@@ -143,6 +150,32 @@ module RabbitSwift
       @res.status
     end
 
+    def upload_manifest(token, end_point, dest_container,input_file_path, manifest_json)
+      #相対パスがきた時のために絶対パスに変換
+      path_name_obj = Pathname.new(input_file_path);
+      file_path = path_name_obj.expand_path.to_s
+      target_url = add_filename_to_url(end_point, file_path)
+
+      manifest_path = File.join(dest_container, File.basename(input_file_path))
+      p end_point
+      p File.basename(input_file_path)
+      p manifest_path
+      p manifest_path.sub!(/^./,'')
+      auth_header =
+            {'X-Auth-Token' => token,
+            'X-Object-Manifest' => manifest_path+'_',
+            'X-STATIC-LARGE-OBJECT' => true,
+            'Content-Type' => 'application/json'
+            }
+      http_client = HTTPClient.new
+      url = URI.parse(URI.encode(target_url + '?multipart-manifest=put'))
+      p url
+      p auth_header
+      response = http_client.put(url, manifest_json, auth_header)
+      p response
+      p response.status
+    end
+
     private
 
     def build_auth_json

data/lib/rabbit_swift/large_object/static_large_object.rb

@@ -0,0 +1,50 @@
+require 'digest/md5'
+require 'rabbit_file_split'
+require 'json'
+
+module RabbitSwift::LargeObject
+  class StaticLargeObject
+
+    LIMIT_FILE_SIZE = 5368709120;
+
+    # 参考　http://blog.bit-isle.jp/bird/2013/06/35
+    # Swift Server SLO https://github.com/openstack/swift/blob/7a9a0e14b1c6a8f51454379beac95cd594a4193b/swift/common/middleware/slo.py
+    # GNU Core Util split http://git.savannah.gnu.org/cgit/coreutils.git/tree/src/split.c
+
+    @file_path = nil
+    @split_file_path = nil
+    @dest_container_path = nil
+    @limit_file_size = nil
+
+    def initialize(file_path, dest_container_path,  split_file_path: nil, limit_file_size: LIMIT_FILE_SIZE)
+      @file_path = file_path
+      @split_file_path = split_file_path
+      @dest_container_path = dest_container_path
+      @limit_file_size = limit_file_size.to_i
+    end
+
+    def split
+      rabbit_file_split = RabbitFileSplit::Bytes.new(@file_path)
+      #p @file_path
+      #p @limit_file_size
+      rabbit_file_split.split(@limit_file_size)
+      rabbit_file_split
+    end
+
+    def create_manifest_list(file_path_list)
+      manifest = []
+
+      file_path_list.each do |file_path|
+        md5 = Digest::MD5.file(file_path).to_s
+        manifest.push(
+        {path:  File.join(@dest_container_path, File.basename(file_path)), etag: md5, size_bytes: File.size(file_path)})
+      end
+      JSON.generate(manifest)
+    end
+
+    def self.is_over_default_limit_object_size(file_size)
+      LIMIT_FILE_SIZE > file_size
+    end
+
+  end
+end

data/lib/rabbit_swift/version.rb

@@ -1,3 +1,3 @@
 module RabbitSwift
-  VERSION = "0.2.8"
+  VERSION = "0.2.9"
 end

data/rabbit_swift.gemspec

@@ -9,8 +9,8 @@ Gem::Specification.new do |spec|
   spec.authors       = ["AKB428"]
   spec.email         = ["otoraru@gmail.com"]
   spec.summary       = %q{OpenStack Swift Simple Client}
-  spec.description   = %q{OpenStack Swift Simple Client. main target for "ConoHa VPS"}
-  spec.homepage      = ""
+  spec.description   = %q{OpenStack Object Storage Simple Client}
+  spec.homepage      = "http://akb428.hatenablog.com/"
   spec.license       = "MIT"
 
   spec.files         = `git ls-files -z`.split("\x0")
@@ -22,4 +22,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_dependency 'httpclient'
   spec.add_dependency 'mime-types'
+  spec.add_dependency 'rabbit_file_split'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rabbit_swift
 version: !ruby/object:Gem::Version
-  version: 0.2.8
+  version: 0.2.9
 platform: ruby
 authors:
 - AKB428
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-01-21 00:00:00.000000000 Z
+date: 2015-02-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -66,10 +66,25 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: OpenStack Swift Simple Client. main target for "ConoHa VPS"
+- !ruby/object:Gem::Dependency
+  name: rabbit_file_split
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: OpenStack Object Storage Simple Client
 email:
 - otoraru@gmail.com
-executables: []
+executables:
+- slo_client.rb
 extensions: []
 extra_rdoc_files: []
 files:
@@ -77,16 +92,19 @@ files:
 - Gemfile
 - LICENSE.txt
 - README.md
+- RELEASE.md
 - Rakefile
+- bin/slo_client.rb
 - lib/rabbit_swift.rb
 - lib/rabbit_swift/account/.gitkeep
 - lib/rabbit_swift/client.rb
 - lib/rabbit_swift/container/.gitkeep
+- lib/rabbit_swift/large_object/static_large_object.rb
 - lib/rabbit_swift/object/delete.rb
 - lib/rabbit_swift/object/head.rb
 - lib/rabbit_swift/version.rb
 - rabbit_swift.gemspec
-homepage: ''
+homepage: http://akb428.hatenablog.com/
 licenses:
 - MIT
 metadata: {}
@@ -111,3 +129,4 @@ signing_key:
 specification_version: 4
 summary: OpenStack Swift Simple Client
 test_files: []
+has_rdoc: