predicsis_ml_sdk 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e55b4f098812ce79e77599cd388e0502cd0d3772
-  data.tar.gz: 7ea659bbdb0850bdddf69c132c03fa5517b76233
+  metadata.gz: b87641f7e72fc46a71a5de47b8af15c5b99b77fd
+  data.tar.gz: 95af5340f3616ea52d27fc6a3ffb49d256a65d98
 SHA512:
-  metadata.gz: 35ebefab33c0d30b75a274fdc719a71b432089cc31e839e6397ff15754c7828823ed8717fbd80c5c5cb0d1aff7bcdbc87a63cc6e58f49b183e5f2fc5b593542c
-  data.tar.gz: 7bfb8c00039e235d5cac7a92132a7b723284a8d4e9e03961b2f3c9bf36dc3ab9483f1948f74b165bb81b3faa5d40b2016913c7226a655b2978183ad536983267
+  metadata.gz: 1e710b701347f0705c8fe52c38f593a3c0e88f844c0487a1701a9c93b8ccbc3b7f959e4c5d27d70a011a379ff97faf194f22aea8343c0f16698a65fe576b816b
+  data.tar.gz: dc432ea91885fdd1a67f86899591b789c25133ce15902fa6084f5d2d570ade11b74d649791608eaf12870d9deaca9f78a59f25fd47a369fe29da8de1c50a909b

data/lib/predicsis_ml_sdk.rb

@@ -6,27 +6,25 @@ require 'predicsis_ml_sdk/configuration'
 require 'predicsis_ml_sdk/core_ext'
 require 'predicsis_ml_sdk/errors'
 require 'predicsis_ml_sdk/util'
-require 'predicsis_ml_sdk/requests'
 
 module PredicsisMlSdk
-  class << self
-    attr_accessor :config
-  end
+  module_function
+  attr_accessor :config
 
-  def self.config
+  def config
     @config ||= Configuration.new
   end
 
-  def self.configure
+  def configure
     yield config
   end
 
-  def self.logger
+  def logger
     config.logger
   end
 
   autoload :AccessToken, 'predicsis_ml_sdk/access_token'
-  autoload :Credentials, 'predicsis_ml_sdk/credentials'
+  autoload :Upload, 'predicsis_ml_sdk/upload'
   autoload :DataFile, 'predicsis_ml_sdk/data_file'
   autoload :Dataset, 'predicsis_ml_sdk/dataset'
   autoload :Dictionary, 'predicsis_ml_sdk/dictionary'
@@ -38,4 +36,5 @@ module PredicsisMlSdk
   autoload :Report, 'predicsis_ml_sdk/report'
   autoload :Source, 'predicsis_ml_sdk/source'
   autoload :Variable, 'predicsis_ml_sdk/variable'
+  autoload :Requests, 'predicsis_ml_sdk/requests'
 end

data/lib/predicsis_ml_sdk/configuration.rb

@@ -1,6 +1,7 @@
 class PredicsisMlSdk::Configuration
   attr_accessor :logger
   attr_accessor :predicsis_api_host
+  attr_accessor :storage_host
   attr_accessor :bearer_token
 
   def initialize

data/lib/predicsis_ml_sdk/data_file.rb

@@ -4,11 +4,6 @@ module PredicsisMlSdk
 
     attr_accessor :id
 
-    def self.upload(params = {})
-      params.delete('expires_at')
-      Requests.upload(params)
-    end
-
     def self.get(id, token = bearer_token)
       new(id, token).get
     end

data/lib/predicsis_ml_sdk/requests.rb

@@ -1,26 +1,29 @@
-require 'httmultiparty'
+require 'rest-client'
 require 'nokogiri'
 
 module PredicsisMlSdk
   class Requests
     extend Util
-    HTTParty::Basement.default_options.update(verify: false)
 
     MODELS = %w(source dataset dictionary preparation_rules_set modalities_set report model data_file)
 
     class << self
       %w(get post patch delete).each do |method|
         define_method("#{method}") do |url, params, headers = {}|
-          options = { body: params, headers: headers }
           with_captured_console do
-            begin
-              response = HTTParty.send(method.to_sym, url, options)
-              if [200, 201, 204].include? response.code
-                return response
-              else
-                fail RequestError.new(response['message'],
-                  JSON.parse(response.body))
-              end
+            uri = URI(PredicsisMlSdk.config.predicsis_api_host + url)
+            res = Net::HTTP.start(uri.host, uri.port) do |http|
+              request = Net::HTTP.const_get(method.capitalize).new(uri, headers)
+              request.body = params if params
+
+              http.request(request)
+            end
+
+            case res
+            when Net::HTTPSuccess, Net::HTTPRedirection
+              return JSON.parse(res.body) unless res.body.empty?
+            else
+              fail RequestError.new(res['message'], res.body.empty? ? res.body : JSON.parse(res.body))
             end
           end
         end
@@ -29,108 +32,108 @@ module PredicsisMlSdk
       def post_token(token, params)
         PredicsisMlSdk.logger.debug 'POST settings/tokens'
         params = { token: params }.to_json
-        response = post("#{api_host}/settings/tokens", params, headers(token))
+        response = post('/settings/tokens', params, headers(token))
         PredicsisMlSdk.logger.debug "Personal Access Token: #{response['token']['token']}"
         response['token']
       end
 
       def get_token(token, id)
         PredicsisMlSdk.logger.debug "GET settings/tokens/#{id}"
-        get("#{api_host}/settings/tokens/#{id}", nil, headers(token))['token']
+        get("/settings/tokens/#{id}", nil, headers(token))['token']
       end
 
       def get_tokens(token)
         PredicsisMlSdk.logger.debug 'GET settings/tokens'
-        get("#{api_host}/settings/tokens", nil, headers(token))['tokens']
+        get('/settings/tokens', nil, headers(token))['tokens']
       end
 
       def delete_token(token, id)
         PredicsisMlSdk.logger.debug "DELETE settings/tokens/#{id}"
-        delete("#{api_host}/settings/tokens/#{id}", nil, headers(token))
+        delete("/settings/tokens/#{id}", nil, headers(token))
       end
 
-      def get_credentials(type, token)
-        PredicsisMlSdk.logger.debug 'GET credentials'
-        get("#{api_host}/sources/credentials/#{type}", nil, headers(token))['credentials']
+      def initiate_upload(token)
+        PredicsisMlSdk.logger.debug 'INIT upload'
+        post('/uploads', nil, headers(token))['upload']
       end
 
-      def upload(params = {})
-        filename = params.delete('file')
-        endpoint = params.delete('signed_url')
-        PredicsisMlSdk.logger.debug "UPLOAD #{filename}"
-        response = HTTParty.put(endpoint, body_stream: File.open(filename),
-                                          headers: { 'Content-Length' => File.size(filename).to_s })
-        if [200, 201, 204].include? response.code
-          params
-        else
-          fail RequestError.new(response['message'], JSON.parse(response.body))
-        end
+      def get_part_url(token, id, part_number, path)
+        get("/uploads/#{id}?part_number=#{part_number}&path=#{path}", nil, headers(token))['upload']
+      end
+
+      def complete_upload(token, id, path)
+        PredicsisMlSdk.logger.debug 'COMPLETE upload'
+        patch("/uploads/#{id}?path=#{path}", nil, headers(token))
+      end
+
+      def upload(endpoint, path, part)
+        RestClient.put(endpoint, File.open(path), { 'Content-Length' => File.size(path).to_s })
       end
 
       def get_job(token, id)
-        get("#{api_host}/jobs/#{id}", nil, headers(token))['job']
+        get("/jobs/#{id}", nil, headers(token))['job']
       end
 
       def get_jobs(token)
         PredicsisMlSdk.logger.debug 'GET jobs'
-        get("#{api_host}/jobs", nil, headers(token))['jobs']
+        get('/jobs', nil, headers(token))['jobs']
       end
 
       def delete_job(token, id)
         PredicsisMlSdk.logger.debug "DELETE job/#{id}"
-        delete("#{api_host}/jobs/#{id}", nil, headers(token))
+        delete("/jobs/#{id}", nil, headers(token))
       end
 
       def get_variables(token, parent_id)
         PredicsisMlSdk.logger.debug "GET dictionaries/#{parent_id}/variables"
-        get("#{api_host}/dictionaries/#{parent_id}/variables", nil, headers(token))['variables']
+        get("/dictionaries/#{parent_id}/variables", nil, headers(token))['variables']
       end
 
       def get_variable(token, id, parent_id)
         PredicsisMlSdk.logger.debug "GET dictionaries/#{parent_id}/variables/#{id}"
-        get("#{api_host}/dictionaries/#{parent_id}/variables/#{id}", nil, headers(token))['variable']
+        get("/dictionaries/#{parent_id}/variables/#{id}", nil, headers(token))['variable']
       end
 
       def patch_variable(token, id, parent_id, params = {})
         PredicsisMlSdk.logger.debug "PATCH dictionaries/#{parent_id}/variables/#{id}"
         params = { variable: params }.to_json
-        patch("#{api_host}/dictionaries/#{parent_id}/variables/#{id}",
+        patch("/dictionaries/#{parent_id}/variables/#{id}",
           params, headers(token))['variable']
       end
 
       def get_signed_url(token, id)
         PredicsisMlSdk.logger.debug "GET data_files/#{id}/signed_url"
-        get("#{api_host}/data_files/#{id}/signed_url", nil, headers(token))['data_file']
+        get("/data_files/#{id}/signed_url", nil, headers(token))['data_file']
       end
 
       MODELS.each do |model|
         define_method("post_#{model}") do |token, params = {}, other_params = {}|
           PredicsisMlSdk.logger.debug "POST #{model.pluralize}"
           params = { :"#{model}" => params }.merge(other_params)
-          response = post("#{api_host}/#{model.pluralize}", params.to_json, headers(token))
+          response = post("/#{model.pluralize}", params.to_json, headers(token))
           PredicsisMlSdk.logger.info("#{model}_id: #{response[model]['id']}")
           response[model]
         end
 
         define_method("delete_#{model}") do |token, id|
           PredicsisMlSdk.logger.debug "DELETE #{model.pluralize}/#{id}"
-          delete("#{api_host}/#{model.pluralize}/#{id}", nil, headers(token))
+          delete("/#{model.pluralize}/#{id}", nil, headers(token))
         end
 
         define_method("get_#{model}") do |token, id|
           PredicsisMlSdk.logger.debug "GET #{model.pluralize}/#{id}"
-          get("#{api_host}/#{model.pluralize}/#{id}", nil, headers(token))[model]
+          get("/#{model.pluralize}/#{id}", nil, headers(token))[model]
         end
 
         define_method("get_#{model.pluralize}") do |token|
           PredicsisMlSdk.logger.debug "GET #{model.pluralize}"
-          get("#{api_host}/#{model.pluralize}", nil, headers(token))[model.pluralize]
+          get("/#{model.pluralize}", nil, headers(token))[model.pluralize]
         end
 
         define_method("patch_#{model}") do |token, id, params = {}|
           params = { :"#{model}" => params }.to_json
           PredicsisMlSdk.logger.debug "PATCH #{model.pluralize}/#{id}"
-          patch("#{api_host}/#{model.pluralize}/#{id}", params, headers(token))[model]
+          patch("/#{model.pluralize}/#{id}", params, headers(token))[model]
         end
       end
 
@@ -143,10 +146,6 @@ module PredicsisMlSdk
           'Authorization' => "Bearer #{token}"
         }
       end
-
-      def api_host
-        PredicsisMlSdk.config.predicsis_api_host
-      end
     end
   end
 end

data/lib/predicsis_ml_sdk/upload.rb

@@ -0,0 +1,70 @@
+require 'predicsis_ml_sdk/upload_helper'
+require 'ruby-progressbar'
+
+module PredicsisMlSdk
+  class Upload
+    include Util
+    include UploadHelper
+
+    def self.start(file, token = bearer_token)
+      Upload.initiate.tap do |u|
+        u.upload(file)
+      end
+    end
+
+    def self.initiate(token = bearer_token)
+      data = Requests.initiate_upload(token)
+      new(data['id'], data['path'], token).tap do |upload|
+        upload.define_attributes(data)
+      end
+    end
+
+    def self.get_part_url(id, path, part_number, token = bearer_token)
+      new(id, path, token).get_part_url(part_number)
+    end
+
+    def self.complete(id, path, token = bearer_token)
+      new(id, path, token).complete
+    end
+
+    def initialize(id, path, token = bearer_token)
+      @id = id
+      @path = path
+      @token = token
+    end
+
+    def upload(file)
+      path = split_into_files(file)
+      nb_of_parts(file).times do |i|
+        tmp_path = path + i.to_s.rjust(5, '0')
+        sleep(1) until exists?(tmp_path, i, file)
+        Requests.upload(get_part_url(i + 1), tmp_path, i + 1)
+        progressbar.increment
+      end
+      Thread.new { FileUtils.rm_rf Dir.glob(path.gsub('part_', '')) }
+      complete
+    end
+
+    def get_part_url(part_number)
+      Requests.get_part_url(token, id, part_number, path)['part_url']
+    end
+
+    def complete
+      Requests.complete_upload(token, id, path)
+    end
+
+    def to_h
+      instance_variables.each_with_object({}) do |var, hash|
+        hash[var.to_s.delete("@")] = instance_variable_get(var) unless var.to_s == '@token'
+      end
+    end
+
+    def path
+      @path ||= defined?(key) ? key : object
+    end
+
+    def progressbar
+      @progressbar ||= ProgressBar.create(format: '|%b>%i| %p%% %t', starting_at: 0, :total => @nb_of_parts)
+    end
+  end
+end

data/lib/predicsis_ml_sdk/upload_helper.rb

@@ -0,0 +1,32 @@
+module PredicsisMlSdk
+  module UploadHelper
+    OFFSET = 52_428_800
+
+    def nb_of_parts(file)
+      @nb_of_parts ||= (File.size(file).to_f / OFFSET).ceil
+    end
+
+    def split_into_files(file_path, offset = OFFSET)
+      dir = "tmp/multiparts/#{Time.now.to_i}"
+      FileUtils.mkdir_p(dir)
+      path = dir + '/part_'
+      Thread.new(abort_on_exception: true) do
+        nb_of_parts(file_path).times do |i|
+          File.open(path + i.to_s.rjust(5, '0'), 'wb') do |tmp_file|
+            tmp_file.write(IO.binread(file_path, offset, offset * i))
+          end
+        end
+      end
+      path
+    end
+
+    def exists?(part_path, part_number, file_path)
+      if @nb_of_parts == part_number + 1
+        File.exist?(part_path) &&
+          File.size(part_path) == File.size(file_path).to_f - ((@nb_of_parts - 1) * OFFSET)
+      else
+        File.exist?(part_path) && File.size(part_path) == OFFSET
+      end
+    end
+  end
+end

data/lib/predicsis_ml_sdk/version.rb

@@ -1,3 +1,3 @@
 module PredicsisMlSdk
-  VERSION = '0.4.0'
+  VERSION = '0.5.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: predicsis_ml_sdk
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Armand
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-01-26 00:00:00.000000000 Z
+date: 2016-02-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json
@@ -51,25 +51,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 2.7.1
 - !ruby/object:Gem::Dependency
-  name: httmultiparty
+  name: rest-client
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.3'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.3.16
+        version: '1.8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.3'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.3.16
+        version: '1.8'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
@@ -90,6 +84,26 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.6.3
+- !ruby/object:Gem::Dependency
+  name: ruby-progressbar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.7.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.7.3
 description: 'PredicSis ML SDK is the official SDK for PredicSis API: http://launch.predicsis.com/'
 email:
 - armand.bourquin@predicsis.com
@@ -102,7 +116,6 @@ files:
 - lib/predicsis_ml_sdk/configuration.rb
 - lib/predicsis_ml_sdk/core_ext.rb
 - lib/predicsis_ml_sdk/core_ext/string.rb
-- lib/predicsis_ml_sdk/credentials.rb
 - lib/predicsis_ml_sdk/data_file.rb
 - lib/predicsis_ml_sdk/dataset.rb
 - lib/predicsis_ml_sdk/dictionary.rb
@@ -115,6 +128,8 @@ files:
 - lib/predicsis_ml_sdk/report.rb
 - lib/predicsis_ml_sdk/requests.rb
 - lib/predicsis_ml_sdk/source.rb
+- lib/predicsis_ml_sdk/upload.rb
+- lib/predicsis_ml_sdk/upload_helper.rb
 - lib/predicsis_ml_sdk/util.rb
 - lib/predicsis_ml_sdk/variable.rb
 - lib/predicsis_ml_sdk/version.rb

data/lib/predicsis_ml_sdk/credentials.rb

@@ -1,25 +0,0 @@
-module PredicsisMlSdk
-  class Credentials
-    include Util
-
-    def self.get(type = 's3', token = bearer_token)
-      new(token).get(type)
-    end
-
-    def initialize(token = bearer_token)
-      @token = token
-    end
-
-    def get(type = 's3')
-      data = Requests.get_credentials(type, token)
-      define_attributes(data)
-      self
-    end
-
-    def to_h
-      instance_variables.each_with_object({}) do |var, hash|
-        hash[var.to_s.delete("@")] = instance_variable_get(var) unless var.to_s == '@token'
-      end
-    end
-  end
-end