deepdetect_ruby 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +50 -0
- data/CODE_OF_CONDUCT.md +49 -0
- data/Gemfile +10 -0
- data/Gemfile.lock +28 -0
- data/LICENSE +21 -0
- data/LICENSE.txt +21 -0
- data/README.md +245 -0
- data/Rakefile +2 -0
- data/bin/console +14 -0
- data/bin/setup +8 -0
- data/deepdetect_ruby.gemspec +33 -0
- data/docs/Finetuning.md +241 -0
- data/docs/convolutional_neural_networks.py +28 -0
- data/docs/dd_client.py +379 -0
- data/docs/finetuning_googlenet.py +40 -0
- data/docs/rsync_for_selfie_f4.sh +23 -0
- data/docs/sync_file.sh +100 -0
- data/docs/sync_mirrorshot.sh +23 -0
- data/lib/configuration.rb +34 -0
- data/lib/dede_server.rb +24 -0
- data/lib/deep_symbolize.rb +60 -0
- data/lib/deepdetect_ruby/version.rb +3 -0
- data/lib/deepdetect_ruby.rb +13 -0
- data/lib/info.rb +42 -0
- data/lib/predict.rb +50 -0
- data/lib/service.rb +228 -0
- data/lib/train.rb +133 -0
- data/test_script.md +87 -0
- metadata +114 -0
@@ -0,0 +1,40 @@
|
|
1
|
+
from dd_client import DD
|
2
|
+
|
3
|
+
model_repo = '/home/me/models/mymodel'
|
4
|
+
height = width = 224
|
5
|
+
nclasses = 300
|
6
|
+
|
7
|
+
# setting up DD client
|
8
|
+
host = '127.0.0.1'
|
9
|
+
sname = 'imgserv'
|
10
|
+
description = 'image classification'
|
11
|
+
mllib = 'caffe'
|
12
|
+
dd = DD(host)
|
13
|
+
dd.set_return_format(dd.RETURN_PYTHON)
|
14
|
+
|
15
|
+
# creating ML service
|
16
|
+
model = {'repository':model_repo}
|
17
|
+
parameters_input = {'connector':'image','width':width,'height':height}
|
18
|
+
parameters_mllib = {'template':'googlenet','nclasses':nclasses,'finetuning':True,'rotate':False,'mirror':True,'weights':'bvlc_googlenet.caffemodel'}
|
19
|
+
parameters_output = {}
|
20
|
+
dd.put_service(sname,model,description,mllib,
|
21
|
+
parameters_input,parameters_mllib,parameters_output)
|
22
|
+
|
23
|
+
# training / finetuning from pre-trained network
|
24
|
+
train_data = ['path/to/train/data']
|
25
|
+
parameters_input = {'test_split':0.1,'shuffle':True}
|
26
|
+
parameters_mllib = {'gpu':True,'net':{'batch_size':64,'test_batch_size':64},'solver':{'test_interval':4000,'iterations':50000,'snapshot':10000,'base_lr':0.01,'solver_type':'NESTEROV','test_initialization':True}}
|
27
|
+
parameters_output = {'measure':['mcll','f1','acc-5']}
|
28
|
+
dd.post_train(sname,train_data,parameters_input,parameters_mllib,parameters_output,async=True)
|
29
|
+
|
30
|
+
time.sleep(1)
|
31
|
+
train_status = ''
|
32
|
+
while True:
|
33
|
+
train_status = dd.get_train(sname,job=1,timeout=10)
|
34
|
+
if train_status['head']['status'] == 'running':
|
35
|
+
print train_status['body']['measure']
|
36
|
+
else:
|
37
|
+
print train_status
|
38
|
+
break
|
39
|
+
|
40
|
+
dd.delete_service(sname)
|
@@ -0,0 +1,23 @@
|
|
1
|
+
# rsync for selfie f40
|
2
|
+
echo "---> sync selfie #0"
|
3
|
+
rsync -arvu --delete /home/ubuntu/projects/deepdetect_u1/models/selfie_f40/ ubuntu@54.229.220.160:/home/ubuntu/projects/deepdetect/models/selfie/
|
4
|
+
|
5
|
+
echo "---> sync selfie #1"
|
6
|
+
rsync -arvu --delete /home/ubuntu/projects/deepdetect_u1/models/selfie_f40/ ubuntu@54.229.220.160:/home/ubuntu/projects/deepdetect/models/selfie_1/
|
7
|
+
|
8
|
+
echo "---> sync selfie #2"
|
9
|
+
rsync -arvu --delete /home/ubuntu/projects/deepdetect_u1/models/selfie_40/ ubuntu@54.229.220.160:/home/ubuntu/projects/deepdetect/models/selfie_2/
|
10
|
+
|
11
|
+
echo "---> sync selfie #3"
|
12
|
+
rsync -arvu --delete /home/ubuntu/projects/deepdetect_u1/models/selfie_f40/ ubuntu@54.229.220.160:/home/ubuntu/projects/deepdetect/models/selfie_3/
|
13
|
+
|
14
|
+
echo "---> sync selfie #4"
|
15
|
+
rsync -arvu --delete /home/ubuntu/projects/deepdetect_u1/models/selfie_f40/ ubuntu@54.229.220.160:/home/ubuntu/projects/deepdetect/models/selfie_4/
|
16
|
+
|
17
|
+
echo "---> sync selfie #5"
|
18
|
+
rsync -arvu --delete /home/ubuntu/projects/deepdetect_u1/models/selfie_f40/ ubuntu@54.229.220.160:/home/ubuntu/projects/deepdetect/models/selfie_5/
|
19
|
+
|
20
|
+
echo "---> sync selfie #reanalyze"
|
21
|
+
rsync -arvu --delete /home/ubuntu/projects/deepdetect_u1/models/selfie_f40/ ubuntu@54.229.220.160:/home/ubuntu/projects/deepdetect/models/selfie_reanalyze/
|
22
|
+
|
23
|
+
echo -e "\nsync fine-tuned selfie f40 done! \n"
|
data/docs/sync_file.sh
ADDED
@@ -0,0 +1,100 @@
|
|
1
|
+
# rsync -avh /home/ubuntu/downloads/test ubuntu@54.229.220.160:/home/ubuntu/downloads/test
|
2
|
+
|
3
|
+
# rsync -arvu --delete /home/ubuntu/projects/deepdetect/models/person_yes_no_f1/ ubuntu@54.229.220.160:/home/ubuntu/projects/deepdetect/models/person_yes_no_f1/
|
4
|
+
|
5
|
+
echo "---> sync person_yes_no"
|
6
|
+
rsync -arvu --delete /home/ubuntu/projects/deepdetect/models/person_yes_no_f1/ ubuntu@54.229.220.160:/home/ubuntu/projects/deepdetect/models/person_yes_no/
|
7
|
+
|
8
|
+
echo "---> sync person_yes_no #1"
|
9
|
+
rsync -arvu --delete /home/ubuntu/projects/deepdetect/models/person_yes_no_f1/ ubuntu@54.229.220.160:/home/ubuntu/projects/deepdetect/models/person_yes_no_1/
|
10
|
+
|
11
|
+
echo "---> sync person_yes_no #2"
|
12
|
+
rsync -arvu --delete /home/ubuntu/projects/deepdetect/models/person_yes_no_f1/ ubuntu@54.229.220.160:/home/ubuntu/projects/deepdetect/models/person_yes_no_2/
|
13
|
+
|
14
|
+
echo "---> sync person_yes_no #3"
|
15
|
+
rsync -arvu --delete /home/ubuntu/projects/deepdetect/models/person_yes_no_f1/ ubuntu@54.229.220.160:/home/ubuntu/projects/deepdetect/models/person_yes_no_3/
|
16
|
+
|
17
|
+
echo "---> sync person_yes_no #4"
|
18
|
+
rsync -arvu --delete /home/ubuntu/projects/deepdetect/models/person_yes_no_f1/ ubuntu@54.229.220.160:/home/ubuntu/projects/deepdetect/models/person_yes_no_4/
|
19
|
+
|
20
|
+
echo "---> sync person_yes_no #5"
|
21
|
+
rsync -arvu --delete /home/ubuntu/projects/deepdetect/models/person_yes_no_f1/ ubuntu@54.229.220.160:/home/ubuntu/projects/deepdetect/models/person_yes_no_5/
|
22
|
+
|
23
|
+
echo "---> sync person_yes_no #reanalyze"
|
24
|
+
rsync -arvu --delete /home/ubuntu/projects/deepdetect/models/person_yes_no_f1/ ubuntu@54.229.220.160:/home/ubuntu/projects/deepdetect/models/person_yes_no_reanalyze/
|
25
|
+
|
26
|
+
# rsync for selfie f1
|
27
|
+
echo "---> sync selfie"
|
28
|
+
rsync -arvu --delete /home/ubuntu/projects/deepdetect/models/selfie_f1/ ubuntu@54.229.220.160:/home/ubuntu/projects/deepdetect/models/selfie/
|
29
|
+
|
30
|
+
echo "---> sync selfie #1"
|
31
|
+
rsync -arvu --delete /home/ubuntu/projects/deepdetect/models/selfie_f1/ ubuntu@54.229.220.160:/home/ubuntu/projects/deepdetect/models/selfie_1/
|
32
|
+
|
33
|
+
echo "---> sync selfie #2"
|
34
|
+
rsync -arvu --delete /home/ubuntu/projects/deepdetect/models/selfie_f1/ ubuntu@54.229.220.160:/home/ubuntu/projects/deepdetect/models/selfie_2/
|
35
|
+
|
36
|
+
echo "---> sync selfie #3"
|
37
|
+
rsync -arvu --delete /home/ubuntu/projects/deepdetect/models/selfie_f1/ ubuntu@54.229.220.160:/home/ubuntu/projects/deepdetect/models/selfie_3/
|
38
|
+
|
39
|
+
echo "---> sync selfie #4"
|
40
|
+
rsync -arvu --delete /home/ubuntu/projects/deepdetect/models/selfie_f1/ ubuntu@54.229.220.160:/home/ubuntu/projects/deepdetect/models/selfie_4/
|
41
|
+
|
42
|
+
echo "---> sync selfie #5"
|
43
|
+
rsync -arvu --delete /home/ubuntu/projects/deepdetect/models/selfie_f1/ ubuntu@54.229.220.160:/home/ubuntu/projects/deepdetect/models/selfie_5/
|
44
|
+
|
45
|
+
echo "---> sync selfie #reanalyze"
|
46
|
+
rsync -arvu --delete /home/ubuntu/projects/deepdetect/models/selfie_f1/ ubuntu@54.229.220.160:/home/ubuntu/projects/deepdetect/models/selfie_reanalyze/
|
47
|
+
|
48
|
+
echo -e "\nsync fine-tuned selfie done! \n"
|
49
|
+
|
50
|
+
# rsync for selfie f2
|
51
|
+
echo "---> sync selfie f2"
|
52
|
+
rsync -arvu --delete /home/ubuntu/projects/deepdetect/models/selfie_f2/ ubuntu@54.229.220.160:/home/ubuntu/projects/deepdetect/models/selfie/
|
53
|
+
|
54
|
+
echo "---> sync selfie #1"
|
55
|
+
rsync -arvu --delete /home/ubuntu/projects/deepdetect/models/selfie_f2/ ubuntu@54.229.220.160:/home/ubuntu/projects/deepdetect/models/selfie_1/
|
56
|
+
|
57
|
+
echo "---> sync selfie #2"
|
58
|
+
rsync -arvu --delete /home/ubuntu/projects/deepdetect/models/selfie_f2/ ubuntu@54.229.220.160:/home/ubuntu/projects/deepdetect/models/selfie_2/
|
59
|
+
|
60
|
+
echo "---> sync selfie #3"
|
61
|
+
rsync -arvu --delete /home/ubuntu/projects/deepdetect/models/selfie_f2/ ubuntu@54.229.220.160:/home/ubuntu/projects/deepdetect/models/selfie_3/
|
62
|
+
|
63
|
+
echo "---> sync selfie #4"
|
64
|
+
rsync -arvu --delete /home/ubuntu/projects/deepdetect/models/selfie_f2/ ubuntu@54.229.220.160:/home/ubuntu/projects/deepdetect/models/selfie_4/
|
65
|
+
|
66
|
+
echo "---> sync selfie #5"
|
67
|
+
rsync -arvu --delete /home/ubuntu/projects/deepdetect/models/selfie_f2/ ubuntu@54.229.220.160:/home/ubuntu/projects/deepdetect/models/selfie_5/
|
68
|
+
|
69
|
+
echo "---> sync selfie #reanalyze"
|
70
|
+
rsync -arvu --delete /home/ubuntu/projects/deepdetect/models/selfie_f2/ ubuntu@54.229.220.160:/home/ubuntu/projects/deepdetect/models/selfie_reanalyze/
|
71
|
+
|
72
|
+
echo -e "\nsync fine-tuned selfie f2 done! \n"
|
73
|
+
|
74
|
+
# --------------------------------------------------
|
75
|
+
# rsync for more_than_one_f1
|
76
|
+
echo "---> sync more_than_one_f1"
|
77
|
+
rsync -arvu --delete /home/ubuntu/projects/deepdetect/models/more_than_one_f1/ ubuntu@54.229.220.160:/home/ubuntu/projects/deepdetect/models/more_than_one/
|
78
|
+
|
79
|
+
echo "---> sync more_than_one #1"
|
80
|
+
rsync -arvu --delete /home/ubuntu/projects/deepdetect/models/more_than_one_f1/ ubuntu@54.229.220.160:/home/ubuntu/projects/deepdetect/models/more_than_one_1/
|
81
|
+
|
82
|
+
echo "---> sync more_than_one #2"
|
83
|
+
rsync -arvu --delete /home/ubuntu/projects/deepdetect/models/more_than_one_f1/ ubuntu@54.229.220.160:/home/ubuntu/projects/deepdetect/models/more_than_one_2/
|
84
|
+
|
85
|
+
echo "---> sync more_than_one #3"
|
86
|
+
rsync -arvu --delete /home/ubuntu/projects/deepdetect/models/more_than_one_f1/ ubuntu@54.229.220.160:/home/ubuntu/projects/deepdetect/models/more_than_one_3/
|
87
|
+
|
88
|
+
echo "---> sync more_than_one #4"
|
89
|
+
rsync -arvu --delete /home/ubuntu/projects/deepdetect/models/more_than_one_f1/ ubuntu@54.229.220.160:/home/ubuntu/projects/deepdetect/models/more_than_one_4/
|
90
|
+
|
91
|
+
echo "---> sync more_than_one #5"
|
92
|
+
rsync -arvu --delete /home/ubuntu/projects/deepdetect/models/more_than_one_f1/ ubuntu@54.229.220.160:/home/ubuntu/projects/deepdetect/models/more_than_one_5/
|
93
|
+
|
94
|
+
echo "---> sync more_than_one #reanalyze"
|
95
|
+
rsync -arvu --delete /home/ubuntu/projects/deepdetect/models/more_than_one_f1/ ubuntu@54.229.220.160:/home/ubuntu/projects/deepdetect/models/more_than_one_reanalyze/
|
96
|
+
|
97
|
+
echo -e "\nsync fine-tuned more_than_one_f1 done! \n"
|
98
|
+
|
99
|
+
|
100
|
+
rsync -arvu --delete /home/ubuntu/projects/deepdetect/models/person_yes_no_f1/ ubuntu@54.229.220.160:/home/ubuntu/projects/deepdetect/models/person_yes_no_1/
|
@@ -0,0 +1,23 @@
|
|
1
|
+
# rsync for mirrorshot f1
|
2
|
+
echo "---> sync mirrorshot #0"
|
3
|
+
rsync -arvu --delete /data/projects/deepdetect_u1/models/mirrorshot_f1/ ubuntu@52.51.246.115:/home/ubuntu/projects/deepdetect/models/mirrorshot/
|
4
|
+
|
5
|
+
echo "---> sync mirrorshot #1"
|
6
|
+
rsync -arvu --delete /data/projects/deepdetect_u1/models/mirrorshot_f1/ ubuntu@52.51.246.115:/home/ubuntu/projects/deepdetect/models/mirrorshot_1/
|
7
|
+
|
8
|
+
echo "---> sync mirrorshot #2"
|
9
|
+
rsync -arvu --delete /data/projects/deepdetect_u1/models/mirrorshot_f1/ ubuntu@52.51.246.115:/home/ubuntu/projects/deepdetect/models/mirrorshot_2/
|
10
|
+
|
11
|
+
echo "---> sync mirrorshot #3"
|
12
|
+
rsync -arvu --delete /data/projects/deepdetect_u1/models/mirrorshot_f1/ ubuntu@52.51.246.115:/home/ubuntu/projects/deepdetect/models/mirrorshot_3/
|
13
|
+
|
14
|
+
echo "---> sync mirrorshot #4"
|
15
|
+
rsync -arvu --delete /data/projects/deepdetect_u1/models/mirrorshot_f1/ ubuntu@52.51.246.115:/home/ubuntu/projects/deepdetect/models/mirrorshot_4/
|
16
|
+
|
17
|
+
echo "---> sync mirrorshot #5"
|
18
|
+
rsync -arvu --delete /data/projects/deepdetect_u1/models/mirrorshot_f1/ ubuntu@52.51.246.115:/home/ubuntu/projects/deepdetect/models/mirrorshot_5/
|
19
|
+
|
20
|
+
echo "---> sync mirrorshot #reanalyze"
|
21
|
+
rsync -arvu --delete /data/projects/deepdetect_u1/models/mirrorshot_f1/ ubuntu@52.51.246.115:/home/ubuntu/projects/deepdetect/models/mirrorshot_reanalyze/
|
22
|
+
|
23
|
+
echo -e "\nsync fine-tuned mirrorshot f1 done! \n"
|
@@ -0,0 +1,34 @@
|
|
1
|
+
require "deepdetect_ruby/version"
|
2
|
+
|
3
|
+
module DeepdetectRuby
|
4
|
+
module Configuration
|
5
|
+
VALID_CONFIG_KEYS = [:host, :host_train, :deepdetect_path, :model_path, :debug, :servers, :is_scaling].freeze
|
6
|
+
DEFAULT_HOST = "http://127.0.0.1:8080"
|
7
|
+
DEFAULT_HOST_TRAIN = "http://127.0.0.1:8080"
|
8
|
+
|
9
|
+
attr_accessor *VALID_CONFIG_KEYS
|
10
|
+
|
11
|
+
def self.extended(base)
|
12
|
+
base.reset
|
13
|
+
end
|
14
|
+
|
15
|
+
def reset
|
16
|
+
self.host = DEFAULT_HOST
|
17
|
+
self.host_train = DEFAULT_HOST_TRAIN
|
18
|
+
self.deepdetect_path = ""
|
19
|
+
self.model_path = ""
|
20
|
+
self.debug = true
|
21
|
+
self.servers = DEFAULT_HOST
|
22
|
+
self.is_scaling = false
|
23
|
+
end
|
24
|
+
|
25
|
+
def configure
|
26
|
+
yield self
|
27
|
+
end
|
28
|
+
|
29
|
+
def options
|
30
|
+
Hash[* VALID_CONFIG_KEYS.map { |key| [key, send(key)] }.flatten]
|
31
|
+
end
|
32
|
+
|
33
|
+
end
|
34
|
+
end
|
data/lib/dede_server.rb
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
module DeepdetectRuby
|
2
|
+
class DedeServer
|
3
|
+
def self.get_server(server_index = 1)
|
4
|
+
debug = DeepdetectRuby.options[:debug]
|
5
|
+
if DeepdetectRuby.options[:is_scaling]
|
6
|
+
servers = DeepdetectRuby.options[:servers].split(",").map{|s| s.strip}
|
7
|
+
puts "\n[get_server] server #{server_index} is running. We have #{servers.count} server working together. \n" if debug
|
8
|
+
if !servers.nil? && servers.present?
|
9
|
+
dede_index = server_index - 1
|
10
|
+
if dede_index >= 0
|
11
|
+
return servers[dede_index]
|
12
|
+
else
|
13
|
+
puts "\n[DedeServer] server_index must be greater than 0 \n" if debug
|
14
|
+
end
|
15
|
+
else
|
16
|
+
puts "\n[DedeServer] hosts must not be empty \n" if debug
|
17
|
+
end
|
18
|
+
else
|
19
|
+
puts "\n[get_server] single server is running \n" if debug
|
20
|
+
return DeepdetectRuby.options[:host]
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,60 @@
|
|
1
|
+
# Symbolizes all of hash's keys and subkeys.
|
2
|
+
# Also allows for custom pre-processing of keys (e.g. downcasing, etc)
|
3
|
+
# if the block is given:
|
4
|
+
#
|
5
|
+
# somehash.deep_symbolize { |key| key.downcase }
|
6
|
+
#
|
7
|
+
# Usage: either include it into global Hash class to make it available to
|
8
|
+
# to all hashes, or extend only your own hash objects with this
|
9
|
+
# module.
|
10
|
+
# E.g.:
|
11
|
+
# 1) class Hash; include DeepSymbolizable; end
|
12
|
+
# 2) myhash.extend DeepSymbolizable
|
13
|
+
|
14
|
+
module DeepSymbolizable
|
15
|
+
def deep_symbolize(&block)
|
16
|
+
method = self.class.to_s.downcase.to_sym
|
17
|
+
syms = DeepSymbolizable::Symbolizers
|
18
|
+
syms.respond_to?(method) ? syms.send(method, self, &block) : self
|
19
|
+
end
|
20
|
+
|
21
|
+
module Symbolizers
|
22
|
+
extend self
|
23
|
+
|
24
|
+
# the primary method - symbolizes keys of the given hash,
|
25
|
+
# preprocessing them with a block if one was given, and recursively
|
26
|
+
# going into all nested enumerables
|
27
|
+
def hash(hash, &block)
|
28
|
+
hash.inject({}) do |result, (key, value)|
|
29
|
+
# Recursively deep-symbolize subhashes
|
30
|
+
value = _recurse_(value, &block)
|
31
|
+
|
32
|
+
# Pre-process the key with a block if it was given
|
33
|
+
key = yield key if block_given?
|
34
|
+
# Symbolize the key string if it responds to to_sym
|
35
|
+
sym_key = key.to_sym rescue key
|
36
|
+
|
37
|
+
# write it back into the result and return the updated hash
|
38
|
+
result[sym_key] = value
|
39
|
+
result
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
# walking over arrays and symbolizing all nested elements
|
44
|
+
def array(ary, &block)
|
45
|
+
ary.map { |v| _recurse_(v, &block) }
|
46
|
+
end
|
47
|
+
|
48
|
+
# handling recursion - any Enumerable elements (except String)
|
49
|
+
# is being extended with the module, and then symbolized
|
50
|
+
def _recurse_(value, &block)
|
51
|
+
if value.is_a?(Enumerable) && !value.is_a?(String)
|
52
|
+
# support for a use case without extended core Hash
|
53
|
+
value.extend DeepSymbolizable unless value.class.include?(DeepSymbolizable)
|
54
|
+
value = value.deep_symbolize(&block)
|
55
|
+
end
|
56
|
+
value
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
require "deepdetect_ruby/version"
|
2
|
+
require "configuration"
|
3
|
+
require "deep_symbolize"
|
4
|
+
require 'httparty'
|
5
|
+
require 'dede_server'
|
6
|
+
require "info"
|
7
|
+
require "service"
|
8
|
+
require "train"
|
9
|
+
require "predict"
|
10
|
+
|
11
|
+
module DeepdetectRuby
|
12
|
+
extend Configuration
|
13
|
+
end
|
data/lib/info.rb
ADDED
@@ -0,0 +1,42 @@
|
|
1
|
+
module DeepdetectRuby
|
2
|
+
def self.info(options = {})
|
3
|
+
begin
|
4
|
+
server_index = options[:server_index] || 1
|
5
|
+
dede_host = DeepdetectRuby::DedeServer.get_server(server_index)
|
6
|
+
info_url = "#{dede_host}/info"
|
7
|
+
dede_info = HTTParty.get(info_url)
|
8
|
+
info = JSON.parse(dede_info.to_json)
|
9
|
+
info.extend DeepSymbolizable
|
10
|
+
return info.deep_symbolize
|
11
|
+
rescue Exception => e
|
12
|
+
puts "\n[DeepdetectRuby Info]. #{e.to_s} \n"
|
13
|
+
return nil
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
def self.status(options = {})
|
18
|
+
info(options)
|
19
|
+
end
|
20
|
+
|
21
|
+
def self.get_server(server_index = 1)
|
22
|
+
debug = DeepdetectRuby.options[:debug]
|
23
|
+
if DeepdetectRuby.options[:is_scaling]
|
24
|
+
servers = DeepdetectRuby.options[:servers].split(",").map{|s| s.strip}
|
25
|
+
puts "\n[get_server] server #{server_index} is running. We have #{servers.count} server working together. \n" if debug
|
26
|
+
if !servers.nil? && servers.present?
|
27
|
+
dede_index = server_index - 1
|
28
|
+
if dede_index >= 0
|
29
|
+
return servers[dede_index]
|
30
|
+
else
|
31
|
+
puts "\n[DedeServer] server_index must be greater than 0 \n" if debug
|
32
|
+
end
|
33
|
+
else
|
34
|
+
puts "\n[DedeServer] hosts must not be empty \n" if debug
|
35
|
+
end
|
36
|
+
else
|
37
|
+
puts "\n[get_server] single server is running \n" if debug
|
38
|
+
return DeepdetectRuby.options[:host]
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
end
|
data/lib/predict.rb
ADDED
@@ -0,0 +1,50 @@
|
|
1
|
+
module DeepdetectRuby
|
2
|
+
class Predict
|
3
|
+
def self.predict(options = {})
|
4
|
+
begin
|
5
|
+
debug = DeepdetectRuby.options[:debug]
|
6
|
+
server_index = options[:server_index] || 1
|
7
|
+
dede_host = DeepdetectRuby::DedeServer.get_server(server_index)
|
8
|
+
dede_api_url = "#{dede_host}/predict"
|
9
|
+
puts "\n-------------> Starting to predict an image #{dede_api_url}....\n" if debug
|
10
|
+
best = options[:best] || 2
|
11
|
+
gpu = options[:gpu] || true
|
12
|
+
width = options[:width] || 224
|
13
|
+
height = options[:height] || 224
|
14
|
+
|
15
|
+
data = {
|
16
|
+
"service" => "#{options[:service]}",
|
17
|
+
"parameters" => {
|
18
|
+
"input" => {
|
19
|
+
"width" => width,
|
20
|
+
"height" => height
|
21
|
+
},
|
22
|
+
"output" => {
|
23
|
+
"best" => best
|
24
|
+
},
|
25
|
+
"mllib" => { "gpu" => gpu }
|
26
|
+
},
|
27
|
+
"data" => [
|
28
|
+
"#{options[:image_url]}"
|
29
|
+
]
|
30
|
+
}
|
31
|
+
if !options[:template].nil?
|
32
|
+
data["parameters"]["output"]["template"] = options[:template]
|
33
|
+
end
|
34
|
+
|
35
|
+
puts "\nData: #{data.to_json} \n" if debug
|
36
|
+
|
37
|
+
dede_body = {
|
38
|
+
:body => data.to_json,
|
39
|
+
:headers => { "Content-Type" => 'application/json' }
|
40
|
+
}
|
41
|
+
object = HTTParty.post(dede_api_url, dede_body)
|
42
|
+
predict_info = JSON.parse(object.body)
|
43
|
+
predict_info.extend DeepSymbolizable
|
44
|
+
return predict_info.deep_symbolize
|
45
|
+
rescue Exception => e
|
46
|
+
puts "\n[DeepdetectRuby Predict - predict]. #{e.to_s} \n"
|
47
|
+
end
|
48
|
+
end # end predict
|
49
|
+
end
|
50
|
+
end
|
data/lib/service.rb
ADDED
@@ -0,0 +1,228 @@
|
|
1
|
+
module DeepdetectRuby
|
2
|
+
class Service
|
3
|
+
def self.create(options = {})
|
4
|
+
begin
|
5
|
+
debug = DeepdetectRuby.options[:debug]
|
6
|
+
server_index = options[:server_index] || 1
|
7
|
+
dede_host = DeepdetectRuby::DedeServer.get_server(server_index)
|
8
|
+
dede_api_url = "#{dede_host}/services/#{options[:name]}"
|
9
|
+
puts "\n-------------> Starting to create DeepDetect service #{dede_api_url}....\n" if debug
|
10
|
+
|
11
|
+
description = "#{options[:name]} classification service"
|
12
|
+
options[:mllib] = !options[:mllib].nil? ? options[:mllib] : "caffe"
|
13
|
+
options[:description] = !options[:description].nil? ? options[:description] : description
|
14
|
+
options[:type] = !options[:type].nil? ? options[:type] : "supervised"
|
15
|
+
options[:connector] = !options[:connector].nil? ? options[:connector] : "image"
|
16
|
+
options[:width] = !options[:width].nil? ? options[:width] : 224
|
17
|
+
options[:height] = !options[:height].nil? ? options[:height] : 224
|
18
|
+
options[:nclasses] = !options[:nclasses].nil? ? options[:nclasses] : 2
|
19
|
+
options[:gpu] = !options[:gpu].nil? ? options[:gpu] : true
|
20
|
+
|
21
|
+
# for finetuning
|
22
|
+
options[:finetuning] = !options[:finetuning].nil? ? options[:finetuning] : false
|
23
|
+
options[:template] = !options[:template].nil? ? options[:template] : "googlenet"
|
24
|
+
options[:weights] = !options[:weights].nil? ? options[:weights] : nil
|
25
|
+
options[:rotate] = !options[:rotate].nil? ? options[:rotate] : false
|
26
|
+
options[:mirror] = !options[:mirror].nil? ? options[:mirror] : false
|
27
|
+
options[:mirror] = options[:finetuning]
|
28
|
+
|
29
|
+
if !DeepdetectRuby.options[:model_path].nil? && !DeepdetectRuby.options[:model_path].empty?
|
30
|
+
options[:repository] = "#{DeepdetectRuby.options[:model_path]}/#{options[:name]}"
|
31
|
+
end
|
32
|
+
|
33
|
+
# for different model name
|
34
|
+
if !options[:model_name].nil? && !options[:model_name].empty?
|
35
|
+
options[:repository] = "#{DeepdetectRuby.options[:model_path]}/#{options[:model_name]}"
|
36
|
+
end
|
37
|
+
|
38
|
+
data = {
|
39
|
+
"mllib" => "#{options[:mllib]}",
|
40
|
+
"description" => "#{options[:description]}",
|
41
|
+
"type" => "#{options[:type]}",
|
42
|
+
"parameters" => {
|
43
|
+
"input" => {
|
44
|
+
"connector" => "#{options[:connector]}",
|
45
|
+
"height" => options[:height],
|
46
|
+
"width" => options[:width]
|
47
|
+
},
|
48
|
+
"mllib" => {
|
49
|
+
"nclasses" => options[:nclasses],
|
50
|
+
"gpu" => options[:gpu]
|
51
|
+
}
|
52
|
+
},
|
53
|
+
"model" => {
|
54
|
+
"repository" => "#{options[:repository]}"
|
55
|
+
}
|
56
|
+
}
|
57
|
+
|
58
|
+
if options[:finetuning] && !options[:weights].nil?
|
59
|
+
data = {
|
60
|
+
"mllib" => "#{options[:mllib]}",
|
61
|
+
"description" => "#{options[:description]}",
|
62
|
+
"type" => "#{options[:type]}",
|
63
|
+
"parameters" => {
|
64
|
+
"input" => {
|
65
|
+
"connector" => "#{options[:connector]}",
|
66
|
+
"height" => options[:height],
|
67
|
+
"width" => options[:width]
|
68
|
+
},
|
69
|
+
"mllib" => {
|
70
|
+
"nclasses" => options[:nclasses],
|
71
|
+
"gpu" => options[:gpu],
|
72
|
+
"weights" => options[:weights],
|
73
|
+
"finetuning" => options[:finetuning],
|
74
|
+
"rotate" => options[:rotate],
|
75
|
+
"mirror" => options[:mirror]
|
76
|
+
}
|
77
|
+
},
|
78
|
+
"model" => {
|
79
|
+
"repository" => "#{options[:repository]}"
|
80
|
+
}
|
81
|
+
}
|
82
|
+
end
|
83
|
+
|
84
|
+
puts "\nparams data: #{data.to_json} \n" if debug
|
85
|
+
dede_body = {
|
86
|
+
:body => data.to_json,
|
87
|
+
:headers => { "Content-Type" => 'application/json' }
|
88
|
+
}
|
89
|
+
object = HTTParty.put(dede_api_url, dede_body)
|
90
|
+
object_info = JSON.parse(object.body)
|
91
|
+
object_info.extend DeepSymbolizable
|
92
|
+
return object_info.deep_symbolize
|
93
|
+
rescue Exception => e
|
94
|
+
puts "\n[DeepdetectRuby Service]. #{e.to_s} \n"
|
95
|
+
end
|
96
|
+
|
97
|
+
end
|
98
|
+
|
99
|
+
def self.get_info(service_name = "", server_index = 1)
|
100
|
+
begin
|
101
|
+
debug = DeepdetectRuby.options[:debug]
|
102
|
+
dede_host = DeepdetectRuby::DedeServer.get_server(server_index)
|
103
|
+
dede_api_url = "#{dede_host}/services/#{service_name}"
|
104
|
+
puts "\n-------------> Starting to get information DeepDetect service #{dede_api_url}....\n" if debug
|
105
|
+
object = HTTParty.get(dede_api_url)
|
106
|
+
object_info = JSON.parse(object.body)
|
107
|
+
object_info.extend DeepSymbolizable
|
108
|
+
return object_info.deep_symbolize
|
109
|
+
rescue Exception => e
|
110
|
+
puts "\n[DeepdetectRuby Service get information]. #{e.to_s} \n"
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
114
|
+
def self.delete(service_name = "", server_index = 1)
|
115
|
+
begin
|
116
|
+
debug = DeepdetectRuby.options[:debug]
|
117
|
+
dede_host = DeepdetectRuby::DedeServer.get_server(server_index)
|
118
|
+
dede_api_url = "#{dede_host}/services/#{service_name}"
|
119
|
+
puts "\n-------------> Starting to delete DeepDetect service #{dede_api_url}....\n" if debug
|
120
|
+
object = HTTParty.delete(dede_api_url)
|
121
|
+
object_info = JSON.parse(object.body)
|
122
|
+
object_info.extend DeepSymbolizable
|
123
|
+
return object_info.deep_symbolize
|
124
|
+
rescue Exception => e
|
125
|
+
puts "\n[DeepdetectRuby delete service]. #{e.to_s} \n"
|
126
|
+
end
|
127
|
+
end
|
128
|
+
|
129
|
+
def self.load_all_models(options = {})
|
130
|
+
debug = DeepdetectRuby.options[:debug]
|
131
|
+
server_index = options[:server_index] || 1
|
132
|
+
dede_host = DeepdetectRuby::DedeServer.get_server(server_index)
|
133
|
+
puts "\n-----------> Starting to load_all_models..." if debug
|
134
|
+
model_path = DeepdetectRuby.options[:model_path]
|
135
|
+
dirs = Dir.glob("#{model_path}/*").select{ |e| File.directory? e }
|
136
|
+
dirs.each_with_index do |dir, index|
|
137
|
+
model_name = File.basename(dir)
|
138
|
+
dede_opts = { repository: dir, name: model_name, server_index: server_index }
|
139
|
+
DeepdetectRuby::Service.create(dede_opts)
|
140
|
+
puts "Index: #{index} - model name: #{model_name} in #{dir}" if debug
|
141
|
+
end
|
142
|
+
puts "\n-----------> End to load_all_models..." if debug
|
143
|
+
end
|
144
|
+
|
145
|
+
def self.reload_models(options = {})
|
146
|
+
debug = DeepdetectRuby.options[:debug]
|
147
|
+
server_index = options[:server_index] || 1
|
148
|
+
dede_host = DeepdetectRuby::DedeServer.get_server(server_index)
|
149
|
+
puts "\n-----------> Starting to reload models..." if debug
|
150
|
+
model_path = DeepdetectRuby.options[:model_path]
|
151
|
+
dirs = Dir.glob("#{model_path}/*").select{ |e| File.directory? e }
|
152
|
+
dirs.each_with_index do |dir, index|
|
153
|
+
model_name = File.basename(dir)
|
154
|
+
DeepdetectRuby::Service.delete(model_name, server_index)
|
155
|
+
end
|
156
|
+
DeepdetectRuby::Service.load_all_models(server_index)
|
157
|
+
puts "\n-----------> End to reload models..." if debug
|
158
|
+
end
|
159
|
+
|
160
|
+
def self.list_models
|
161
|
+
debug = DeepdetectRuby.options[:debug]
|
162
|
+
puts "\n-----------> Starting to reload models..." if debug
|
163
|
+
model_path = DeepdetectRuby.options[:model_path]
|
164
|
+
dirs = Dir.glob("#{model_path}/*").select{ |e| File.directory? e }
|
165
|
+
list = []
|
166
|
+
dirs.each_with_index do |dir, index|
|
167
|
+
model_name = File.basename(dir)
|
168
|
+
list << model_name
|
169
|
+
end
|
170
|
+
puts "\nList models: #{list.to_json}" if debug
|
171
|
+
return list
|
172
|
+
end
|
173
|
+
|
174
|
+
def self.list_models_running(options = {})
|
175
|
+
dede = DeepdetectRuby.info(options)
|
176
|
+
services = dede[:head][:services]
|
177
|
+
return services
|
178
|
+
end
|
179
|
+
|
180
|
+
def self.finetuning(options = {})
|
181
|
+
puts "\n-----> Starting to create service and train for finetuning \n"
|
182
|
+
server_index = options[:server_index] || 1
|
183
|
+
dede_host = DeepdetectRuby::DedeServer.get_server(server_index)
|
184
|
+
options[:name] = !options[:name].nil? ? options[:name] : ""
|
185
|
+
# weights is called the caffe filename model, example: model_iter_16368.caffemodel
|
186
|
+
options[:weights] = !options[:weights].nil? ? options[:weights] : ""
|
187
|
+
options[:repository] = !options[:repository].nil? ? options[:repository] : ""
|
188
|
+
options[:gpu] = !options[:gpu].nil? ? options[:gpu] : true
|
189
|
+
options[:iterations] = !options[:iterations].nil? ? options[:iterations] : 10000
|
190
|
+
options[:test_interval] = !options[:test_interval].nil? ? options[:test_interval] : 500
|
191
|
+
options[:nclasses] = !options[:nclasses].nil? ? options[:nclasses] : 2
|
192
|
+
options[:measure_index] = !options[:measure_index].nil? ? options[:measure_index] : 1
|
193
|
+
options[:batch_size] = !options[:batch_size].nil? ? options[:batch_size] : 32
|
194
|
+
options[:test_batch_size] = !options[:test_batch_size].nil? ? options[:test_batch_size] : 32
|
195
|
+
|
196
|
+
if !options[:name].empty? && !options[:weights].empty? && !options[:repository].empty?
|
197
|
+
# Create a service for finetuning
|
198
|
+
service_options = { name: "#{options[:name]}",
|
199
|
+
finetuning: true,
|
200
|
+
weights: "#{options[:weights]}",
|
201
|
+
nclasses: options[:nclasses],
|
202
|
+
server_index: server_index
|
203
|
+
}
|
204
|
+
DeepdetectRuby::Service.create(service_options)
|
205
|
+
|
206
|
+
puts "\nWaiting to start function fine tune..."
|
207
|
+
sleep(3)
|
208
|
+
|
209
|
+
# Train a new updated service for finetuning
|
210
|
+
train_options = { service: "#{options[:name]}",
|
211
|
+
repository: "#{options[:repository]}",
|
212
|
+
gpu: true,
|
213
|
+
iterations: options[:iterations],
|
214
|
+
test_interval: options[:test_interval],
|
215
|
+
finetuning: true,
|
216
|
+
measure_index: options[:measure_index],
|
217
|
+
batch_size: options[:batch_size],
|
218
|
+
test_batch_size: options[:test_batch_size],
|
219
|
+
server_index: server_index
|
220
|
+
}
|
221
|
+
DeepdetectRuby::Train.launch(train_options)
|
222
|
+
else
|
223
|
+
puts "\nMissing options :name or :weights or :repository. Please check. :) \n"
|
224
|
+
end
|
225
|
+
end
|
226
|
+
|
227
|
+
end
|
228
|
+
end
|