filecluster 0.2.9 → 0.3.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (14) hide show
  1. data/README.md +1 -1
  2. data/lib/daemon/global_daemon_thread.rb +4 -4
  3. data/lib/fc/db.rb +6 -13
  4. data/lib/fc/policy.rb +4 -37
  5. data/lib/fc/storage.rb +22 -2
  6. data/lib/fc/version.rb +1 -1
  7. data/lib/manage/policies.rb +2 -10
  8. data/lib/manage/storages.rb +41 -28
  9. data/test/daemon_test.rb +5 -5
  10. data/test/db_test.rb +5 -13
  11. data/test/policy_test.rb +5 -51
  12. data/test/{storage_sync.rb → storage_sync_test.rb} +9 -5
  13. data/test/storage_test.rb +45 -1
  14. metadata +6 -6
data/README.md CHANGED
@@ -31,7 +31,7 @@ Selecting available storage to store item by policy.create_storages (from left t
31
31
 
32
32
  ## Copy policy
33
33
 
34
- Selecting available storage to copy item by policy.copy_storages (from left to tight) with the nearest copy_id.
34
+ Selecting available storage to copy item by storage.copy_storages (from left to right).
35
35
 
36
36
  ## Variables
37
37
 
data/lib/daemon/global_daemon_thread.rb CHANGED
@@ -30,11 +30,11 @@ class GlobalDaemonThread < BaseThread
30
30
  all_policies = FC::Policy.where
31
31
 
32
32
  # policies.get_storages => all_policies.select
33
- all_policies.each do |policy|
34
- metaclass = class << policy; self; end
33
+ all_storages.each do |storage|
34
+ metaclass = class << storage; self; end
35
35
  metaclass.send(:define_method, :get_copy_storages) do
36
36
  self.class.get_copy_storages_mutex.synchronize do
37
- @copy_storages_cache ||= self.copy_storages.split(',').map{|storage_name| all_storages.detect{|s| storage_name == s.name} }
37
+ @copy_storages_cache ||= self.copy_storages.to_s.split(',').map{|storage_name| all_storages.detect{|s| storage_name == s.name} }
38
38
  end
39
39
  @copy_storages_cache
40
40
  end
@@ -57,7 +57,7 @@ class GlobalDaemonThread < BaseThread
57
57
  $log.warn("GlobalDaemonThread: ItemStorage count >= policy.copies for item #{row['item_id']}")
58
58
  else
59
59
  src_storage = all_storages.detect{|s| item_storages.first == s.name}
60
- storage = policy.get_proper_storage_for_copy(row['size'], src_storage.copy_id, item_storages) if src_storage && policy
60
+ storage = src_storage.get_proper_storage_for_copy(row['size'], item_storages) if src_storage
61
61
  if storage
62
62
  FC::Item.new(:id => row['item_id']).make_item_storage(storage, 'copy')
63
63
  else
data/lib/fc/db.rb CHANGED
@@ -85,21 +85,17 @@ module FC
85
85
  size bigint NOT NULL DEFAULT 0,
86
86
  size_limit bigint NOT NULL DEFAULT 0,
87
87
  check_time int DEFAULT NULL,
88
- copy_id int NOT NULL DEFAULT 0,
88
+ copy_storages text NOT NULL DEFAULT '',
89
89
  PRIMARY KEY (id), UNIQUE KEY (name), KEY (host)
90
90
  ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
91
91
  })
92
92
  proc = %{
93
- # update policy.create_storages and policy.copy_storages on storage delete and update
93
+ # update policy.create_storages on storage delete and update
94
94
  UPDATE #{@prefix}policies,
95
95
  (SELECT #{@prefix}policies.id, GROUP_CONCAT(#{@prefix}storages.name ORDER BY FIND_IN_SET(#{@prefix}storages.name, create_storages)) as storages FROM #{@prefix}policies LEFT JOIN #{@prefix}storages ON
96
- FIND_IN_SET(#{@prefix}storages.name, create_storages) GROUP BY #{@prefix}policies.id) as policy_create,
97
- (SELECT #{@prefix}policies.id, GROUP_CONCAT(#{@prefix}storages.name ORDER BY FIND_IN_SET(#{@prefix}storages.name, copy_storages)) as storages FROM #{@prefix}policies LEFT JOIN #{@prefix}storages ON
98
- FIND_IN_SET(#{@prefix}storages.name, copy_storages) GROUP BY #{@prefix}policies.id) as policy_copy
99
- SET
100
- #{@prefix}policies.create_storages = policy_create.storages,
101
- #{@prefix}policies.copy_storages = policy_copy.storages
102
- WHERE policy_create.id = #{@prefix}policies.id AND policy_copy.id = #{@prefix}policies.id;
96
+ FIND_IN_SET(#{@prefix}storages.name, create_storages) GROUP BY #{@prefix}policies.id) as policy_create
97
+ SET #{@prefix}policies.create_storages = policy_create.storages
98
+ WHERE policy_create.id = #{@prefix}policies.id;
103
99
  }
104
100
  proc_update = %{
105
101
  IF OLD.name <> NEW.name THEN
@@ -114,17 +110,14 @@ module FC
114
110
  id int NOT NULL AUTO_INCREMENT,
115
111
  name varchar(255) NOT NULL DEFAULT '',
116
112
  create_storages text NOT NULL DEFAULT '',
117
- copy_storages text NOT NULL DEFAULT '',
118
113
  copies int NOT NULL DEFAULT 0,
119
114
  PRIMARY KEY (id), UNIQUE KEY (name)
120
115
  ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
121
116
  })
122
117
  proc = %{
123
- # update policy.create_storages and policy.copy_storages on policy change - guarantee valid policy.storages
118
+ # update policy.create_storages on policy change - guarantee valid policy.storages
124
119
  SELECT GROUP_CONCAT(name ORDER BY FIND_IN_SET(name, NEW.create_storages)) INTO @create_storages_list FROM #{@prefix}storages WHERE FIND_IN_SET(name, NEW.create_storages);
125
- SELECT GROUP_CONCAT(name ORDER BY FIND_IN_SET(name, NEW.copy_storages)) INTO @copy_storages_list FROM #{@prefix}storages WHERE FIND_IN_SET(name, NEW.copy_storages);
126
120
  SET NEW.create_storages = @create_storages_list;
127
- SET NEW.copy_storages = @copy_storages_list;
128
121
  }
129
122
  FC::DB.connect.query("CREATE TRIGGER fc_policies_before_insert BEFORE INSERT on #{@prefix}policies FOR EACH ROW BEGIN #{proc} END")
130
123
  FC::DB.connect.query("CREATE TRIGGER fc_policies_before_update BEFORE UPDATE on #{@prefix}policies FOR EACH ROW BEGIN #{proc} END")
data/lib/fc/policy.rb CHANGED
@@ -2,14 +2,13 @@
2
2
 
3
3
  module FC
4
4
  class Policy < DbBase
5
- set_table :policies, 'name, create_storages, copy_storages, copies'
5
+ set_table :policies, 'name, create_storages, copies'
6
6
 
7
7
  class << self
8
- attr_accessor :storages_cache_time, :get_create_storages_mutex, :get_copy_storages_mutex
8
+ attr_accessor :storages_cache_time, :get_create_storages_mutex
9
9
  end
10
10
  @storages_cache_time = 20 # ttl for storages cache
11
11
  @get_create_storages_mutex = Mutex.new
12
- @get_copy_storages_mutex = Mutex.new
13
12
 
14
13
  def self.filter_by_host(host = nil)
15
14
  host = FC::Storage.curr_host unless host
@@ -22,50 +21,18 @@ module FC
22
21
  self.class.get_create_storages_mutex.synchronize do
23
22
  unless @create_storages_cache && Time.new.to_i - @get_create_storages_time.to_i < self.class.storages_cache_time
24
23
  @get_create_storages_time = Time.new.to_i
25
- names = create_storages.split(',').map{|s| "'#{s}'"}.join(',')
26
- @create_storages_cache = FC::Storage.where("name IN (#{names}) ORDER BY FIELD(name, #{names})")
24
+ names = create_storages.to_s.split(',').map{|s| "'#{s}'"}.join(',')
25
+ @create_storages_cache = names.empty? ? [] : FC::Storage.where("name IN (#{names}) ORDER BY FIELD(name, #{names})")
27
26
  end
28
27
  end
29
28
  @create_storages_cache
30
29
  end
31
30
 
32
- def get_copy_storages
33
- self.class.get_copy_storages_mutex.synchronize do
34
- unless @copy_storages_cache && Time.new.to_i - @get_copy_storages_time.to_i < self.class.storages_cache_time
35
- @get_copy_storages_time = Time.new.to_i
36
- names = copy_storages.split(',').map{|s| "'#{s}'"}.join(',')
37
- @copy_storages_cache = FC::Storage.where("name IN (#{names}) ORDER BY FIELD(name, #{names})")
38
- end
39
- end
40
- @copy_storages_cache
41
- end
42
-
43
31
  # get available storage for create by size
44
32
  def get_proper_storage_for_create(size, exclude = [])
45
33
  get_create_storages.detect do |storage|
46
34
  !exclude.include?(storage.name) && storage.up? && storage.size + size < storage.size_limit
47
35
  end
48
36
  end
49
-
50
- # get available storage for copy by copy_id and size
51
- def get_proper_storage_for_copy(size, copy_id = nil, exclude = [])
52
- storages = get_copy_storages
53
- storage_index = 0
54
- storage_host = nil
55
- while s = storages[storage_index]
56
- if copy_id.to_i == s.copy_id.to_i && !exclude.include?(s.name)
57
- storage_index -= 1 while storage_index > 0 && storages[storage_index-1].host == s.host
58
- storage_host = s.host
59
- break
60
- end
61
- storage_index += 1
62
- end
63
- storages = (storages[storage_index..-1]+storages[0..storage_index-1]).select do |s|
64
- !exclude.include?(s.name) && s.up? && s.size + size < s.size_limit
65
- end
66
- storage = storages.select{|s| storage_host == s.host}.sort{|a,b| (copy_id.to_i - a.copy_id.to_i).abs <=> (copy_id.to_i - b.copy_id.to_i).abs}.first
67
- storage = storages.sort{|a,b| (copy_id.to_i - a.copy_id.to_i).abs <=> (copy_id.to_i - b.copy_id.to_i).abs}.first unless storage
68
- storage
69
- end
70
37
  end
71
38
  end
data/lib/fc/storage.rb CHANGED
@@ -3,12 +3,14 @@ require 'shellwords'
3
3
 
4
4
  module FC
5
5
  class Storage < DbBase
6
- set_table :storages, 'name, host, path, url, size, size_limit, check_time, copy_id'
6
+ set_table :storages, 'name, host, path, url, size, size_limit, check_time, copy_storages'
7
7
 
8
8
  class << self
9
- attr_accessor :check_time_limit
9
+ attr_accessor :check_time_limit, :storages_cache_time, :get_copy_storages_mutex
10
10
  end
11
11
  @check_time_limit = 120 # ttl for up status check
12
+ @storages_cache_time = 20 # ttl for storages cache
13
+ @get_copy_storages_mutex = Mutex.new
12
14
 
13
15
  def self.curr_host
14
16
  @uname || @uname = `uname -n`.chomp
@@ -24,6 +26,17 @@ module FC
24
26
  super params
25
27
  end
26
28
 
29
+ def get_copy_storages
30
+ self.class.get_copy_storages_mutex.synchronize do
31
+ unless @copy_storages_cache && Time.new.to_i - @get_copy_storages_time.to_i < self.class.storages_cache_time
32
+ @get_copy_storages_time = Time.new.to_i
33
+ names = copy_storages.to_s.split(',').map{|s| "'#{s}'"}.join(',')
34
+ @copy_storages_cache = names.empty? ? [] : FC::Storage.where("name IN (#{names}) ORDER BY FIELD(name, #{names})")
35
+ end
36
+ end
37
+ @copy_storages_cache
38
+ end
39
+
27
40
  def update_check_time
28
41
  self.check_time = Time.new.to_i
29
42
  save
@@ -94,5 +107,12 @@ module FC
94
107
  raise r if $?.exitstatus != 0
95
108
  r.to_i
96
109
  end
110
+
111
+ # get available storage for copy by size
112
+ def get_proper_storage_for_copy(size, exclude = [])
113
+ get_copy_storages.select do |storage|
114
+ !exclude.include?(storage.name) && storage.up? && storage.size + size < storage.size_limit
115
+ end.first
116
+ end
97
117
  end
98
118
  end
data/lib/fc/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module FC
2
- VERSION = "0.2.9"
2
+ VERSION = "0.3.0"
3
3
  end
data/lib/manage/policies.rb CHANGED
@@ -4,7 +4,7 @@ def policies_list
4
4
  puts "No storages."
5
5
  else
6
6
  policies.each do |policy|
7
- puts "##{policy.id} #{policy.name}, create storages: #{policy.create_storages}, copy storages: #{policy.copy_storages}, copies: #{policy.copies}"
7
+ puts "##{policy.id} #{policy.name}, create storages: #{policy.create_storages}, copies: #{policy.copies}"
8
8
  end
9
9
  end
10
10
  end
@@ -16,7 +16,6 @@ def policies_show
16
16
  ID: #{policy.id}
17
17
  Name: #{policy.name}
18
18
  Create storages: #{policy.create_storages}
19
- Copy storages: #{policy.copy_storages}
20
19
  Copies: #{policy.copies}
21
20
  Items: #{count}}
22
21
  end
@@ -26,15 +25,13 @@ def policies_add
26
25
  puts "Add Policy"
27
26
  name = stdin_read_val('Name')
28
27
  create_storages = stdin_read_val('Create storages')
29
- copy_storages = stdin_read_val('Copy storages')
30
28
  copies = stdin_read_val('Copies').to_i
31
29
 
32
30
  storages = FC::Storage.where.map(&:name)
33
31
  create_storages = create_storages.split(',').select{|s| storages.member?(s.strip)}.join(',').strip
34
- copy_storages = copy_storages.split(',').select{|s| storages.member?(s.strip)}.join(',').strip
35
32
 
36
33
  begin
37
- policy = FC::Policy.new(:name => name, :create_storages => create_storages, :copy_storages => copy_storages, :copies => copies)
34
+ policy = FC::Policy.new(:name => name, :create_storages => create_storages, :copies => copies)
38
35
  rescue Exception => e
39
36
  puts "Error: #{e.message}"
40
37
  exit
@@ -42,7 +39,6 @@ def policies_add
42
39
  puts %Q{\nPolicy
43
40
  Name: #{name}
44
41
  Create storages: #{create_storages}
45
- Copy storages: #{copy_storages}
46
42
  Copies: #{copies}}
47
43
  s = Readline.readline("Continue? (y/n) ", false).strip.downcase
48
44
  puts ""
@@ -77,22 +73,18 @@ def policies_change
77
73
  puts "Change policy ##{policy.id} #{policy.name}"
78
74
  name = stdin_read_val("Name (now #{policy.name})", true)
79
75
  create_storages = stdin_read_val("Create storages (now #{policy.create_storages})", true)
80
- copy_storages = stdin_read_val("Copy storages (now #{policy.copy_storages})", true)
81
76
  copies = stdin_read_val("Copies (now #{policy.copies})", true)
82
77
 
83
78
  storages = FC::Storage.where.map(&:name)
84
79
  create_storages = create_storages.split(',').select{|s| storages.member?(s.strip)}.join(',').strip unless create_storages.empty?
85
- copy_storages = copy_storages.split(',').select{|s| storages.member?(s.strip)}.join(',').strip unless copy_storages.empty?
86
80
 
87
81
  policy.name = name unless name.empty?
88
82
  policy.create_storages = create_storages unless create_storages.empty?
89
- policy.copy_storages = copy_storages unless copy_storages.empty?
90
83
  policy.copies = copies.to_i unless copies.empty?
91
84
 
92
85
  puts %Q{\nStorage
93
86
  Name: #{policy.name}
94
87
  Create storages: #{policy.create_storages}
95
- Copy storages: #{policy.copy_storages}
96
88
  Copies: #{policy.copies}}
97
89
  s = Readline.readline("Continue? (y/n) ", false).strip.downcase
98
90
  puts ""
data/lib/manage/storages.rb CHANGED
@@ -25,7 +25,7 @@ def storages_show
25
25
  Url: #{storage.url}
26
26
  Size: #{size_to_human storage.size}
27
27
  Size limit: #{size_to_human storage.size_limit}
28
- Copy id: #{storage.copy_id}
28
+ Copy storages: #{storage.copy_storages}
29
29
  Check time: #{storage.check_time ? "#{Time.at(storage.check_time)} (#{storage.check_time_delay} seconds ago)" : ''}
30
30
  Status: #{storage.up? ? colorize_string('UP', :green) : colorize_string('DOWN', :red)}
31
31
  Items storages: #{count}}
@@ -39,11 +39,13 @@ def storages_add
39
39
  path = stdin_read_val('Path')
40
40
  url = stdin_read_val('Url')
41
41
  size_limit = human_to_size stdin_read_val('Size limit') {|val| "Size limit not is valid size." unless human_to_size(val)}
42
- copy_id = stdin_read_val('Copy id').to_i
42
+ copy_storages = stdin_read_val('Copy storages')
43
+ storages = FC::Storage.where.map(&:name)
44
+ copy_storages = copy_storages.split(',').select{|s| storages.member?(s.strip)}.join(',').strip
43
45
  begin
44
46
  path = path +'/' unless path[-1] == '/'
45
47
  path = '/' + path unless path[0] == '/'
46
- storage = FC::Storage.new(:name => name, :host => host, :path => path, :url => url, :size_limit => size_limit, :copy_id => copy_id)
48
+ storage = FC::Storage.new(:name => name, :host => host, :path => path, :url => url, :size_limit => size_limit, :copy_storages => copy_storages)
47
49
  print "Calc current size.. "
48
50
  size = storage.file_size('', true)
49
51
  puts "ok"
@@ -52,13 +54,13 @@ def storages_add
52
54
  exit
53
55
  end
54
56
  puts %Q{\nStorage
55
- Name: #{name}
56
- Host: #{host}
57
- Path: #{path}
58
- Url: #{url}
59
- Size: #{size_to_human size}
60
- Size limit: #{size_to_human size_limit}
61
- Copy id: #{copy_id}}
57
+ Name: #{name}
58
+ Host: #{host}
59
+ Path: #{path}
60
+ Url: #{url}
61
+ Size: #{size_to_human size}
62
+ Size limit: #{size_to_human size_limit}
63
+ Copy storages #{copy_storages}}
62
64
  s = Readline.readline("Continue? (y/n) ", false).strip.downcase
63
65
  puts ""
64
66
  if s == "y" || s == "yes"
@@ -90,9 +92,11 @@ end
90
92
 
91
93
  def storages_update_size
92
94
  if storage = find_storage
95
+ FC::DB.close
93
96
  print "Calc current size.. "
94
97
  size = storage.file_size('', true)
95
98
  storage.size = size
99
+ FC::DB.connect
96
100
  begin
97
101
  storage.save
98
102
  rescue Exception => e
@@ -110,7 +114,7 @@ def storages_change
110
114
  path = stdin_read_val("Path (now #{storage.path})", true)
111
115
  url = stdin_read_val("Url (now #{storage.url})", true)
112
116
  size_limit = stdin_read_val("Size (now #{size_to_human(storage.size_limit)})", true) {|val| "Size limit not is valid size." if !val.empty? && !human_to_size(val)}
113
- copy_id = stdin_read_val("Copy id (now #{storage.copy_id})", true)
117
+ copy_storages = stdin_read_val("Copy storages (now #{storage.copy_storages})", true)
114
118
 
115
119
  storage.host = host unless host.empty?
116
120
  if !path.empty? && path != storage.path
@@ -123,16 +127,17 @@ def storages_change
123
127
  end
124
128
  storage.url = url unless url.empty?
125
129
  storage.size_limit = human_to_size(size_limit) unless size_limit.empty?
126
- storage.copy_id = copy_id.to_i unless copy_id.empty?
130
+ storages = FC::Storage.where.map(&:name)
131
+ copy_storages = copy_storages.split(',').select{|s| storages.member?(s.strip)}.join(',').strip unless copy_storages.empty?
127
132
 
128
133
  puts %Q{\nStorage
129
- Name: #{storage.name}
130
- Host: #{storage.host}
131
- Path: #{storage.path}
132
- Url: #{storage.url}
133
- Size: #{size_to_human storage.size}
134
- Size limit: #{size_to_human storage.size_limit}
135
- Copy id: #{copy_id}}
134
+ Name: #{storage.name}
135
+ Host: #{storage.host}
136
+ Path: #{storage.path}
137
+ Url: #{storage.url}
138
+ Size: #{size_to_human storage.size}
139
+ Size limit: #{size_to_human storage.size_limit}
140
+ Copy storages: #{copy_storages}}
136
141
  s = Readline.readline("Continue? (y/n) ", false).strip.downcase
137
142
  puts ""
138
143
  if s == "y" || s == "yes"
@@ -167,6 +172,7 @@ def storages_sync
167
172
  if s == "y" || s == "yes"
168
173
  make_storages_sync(storage, true)
169
174
  puts "Synchronize done."
175
+ FC::DB.connect
170
176
  storages_update_size
171
177
  else
172
178
  puts "Canceled."
@@ -183,27 +189,30 @@ def find_storage
183
189
  storage
184
190
  end
185
191
 
186
- def make_storages_sync(storage, make_delete, silent = false)
192
+ def make_storages_sync(storage, make_delete, silent = false, no_reconnect = false)
187
193
  # get all items for storage
194
+ puts "Getting all items from DB" unless silent
188
195
  db_items = {}
189
- FC::DB.query("SELECT i.name, ist.id FROM #{FC::Item.table_name} as i, #{FC::ItemStorage.table_name} as ist WHERE ist.item_id = i.id AND ist.storage_name = '#{storage.name}'").each do |row|
196
+ FC::DB.query("SELECT i.name, ist.id, ist.status FROM #{FC::Item.table_name} as i, #{FC::ItemStorage.table_name} as ist WHERE ist.item_id = i.id AND ist.storage_name = '#{storage.name}'").each do |row|
190
197
  name = row['name'].sub(/\/$/, '').sub(/^\//, '').strip
191
198
  path = ''
192
199
  name.split('/').each do |dir|
193
200
  path << dir
194
- db_items[path] = [false, path == name ? row['id'].to_i : nil]
201
+ db_items[path] = [false, path == name ? row['id'].to_i : nil, row['status']]
195
202
  path << '/'
196
203
  end
197
204
  end
205
+ FC::DB.close unless no_reconnect
198
206
 
199
207
  # walk on all storage folders and files
208
+ puts "Getting all files" unless silent
200
209
  delete_files = []
201
210
  process_storage_dir_sync = lambda do |dir = ''|
202
211
  Dir.glob(storage.path+dir+'*').each do |f|
203
212
  path = f.sub(storage.path, '')
204
213
  if db_items[path]
205
214
  db_items[path][0] = true
206
- next if db_items[path][1]
215
+ next if db_items[path][1] && db_items[path][2] != 'delete'
207
216
  end
208
217
  delete_files << path if File.file?(f)
209
218
  process_storage_dir_sync.call(path+'/') if File.directory?(f)
@@ -212,24 +221,28 @@ def make_storages_sync(storage, make_delete, silent = false)
212
221
  process_storage_dir_sync.call
213
222
 
214
223
  # rm delete_files
224
+ FC::DB.connect unless no_reconnect
215
225
  if make_delete
226
+ puts "Deleting files" unless silent
216
227
  delete_files.each do |f|
217
228
  # check in DB again
218
- next if FC::DB.query("SELECT ist.id FROM #{FC::Item.table_name} as i, #{FC::ItemStorage.table_name} as ist WHERE ist.item_id = i.id AND ist.storage_name = '#{storage.name}' AND i.name='#{f}'").first
229
+ next if FC::DB.query("SELECT ist.id FROM #{FC::Item.table_name} as i, #{FC::ItemStorage.table_name} as ist WHERE ist.item_id = i.id AND ist.storage_name = '#{storage.name}' AND i.name='#{f}' AND ist.status<>'delete'").first
219
230
  path = storage.path+f
220
231
  File.delete(path) rescue nil
221
232
  end
233
+ puts "Deleted #{delete_files.count} files" unless silent
222
234
  end
223
- puts "Deleted #{delete_files.count} files" unless silent
224
235
 
225
236
  # delete non synchronize items_storages
237
+ puts "Deleting items from DB" unless silent
226
238
  count = 0
227
239
  db_items.values.each do |item|
228
- if !item[0] && item[1]
240
+ if !item[0] && item[1] || item[2] == 'delete' && item[1]
229
241
  count += 1
230
242
  FC::DB.query("DELETE FROM #{FC::ItemStorage.table_name} WHERE id=#{item[1]}") if make_delete
231
243
  end
232
244
  end
245
+ FC::DB.close unless no_reconnect
233
246
  puts "Deleted #{count} items_storages" unless silent
234
247
 
235
248
  # delete empty folders
@@ -249,9 +262,9 @@ def make_storages_sync(storage, make_delete, silent = false)
249
262
  if (ARGV[4])
250
263
  File.open(ARGV[4], 'w') do |file|
251
264
  db_items.values.each do |item|
252
- file.puts item[1] if !item[0] && item[1]
265
+ file.puts item[1] if !item[0] && item[1] || item[2] == 'delete' && item[1]
253
266
  end
254
267
  end
255
268
  puts "Save deleted items_storages to #{ARGV[4]}" unless silent
256
269
  end
257
- end
270
+ end
data/test/daemon_test.rb CHANGED
@@ -45,12 +45,12 @@ class DaemonTest < Test::Unit::TestCase
45
45
  `cp #{@@test_file_path} #{@@test_dir_path}/bbb/test2`
46
46
 
47
47
  @@storages = []
48
- @@storages << FC::Storage.new(:name => 'host1-sda', :host => 'host1', :path => '/tmp/host1-sda/', :copy_id => 1, :size_limit => 1000000000)
49
- @@storages << FC::Storage.new(:name => 'host1-sdb', :host => 'host1', :path => '/tmp/host1-sdb/', :copy_id => 2, :size_limit => 1000000000)
50
- @@storages << FC::Storage.new(:name => 'host1-sdc', :host => 'host1', :path => '/tmp/host1-sdc/', :copy_id => 1, :size_limit => 1000000000)
48
+ @@storages << FC::Storage.new(:name => 'host1-sda', :host => 'host1', :path => '/tmp/host1-sda/', :copy_storages => 'host1-sdb,host1-sdc', :size_limit => 1000000000)
49
+ @@storages << FC::Storage.new(:name => 'host1-sdb', :host => 'host1', :path => '/tmp/host1-sdb/', :copy_storages => 'host1-sda,host1-sdc', :size_limit => 1000000000)
50
+ @@storages << FC::Storage.new(:name => 'host1-sdc', :host => 'host1', :path => '/tmp/host1-sdc/', :copy_storages => 'host1-sda,host1-sdb', :size_limit => 1000000000)
51
51
  @@storages.each { |storage| storage.save}
52
52
 
53
- @@policy = FC::Policy.new(:create_storages => 'host1-sda,host1-sdb,host1-sdc', :copy_storages => 'host1-sda,host1-sdb,host1-sdc', :copies => 2, :name => 'policy 1')
53
+ @@policy = FC::Policy.new(:create_storages => 'host1-sda,host1-sdb,host1-sdc', :copies => 2, :name => 'policy 1')
54
54
  @@policy.save
55
55
 
56
56
  # wait for running fc-daemon
@@ -72,7 +72,7 @@ class DaemonTest < Test::Unit::TestCase
72
72
  `rm -rf #{@@test_dir_path}`
73
73
  end
74
74
  end
75
-
75
+
76
76
  should "daemon_all" do
77
77
  @@storages.each {|storage| storage.reload}
78
78
  assert @@storages[0].up?, "Storage #{@@storages[0].name} down"
data/test/db_test.rb CHANGED
@@ -15,9 +15,9 @@ class DbTest < Test::Unit::TestCase
15
15
  @@storages_ids = storages.map{|storage| storage.save; storage.id }
16
16
 
17
17
  policies = []
18
- policies << FC::Policy.new(:create_storages => 'rec1-sda,rec1-sdd', :copy_storages => 'rec2-sda,rec2-sdd', :copies => 2, :name => 'policy 1')
19
- policies << FC::Policy.new(:create_storages => 'rec1-sda,bla,rec2-sdd', :copy_storages => 'rec2-sda,bla,rec1-sdd', :copies => 3, :name => 'policy 2')
20
- policies << FC::Policy.new(:create_storages => 'bla,rec1-sda,test', :copy_storages => 'bla,rec2-sda,test', :copies => 4, :name => 'policy 3')
18
+ policies << FC::Policy.new(:create_storages => 'rec1-sda,rec1-sdd', :copies => 2, :name => 'policy 1')
19
+ policies << FC::Policy.new(:create_storages => 'rec1-sda,bla,rec2-sdd', :copies => 3, :name => 'policy 2')
20
+ policies << FC::Policy.new(:create_storages => 'bla,rec1-sda,test', :copies => 4, :name => 'policy 3')
21
21
  @@policies_ids = policies.map{|policy| policy.save; policy.id }
22
22
 
23
23
  items = []
@@ -78,13 +78,9 @@ class DbTest < Test::Unit::TestCase
78
78
  assert_equal 'rec1-sda,rec1-sdd', @policies[0].create_storages, "Policy (id=#{@policies[0].id}) incorrect create_storages"
79
79
  assert_equal 'rec1-sda,rec2-sdd', @policies[1].create_storages, "Policy (id=#{@policies[0].id}) incorrect create_storages"
80
80
  assert_equal 'rec1-sda', @policies[2].create_storages, "Policy (id=#{@policies[0].id}) incorrect create_storages"
81
- assert_equal 'rec2-sda,rec2-sdd', @policies[0].copy_storages, "Policy (id=#{@policies[0].id}) incorrect copy_storages"
82
- assert_equal 'rec2-sda,rec1-sdd', @policies[1].copy_storages, "Policy (id=#{@policies[0].id}) incorrect copy_storages"
83
- assert_equal 'rec2-sda', @policies[2].copy_storages, "Policy (id=#{@policies[0].id}) incorrect copy_storages"
84
81
 
85
- assert_raise(Mysql2::Error, 'Create policy with uniq name') { FC::Policy.new(:create_storages => 'bla,test', :copy_storages => 'rec2-sda', :name => 'policy 1').save }
86
- assert_raise(Mysql2::Error, 'Create policy with incorrect create_storages') { FC::Policy.new(:create_storages => 'bla,test', :copy_storages => 'rec2-sda', :name => 'new policy').save }
87
- assert_raise(Mysql2::Error, 'Create policy with incorrect copy_storages') { FC::Policy.new(:create_storages => 'rec1-sda', :copy_storages => 'bla,test', :name => 'new policy').save }
82
+ assert_raise(Mysql2::Error, 'Create policy with uniq name') { FC::Policy.new(:create_storages => 'bla,test', :name => 'policy 1').save }
83
+ assert_raise(Mysql2::Error, 'Create policy with incorrect create_storages') { FC::Policy.new(:create_storages => 'bla,test', :name => 'new policy').save }
88
84
 
89
85
  assert_raise(Mysql2::Error, 'Change storage name with linked polices') { @storages[0].name = 'blabla'; @storages[0].save }
90
86
  assert_raise(Mysql2::Error, 'Delete storage name with linked polices') { @storages[0].delete }
@@ -94,19 +90,15 @@ class DbTest < Test::Unit::TestCase
94
90
  @policies[0].reload
95
91
  @policies[1].reload
96
92
  assert_equal 'rec1-sda', @policies[0].create_storages, "Policy (id=#{@policies[0].id}) incorrect create_storages after storage change"
97
- assert_equal 'rec2-sda', @policies[1].copy_storages, "Policy (id=#{@policies[1].id}) incorrect copy_storages after storage change"
98
93
  @storages[7].delete #rec2-sdd
99
94
  @@storages_ids.delete(@storages[7].id)
100
95
  @policies[0].reload
101
96
  @policies[1].reload
102
97
  assert_equal 'rec1-sda', @policies[1].create_storages, "Policy (id=#{@policies[1].id}) incorrect create_storages after storage delete"
103
- assert_equal 'rec2-sda', @policies[0].copy_storages, "Policy (id=#{@policies[0].id}) incorrect copy_storages after storage delete"
104
98
  @policies[0].create_storages = 'rec2-sda,rec1-sda,bla bla'
105
- @policies[0].copy_storages = 'rec1-sdb,rec2-sda,bla bla'
106
99
  @policies[0].save
107
100
  @policies[0].reload
108
101
  assert_equal 'rec2-sda,rec1-sda', @policies[0].create_storages, "Policy (id=#{@policies[0].id}) incorrect create_storages after change"
109
- assert_equal 'rec1-sdb,rec2-sda', @policies[0].copy_storages, "Policy (id=#{@policies[0].id}) incorrect copy_storages after change"
110
102
  assert_raise(Mysql2::Error, 'Save empty policy storage') { @policies[0].create_storages = 'blabla'; @policies[0].save }
111
103
  end
112
104
 
data/test/policy_test.rb CHANGED
@@ -4,14 +4,14 @@ class PolicyTest < Test::Unit::TestCase
4
4
  class << self
5
5
  def startup
6
6
  @@storages = []
7
- @@storages << FC::Storage.new(:name => 'rec1-sda', :host => 'rec1', :size => 0, :copy_id => 1, :size_limit => 10)
8
- @@storages << FC::Storage.new(:name => 'rec2-sda', :host => 'rec2', :size => 0, :copy_id => 2 , :size_limit => 100)
9
- @@storages << FC::Storage.new(:name => 'rec2-sdb', :host => 'rec2', :size => 0, :copy_id => 4 , :size_limit => 100)
7
+ @@storages << FC::Storage.new(:name => 'rec1-sda', :host => 'rec1', :size => 0, :size_limit => 10)
8
+ @@storages << FC::Storage.new(:name => 'rec2-sda', :host => 'rec2', :size => 0, :size_limit => 100)
9
+ @@storages << FC::Storage.new(:name => 'rec2-sdb', :host => 'rec2', :size => 0, :size_limit => 100)
10
10
  @@storages.each {|storage| storage.save}
11
- @@storage3 = FC::Storage.new(:name => 'rec3-sda', :host => 'rec3', :size => 8, :copy_id => 3 , :size_limit => 10)
11
+ @@storage3 = FC::Storage.new(:name => 'rec3-sda', :host => 'rec3', :size => 8, :size_limit => 10)
12
12
  @@storage3.save
13
13
 
14
- @@policy = FC::Policy.new(:create_storages => 'rec1-sda,rec2-sda,rec2-sdb', :copy_storages => 'rec1-sda,rec2-sda', :copies => 1, :name => 'policy 1')
14
+ @@policy = FC::Policy.new(:create_storages => 'rec1-sda,rec2-sda,rec2-sdb', :copies => 1, :name => 'policy 1')
15
15
  @@policy.save
16
16
  end
17
17
  def shutdown
@@ -30,18 +30,6 @@ class PolicyTest < Test::Unit::TestCase
30
30
  assert_equal 2, @@policy.get_create_storages.size
31
31
  end
32
32
 
33
- should "get_copy_storages" do
34
- @@policy.copy_storages = 'rec1-sda,rec2-sda,rec2-sdb'
35
- @@policy.save
36
- FC::Policy.storages_cache_time = 10
37
- assert_same_elements @@storages.map(&:id), @@policy.get_copy_storages.map(&:id)
38
- @@policy.copy_storages = 'rec1-sda,rec2-sda'
39
- @@policy.save
40
- assert_equal @@storages.size, @@policy.get_copy_storages.size
41
- FC::Policy.storages_cache_time = 0
42
- assert_equal 2, @@policy.get_copy_storages.size
43
- end
44
-
45
33
  should "get_proper_storage_for_create" do
46
34
  @@storages.each {|storage| storage.check_time = 0; storage.save}
47
35
  @@storage3.check_time = 0
@@ -56,40 +44,6 @@ class PolicyTest < Test::Unit::TestCase
56
44
  assert_nil @@policy.get_proper_storage_for_create(1000), 'all storages full'
57
45
  end
58
46
 
59
- should "get_proper_storage_for_copy" do
60
- @@storages.each {|storage| storage.check_time = 0; storage.save}
61
- FC::Policy.storages_cache_time = 0
62
- assert_nil @@policy.get_proper_storage_for_copy(1), 'all storages down'
63
- @@storages[0].update_check_time
64
- @@storage3.update_check_time
65
- assert_equal @@storages[0].id, @@policy.get_proper_storage_for_copy(5).id, 'first storages up'
66
- assert_nil @@policy.get_proper_storage_for_copy(20), 'first storage full'
67
- @@storages[1].update_check_time
68
- assert_equal @@storages[1].id, @@policy.get_proper_storage_for_copy(20).id, 'second storages up'
69
- assert_nil @@policy.get_proper_storage_for_copy(1000), 'all storages full'
70
-
71
- @@policy.copy_storages = 'rec3-sda,rec1-sda,rec2-sda'
72
- @@policy.save
73
- assert_equal 'rec1-sda', @@policy.get_proper_storage_for_copy(1).name, 'storage with min copy_id in copy_storages'
74
- assert_equal 'rec2-sda', @@policy.get_proper_storage_for_copy(1, 2).name, 'storage by copy_id'
75
- @@policy.copy_storages = 'rec1-sda,rec3-sda'
76
- @@policy.save
77
- assert_equal 'rec1-sda', @@policy.get_proper_storage_for_copy(1, 2).name, 'storage by copy_id'
78
- @@policy.copy_storages = 'rec3-sda,rec1-sda,rec2-sda,rec2-sdb'
79
- @@policy.save
80
- assert_equal 'rec2-sda', @@policy.get_proper_storage_for_copy(1, 4).name, 'storage by copy_id'
81
- @@policy.copy_storages = 'rec2-sda,rec3-sda,rec1-sda,rec2-sdb'
82
- @@policy.save
83
- @@storages[2].update_check_time
84
- assert_equal 'rec2-sdb', @@policy.get_proper_storage_for_copy(1, 4).name, 'storage by copy_id'
85
-
86
- @@policy.copy_storages = 'rec3-sda,rec1-sda,rec2-sda'
87
- @@policy.save
88
- @@storages[0].check_time = 0
89
- @@storages[0].save
90
- assert_equal 'rec2-sda', @@policy.get_proper_storage_for_copy(1, 1).name, 'storage by copy_id'
91
- end
92
-
93
47
  should "filter_by_host" do
94
48
  FC::Storage.stubs(:curr_host).returns('rec2')
95
49
  FC::Policy.new(:create_storages => 'rec3-sda,rec2-sda', :copy_storages => 'rec1-sda,rec2-sda', :copies => 1, :name => 'policy 2').save
data/test/{storage_sync.rb → storage_sync_test.rb} RENAMED
@@ -1,7 +1,7 @@
1
1
  require 'helper'
2
2
  require 'manage'
3
3
 
4
- class DaemonTest < Test::Unit::TestCase
4
+ class StorageSyncTest < Test::Unit::TestCase
5
5
  class << self
6
6
  def startup
7
7
  # tmp fake storage dir
@@ -12,9 +12,9 @@ class DaemonTest < Test::Unit::TestCase
12
12
  @@test_file_path = '/tmp/fc_test_file'
13
13
  `dd if=/dev/urandom of=#{@@test_file_path} bs=1M count=1 2>&1`
14
14
 
15
- @@storage = FC::Storage.new(:name => 'host1-sda', :host => 'host1', :path => '/tmp/host1-sda/', :copy_id => 1, :size_limit => 1000000000, :check_time => Time.new.to_i)
15
+ @@storage = FC::Storage.new(:name => 'host1-sda', :host => 'host1', :path => '/tmp/host1-sda/', :size_limit => 1000000000, :check_time => Time.new.to_i)
16
16
  @@storage.save
17
- @@policy = FC::Policy.new(:create_storages => 'host1-sda', :copy_storages => 'host1-sda', :copies => 1, :name => 'policy 1')
17
+ @@policy = FC::Policy.new(:create_storages => 'host1-sda', :copies => 1, :name => 'policy 1')
18
18
  @@policy.save
19
19
  end
20
20
 
@@ -38,13 +38,15 @@ class DaemonTest < Test::Unit::TestCase
38
38
  assert_nothing_raised { @item2 = FC::Item.create_from_local(@@test_file_path, 'a/b/test2', @@policy, {:tag => 'test'}) }
39
39
  assert_nothing_raised { @item3 = FC::Item.create_from_local(@@test_file_path, 'a/b/c/test3', @@policy, {:tag => 'test'}) }
40
40
  assert_nothing_raised { @item4 = FC::Item.create_from_local(@@test_file_path, 'a/b/c/d/test4', @@policy, {:tag => 'test'}) }
41
+ assert_nothing_raised { @item5 = FC::Item.create_from_local(@@test_file_path, 'a/del_file', @@policy, {:tag => 'test'}) }
42
+ @item5.mark_deleted
41
43
  `mv /tmp/host1-sda/a/test1 /tmp/host1-sda/test1`
42
44
  `mv /tmp/host1-sda/a/b/c/d/test4 /tmp/host1-sda/a/b/c/d/test5`
43
45
  `mkdir /tmp/host1-sda/test_dir`
44
46
  `cp #{@@test_file_path} /tmp/host1-sda/test_dir/t1`
45
47
  `cp #{@@test_file_path} /tmp/host1-sda/test_dir/t2`
46
48
 
47
- make_storages_sync(@@storage, true, true)
49
+ make_storages_sync(@@storage, true, true, true)
48
50
 
49
51
  @item1.reload
50
52
  @item2.reload
@@ -60,6 +62,8 @@ class DaemonTest < Test::Unit::TestCase
60
62
  assert_equal size, `du -sb /tmp/host1-sda/a/b/c/test3 2>&1`.to_i
61
63
  assert_equal 0, `du -sb /tmp/host1-sda/a/b/c/d/test5 2>&1`.to_i
62
64
  assert_equal 0, `du -sb /tmp/host1-sda/a/b/c/d 2>&1`.to_i
63
- assert_equal 0, `du -sb tmp/host1-sda/test_dir 2>&1`.to_i
65
+ assert_equal 0, `du -sb /tmp/host1-sda/test_dir 2>&1`.to_i
66
+ assert_equal 0, @item5.get_item_storages.size
67
+ assert_equal 0, `du -sb /tmp/host1-sda/a/del_file 2>&1`.to_i
64
68
  end
65
69
  end
data/test/storage_test.rb CHANGED
@@ -1,6 +1,24 @@
1
1
  require 'helper'
2
2
 
3
3
  class StorageTest < Test::Unit::TestCase
4
+ class << self
5
+ def startup
6
+ @@storages = []
7
+ @@storages << FC::Storage.new(:name => 'rec1-sda', :host => 'rec1', :size => 0, :copy_storages => 'rec2-sda,rec3-sda', :size_limit => 10)
8
+ @@storages << FC::Storage.new(:name => 'rec2-sda', :host => 'rec2', :size => 0, :copy_storages => 'rec1-sda,rec3-sda', :size_limit => 100)
9
+ @@storages << FC::Storage.new(:name => 'rec3-sda', :host => 'rec3', :size => 8, :copy_storages => 'rec1-sda,rec2-sda', :size_limit => 10)
10
+ @@storages << FC::Storage.new(:name => 'rec2-sdb', :host => 'rec2', :size => 0, :size_limit => 100)
11
+ @@storages.each {|storage| storage.save}
12
+
13
+ @@policy = FC::Policy.new(:create_storages => 'rec1-sda,rec2-sda,rec2-sdb', :copies => 1, :name => 'policy 1')
14
+ @@policy.save
15
+ end
16
+ def shutdown
17
+ FC::DB.query("DELETE FROM policies")
18
+ FC::DB.query("DELETE FROM storages")
19
+ end
20
+ end
21
+
4
22
  should "curr_host" do
5
23
  assert ! FC::Storage.curr_host.to_s.empty?
6
24
  end
@@ -12,7 +30,7 @@ class StorageTest < Test::Unit::TestCase
12
30
  end
13
31
 
14
32
  should "update_check_time and up?" do
15
- storage = FC::Storage.new(:name => 'rec1-sda', :host => 'rec1')
33
+ storage = FC::Storage.new(:name => 'rec1-test', :host => 'rec1')
16
34
  storage.save
17
35
  assert_equal false, storage.up?
18
36
  storage.update_check_time
@@ -25,4 +43,30 @@ class StorageTest < Test::Unit::TestCase
25
43
  storage.delete
26
44
  end
27
45
 
46
+ should "get_copy_storages" do
47
+ @@storages[2].copy_storages = 'rec1-sda,rec2-sda,rec2-sdb'
48
+ @@storages[2].save
49
+ FC::Storage.storages_cache_time = 10
50
+ assert_equal 'rec1-sda,rec2-sda,rec2-sdb', @@storages[2].get_copy_storages.map(&:name).join(',')
51
+ @@storages[2].copy_storages = 'rec1-sda,rec2-sda'
52
+ @@storages[2].save
53
+ assert_equal 'rec1-sda,rec2-sda,rec2-sdb', @@storages[2].get_copy_storages.map(&:name).join(',')
54
+ FC::Storage.storages_cache_time = 0
55
+ assert_equal 'rec1-sda,rec2-sda', @@storages[2].get_copy_storages.map(&:name).join(',')
56
+ end
57
+
58
+ should "get_proper_storage_for_copy" do
59
+ @@storages.each {|storage| storage.check_time = 0; storage.save}
60
+ @@storages[3].update_check_time
61
+ assert_nil @@storages[3].get_proper_storage_for_copy(1), 'empty copy_storages'
62
+ assert_nil @@storages[2].get_proper_storage_for_copy(1), 'all storages down'
63
+
64
+ @@storages[0].update_check_time
65
+ assert_equal 'rec1-sda', @@storages[2].get_proper_storage_for_copy(5).name, 'first storages up'
66
+ assert_nil @@storages[2].get_proper_storage_for_copy(20), 'first storage full'
67
+ @@storages[1].update_check_time
68
+ assert_equal 'rec1-sda', @@storages[2].get_proper_storage_for_copy(5).name, 'second storages up, small file'
69
+ assert_equal 'rec2-sda', @@storages[2].get_proper_storage_for_copy(20).name, 'second storages up, big file'
70
+ assert_nil @@storages[2].get_proper_storage_for_copy(1000), 'second storages up, huge file'
71
+ end
28
72
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: filecluster
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.9
4
+ version: 0.3.0
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2013-10-11 00:00:00.000000000 Z
12
+ date: 2013-11-11 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: rb-readline
@@ -174,7 +174,7 @@ files:
174
174
  - test/helper.rb
175
175
  - test/item_test.rb
176
176
  - test/policy_test.rb
177
- - test/storage_sync.rb
177
+ - test/storage_sync_test.rb
178
178
  - test/storage_test.rb
179
179
  - test/var_test.rb
180
180
  - test/version_test.rb
@@ -192,7 +192,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
192
192
  version: '0'
193
193
  segments:
194
194
  - 0
195
- hash: -3823121111064236338
195
+ hash: 2037414088552914900
196
196
  required_rubygems_version: !ruby/object:Gem::Requirement
197
197
  none: false
198
198
  requirements:
@@ -201,7 +201,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
201
201
  version: '0'
202
202
  segments:
203
203
  - 0
204
- hash: -3823121111064236338
204
+ hash: 2037414088552914900
205
205
  requirements: []
206
206
  rubyforge_project:
207
207
  rubygems_version: 1.8.24
@@ -217,7 +217,7 @@ test_files:
217
217
  - test/helper.rb
218
218
  - test/item_test.rb
219
219
  - test/policy_test.rb
220
- - test/storage_sync.rb
220
+ - test/storage_sync_test.rb
221
221
  - test/storage_test.rb
222
222
  - test/var_test.rb
223
223
  - test/version_test.rb