filecluster 0.5.16 → 0.5.17

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 04a6d63f3d0df33e01e31f154736866746ffc579
4
- data.tar.gz: f0525db53c050a1413c4995b4f87a12b56f6823c
3
+ metadata.gz: 9e81ff8dd657f27201d09032d872bfe8425cf65d
4
+ data.tar.gz: 8659faa5b1949c5f0759f73ca499ae07432eac92
5
5
  SHA512:
6
- metadata.gz: 32c4794f66a75ee6757eb6d481346abed46ea4a932220f806c2b64620c26206c101de322977784114fda6bf037008eea4edac57480faa21c2b6933fc3e34f61f
7
- data.tar.gz: f57b788b11400354ec265e9f1ca40bdd5ae2074412c25c459ad0223ee27984d7cfdc933ef8708db49fc423885c7ce7c5bedfa50f2d217c827f3510dadafe2e15
6
+ metadata.gz: 0f8aa7d24c0fa3ed9fdccc5ed60d6fc3c48daf7bdfb190e688685a19beb42ee84a948182122502a5542b9c28c18ac3bbc442cf49dd60b7800c3b048f137c77f7
7
+ data.tar.gz: a5b54d47f2c000a947a961f2f5ef59941eaa1fa104a134d6ba89d78c87600c7dc0f143e64e283939851b16f0e27ef02849abc2dddedd5606ff578c92289c7d2d
@@ -89,7 +89,16 @@ Command:
89
89
  add_local <storage name> <path> <policy id/name> <tag> <outer_id> add file in local storage as item (as <path> name) with policy, tag & outer_id is not required
90
90
  add <path> <name> <policy id/name> add file as item <name> with policy
91
91
  rm <name> delete item
92
- }]
92
+ }],
93
+ 'schema' => [
94
+ 'use external configuration to manage schema definition (yaml formatted)',
95
+ %q{Usage: fc-manage schema <command>
96
+ Command:
97
+ create [options] <config file>.yml creates storages/policies/db connection, fails if any already exists
98
+ Options: force force to warn and replace objects instead of fail
99
+ apply <config file>.yml applying storages/policies properties, warn for missing objects
100
+ dump <target file>.yml loads out existing storages/policies into single target file
101
+ }]
93
102
  }
94
103
  desc = %q{Get info and manage for storages, policies and items.
95
104
  Usage: fc-manage [options] <command> [<args>]
@@ -100,5 +100,30 @@ module FC
100
100
  FC::DB.query("DELETE FROM #{self.class.table_name} WHERE id=#{@id.to_i}") if @id
101
101
  end
102
102
 
103
+ def self.apply!(data: {}, key_field: :name, update_only: false)
104
+ return nil unless data[key_field]
105
+ obj_instance = where("#{key_field} = ?", data[key_field]).first
106
+ return "#{name} \"#{data[key_field]}\" not found" if obj_instance.nil? && update_only
107
+ obj_instance ||= new
108
+ obj_instance.load(data: data)
109
+ obj_instance.save
110
+ obj_instance
111
+ end
112
+
113
+ def load(data: {})
114
+ self.class.table_fields.each do |field|
115
+ send("#{field}=", data[field.to_sym]) if data[field.to_sym]
116
+ end
117
+ end
118
+
119
+ def dump(exclude = [])
120
+ obj_dump = {}
121
+ self.class.table_fields.each do |field|
122
+ sym_field = field.to_sym
123
+ next if exclude.include?(sym_field)
124
+ obj_dump[sym_field] = database_fields[field]
125
+ end
126
+ obj_dump
127
+ end
103
128
  end
104
129
  end
@@ -56,7 +56,7 @@ module FC
56
56
  $SELF = 0
57
57
  r ? true : false
58
58
  end
59
-
59
+
60
60
  def test
61
61
  storage = FC::Storage.new(
62
62
  :id => 1,
@@ -195,5 +195,17 @@ module FC
195
195
  def get_proper_storage_for_copy(size, exclude = [])
196
196
  FC::Storage.select_proper_storage_for_create(get_copy_storages, size, exclude)
197
197
  end
198
+
199
+ def dump
200
+ super(%i[check_time autosync_at])
201
+ end
202
+
203
+ def load(data: {})
204
+ if name
205
+ data = data.clone
206
+ data.delete(:host)
207
+ end
208
+ super(data: data)
209
+ end
198
210
  end
199
211
  end
@@ -1,3 +1,3 @@
1
1
  module FC
2
- VERSION = '0.5.16'.freeze
2
+ VERSION = '0.5.17'.freeze
3
3
  end
@@ -7,4 +7,5 @@ require "manage/var"
7
7
  require "manage/item"
8
8
  require "manage/copy_speed"
9
9
  require "manage/autosync"
10
+ require "manage/schema"
10
11
  require 'autosync'
@@ -0,0 +1,114 @@
1
+ require 'psych'
2
+ def schema_create
3
+ unless ARGV[2]
4
+ puts 'config file not set'
5
+ exit(1)
6
+ end
7
+ force = ARGV[3].to_s == 'force'
8
+ schema = load_schema_file
9
+
10
+ apply_connection(schema[:connection])
11
+ # check existing objects
12
+ unless force
13
+ errors = ''
14
+ schema[:storages].each do |cs|
15
+ errors << "Storage \"#{cs[:name]}\" already exists\n" if FC::Storage.where('name = ?', cs[:name]).any?
16
+ end
17
+ schema[:policies].each do |pl|
18
+ errors << "Policy \"#{pl[:name]}\" already exists\n" if FC::Policy.where('name = ?', pl[:name]).any?
19
+ end
20
+ schema[:copy_rules].each do |pl|
21
+ errors << "CopyRule \"#{pl[:rule]}\" already exists\n" if FC::CopyRule.where('rule = ?', pl[:rule]).any?
22
+ end
23
+ existing_vars = FC::Var.get_all
24
+ schema[:vars].keys.each do |v|
25
+ errors << "Var \"#{v}\" already exists\n" if existing_vars.include?(v)
26
+ end
27
+
28
+ unless errors.empty?
29
+ puts errors
30
+ exit(1)
31
+ end
32
+ end
33
+ errors = create_or_update_schema(schema: schema, update: false)
34
+ puts errors.join("\n") if errors && errors.any?
35
+ end
36
+
37
+ def schema_apply
38
+ schema = load_schema_file
39
+ apply_connection(schema[:connection])
40
+ errors = create_or_update_schema(schema: schema, update: true)
41
+ puts errors.join("\n") if errors && errors.any?
42
+ end
43
+
44
+ def schema_dump
45
+ dump = create_dump
46
+ if ARGV[2]
47
+ File.write ARGV[2], Psych.dump(dump)
48
+ else
49
+ puts Psych.dump(dump)
50
+ end
51
+ end
52
+
53
+ def create_or_update_schema(schema:, update:)
54
+ errors = []
55
+ errors << apply_entity(klass: FC::Storage, items: schema[:storages], update_only: update)
56
+ errors << apply_entity(klass: FC::Policy, items: schema[:policies], update_only: update)
57
+ errors << apply_entity(klass: FC::CopyRule, items: schema[:copy_rules], key_field: :rule, update_only: update)
58
+ errors << apply_vars(vars: schema[:vars], update_only: update)
59
+ errors.flatten
60
+ end
61
+
62
+ def create_dump
63
+ schema = { }
64
+ schema[:storages] = FC::Storage.where.map(&:dump)
65
+ schema[:policies] = FC::Policy.where.map(&:dump)
66
+ schema[:copy_rules] = FC::CopyRule.where.map(&:dump)
67
+ schema[:vars] = FC::Var.get_all.select { |k, _| k.is_a? Symbol }
68
+ schema[:connection] = FC::DB.instance_variable_get('@options')
69
+ schema
70
+ end
71
+
72
+ def load_schema_file
73
+ schema = Psych.load(File.read(ARGV[2]))
74
+ schema[:storages] ||= []
75
+ schema[:policies] ||= []
76
+ schema[:copy_rules] ||= []
77
+ schema[:vars] ||= {}
78
+ schema
79
+ end
80
+
81
+ def apply_connection(options)
82
+ return unless options
83
+ FC::DB.close
84
+ FC::DB.connect_by_config(options)
85
+ FC::DB.connect.ping
86
+ File.write(FC::DB.options_yml_path, Psych.dump(options))
87
+ end
88
+
89
+ def apply_vars(vars:, update_only: false)
90
+ errors = []
91
+ existing_vars = FC::Var.get_all
92
+ vars.each do |k, v|
93
+ if !existing_vars.keys.include?(k) && update_only
94
+ errors << "FC::Var \"#{k}\" not found"
95
+ next
96
+ end
97
+ FC::Var.set(k, v)
98
+ end
99
+ errors
100
+ end
101
+
102
+ def apply_entity(klass:, items:, update_only: false, key_field: :name)
103
+ errors = []
104
+ return errors unless items
105
+ items.each do |item|
106
+ begin
107
+ result = klass.apply!(data: item, update_only: update_only, key_field: key_field)
108
+ errors << result if result.is_a? String
109
+ rescue => save_error
110
+ errors << "Error while saving #{klass} \"#{item[key_field]}\": #{save_error}"
111
+ end
112
+ end
113
+ errors
114
+ end
@@ -0,0 +1,184 @@
1
+ require 'helper'
2
+ require 'manage/schema'
3
+ class SchemaTest < Test::Unit::TestCase
4
+ class << self
5
+ def startup
6
+ FC::DB.query("DELETE FROM policies")
7
+ FC::DB.query("DELETE FROM vars")
8
+ FC::DB.query("DELETE FROM copy_rules")
9
+ FC::DB.query("DELETE FROM storages")
10
+ end
11
+ end
12
+
13
+ def teardown
14
+ FC::DB.query("DELETE FROM policies")
15
+ FC::DB.query("DELETE FROM vars")
16
+ FC::DB.query("DELETE FROM copy_rules")
17
+ FC::DB.query("DELETE FROM storages")
18
+ end
19
+
20
+ should 'create dump' do
21
+ rule = FC::CopyRule.new(copy_storages: 'rec2-sda,rec3-sda', rule: 'size < 100')
22
+ rule.save
23
+ rule.reload
24
+ storage = FC::Storage.new(
25
+ name: 'rec1-sda',
26
+ host: 'rec1',
27
+ size: 0,
28
+ size_limit: 100,
29
+ dc: 'test',
30
+ copy_storages: 'rec2-sda',
31
+ url: '//host/disk/',
32
+ url_weight: 1,
33
+ write_weight: 2
34
+ )
35
+ storage.save
36
+ storage.reload
37
+
38
+ policy = FC::Policy.new(name: 'src', create_storages: 'rec1-sda', copies: 1, delete_deferred_time: 86400);
39
+ policy.save
40
+ policy.reload
41
+
42
+ FC::Var.set('some_variable', 'value sample')
43
+ FC::Var.set('awesome variable', 'awesome value')
44
+
45
+ dump = create_dump
46
+ assert_equal 1, dump[:storages].size
47
+ FC::Storage.table_fields.each do |field|
48
+ next if %w[check_time autosync_at].include? field # not dumping attrs
49
+ assert_equal storage.send(field), dump[:storages][0][field.to_sym], "field \"#{field}\" not dumped correctly"
50
+ end
51
+ assert_equal 1, dump[:copy_rules].size
52
+ FC::CopyRule.table_fields.each do |field|
53
+ assert_equal rule.send(field), dump[:copy_rules][0][field.to_sym], "field \"#{field}\" not dumped correctly"
54
+ end
55
+ assert_equal 1, dump[:policies].size
56
+ FC::Policy.table_fields.each do |field|
57
+ assert_equal policy.send(field), dump[:policies][0][field.to_sym], "field \"#{field}\" not dumped correctly"
58
+ end
59
+ assert_equal 'value sample', dump[:vars][:some_variable]
60
+ assert_equal 'awesome value', dump[:vars][:"awesome variable"]
61
+ end
62
+
63
+ should 'create db objects from dump' do
64
+ schema = test_schema
65
+ errors = create_or_update_schema(schema: schema, update: false)
66
+ assert_equal 0, errors.size
67
+ storage = FC::Storage.where.first
68
+ policy = FC::Policy.where.first
69
+ rule = FC::CopyRule.where.first
70
+ FC::Storage.table_fields.each do |field|
71
+ next if %w[check_time autosync_at].include? field # not dumping attrs
72
+ assert_equal storage.send(field), schema[:storages][0][field.to_sym], "field \"#{field}\" not loaded correctly"
73
+ end
74
+ FC::CopyRule.table_fields.each do |field|
75
+ assert_equal rule.send(field), schema[:copy_rules][0][field.to_sym], "field \"#{field}\" not dumped correctly"
76
+ end
77
+ FC::Policy.table_fields.each do |field|
78
+ assert_equal policy.send(field), schema[:policies][0][field.to_sym], "field \"#{field}\" not dumped correctly"
79
+ end
80
+ assert_equal FC::Var.get('awesome variable'), schema[:vars][:"awesome variable"]
81
+ assert_equal FC::Var.get(:some_variable), schema[:vars][:some_variable]
82
+ end
83
+
84
+ should 'apply props to objects from dump' do
85
+ # create some objects
86
+ rule = FC::CopyRule.new(copy_storages: 'rec2-sd1', rule: 'size < 100')
87
+ rule.save
88
+
89
+ storage = FC::Storage.new(
90
+ name: 'rec1-sda_old',
91
+ host: 'rec1',
92
+ size: 10,
93
+ size_limit: 200,
94
+ dc: 'test_old',
95
+ copy_storages: 'rec2-sda_old',
96
+ url: '//host/disk_old/',
97
+ url_weight: 3,
98
+ write_weight: 4
99
+ )
100
+ storage.save
101
+
102
+ policy = FC::Policy.new(name: 'src_old', create_storages: 'rec1-sda_old', copies: 1, delete_deferred_time: 86400);
103
+ policy.save
104
+
105
+ FC::Var.set('some_variable', 'original value')
106
+ FC::Var.set('should fail variable', 'value') # this should be in errors
107
+
108
+ schema = test_schema
109
+ errors = create_or_update_schema(schema: schema, update: true)
110
+
111
+ assert_equal 3, errors.size # storage, policy, var
112
+ assert errors.include?("FC::Storage \"#{schema[:storages][0][:name]}\" not found")
113
+ assert errors.include?('FC::Var "awesome variable" not found')
114
+ assert errors.include?("FC::Policy \"#{schema[:policies][0][:name]}\" not found")
115
+
116
+ # restore right names
117
+ storage = FC::Storage.new(
118
+ name: 'rec1-sda',
119
+ host: 'rec1',
120
+ size: 10,
121
+ size_limit: 200,
122
+ dc: 'test_old',
123
+ copy_storages: 'rec2-sda_old',
124
+ url: '//host/disk_old/',
125
+ url_weight: 3,
126
+ write_weight: 4
127
+ )
128
+ storage.save
129
+
130
+ policy.name = 'src'
131
+ policy.save
132
+
133
+ FC::Var.set('awesome variable', 'old awesome value')
134
+
135
+ errors = create_or_update_schema(schema: schema, update: true)
136
+
137
+ assert_equal 0, errors.size
138
+
139
+ storage.reload
140
+ policy.reload
141
+ rule.reload
142
+
143
+ FC::Storage.table_fields.each do |field|
144
+ next if %w[check_time autosync_at].include? field # not dumping attrs
145
+ assert_equal storage.send(field), schema[:storages][0][field.to_sym], "field \"#{field}\" not loaded correctly"
146
+ end
147
+ FC::CopyRule.table_fields.each do |field|
148
+ assert_equal rule.send(field), schema[:copy_rules][0][field.to_sym], "field \"#{field}\" not dumped correctly"
149
+ end
150
+ FC::Policy.table_fields.each do |field|
151
+ assert_equal policy.send(field), schema[:policies][0][field.to_sym], "field \"#{field}\" not dumped correctly"
152
+ end
153
+ assert_equal FC::Var.get('awesome variable'), schema[:vars][:"awesome variable"]
154
+ end
155
+
156
+ def test_schema
157
+ yaml = %(
158
+ ---
159
+ :storages:
160
+ - :name: rec1-sda
161
+ :host: rec1
162
+ :dc: test
163
+ :path: ''
164
+ :url: "//host/disk/"
165
+ :size: 0
166
+ :size_limit: 100
167
+ :copy_storages: rec2-sda
168
+ :url_weight: 1
169
+ :write_weight: 2
170
+ :auto_size: 0
171
+ :policies:
172
+ - :name: src
173
+ :create_storages: rec1-sda
174
+ :copies: 1
175
+ :delete_deferred_time: 86400
176
+ :copy_rules:
177
+ - :rule: size < 100
178
+ :copy_storages: rec2-sda,rec3-sda
179
+ :vars:
180
+ :awesome variable: 'awesome value'
181
+ :some_variable: 'value sample')
182
+ Psych.load(yaml)
183
+ end
184
+ end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: filecluster
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.5.16
4
+ version: 0.5.17
5
5
  platform: ruby
6
6
  authors:
7
7
  - sh
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-05-24 00:00:00.000000000 Z
11
+ date: 2018-06-19 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: mysql2
@@ -178,6 +178,7 @@ files:
178
178
  - lib/manage/copy_speed.rb
179
179
  - lib/manage/item.rb
180
180
  - lib/manage/policies.rb
181
+ - lib/manage/schema.rb
181
182
  - lib/manage/show.rb
182
183
  - lib/manage/storages.rb
183
184
  - lib/manage/var.rb
@@ -192,6 +193,7 @@ files:
192
193
  - test/helper.rb
193
194
  - test/item_test.rb
194
195
  - test/policy_test.rb
196
+ - test/schema_test.rb
195
197
  - test/storage_sync_test.rb
196
198
  - test/storage_test.rb
197
199
  - test/var_test.rb
@@ -230,6 +232,7 @@ test_files:
230
232
  - test/helper.rb
231
233
  - test/item_test.rb
232
234
  - test/policy_test.rb
235
+ - test/schema_test.rb
233
236
  - test/storage_sync_test.rb
234
237
  - test/storage_test.rb
235
238
  - test/var_test.rb