filecluster 0.5.16 → 0.5.17

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 04a6d63f3d0df33e01e31f154736866746ffc579
-  data.tar.gz: f0525db53c050a1413c4995b4f87a12b56f6823c
+  metadata.gz: 9e81ff8dd657f27201d09032d872bfe8425cf65d
+  data.tar.gz: 8659faa5b1949c5f0759f73ca499ae07432eac92
 SHA512:
-  metadata.gz: 32c4794f66a75ee6757eb6d481346abed46ea4a932220f806c2b64620c26206c101de322977784114fda6bf037008eea4edac57480faa21c2b6933fc3e34f61f
-  data.tar.gz: f57b788b11400354ec265e9f1ca40bdd5ae2074412c25c459ad0223ee27984d7cfdc933ef8708db49fc423885c7ce7c5bedfa50f2d217c827f3510dadafe2e15
+  metadata.gz: 0f8aa7d24c0fa3ed9fdccc5ed60d6fc3c48daf7bdfb190e688685a19beb42ee84a948182122502a5542b9c28c18ac3bbc442cf49dd60b7800c3b048f137c77f7
+  data.tar.gz: a5b54d47f2c000a947a961f2f5ef59941eaa1fa104a134d6ba89d78c87600c7dc0f143e64e283939851b16f0e27ef02849abc2dddedd5606ff578c92289c7d2d

data/bin/fc-manage

@@ -89,7 +89,16 @@ Command:
    add_local <storage name> <path> <policy id/name> <tag> <outer_id>  add file in local storage as item (as <path> name) with policy, tag & outer_id is not required 
    add <path> <name> <policy id/name>                                 add file as item <name> with policy
    rm <name>                                                          delete item
-}]
+}],
+  'schema' => [
+    'use external configuration to manage schema definition (yaml formatted)',
+    %q{Usage: fc-manage schema <command>
+  Command:
+    create  [options] <config file>.yml       creates storages/policies/db connection, fails if any already exists
+      Options: force                          force to warn and replace objects instead of fail
+    apply  <config file>.yml                  applying storages/policies properties, warn for missing objects
+    dump   <target file>.yml                  loads out existing storages/policies into single target file
+  }]
 }
 desc = %q{Get info and manage for storages, policies and items.
 Usage: fc-manage [options] <command> [<args>]

data/lib/fc/base.rb

@@ -100,5 +100,30 @@ module FC
       FC::DB.query("DELETE FROM #{self.class.table_name} WHERE id=#{@id.to_i}") if @id
     end
     
+    def self.apply!(data: {}, key_field: :name, update_only: false)
+      return nil unless data[key_field]
+      obj_instance = where("#{key_field} = ?", data[key_field]).first
+      return "#{name} \"#{data[key_field]}\" not found" if obj_instance.nil? && update_only
+      obj_instance ||= new
+      obj_instance.load(data: data)
+      obj_instance.save
+      obj_instance
+    end
+
+    def load(data: {})
+      self.class.table_fields.each do |field|
+        send("#{field}=", data[field.to_sym]) if data[field.to_sym]
+      end
+    end
+
+    def dump(exclude = [])
+      obj_dump = {}
+      self.class.table_fields.each do |field|
+        sym_field = field.to_sym
+        next if exclude.include?(sym_field)
+        obj_dump[sym_field] = database_fields[field]
+      end
+      obj_dump
+    end
   end
 end

data/lib/fc/copy_rule.rb

@@ -56,7 +56,7 @@ module FC
       $SELF = 0
       r ? true : false
     end
-    
+
     def test
       storage = FC::Storage.new(
         :id => 1,

data/lib/fc/storage.rb

@@ -195,5 +195,17 @@ module FC
     def get_proper_storage_for_copy(size, exclude = [])
       FC::Storage.select_proper_storage_for_create(get_copy_storages, size, exclude)
     end
+
+    def dump
+      super(%i[check_time autosync_at])
+    end
+
+    def load(data: {})
+      if name
+        data = data.clone
+        data.delete(:host)
+      end
+      super(data: data)
+    end
   end
 end

data/lib/fc/version.rb

@@ -1,3 +1,3 @@
 module FC
-  VERSION = '0.5.16'.freeze
+  VERSION = '0.5.17'.freeze
 end

data/lib/manage.rb

@@ -7,4 +7,5 @@ require "manage/var"
 require "manage/item"
 require "manage/copy_speed"
 require "manage/autosync"
+require "manage/schema"
 require 'autosync'

data/lib/manage/schema.rb

@@ -0,0 +1,114 @@
+require 'psych'
+def schema_create
+  unless ARGV[2]
+    puts 'config file not set'
+    exit(1)
+  end
+  force = ARGV[3].to_s == 'force'
+  schema = load_schema_file
+
+  apply_connection(schema[:connection])
+  # check existing objects
+  unless force
+    errors = ''
+    schema[:storages].each do |cs|
+      errors << "Storage \"#{cs[:name]}\" already exists\n" if FC::Storage.where('name = ?', cs[:name]).any?
+    end
+    schema[:policies].each do |pl|
+      errors << "Policy \"#{pl[:name]}\" already exists\n" if FC::Policy.where('name = ?', pl[:name]).any?
+    end
+    schema[:copy_rules].each do |pl|
+      errors << "CopyRule \"#{pl[:rule]}\" already exists\n" if FC::CopyRule.where('rule = ?', pl[:rule]).any?
+    end
+    existing_vars = FC::Var.get_all
+    schema[:vars].keys.each do |v|
+      errors << "Var \"#{v}\" already exists\n" if existing_vars.include?(v)
+    end
+
+    unless errors.empty?
+      puts errors
+      exit(1)
+    end
+  end
+  errors = create_or_update_schema(schema: schema, update: false)
+  puts errors.join("\n") if errors && errors.any?
+end
+
+def schema_apply
+  schema = load_schema_file
+  apply_connection(schema[:connection])
+  errors = create_or_update_schema(schema: schema, update: true)
+  puts errors.join("\n") if errors && errors.any?
+end
+
+def schema_dump
+  dump = create_dump
+  if ARGV[2]
+    File.write ARGV[2], Psych.dump(dump)
+  else
+    puts Psych.dump(dump)
+  end
+end
+
+def create_or_update_schema(schema:, update:)
+  errors = []
+  errors << apply_entity(klass: FC::Storage, items: schema[:storages], update_only: update)
+  errors << apply_entity(klass: FC::Policy, items: schema[:policies], update_only: update)
+  errors << apply_entity(klass: FC::CopyRule, items: schema[:copy_rules], key_field: :rule, update_only: update)
+  errors << apply_vars(vars: schema[:vars], update_only: update)
+  errors.flatten
+end
+
+def create_dump
+  schema = { }
+  schema[:storages] = FC::Storage.where.map(&:dump)
+  schema[:policies] = FC::Policy.where.map(&:dump)
+  schema[:copy_rules] = FC::CopyRule.where.map(&:dump)
+  schema[:vars] = FC::Var.get_all.select { |k, _| k.is_a? Symbol }
+  schema[:connection] = FC::DB.instance_variable_get('@options')
+  schema
+end
+
+def load_schema_file
+  schema = Psych.load(File.read(ARGV[2]))
+  schema[:storages] ||= []
+  schema[:policies] ||= []
+  schema[:copy_rules] ||= []
+  schema[:vars] ||= {}
+  schema
+end
+
+def apply_connection(options)
+  return unless options
+  FC::DB.close
+  FC::DB.connect_by_config(options)
+  FC::DB.connect.ping
+  File.write(FC::DB.options_yml_path, Psych.dump(options))
+end
+
+def apply_vars(vars:, update_only: false)
+  errors = []
+  existing_vars = FC::Var.get_all
+  vars.each do |k, v|
+    if !existing_vars.keys.include?(k) && update_only
+      errors << "FC::Var \"#{k}\" not found"
+      next
+    end
+    FC::Var.set(k, v)
+  end
+  errors
+end
+
+def apply_entity(klass:, items:, update_only: false, key_field: :name)
+  errors = []
+  return errors unless items
+  items.each do |item|
+    begin
+      result = klass.apply!(data: item, update_only: update_only, key_field: key_field)
+      errors << result if result.is_a? String
+    rescue => save_error
+      errors << "Error while saving #{klass} \"#{item[key_field]}\": #{save_error}"
+    end
+  end
+  errors
+end

data/test/schema_test.rb

@@ -0,0 +1,184 @@
+require 'helper'
+require 'manage/schema'
+class SchemaTest < Test::Unit::TestCase
+  class << self
+    def startup
+      FC::DB.query("DELETE FROM policies")
+      FC::DB.query("DELETE FROM vars")
+      FC::DB.query("DELETE FROM copy_rules")
+      FC::DB.query("DELETE FROM storages")
+    end
+  end
+
+  def teardown
+    FC::DB.query("DELETE FROM policies")
+    FC::DB.query("DELETE FROM vars")
+    FC::DB.query("DELETE FROM copy_rules")
+    FC::DB.query("DELETE FROM storages")
+  end
+
+  should 'create dump' do
+    rule = FC::CopyRule.new(copy_storages: 'rec2-sda,rec3-sda', rule: 'size < 100')
+    rule.save
+    rule.reload
+    storage = FC::Storage.new(
+      name: 'rec1-sda',
+      host: 'rec1',
+      size: 0,
+      size_limit: 100,
+      dc: 'test',
+      copy_storages: 'rec2-sda',
+      url: '//host/disk/',
+      url_weight: 1,
+      write_weight: 2
+    )
+    storage.save
+    storage.reload
+
+    policy = FC::Policy.new(name: 'src', create_storages: 'rec1-sda', copies: 1, delete_deferred_time: 86400);
+    policy.save
+    policy.reload
+
+    FC::Var.set('some_variable', 'value sample')
+    FC::Var.set('awesome variable', 'awesome value')
+
+    dump = create_dump
+    assert_equal 1, dump[:storages].size
+    FC::Storage.table_fields.each do |field|
+      next if %w[check_time autosync_at].include? field # not dumping attrs
+      assert_equal storage.send(field), dump[:storages][0][field.to_sym], "field \"#{field}\" not dumped correctly"
+    end
+    assert_equal 1, dump[:copy_rules].size
+    FC::CopyRule.table_fields.each do |field|
+      assert_equal rule.send(field), dump[:copy_rules][0][field.to_sym], "field \"#{field}\" not dumped correctly"
+    end
+    assert_equal 1, dump[:policies].size
+    FC::Policy.table_fields.each do |field|
+      assert_equal policy.send(field), dump[:policies][0][field.to_sym], "field \"#{field}\" not dumped correctly"
+    end
+    assert_equal 'value sample', dump[:vars][:some_variable]
+    assert_equal 'awesome value', dump[:vars][:"awesome variable"]
+  end
+
+  should 'create db objects from dump' do
+    schema = test_schema
+    errors = create_or_update_schema(schema: schema, update: false)
+    assert_equal 0, errors.size
+    storage = FC::Storage.where.first
+    policy = FC::Policy.where.first
+    rule = FC::CopyRule.where.first
+    FC::Storage.table_fields.each do |field|
+      next if %w[check_time autosync_at].include? field # not dumping attrs
+      assert_equal storage.send(field), schema[:storages][0][field.to_sym], "field \"#{field}\" not loaded correctly"
+    end
+    FC::CopyRule.table_fields.each do |field|
+      assert_equal rule.send(field), schema[:copy_rules][0][field.to_sym], "field \"#{field}\" not dumped correctly"
+    end
+    FC::Policy.table_fields.each do |field|
+      assert_equal policy.send(field), schema[:policies][0][field.to_sym], "field \"#{field}\" not dumped correctly"
+    end
+    assert_equal FC::Var.get('awesome variable'), schema[:vars][:"awesome variable"]
+    assert_equal FC::Var.get(:some_variable), schema[:vars][:some_variable]
+  end
+
+  should 'apply props to objects from dump' do
+    # create some objects
+    rule = FC::CopyRule.new(copy_storages: 'rec2-sd1', rule: 'size < 100')
+    rule.save
+
+    storage = FC::Storage.new(
+      name: 'rec1-sda_old',
+      host: 'rec1',
+      size: 10,
+      size_limit: 200,
+      dc: 'test_old',
+      copy_storages: 'rec2-sda_old',
+      url: '//host/disk_old/',
+      url_weight: 3,
+      write_weight: 4
+    )
+    storage.save
+
+    policy = FC::Policy.new(name: 'src_old', create_storages: 'rec1-sda_old', copies: 1, delete_deferred_time: 86400);
+    policy.save
+
+    FC::Var.set('some_variable', 'original value')
+    FC::Var.set('should fail variable', 'value') # this should be in errors
+
+    schema = test_schema
+    errors = create_or_update_schema(schema: schema, update: true)
+
+    assert_equal 3, errors.size # storage, policy, var
+    assert errors.include?("FC::Storage \"#{schema[:storages][0][:name]}\" not found")
+    assert errors.include?('FC::Var "awesome variable" not found')
+    assert errors.include?("FC::Policy \"#{schema[:policies][0][:name]}\" not found")
+    
+    # restore right names
+    storage = FC::Storage.new(
+      name: 'rec1-sda',
+      host: 'rec1',
+      size: 10,
+      size_limit: 200,
+      dc: 'test_old',
+      copy_storages: 'rec2-sda_old',
+      url: '//host/disk_old/',
+      url_weight: 3,
+      write_weight: 4
+    )
+    storage.save
+
+    policy.name = 'src'
+    policy.save
+
+    FC::Var.set('awesome variable', 'old awesome value')
+
+    errors = create_or_update_schema(schema: schema, update: true)
+
+    assert_equal 0, errors.size
+    
+    storage.reload
+    policy.reload
+    rule.reload
+
+    FC::Storage.table_fields.each do |field|
+      next if %w[check_time autosync_at].include? field # not dumping attrs
+      assert_equal storage.send(field), schema[:storages][0][field.to_sym], "field \"#{field}\" not loaded correctly"
+    end
+    FC::CopyRule.table_fields.each do |field|
+      assert_equal rule.send(field), schema[:copy_rules][0][field.to_sym], "field \"#{field}\" not dumped correctly"
+    end
+    FC::Policy.table_fields.each do |field|
+      assert_equal policy.send(field), schema[:policies][0][field.to_sym], "field \"#{field}\" not dumped correctly"
+    end
+    assert_equal FC::Var.get('awesome variable'), schema[:vars][:"awesome variable"]
+  end
+
+  def test_schema
+    yaml = %(
+---
+:storages:
+- :name: rec1-sda
+  :host: rec1
+  :dc: test
+  :path: ''
+  :url: "//host/disk/"
+  :size: 0
+  :size_limit: 100
+  :copy_storages: rec2-sda
+  :url_weight: 1
+  :write_weight: 2
+  :auto_size: 0
+:policies:
+- :name: src
+  :create_storages: rec1-sda
+  :copies: 1
+  :delete_deferred_time: 86400
+:copy_rules:
+- :rule: size < 100
+  :copy_storages: rec2-sda,rec3-sda
+:vars:
+  :awesome variable: 'awesome value'
+  :some_variable: 'value sample')
+    Psych.load(yaml)
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: filecluster
 version: !ruby/object:Gem::Version
-  version: 0.5.16
+  version: 0.5.17
 platform: ruby
 authors:
 - sh
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-05-24 00:00:00.000000000 Z
+date: 2018-06-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mysql2
@@ -178,6 +178,7 @@ files:
 - lib/manage/copy_speed.rb
 - lib/manage/item.rb
 - lib/manage/policies.rb
+- lib/manage/schema.rb
 - lib/manage/show.rb
 - lib/manage/storages.rb
 - lib/manage/var.rb
@@ -192,6 +193,7 @@ files:
 - test/helper.rb
 - test/item_test.rb
 - test/policy_test.rb
+- test/schema_test.rb
 - test/storage_sync_test.rb
 - test/storage_test.rb
 - test/var_test.rb
@@ -230,6 +232,7 @@ test_files:
 - test/helper.rb
 - test/item_test.rb
 - test/policy_test.rb
+- test/schema_test.rb
 - test/storage_sync_test.rb
 - test/storage_test.rb
 - test/var_test.rb