db_to_file 1.2.4

data/lib/db_to_file/values_normalizer/object_to_hash.rb

@@ -0,0 +1,46 @@
+module DbToFile
+  module ValuesNormalizer
+    class ObjectToHash
+      def initialize(object)
+        @object = object
+      end
+
+      def normalize
+        @hash = {}
+        @object.attributes.each do |field, value|
+          @current_field = field
+          @hash[field] = normalize_field_value(value)
+        end
+        @hash
+      end
+
+      private
+        def normalize_field_value(value)
+          value = convert_yaml(value)
+          value = convert_nil_value(value)
+          value = convert_integer_to_string(value)
+          value = add_trailing_newline(value)
+        end
+
+        def convert_yaml(value)
+          (!value.nil? && serialized_attribute?(@current_field)) ? value.to_yaml : value
+        end
+
+        def serialized_attribute?(field)
+          @object.class.serialized_attributes.keys.include?(field.to_s)
+        end
+
+        def convert_nil_value(value)
+          (value.nil?) ? '<NULL>' : value
+        end
+
+        def convert_integer_to_string(integer)
+          "#{integer}"
+        end
+
+        def add_trailing_newline(string)
+          (string[-1] == "\n") ? string : "#{string}\n"
+        end
+    end
+  end
+end

data/lib/db_to_file/values_normalizer/value_into_object.rb

@@ -0,0 +1,40 @@
+module DbToFile
+  module ValuesNormalizer
+    class ValueIntoObject
+      def initialize(object)
+        @object = object
+      end
+
+      def normalize(fieldname, value)
+        if @object.respond_to?(fieldname.to_sym)
+          normalized_value = normalize_field_value(fieldname, value)
+          @object.send("#{fieldname}=", normalized_value)
+        end
+      end
+
+      private
+        def normalize_field_value(fieldname, value)
+          @current_field = fieldname
+          value = strip_trailing_newline(value)
+          value = convert_nil_value(value)
+          value = convert_yaml(value)
+        end
+
+        def strip_trailing_newline(text)
+          (text[-1] == "\n") ? text[0..-2] : text
+        end
+
+        def convert_nil_value(value)
+          (value == '<NULL>') ? nil : value
+        end
+
+        def convert_yaml(value)
+          (!value.nil? && serialized_attribute?(@current_field)) ? YAML.load(value) : value
+        end
+
+        def serialized_attribute?(field)
+          @object.class.serialized_attributes.keys.include?(field.to_s)
+        end
+    end
+  end
+end

data/lib/db_to_file/version.rb

@@ -0,0 +1,3 @@
+module DbToFile
+  VERSION = "1.2.4"
+end

data/lib/db_to_file/version_controller.rb

@@ -0,0 +1,78 @@
+module DbToFile
+  class VersionController
+    def prepare_code_version
+      SystemExecuter.new("git stash save 'db-to-file'").execute
+      SystemExecuter.new('git pull').execute
+      FileUtils.rm_rf('db/db_to_file')
+    end
+
+    def update_code_version(commit_message = nil)
+      update_commit_stash
+      commit_changes(commit_message) if commitable_files_present?
+    end
+
+    def restore_local_stash
+      restore_stash
+    end
+
+    def merge_conflicts_present?
+      out = SystemExecuter.new('git status --porcelain').execute.split("\n")
+      out.any?{|line| line[0..0] == 'U'}
+    end
+
+    def get_modified_file_list
+      modified_files
+    end
+
+    private
+      DEFAULT_COMMIT_MESSAGE = 'DbToFile: changes by customer'
+
+      def update_commit_stash
+        SystemExecuter.new('git status --porcelain').execute
+        new_files.each do |file|
+          SystemExecuter.new("git add #{file}").execute if table_file?(file)
+        end
+        modified_files.each do |file|
+          SystemExecuter.new("git add #{file}").execute if table_file?(file)
+        end
+        deleted_files.each do |file|
+          SystemExecuter.new("git rm #{file}").execute if table_file?(file)
+        end
+      end
+
+      def commitable_files_present?
+        out = SystemExecuter.new('git status --porcelain').execute
+        out.split("\n").reject{|line| [' ', '?'].include?(line[0])}.any?
+      end
+
+      def commit_changes(commit_message)
+        puts 'Commit changes'
+        message = commit_message || DEFAULT_COMMIT_MESSAGE
+        git.commit(message)
+      end
+
+      def restore_stash
+        SystemExecuter.new('git stash pop').execute
+      end
+
+      def new_files
+        git.status.untracked.map{|file, git_object| file}
+      end
+
+      def modified_files
+        git.status.changed.map{|file, git_object| file}
+      end
+
+      def deleted_files
+        git.status.deleted.map{|file, git_object| file}
+      end
+
+      def table_file?(file)
+        file.index('db/db_to_file') === 0
+      end
+
+      def git
+        @git ||= Git.open(Dir.pwd)
+      end
+  end
+end

data/lib/tasks/unloader.rake

@@ -0,0 +1,18 @@
+namespace :db_to_file do
+  class WrongEnvironmentGiven < Exception; end
+
+  desc 'Unload tables to file system'
+  task :unload => :environment do |args|
+    # wrong_environment(:environment) unless Rails.env.production?
+    unload_tables
+  end
+
+  private
+    def unload_tables
+      DbToFile::Unloader.new.unload
+    end
+
+    def wrong_environment(environment)
+      raise WrongEnvironmentGiven, "Environment should be 'production'. '#{Rails.env}' is used"
+    end
+end

data/lib/tasks/uploader.rake

@@ -0,0 +1,37 @@
+namespace :db_to_file do
+  class NoCommitMessage < Exception; end
+
+  desc "Upload files to the database (set commit message via command 'rake db_to_file:unload['<commit message>'])"
+  task :upload, [:commit_message] => :environment do |t, args|
+    no_commit_message_error if args[:commit_message].blank?
+
+    upload_files(args[:commit_message])
+  end
+
+  desc "Force uploading to the database, without checks"
+  task :force_upload => :environment do |t, args|
+    force_upload_files
+  end
+
+  desc 'Force uploading only changed files in database, without checks'
+  task :force_changed => :environment do |t, args|
+    force_upload_changed_files
+  end
+
+  private
+    def upload_files(commit_message)
+      DbToFile::Uploader.new.upload(commit_message)
+    end
+
+    def force_upload_files
+      DbToFile::Uploader.new.force_upload
+    end
+
+    def force_upload_changed_files
+      DbToFile::Uploader.new.force_upload
+    end
+
+    def no_commit_message_error
+      raise NoCommitMessage, "invoke via 'rake db_to_file:unload['<commit message>']'"
+    end
+end

data/test/fixtures/config.yml

@@ -0,0 +1,6 @@
+tables:
+  users:
+    directory_prefix: name
+    field_extensions:
+      name: 'txt'
+  settings:

data/test/lib/db_to_file/unloader_test.rb

@@ -0,0 +1,137 @@
+require_relative '../../test_helper'
+require 'fileutils'
+
+describe DbToFile::Unloader do
+  let(:unloader) { DbToFile::Unloader.new }
+
+  before do
+    DbToFile::Config.any_instance.stubs(:config_file).returns('test/fixtures/config.yml')
+  end
+
+  describe 'configuration-file' do
+    it 'can be parsed' do
+      unloader.send(:tables).must_equal(['users', 'settings'])
+    end
+
+    it 'can read directory prefix' do
+      @table  = DbToFile::Unloader::Table.new('users', @unloader)
+      @record = DbToFile::Unloader::Table::Record.new(User.first, @table)
+
+      @record.send(:config_directory_prefix).must_equal('name')
+
+      @table  = DbToFile::Unloader::Table.new('settings', @unloader)
+      @record = DbToFile::Unloader::Table::Record.new(Setting.first, @table)
+
+      @record.send(:config_directory_prefix).must_equal(nil)
+    end
+
+    it 'can read file-extensions' do
+      @table      = DbToFile::Unloader::Table.new('users', @unloader)
+      @record     = DbToFile::Unloader::Table::Record.new(User.first, @table)
+      @field_name = DbToFile::Unloader::Table::Record::Field.new('name', @record)
+      @field_id   = DbToFile::Unloader::Table::Record::Field.new('id', @record)
+
+      DbToFile::Config.instance.stub(:config_file, 'test/fixtures/config.yml') do
+        @field_name.send(:config_field_extension).must_equal('txt')
+        @field_id.send(:config_field_extension).must_equal(nil)
+      end
+    end
+  end
+
+  describe 'unloading' do
+    it 'calls the functions' do
+      unloader.expects(:prepare_code_version)
+      unloader.expects(:unload_tables)
+      unloader.expects(:update_code_version)
+      unloader.expects(:restore_local_stash)
+
+      unloader.unload
+    end
+  end
+
+  describe 'build directory for users with prefix' do
+    before do
+      unloader.stubs(:config_directory_prefix).returns('name')
+      DbToFile::Config.instance.stub(:config_file, 'test/fixtures/config.yml') do
+        unloader.stub(:tables, ['users']) do
+          unloader.send(:unload_tables)
+        end
+      end
+    end
+
+    after do
+      FileUtils.rm_rf('db/db_to_file')
+    end
+
+    it 'for the users' do
+      File.directory?('db/db_to_file/users').must_equal true
+    end
+
+    it 'builds the directory for the records' do
+      File.directory?('db/db_to_file/users/ewout-quax_1').must_equal true
+      File.directory?('db/db_to_file/users/test-example_2').must_equal true
+      File.directory?('db/db_to_file/users/3').must_equal true
+    end
+
+    it 'builds the files for the record-files' do
+      File.file?('db/db_to_file/users/ewout-quax_1/id').must_equal true
+      File.file?('db/db_to_file/users/ewout-quax_1/name.txt').must_equal true
+
+      File.read('db/db_to_file/users/ewout-quax_1/name.txt').must_equal "Ewout Quax\n"
+    end
+  end
+
+  describe 'build directory for users without prefix' do
+    before do
+      unloader.stubs(:config_directory_prefix).returns(nil)
+      unloader.stubs(:config_ignore_columns).returns(nil)
+      unloader.stubs(:config_field_extension).returns(nil)
+      unloader.stub(:tables, ['settings']) do
+        unloader.send(:unload_tables)
+      end
+    end
+
+    after do
+      FileUtils.rm_rf('db/db_to_file')
+    end
+
+    it 'for the settings' do
+      File.directory?('db/db_to_file/settings').must_equal true
+    end
+
+    it 'builds the directory for the records' do
+      File.directory?('db/db_to_file/settings/1').must_equal true
+      File.directory?('db/db_to_file/settings/2').must_equal true
+    end
+
+    it 'builds the files for the record-files' do
+      File.file?('db/db_to_file/settings/2/key').must_equal true
+      File.file?('db/db_to_file/settings/2/value').must_equal true
+
+      File.read('db/db_to_file/settings/2/value').must_equal "<NULL>\n"
+    end
+  end
+
+  describe 'prepare code-versioning' do
+    it 'invokes the version-controller' do
+      controller = Minitest::Mock.new
+      DbToFile::VersionController.expects(:new).returns(controller)
+
+      controller.expect(:prepare_code_version, nil)
+      unloader.send(:prepare_code_version)
+      controller.verify
+    end
+  end
+
+  describe 'update_code_version' do
+    it 'invokes all the functions' do
+      controller = Minitest::Mock.new
+
+      unloader.expects(:version_controller).returns(controller)
+
+      controller.expect(:update_code_version, nil)
+      unloader.send(:update_code_version)
+      controller.verify
+    end
+  end
+end

data/test/lib/db_to_file/uploader_test.rb

@@ -0,0 +1,145 @@
+require_relative '../../test_helper'
+require 'fileutils'
+
+describe DbToFile::Uploader do
+  it 'invokes all the functions' do
+    uploader = DbToFile::Uploader.new
+
+    uploader.expects(:can_continue?).returns(true).times(2)
+    uploader.expects(:invoke_unloader)
+    uploader.expects(:write_objects_to_db)
+    uploader.expects(:update_code_version)
+
+    uploader.upload('uploader_test')
+  end
+
+  describe 'can_continue' do
+    let(:uploader) { DbToFile::Uploader.new }
+
+    it 'return false, if merge conflicts are found' do
+      uploader.stub(:merge_conflicts_present?, true) do
+        uploader.send(:can_continue?).must_equal(false)
+      end
+    end
+
+    it 'return true, if no merge conflicts are found' do
+      uploader.stub(:merge_conflicts_present?, false) do
+        uploader.send(:can_continue?).must_equal(true)
+      end
+    end
+  end
+
+  describe 'merge_conflicts_present' do
+    let(:uploader) { DbToFile::Uploader.new }
+    let(:controller) { Minitest::Mock.new }
+
+    it 'invokes the version controller' do
+      controller.expect(:merge_conflicts_present?, true)
+      uploader.stub(:version_controller, controller) do
+        uploader.send(:merge_conflicts_present?).must_equal(true)
+      end
+      controller.verify
+    end
+  end
+
+  describe 'objects' do
+    after do
+      User.last.delete
+    end
+
+    it 'are saved to the db' do
+      user_1 = User.new(name: 'Test UserName')
+      uploader = DbToFile::Uploader.new
+      uploader.expects(:build_objects).returns([user_1])
+
+      uploader.send(:write_objects_to_db)
+
+      User.last.must_equal(user_1)
+    end
+  end
+
+  describe 'unloaded files' do
+    before do
+      write_file('db/db_to_file/users/ewout-quax_1', 'id', '1')
+      write_file('db/db_to_file/users/ewout-quax_1', 'name.txt', 'Ewout Quax')
+      write_file('db/db_to_file/users/test-example_2', 'id', '2')
+      write_file('db/db_to_file/users/test-example_2', 'name.txt', 'Test Example')
+    end
+
+    after do
+      FileUtils.rm_rf('db/db_to_file')
+    end
+
+    it 'can be read' do
+      files = DbToFile::Uploader.new.send(:read_files)
+      files.include?('db/db_to_file/users/ewout-quax_1/id').must_equal true
+      files.include?('db/db_to_file/users/ewout-quax_1/name.txt').must_equal true
+      files.include?('db/db_to_file/users/test-example_2/id').must_equal true
+      files.include?('db/db_to_file/users/test-example_2/name.txt').must_equal true
+    end
+
+    it 'can be builded into models' do
+      models = DbToFile::Uploader.new.send(:build_objects)
+      models.size.must_equal 2
+      models.first.class.must_equal(User)
+      models.last.class.must_equal(User)
+
+      user_1 = models.detect{|m| m.id == 1}
+      user_2 = models.detect{|m| m.id == 2}
+      user_1.name.must_equal 'Ewout Quax'
+      user_2.name.must_equal 'Test Example'
+    end
+  end
+
+  describe 'update_code_version' do
+    let(:uploader) { DbToFile::Uploader.new }
+    let(:controller) { Minitest::Mock.new }
+
+    it 'invokes the version-controller, with a commit message' do
+      uploader.expects(:version_controller).returns(controller)
+
+      controller.expect(:update_code_version, true, ['commit via test'])
+      uploader.send(:update_code_version, 'commit via test')
+      controller.verify
+    end
+  end
+
+  describe 'update_object_with_field_value' do
+    let(:uploader) { DbToFile::Uploader.new }
+
+    it 'with field value' do
+      uploader.expects(:file_value).with('db/db_to_file/users/1/name').returns('Bruce Wayne')
+
+      object = User.new
+      uploader.send(:update_object_with_field_value, object, 'name', 'db/db_to_file/users/1/name')
+      object.name.must_equal('Bruce Wayne')
+    end
+
+    it 'strips newline at end of field-value' do
+      uploader.expects(:file_value).with('db/db_to_file/users/1/name').returns("Bruce Wayne\n")
+
+      object = User.new
+      uploader.send(:update_object_with_field_value, object, 'name', 'db/db_to_file/users/1/name')
+      object.name.must_equal('Bruce Wayne')
+    end
+
+    it 'converts <NULL> to nil-value' do
+      uploader.expects(:file_value).with('db/db_to_file/users/1/name').returns("<NULL>\n")
+
+      object = User.new
+      uploader.send(:update_object_with_field_value, object, 'name', 'db/db_to_file/users/1/name')
+      object.name.must_equal(nil)
+    end
+  end
+
+  describe 'extract_data_segments' do
+    it 'splits the full-file-path into segments' do
+      uploader = DbToFile::Uploader.new
+      segments = uploader.send(:extract_data_segments, '/db/db_to_files/users/ewout-quax_1/name.txt')
+
+      segments[:model].must_equal User
+      segments[:id].must_equal 1
+      segments[:field].must_equal 'name'
+    end
+  end
+end