tapsoob 0.2.7-java

data/lib/tapsoob/schema.rb

@@ -0,0 +1,91 @@
+# -*- encoding : utf-8 -*-
+require 'sequel'
+require 'sequel/extensions/schema_dumper'
+require 'sequel/extensions/migration'
+require 'json'
+
+module Tapsoob
+  module Schema
+    extend self
+
+    def dump(database_url)
+      db = Sequel.connect(database_url)
+      db.extension :schema_dumper
+      db.dump_schema_migration(:indexes => false)
+    end
+
+    def dump_table(database_url, table)
+      table = table.to_sym
+      Sequel.connect(database_url) do |db|
+        db.extension :schema_dumper
+        <<END_MIG
+Class.new(Sequel::Migration) do
+  def up
+    #{db.dump_table_schema(table, :indexes => false)}
+  end
+
+  def down
+    drop_table("#{table}") if @db.table_exists?("#{table}")
+  end
+end
+END_MIG
+      end
+    end
+
+    def indexes(database_url)
+      db = Sequel.connect(database_url)
+      db.extension :schema_dumper
+      db.dump_indexes_migration
+    end
+
+    def indexes_individual(database_url)
+      idxs = {}
+      Sequel.connect(database_url) do |db|
+        db.extension :schema_dumper
+
+        tables = db.tables
+        tables.each do |table|
+          idxs[table] = db.send(:dump_table_indexes, table, :add_index, {}).split("\n")
+        end
+      end
+
+      idxs.each do |table, indexes|
+        idxs[table] = indexes.map do |idx|
+          <<END_MIG
+Class.new(Sequel::Migration) do
+  def up
+    #{idx}
+  end
+end
+END_MIG
+        end
+      end
+      JSON.generate(idxs)
+    end
+
+    def load(database_url, schema)
+      Sequel.connect(database_url) do |db|
+        db.extension :schema_dumper
+        klass = eval(schema)
+        klass.apply(db, :down)
+        klass.apply(db, :up)
+      end
+    end
+
+    def load_indexes(database_url, indexes)
+      Sequel.connect(database_url) do |db|
+        db.extension :schema_dumper
+        eval(indexes).apply(db, :up)
+      end
+    end
+
+    def reset_db_sequences(database_url)
+      db = Sequel.connect(database_url)
+      db.extension :schema_dumper
+      return unless db.respond_to?(:reset_primary_key_sequence)
+      db.tables.each do |table|
+        db.reset_primary_key_sequence(table)
+      end
+    end
+  end
+end

data/lib/tapsoob/utils.rb

@@ -0,0 +1,184 @@
+# -*- encoding : utf-8 -*-
+require 'zlib'
+
+require 'tapsoob/errors'
+require 'tapsoob/chunksize'
+require 'tapsoob/schema'
+
+module Tapsoob
+  module Utils
+    extend self
+
+    def windows?
+      return @windows if defined?(@windows)
+      require 'rbconfig'
+      @windows = !!(::RbConfig::CONFIG['host_os'] =~ /mswin|mingw/)
+    end
+
+    def bin(cmd)
+      cmd = "#{cmd}.cmd" if windows?
+      cmd
+    end
+
+    def checksum(data)
+      Zlib.crc32(data)
+    end
+
+    def valid_data?(data, crc32)
+      Zlib.crc32(data) == crc32.to_i
+    end
+
+    def base64encode(data)
+      [data].pack("m")
+    end
+
+    def base64decode(data)
+      data.unpack("m").first
+    end
+
+    def format_data(data, opts = {})
+      return {} if data.size == 0
+      string_columns = opts[:string_columns] || []
+      schema = opts[:schema] || []
+      table = opts[:table]
+
+      max_lengths = schema.inject({}) do |hash, (column, meta)|
+        if meta[:db_type] =~ /^varchar\((\d+)\)/
+          hash.update(column => $1.to_i)
+        end
+        hash
+      end
+
+      header = data[0].keys
+      only_data = data.collect do |row|
+        row = blobs_to_string(row, string_columns)
+        row.each do |column, data|
+          if data.to_s.length > (max_lengths[column] || data.to_s.length)
+            raise Tapsoob::InvalidData.new(<<-ERROR)
+Detected data that exceeds the length limitation of its column. This is
+generally due to the fact that SQLite does not enforce length restrictions.
+
+Table : #{table}
+Column : #{column}
+Type : #{schema.detect{|s| s.first == column}.last[:db_type]}
+Data : #{data}
+            ERROR
+          end
+
+          # Type conversion
+          row[column] = data.strftime('%Y-%m-%d %H:%M:%S') if data.is_a?(Time)
+        end
+        header.collect { |h| row[h] }
+      end
+      { :header => header, :data => only_data }
+    end
+
+    # mysql text and blobs fields are handled the same way internally
+    # this is not true for other databases so we must check if the field is
+    # actually text and manually convert it back to a string
+    def incorrect_blobs(db, table)
+      return [] if (db.url =~ /mysql:\/\//).nil?
+
+      columns = []
+      db.schema(table).each do |data|
+        column, cdata = data
+        columns << column if cdata[:db_type] =~ /text/
+      end
+      columns
+    end
+
+    def blobs_to_string(row, columns)
+      return row if columns.size == 0
+      columns.each do |c|
+        row[c] = row[c].to_s if row[c].kind_of?(Sequel::SQL::Blob)
+      end
+      row
+    end
+
+    def calculate_chunksize(old_chunksize)
+      c = Tapsoob::Chunksize.new(old_chunksize)
+
+      begin
+        c.start_time = Time.now
+        c.time_in_db = yield c
+      rescue Errno::EPIPE
+        c.retries += 1
+        raise if c.retries > 2
+
+        # we got disconnected, the chunksize could be too large
+        # reset the chunksize based on the number of retries
+        c.reset_chunksize
+        retry
+      end
+
+      c.end_time = Time.now
+      c.calc_new_chunksize
+    end
+
+    def export_schema(dump_path, table, schema_data)
+      File.open(File.join(dump_path, "schemas", "#{table}.rb"), 'w') do |file|
+        file.write(schema_data)
+      end
+    end
+
+    def export_indexes(dump_path, table, index_data)
+      data = [index_data]
+      if File.exists?(File.join(dump_path, "indexes", "#{table}.json"))
+        previous_data = JSON.parse(File.read(File.join(dump_path, "indexes", "#{table}.json")))
+        data = data + previous_data
+      end
+
+      File.open(File.join(dump_path, "indexes", "#{table}.json"), 'w') do |file|
+        file.write(JSON.generate(data))
+      end
+    end
+
+    def export_rows(dump_path, table, row_data)
+      data = row_data
+      if File.exists?(File.join(dump_path, "data", "#{table}.json"))
+        previous_data = JSON.parse(File.read(File.join(dump_path, "data", "#{table}.json")))
+        data[:data] = previous_data["data"] + row_data[:data]
+      end
+
+      File.open(File.join(dump_path, "data", "#{table}.json"), 'w') do |file|
+        file.write(JSON.generate(data))
+      end
+    end
+
+    def load_schema(dump_path, database_url, table)
+      schema = File.join(dump_path, "schemas", "#{table}.rb")
+      schema_bin(:load, database_url, schema.to_s)
+    end
+
+    def load_indexes(database_url, index)
+      Tapsoob::Schema.load_indexes(database_url, index)
+    end
+
+    def schema_bin(command, *args)
+      require 'tapsoob/cli'
+      subcommand = "schema"
+      script = Tapsoob::CLI::Schema.new
+      script.invoke(command, args.map { |a| "#{a}" })
+    end
+
+    def primary_key(db, table)
+      db.schema(table).select { |c| c[1][:primary_key] }.map { |c| c[0] }
+    end
+
+    def single_integer_primary_key(db, table)
+      table = table.to_sym unless table.kind_of?(Sequel::SQL::Identifier)
+      keys = db.schema(table).select { |c| c[1][:primary_key] and c[1][:type] == :integer }
+      not keys.nil? and keys.size == 1
+    end
+
+    def order_by(db, table)
+      pkey = primary_key(db, table)
+      if pkey
+        pkey.kind_of?(Array) ? pkey : [pkey.to_sym]
+      else
+        table = table.to_sym unless table.kind_of?(Sequel::SQL::Identifier)
+        db[table].columns
+      end
+    end
+  end
+end

data/lib/tapsoob/version.rb

@@ -0,0 +1,4 @@
+# -*- encoding : utf-8 -*-
+module Tapsoob
+  VERSION = "0.2.7".freeze
+end

data/lib/tapsoob.rb

@@ -0,0 +1,6 @@
+# -*- encoding : utf-8 -*-
+require 'tapsoob/operation'
+
+module Tapsoob
+  require 'tapsoob/railtie' if defined?(Rails)
+end

data/lib/tasks/tapsoob.rake

@@ -0,0 +1,95 @@
+namespace :tapsoob do
+  desc "Pulls a database to your filesystem"
+  task :pull => :environment do
+    # Default options
+    opts={:default_chunksize => 1000, :debug => false, :resume_filename => nil, :disable_compression => false, :indexes_first => false}
+
+    # Get the dump_path
+    dump_path = File.expand_path(Rails.root.join("db", Time.now.strftime("%Y%m%d%I%M%S%p"))).to_s
+
+    # Create paths
+    FileUtils.mkpath "#{dump_path}/schemas"
+    FileUtils.mkpath "#{dump_path}/data"
+    FileUtils.mkpath "#{dump_path}/indexes"
+
+    # Run operation
+    Tapsoob::Operation.factory(:pull, database_uri, dump_path, opts).run
+
+    # Invoke cleanup task
+    Rake::Task["tapsoob:clean"].reenable
+    Rake::Task["tapsoob:clean"].invoke
+  end
+
+  desc "Push a compatible dump on your filesystem to a database"
+  task :push, [:timestamp] => :environment do |t, args|
+    # Default options
+    opts={:default_chunksize => 1000, :debug => false, :resume_filename => nil, :disable_compression => false, :indexes_first => false}
+
+    # Get the dumps
+    dumps = Dir[Rails.root.join("db", "*/")].select { |e| e =~ /([0-9]{14})([A-Z]{2})/ }.sort
+
+    # In case a timestamp argument try to use it instead of using the last dump
+    dump_path = dumps.last
+    unless args[:timestamp].nil?
+      timestamps = dumps.map { |dump| File.basename(dump) }
+
+      # Check that the dump_path exists
+      raise Exception.new "Invalid or non existent timestamp: '#{args[:timestamp]}'" unless timestamps.include?(args[:timestamp])
+
+      # Select dump_path
+      dump_path = Rails.root.join("db", args[:timestamp])
+    end
+
+    # Run operation
+    Tapsoob::Operation.factory(:push, database_uri, dump_path, opts).run
+  end
+
+  desc "Cleanup old dumps"
+  task :clean, [:keep] => :environment do |t, args|
+    # Number of dumps to keep
+    keep = ((args[:keep] =~ /\A[0-9]+\z/).nil? ? 5 : args[:keep].to_i)
+
+    # Get all the dump folders
+    dumps = Dir[Rails.root.join("db", "*/")].select { |e| e =~ /([0-9]{14})([A-Z]{2})/ }.sort
+
+    # Delete old dumps only if there more than we want to keep
+    if dumps.count > keep
+      old_dumps = dumps - dumps.reverse[0..(keep - 1)]
+      old_dumps.each do |dir|
+        if Dir.exists?(dir)
+          puts "Deleting old dump directory ('#{dir}')"
+          FileUtils.remove_entry_secure(dir)
+        end
+      end
+    end
+  end
+
+  private
+    def database_uri
+      uri               = ""
+      connection_config = YAML::load(ERB.new(Rails.root.join("config", "database.yml").read).result)[Rails.env]
+
+      case connection_config['adapter']
+      when "mysql", "mysql2"
+        uri = "#{connection_config['adapter']}://#{connection_config['host']}/#{connection_config['database']}?user=#{connection_config['username']}&password=#{connection_config['password']}"
+      when "postgresql", "postgres", "pg"
+        uri = "://#{connection_config['host']}/#{connection_config['database']}?user=#{connection_config['username']}&password=#{connection_config['password']}"
+        uri = ((RUBY_PLATFORM =~ /java/).nil? ? "postgres" : "postgresql") + uri
+      when "oracle_enhanced"
+        if (RUBY_PLATFORM =~ /java/).nil?
+          uri = "oracle://#{connection_config['host']}/#{connection_config['database']}?user=#{connection_config['username']}&password=#{connection_config['password']}"
+        else
+          uri = "oracle:thin:#{connection_config['username']}/#{connection_config['password']}@#{connection_config['host']}:1521:#{connection_config['database']}"
+        end
+      when "sqlite3", "sqlite"
+        uri = "sqlite://#{connection_config['database']}"
+      else
+        raise Exception, "Unsupported database adapter."
+        #uri = "#{connection_config['adapter']}://#{connection_config['host']}/#{connection_config['database']}?user=#{connection_config['username']}&password=#{connection_config['password']}"
+      end
+
+      uri = "jdbc:#{uri}" unless (RUBY_PLATFORM =~ /java/).nil?
+
+      uri
+    end
+end

data/spec/lib/tapsoob/chunksize_spec.rb

@@ -0,0 +1,92 @@
+require 'spec_helper'
+require 'tapsoob/chunksize'
+
+describe Tapsoob::Chunksize do
+  subject(:tapsoob) { Tapsoob::Chunksize.new(1) }
+  let(:chunksize) { double('chunksize') }
+  chunksize = Tapsoob::Chunksize.new(chunksize)
+  describe '#new' do
+    it 'works' do
+      result = Tapsoob::Chunksize.new(chunksize)
+      expect(result).not_to be_nil
+    end
+
+    describe '#initialize' do
+      it { should respond_to :chunksize }
+      it { should respond_to :idle_secs }
+      it { should respond_to :retries }
+    end
+  end
+
+  describe '#to_i' do
+
+    it { expect(tapsoob.to_i).to eq(1) }
+    it { expect(tapsoob.to_i).to be_a(Integer) }
+    it 'works' do
+      chunksize = Tapsoob::Chunksize.new(chunksize)
+      result = chunksize.to_i
+      expect(result).not_to be_nil
+    end
+
+    context 'converts to type integer' do
+      it { expect(tapsoob.to_i).to eq(1) }
+      it { expect(tapsoob.to_i).to be_an(Integer) }
+    end
+  end
+
+  describe '#reset_chunksize' do
+
+    context 'retries <= 1' do
+      it { expect(tapsoob.retries).to eq(0) }
+      it { expect(tapsoob.reset_chunksize).to eq(10) }
+    end
+
+    it 'works' do
+      chunksize = Tapsoob::Chunksize.new(chunksize)
+      result = chunksize.reset_chunksize
+      expect(result).not_to be_nil
+    end
+  end
+
+
+  describe '#diff' do
+    it 'works' do
+      chunksize = Tapsoob::Chunksize.new(chunksize)
+      chunksize.start_time = 1
+      chunksize.end_time = 10
+      chunksize.time_in_db = 2
+      chunksize.idle_secs = 3
+      result = chunksize.diff
+      expect(result).not_to be_nil
+    end
+  end
+
+  describe '#time_in_db=' do
+    it 'works' do
+      chunksize = Tapsoob::Chunksize.new(chunksize)
+      result = chunksize.time_in_db = (1)
+      expect(result).not_to be_nil
+    end
+  end
+
+  describe '#time_delta' do
+    it 'works' do
+      chunksize = double('chunksize')
+      chunksize = Tapsoob::Chunksize.new(chunksize)
+      result = chunksize.time_delta
+      expect(result).not_to be_nil
+    end
+  end
+
+  describe '#calc_new_chunksize' do
+    it 'works' do
+      chunksize = Tapsoob::Chunksize.new(1)
+      chunksize.start_time = 1
+      chunksize.end_time = 10
+      chunksize.time_in_db = 2
+      chunksize.idle_secs = 3
+      result = chunksize.calc_new_chunksize
+      expect(result).not_to be_nil
+    end
+  end
+end

data/spec/lib/tapsoob/version_spec.rb

@@ -0,0 +1,7 @@
+require 'spec_helper'
+
+describe Tapsoob::Chunksize do
+  it 'has a version number' do
+    expect(Tapsoob::Chunksize).not_to be nil
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,91 @@
+require 'simplecov'
+SimpleCov.start
+
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# The generated `.rspec` file contains
+# `--require spec_helper` which will cause this
+# file to always be loaded, without
+# a need to explicitly require it in any files.
+#
+# Given that it is always loaded, you are encouraged to keep this file as
+# light-weight as possible. Requiring heavyweight dependencies from this file
+# will add to the boot time of your test suite on EVERY test run, even for an
+# individual file that may not need all of that loaded. Instead, consider making
+# a separate helper file that requires the additional dependencies and performs
+# the additional setup,
+# and require it from the spec files that actually need it.
+#
+# The `.rspec` file also contains a few flags that are not defaults but that
+# users commonly want.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+RSpec.configure do |config|
+  # rspec-expectations config goes here. You can use an alternate
+  # assertion/expectation library such as wrong or the stdlib/minitest
+  # assertions if you prefer.
+  config.expect_with :rspec do |expectations|
+    # This option will default to `true` in RSpec 4. It makes the `description`
+    # and `failure_message` of custom matchers include text for helper methods
+    # defined using `chain`, e.g.:
+    # be_bigger_than(2).and_smaller_than(4).description
+    #   # => "be bigger than 2 and smaller than 4"
+    # ...rather than:
+    #   # => "be bigger than 2"
+    expectations.include_chain_clauses_in_custom_matcher_descriptions = true
+  end
+
+  # rspec-mocks config goes here. You can use an alternate test double
+  # library (such as bogus or mocha) by changing the `mock_with` option here.
+  config.mock_with :rspec do |mocks|
+    # Prevents you from mocking or stubbing a method that does not exist on
+    # a real object. This is generally recommended, and will default to
+    # `true` in RSpec 4.
+    mocks.verify_partial_doubles = true
+  end
+
+  # The settings below are suggested to provide a good initial experience
+  # with RSpec, but feel free to customize to your heart's content.
+  #   # These two settings work together to allow you to limit a spec run
+  #   # to individual examples or groups you care about by tagging them with
+  #   # `:focus` metadata. When nothing is tagged with `:focus`, all examples
+  #   # get run.
+  #   config.filter_run :focus
+  #   config.run_all_when_everything_filtered = true
+  #
+
+  #   config.disable_monkey_patching!
+  #
+  #   # This setting enables warnings. It's recommended, but in some cases may
+  #   # be too noisy due to issues in dependencies.
+  #   config.warnings = true
+  #
+  #   # Many RSpec users commonly either run the entire suite or an individual
+  #   # file, and it's useful to allow more verbose output when running an
+  #   # individual spec file.
+  #   if config.files_to_run.one?
+  #     # Use the documentation formatter for detailed output,
+  #     # unless a formatter has already been configured
+  #     # (e.g. via a command-line flag).
+  #     config.default_formatter = 'doc'
+  #   end
+  #
+  #   # Print the 10 slowest examples and example groups at the
+  #   # end of the spec run, to help surface which specs are running
+  #   # particularly slow.
+  #   config.profile_examples = 10
+  #
+  #   # Run specs in random order to surface order dependencies. If you find an
+  #   # order dependency and want to debug it,
+  #   # you can fix the order by providing
+  #   # the seed, which is printed after each run.
+  #   #     --seed 1234
+  #   config.order = :random
+  #
+  #   # Seed global randomization in this process using the `--seed` CLI option.
+  #   # Setting this allows you to use `--seed` to deterministically reproduce
+  #   # test failures
+  #   # related to randomization by passing the same `--seed` value
+  #   # as the one that triggered the failure.
+  #   Kernel.srand config.seed
+end

data/tapsoob.gemspec

@@ -0,0 +1,37 @@
+# -*- encoding: utf-8 -*-
+require "./lib/tapsoob/version" unless defined? Tapsoob::VERSION
+
+Gem::Specification.new do |s|
+  # Metadata
+  s.name        = "tapsoob"
+  s.version     = Tapsoob::VERSION.dup
+  s.authors     = ["Félix Bellanger", "Michael Chrisco"]
+  s.email       = "felix.bellanger@faveod.com"
+  s.homepage    = "https://github.com/Keeguon/tapsoob"
+  s.summary     = "Simple tool to import/export databases."
+  s.description = "Simple tool to import/export databases inspired by taps but OOB, meaning databases are imported/exported from the filesystem."
+  s.license     = "MIT"
+
+  # Manifest
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.bindir        = 'bin'
+  s.require_paths = ["lib"]
+
+  # Dependencies
+  s.add_dependency "sequel", "~> 5.0.0"
+  s.add_dependency "thor", "~> 0.20.0"
+
+  if (RUBY_PLATFORM =~ /java/).nil?
+    s.add_development_dependency "mysql2",  "~> 0.4.10"
+    s.add_development_dependency "pg",      "~> 0.21.0"
+    s.add_development_dependency "sqlite3", "~> 1.3.11"
+  else
+    s.platform = 'java'
+
+    s.add_dependency "jdbc-mysql",    "~> 5.1.44"
+    s.add_dependency "jdbc-postgres", "~> 42.1.4"
+    s.add_dependency "jdbc-sqlite3",  "~> 3.20.1"
+  end
+end