tapsoob 0.2.7-java

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 5e733ad5dfac7d36dda9f7d5447a437454d0224d6743041545e3a75f5b65b7d4
+  data.tar.gz: f073d871bf3e188c97ea50411c58da41b45709913a72582443f3ffc0c06a34ec
+SHA512:
+  metadata.gz: df1e0f59bfe3cce43db3be4f716b922e1f11e56e9fbd6bc2a669340478d19eec1936a6d26eab6c67adaa582e030a0a3010d0ed1cd295e767c4b1124f5e6f3a81
+  data.tar.gz: b6f09e52587bc528b3d9bc8b8292e3f63927b51cb6c9fb58b3128b7e266cfbb6215349a9bc237bdbb6b1ea34e142bcfbaa896d09199512fab1f8f198d46f099e

data/.gitignore

@@ -0,0 +1,10 @@
+# Ignore dev stuffs
+config/
+coverage/
+db/
+dist/
+lib/java/
+*.dat
+*.gem
+*.lock
+.ruby-version

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--require spec_helper

data/Gemfile

@@ -0,0 +1,44 @@
+source "http://rubygems.org"
+
+# monkey patching to support dual booting
+module Bundler::SharedHelpers
+  def default_lockfile=(path)
+    @default_lockfile = path
+  end
+  def default_lockfile
+    @default_lockfile ||= Pathname.new("#{default_gemfile}.lock")
+  end
+end
+
+module ::Kernel
+  def jruby?
+    !(RUBY_PLATFORM =~ /java/).nil?
+  end
+end
+
+if jruby?
+  Bundler::SharedHelpers.default_lockfile = Pathname.new("#{Bundler::SharedHelpers.default_gemfile}_jruby.lock")
+
+  # Bundler::Dsl.evaluate already called with an incorrect lockfile ... fix it
+  class Bundler::Dsl
+    # A bit messy, this can be called multiple times by bundler, avoid blowing the stack
+    unless self.method_defined? :to_definition_unpatched
+      alias_method :to_definition_unpatched, :to_definition
+    end
+    def to_definition(bad_lockfile, unlock)
+      to_definition_unpatched(Bundler::SharedHelpers.default_lockfile, unlock)
+    end
+  end
+end
+
+# gemspec
+gemspec
+
+group :development do
+  gem 'warbler', '~> 2.0.4', platform: :jruby, require: false
+end
+
+group :test do
+  gem 'rspec', '~> 3.2.0'
+  gem 'simplecov', '~> 0.9.2'
+end

data/README.md

@@ -0,0 +1,73 @@
+# Tapsoob
+
+Tapsoob is a simple tool to export and import databases. It is inspired by <https://github.com/ricardochimal/taps> but instead of having to rely on a server and a client, databases are exported to the filesystem or imported from a previous export, hence the OOB (out-of-band). This was done in order to avoid exposing a whole database using the HyperText Protocol for security reasons.
+
+Although the code is not quite perfect yet and some improvement will probably be made in the future it's usable. However if you notice an issue don't hesitate to report it.
+
+
+## Database support
+
+Tapsoob currently rely on the Sequel ORM (<http://sequel.rubyforge.org/>) so we currently support every database that Sequel supports. If additionnal support is required in the future (NoSQL databases) I'll make my best to figure out a way to bring that in my ToDo list.
+
+### Oracle
+
+If you're using either Oracle or Oracle XE you will need some extra requirements. If you're using Ruby you'll need to have your ORACLE_HOME environnement variable set properly and the `ruby-oci8` gem installed. However if you're using jRuby you'll need to have the official Oracle JDBC driver (see here for more informations: <http://www.oracle.com/technetwork/articles/dsl/jruby-oracle11g-330825.html>) and it should be loaded prior to using Tapsoob otherwise you won't be able to connect the database.
+
+
+## Exporting your data
+
+    tapsoob pull [OPTIONS] <dump_path> <database_url>
+
+The `dump_path` is the location on the filesystem where you want your database exported and the `database_url` is an URL looking like this `postgres://user:password@localhost/blog`, you can find out more about how to connect to your RDBMS by refering yourself to this page: <http://sequel.rubyforge.org/rdoc/files/doc/opening_databases_rdoc.html> on the Sequel documentation.
+
+Regarding options a complete list of options can be displayed using the following command:
+
+    tapsoob pull -h
+
+
+## Importing your data
+
+    tapsoob push [OPTIONS] <dump_path> <database_url>
+
+As for exporting the `dump_path` is the path you previously exported a database you now wish to import and the `database_url` should conform to the same format described before.
+
+You can list all available options using the command:
+
+    tapsoob push -h
+
+
+## Integration with Rails
+
+If you're using Rails, there's also two Rake tasks provided:
+
+* `tapsoob:pull` which dumps the database into a new folder under the `db` folder
+* `tapsoob:push` which reads the last dump you made from `tapsoob:pull` from the `db` folder
+
+
+## Notes
+
+Your exports can be moved from one machine to another for backups or replication, you can also use Tapsoob to switch your RDBMS from one of the supported system to another.
+
+
+## ToDo
+
+* Add a compression layer
+* Tests (in progress)
+
+
+## Contributors
+
+* Félix Bellanger <felix.bellanger@gmail.com>
+* Michael Chrisco <michaelachrisco@gmail.com>
+
+
+## License
+
+The MIT License (MIT)
+Copyright © 2015 Félix Bellanger <felix.bellanger@gmail.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,4 @@
+unless (RUBY_PLATFORM =~ /java/).nil?
+  require 'warbler'
+  Warbler::Task.new
+end

data/bin/tapsoob

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+$:.unshift File.dirname(__FILE__) + '/../lib'
+require 'tapsoob/cli'
+
+Tapsoob::CLI::Root.start(ARGV)

data/lib/tapsoob/chunksize.rb

@@ -0,0 +1,53 @@
+# -*- encoding : utf-8 -*-
+require 'tapsoob/errors'
+
+class Tapsoob::Chunksize
+  attr_accessor :idle_secs, :time_in_db, :start_time, :end_time, :retries
+  attr_reader :chunksize
+
+  def initialize(chunksize)
+    @chunksize = chunksize
+    @idle_secs = 0.0
+    @retries = 0
+  end
+
+  def to_i
+    chunksize
+  end
+
+  def reset_chunksize
+    @chunksize = (retries <= 1) ? 10 : 1
+  end
+
+  def diff
+    end_time - start_time - time_in_db - idle_secs
+  end
+
+  def time_in_db=(t)
+    @time_in_db = t
+    @time_in_db = @time_in_db.to_f rescue 0.0
+  end
+
+  def time_delta
+    t1 = Time.now
+    yield if block_given?
+    t2 = Time.now
+    t2 - t1
+  end
+
+  def calc_new_chunksize
+    new_chunksize = if retries > 0
+      chunksize
+    elsif diff > 3.0
+      (chunksize / 3).ceil
+    elsif diff > 1.1
+      chunksize - 100
+    elsif diff < 0.8
+      chunksize * 2
+    else
+      chunksize + 100
+    end
+    new_chunksize = 1 if new_chunksize < 1
+    new_chunksize
+  end
+end

data/lib/tapsoob/cli.rb

@@ -0,0 +1,192 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'fileutils'
+require 'sequel'
+require 'tempfile'
+require 'thor'
+
+# tapsoob deps
+require 'tapsoob/config'
+require 'tapsoob/log'
+require 'tapsoob/operation'
+require 'tapsoob/schema'
+require 'tapsoob/version'
+
+Tapsoob::Config.tapsoob_database_url = ENV['TAPSOOB_DATABASE_URL'] || begin
+  # this is dirty but it solves a weird problem where the tempfile disappears mid-process
+  #require ((RUBY_PLATFORM =~ /java/).nil? ? 'sqlite3' : 'jdbc-sqlite3')
+  $__taps_database = Tempfile.new('tapsoob.db')
+  $__taps_database.open()
+  "sqlite://#{$__taps_database.path}"
+end
+
+module Tapsoob
+  module CLI
+    class Schema < Thor
+      desc "console DATABASE_URL", "Create an IRB REPL connected to a database"
+      def console(database_url)
+        $db = Sequel.connect(database_url)
+        require 'irb'
+        require 'irb/completion'
+        IRB.start
+      end
+
+      desc "dump DATABASE_URL", "Dump a database using a database URL"
+      def dump(database_url)
+        puts Tapsoob::Schema.dump(database_url)
+      end
+
+      desc "dump_table DATABASE_URL TABLE", "Dump a table from a database using a database URL"
+      def dump_table(database_url, table)
+        puts Tapsoob::Schema.dump_table(database_url, table)
+      end
+
+      desc "indexes DATABASE_URL", "Dump indexes from a database using a database URL"
+      def indexes(database_url)
+        puts Tapsoob::Schema.indexes(database_url)
+      end
+
+      desc "indexes_individual DATABASE_URL", "Dump indexes per table individually using a database URL"
+      def indexes_individual(database_url)
+        puts Tapsoob::Schema.indexes_individual(database_url)
+      end
+
+      desc "reset_db_sequences DATABASE_URL", "Reset database sequences using a database URL"
+      def reset_db_sequences(database_url)
+        Tapsoob::Schema.reset_db_sequences(database_url)
+      end
+
+      desc "load DATABASE_URL FILENAME", "Load a database schema from a file to a database using a database URL"
+      def load(database_url, filename)
+        schema = File.read(filename) rescue help
+        Tapsoob::Schema.load(database_url, schema)
+      end
+
+      desc "load_indexes DATABASE_URL FILENAME", "Load indexes from a file to a database using a database URL"
+      def load_indexes(database_url, filename)
+        indexes = File.read(filename) rescue help
+        Tapsoob::Schema.load_indexes(database_url, indexes)
+      end
+    end
+
+    class Root < Thor
+      desc "pull DUMP_PATH DATABASE_URL", "Pull a dump from a database to a folder"
+      option :"skip-schema", desc: "Don't transfer the schema just data", default: false, type: :boolean, aliases: "-s"
+      option :"indexes-first", desc: "Transfer indexes first before data", default: false, type: :boolean, aliases: "-i"
+      option :resume, desc: "Resume a Tapsoob Session from a stored file", type: :string, aliases: "-r"
+      option :chunksize, desc: "Initial chunksize", default: 1000, type: :numeric, aliases: "-c"
+      option :"disable-compression", desc: "Disable Compression", default: false, type: :boolean, aliases: "-g"
+      option :filter, desc: "Regex Filter for tables", type: :string, aliases: "-f"
+      option :tables, desc: "Shortcut to filter on a list of tables", type: :array, aliases: "-t"
+      option :"exclude-tables", desc: "Shortcut to exclude a list of tables", type: :array, aliases: "-e"
+      option :debug, desc: "Enable debug messages", default: false, type: :boolean, aliases: "-d"
+      def pull(dump_path, database_url)
+        opts = parse_opts(options)
+        Tapsoob.log.level = Logger::DEBUG if opts[:debug]
+        if opts[:resume_filename]
+          clientresumexfer(:pull, dump_path, database_url, opts)
+        else
+          clientxfer(:pull, dump_path, database_url, opts)
+        end
+      end
+
+      desc "push DUMP_PATH DATABASE_URL", "Push a previously tapsoob dump to a database"
+      option :"skip-schema", desc: "Don't transfer the schema just data", default: false, type: :boolean, aliases: "-s"
+      option :"indexes-first", desc: "Transfer indexes first before data", default: false, type: :boolean, aliases: "-i"
+      option :resume, desc: "Resume a Tapsoob Session from a stored file", type: :string, aliases: "-r"
+      option :chunksize, desc: "Initial chunksize", default: 1000, type: :numeric, aliases: "-c"
+      option :"disable-compression", desc: "Disable Compression", default: false, type: :boolean, aliases: "-g"
+      option :filter, desc: "Regex Filter for tables", type: :string, aliases: "-f"
+      option :tables, desc: "Shortcut to filter on a list of tables", type: :array, aliases: "-t"
+      option :"exclude-tables", desc: "Shortcut to exclude a list of tables", type: :array, aliases: "-e"
+      option :debug, desc: "Enable debug messages", default: false, type: :boolean, aliases: "-d"
+      def push(dump_path, database_url)
+        opts = parse_opts(options)
+        Tapsoob.log.level = Logger::DEBUG if opts[:debug]
+        if opts[:resume_filename]
+          clientresumexfer(:push, dump_path, database_url, opts)
+        else
+          clientxfer(:push, dump_path, database_url, opts)
+        end
+      end
+
+      desc "version", "Show tapsoob version"
+      def version
+        puts Tapsoob::VERSION.dup
+      end
+
+      desc "schema SUBCOMMAND ...ARGS", "Direct access to Tapsoob::Schema class methods"
+      subcommand "schema", Schema
+
+      private
+        def parse_opts(options)
+          # Default options
+          opts = {
+            skip_schema: options[:"skip-schema"],
+            indexes_first: options[:"indexes_first"],
+            disable_compression: options[:"disable-compression"],
+            debug: options[:debug]
+          }
+
+          # Resume
+          if options[:resume]
+            if File.exists?(options[:resume])
+              opts[:resume_file] = options[:resume]
+            else
+              raise "Unable to find resume file."
+            end
+          end
+
+          # Default chunksize
+          if options[:chunksize]
+            opts[:default_chunksize] = (options[:chunksize] < 10 ? 10 : options[:chunksize])
+          end
+
+          # Regex filter
+          opts[:table_filter] = options[:filter] if options[:filter]
+
+          # Table filter
+          if options[:tables]
+            r_tables = options[:tables].collect { |t| "^#{t}" }.join("|")
+            opts[:table_filter] = "#{r_tables}"
+          end
+
+          # Exclude tables
+          opts[:exclude_tables] = options[:"exclude-tables"] if options[:"exclude-tables"]
+
+          opts
+        end
+
+        def clientxfer(method, dump_path, database_url, opts)
+          Tapsoob::Config.verify_database_url(database_url)
+
+          FileUtils.mkpath "#{dump_path}/schemas"
+          FileUtils.mkpath "#{dump_path}/data"
+          FileUtils.mkpath "#{dump_path}/indexes"
+
+          require 'tapsoob/operation'
+
+          Tapsoob::Operation.factory(method, database_url, dump_path, opts).run
+        end
+
+        def clientresumexfer(method, dump_path, database_url, opts)
+          session = JSON.parse(File.read(opts.delete(:resume_filename)))
+          session.symbolize_recursively!
+
+          dump_path = dump_path || session.delete(:dump_path)
+
+          require 'taps/operation'
+
+          newsession = session.merge({
+            :default_chunksize => opts[:default_chunksize],
+            :disable_compression => opts[:disable_compression],
+            :resume => true
+          })
+
+          Tapsoob::Operation.factory(method, database_url, dump_path, newsession).run
+        end
+    end
+  end
+end
+

data/lib/tapsoob/config.rb

@@ -0,0 +1,33 @@
+# -*- encoding : utf-8 -*-
+require 'sequel'
+require 'tapsoob/version'
+
+Sequel.datetime_class = DateTime
+
+module Tapsoob
+  def self.exiting=(val)
+    @@exiting = val
+  end
+
+  def exiting?
+    (@@exiting ||= false) == true
+  end
+
+  class Config
+    class << self
+      attr_accessor :tapsoob_database_url
+      attr_accessor :login, :password, :database_url, :dump_path
+      attr_accessor :chunksize
+
+      def verify_database_url(db_url=nil)
+        db_url ||= self.database_url
+        db = Sequel.connect(db_url)
+        db.tables
+        db.disconnect
+      rescue Object => e
+        puts "Failed to connect to database:\n  #{e.class} -> #{e}"
+        exit 1
+      end
+    end
+  end
+end