cassandra_backup 0.0.6 → 0.1.0

data/Gemfile

@@ -1,3 +1,5 @@
 source :rubygems
+gemspec
 
-gemspec
+gem 'rake'
+gem 'mocha'

data/README.rdoc

@@ -6,16 +6,25 @@ CassandraBackup is a command line tool for dumping and restoring a Cassandra key
 
 Run cassandra_dump, specifying the keyspace:
 
-  cassandra_dump depot_production > backup.json
+  cassandra_dump depot_production > backup.cql
 
 == Restoring a Keyspace
 
 Run cassandra_import, specifying the keyspace:
 
-  cassandra_import depot_production < backup.json
+  cassandra_import depot_production < backup.cql
 
 == Options
 
-  --servers:: A comma delimited list of servers. Defaults to 127.0.0.1:9160. (--servers 127.0.0.14:9169,127.0.1.19:9100)
-  --version:: Set cassandra version. Defaults to 1.0. (--version 0.6)
-  --columns:: The column families to dump. Defaults to all. (--columns Widgets --columns People)
+  --servers: A list of servers. Defaults to 127.0.0.1:9160. (--servers 127.0.0.14:9169,127.0.1.19:9100)
+  --version: Set cassandra version. Defaults to latest. (--version 1.0)
+  --tables: The tables to dump. Defaults to all. (--tables widgets,people)
+  --where: A condition to apply to all queries (--where="state='WA'")
+
+== Notes on options
+
+Options such as --servers and --tables support arrays in multiple formats. All of the following are the same:
+
+  --tables Widgets,People
+  --tables "Widgets People"
+  --tables Widgets --tables People

data/Rakefile

@@ -0,0 +1,12 @@
+require 'bundler/setup'
+require 'rake'
+require 'rake/testtask'
+
+task default: :test
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'lib'
+  t.libs << 'test'
+  t.pattern = 'test/cassandra_backup/**/*_test.rb'
+  t.verbose = true
+end

data/bin/cassandra_dump

@@ -1,4 +1,4 @@
 #!/usr/bin/env ruby
 
 require File.expand_path('../../lib/cassandra_backup',  __FILE__)
-CassandraBackup::Dumper.run!
+CassandraBackup::Dumper.run_command CassandraBackup::Command.new(ARGV)

data/bin/cassandra_import

@@ -1,4 +1,4 @@
 #!/usr/bin/env ruby
 
 require File.expand_path('../../lib/cassandra_backup',  __FILE__)
-CassandraBackup::Importer.run!
+CassandraBackup::Importer.run_command CassandraBackup::Command.new(ARGV)

data/cassandra_backup.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |s|
   s.name = 'cassandra_backup'
-  s.version = '0.0.6'
+  s.version = '0.1.0'
   s.description = 'Cassandra Backup'
   s.summary = 'Cassandra Backup'
   s.authors = ['Infogroup']
@@ -16,7 +16,5 @@ Gem::Specification.new do |s|
   s.files       = `git ls-files`.split("\n")
   s.test_files  = `git ls-files -- {test}/*`.split("\n")
 
-  s.add_runtime_dependency('mcmire-cassandra', ">= 0.12.3")
-  s.add_runtime_dependency('yajl-ruby')
-  s.add_development_dependency('bundler')
+  s.add_dependency 'cassandra-cql'
 end

data/lib/cassandra_backup.rb

@@ -1,4 +1,6 @@
-require 'yajl'
-
+require 'cassandra-cql'
+require 'cassandra_backup/command'
+require 'cassandra_backup/config'
+require 'cassandra_backup/cqlsh'
 require 'cassandra_backup/dumper'
 require 'cassandra_backup/importer'

data/lib/cassandra_backup/command.rb

@@ -3,9 +3,9 @@ require 'optparse'
 module CassandraBackup
   class Command
 
-    attr_accessor :options
+    attr_reader :options
     def initialize(args)
-      self.options = {}
+      @options = {}
 
       if args.first
         options[:keyspace] = args.first
@@ -15,48 +15,33 @@ module CassandraBackup
 
       OptionParser.new do |opts|
         opts.banner = "Usage: keyspace [options]"
-        opts.on('-s', '--servers SERVERS', 'Set server list. Default is 127.0.0.1:9160.') do |v|
-          options[:servers] = v.split(/\s|,/)
+        opts.on('-s', '--servers servers', 'Set server list. Default is 127.0.0.1:9160.') do |v|
+          store_array_option :servers, v
         end
-        opts.on('-v', '--version VERSION', 'Set cassandra version. Default is 1.0.') do |v|
+        opts.on('-c', '--tables tables', 'Set tables. Defaults to all') do |v|
+          store_array_option :tables, v
+        end
+        opts.on('-v', '--version VERSION', 'Set cassandra version. Defaults to latest.') do |v|
           options[:version] = v
         end
-        opts.on('-c', '--columns Columns', 'Set column families. Defaults to all') do |v|
-          (options[:columns] ||= []).concat v.split(/\s|,/)
+        opts.on('-w', '--where=\'where_conditions\'') do |v|
+          options[:where] = v
         end
         opts.on('-h', 'Show this help message.') do
           $stdout.puts opts; exit
         end
-        opts.parse!(args)
-      end
-
-      require required_version
-    end
-
-    def keyspace
-      options[:keyspace]
-    end
 
-    def servers
-      if options[:servers]
-        options[:servers]
-      else
-        ['127.0.0.1:9160']
+        opts.parse!(args)
       end
     end
 
-    def columns
-      if options[:columns]
-        options[:columns]
-      end
+    def config
+      CassandraBackup::Config.new(options)
     end
 
-    def required_version
-      if options[:version]
-        "cassandra/#{options[:version]}"
-      else
-        "cassandra/1.0"
+    private
+      def store_array_option(name, value)
+        (options[name] ||= []).concat value.split(/\s|,/)
       end
-    end
   end
 end

data/lib/cassandra_backup/config.rb

@@ -0,0 +1,53 @@
+module CassandraBackup
+  class Config
+    attr_reader :options
+    def initialize(options = {})
+      @options = options
+
+      require_version!
+    end
+
+    def servers
+      if options[:servers]
+        options[:servers]
+      else
+        ['127.0.0.1:9160']
+      end
+    end
+
+    def input
+      options[:input] || STDIN
+    end
+
+    def output
+      options[:output] || STDOUT
+    end
+
+    def require_version!
+      if options[:version]
+        require "cassandra-cql/#{options[:version]}"
+      end
+    end
+
+    def keyspace
+      options[:keyspace]
+    end
+
+    def tables
+      options[:tables]
+    end    
+
+    def where
+      options[:where]
+    end
+
+    def connection
+      if keyspace
+        options = {keyspace: keyspace}
+      else
+        options = {}
+      end
+      @connection ||= CassandraCQL::Database.new(servers, options)
+    end
+  end
+end

data/lib/cassandra_backup/cqlsh.rb

@@ -0,0 +1,27 @@
+module CassandraBackup
+  class Cqlsh
+
+    attr_reader :config
+
+    def initialize(config)
+      @config = config
+    end
+
+    def exec(command)
+      `echo "#{command};" | #{cqlsh} -k #{keyspace} #{server}`.sub(/^(.*)$/, '').strip
+    end
+
+    private
+      def cqlsh
+        ENV['CQLSH'] || 'cqlsh'
+      end
+
+      def keyspace
+        config.keyspace
+      end
+
+      def server
+        config.servers.first.gsub(/:.*/, '')
+      end
+  end
+end

data/lib/cassandra_backup/dumper.rb

@@ -1,26 +1,79 @@
-require 'cassandra_backup/backup_process'
-
 module CassandraBackup
-  class Dumper < BackupProcess
+  class Dumper
+    def self.run_command(command)
+      new(command.config).run
+    end
+
+    attr_reader :config
+    attr_reader :cqlsh
+    attr_reader :primary_keys
+    def initialize(config)
+      @config = config
+      @cqlsh = CassandraBackup::Cqlsh.new(config)
+      @primary_keys = {}
+    end
+
     def run
-      column_family_names.each do |column_family|
-        connection.each(column_family) do |key, attributes|
-          output_io.puts encoder.encode([column_family, key, attributes])
-        end
+      table_names.each { |table_name| dump_table_schema(table_name) }
+      table_names.each { |table_name| dump_table_rows(table_name) }
+    end
+
+    def dump_table_schema(table_name)
+      table_description = cqlsh.exec("DESCRIBE COLUMNFAMILY #{table_name}")
+      capture_primary_key(table_name, table_description)
+      config.output.puts(table_description)
+      config.output.puts
+    end
+
+    def dump_table_rows(table_name)
+      primary_key = primary_key_for(table_name)
+      each_row(table_name) do |row|
+        next if row.keys.size == 1 && row.keys.first == primary_key
+
+        colummns = CassandraCQL::Statement.quote(row.keys)
+        values = CassandraCQL::Statement.quote(row.values.map { |v| CassandraCQL::Statement.escape(v) })
+        
+        config.output.puts "INSERT INTO #{table_name} (#{colummns}) VALUES (#{values});"
       end
     end
 
+    def table_names
+      if config.tables
+        config.tables
+      else
+        cqlsh.exec('DESCRIBE COLUMNFAMILIES').split.sort
+      end
+    end
+
+    def execute(cql)
+      config.connection.execute(cql)
+    end
+
     private
-      def column_family_names
-        if command.columns
-          command.columns
-        else
-          connection.column_families.keys - ['schema_migrations']
+      def primary_key_for(table_name)
+        primary_keys[table_name] || 'id'
+      end
+    
+      def capture_primary_key(table_name, table_description)
+        if table_description =~ /^\s*(\w+).*PRIMARY KEY/
+          primary_keys[table_name] = $1
         end
       end
 
-      def encoder
-        @encoder ||= Yajl::Encoder.new
+      def each_row(table_name)
+        primary_key = primary_key_for(table_name)
+        cassandra_result = execute "SELECT * FROM #{table_name}"
+
+        while cassandra_result.rows > 0
+          last_fetch = nil
+          cassandra_result.fetch_hash do |hash|
+            yield(hash)
+            last_fetch = hash
+          end
+
+          where_cql = "'#{primary_key}' > '#{last_fetch[primary_key]}'"
+          cassandra_result = execute "SELECT * FROM #{table_name} WHERE #{where_cql}"
+        end
       end
   end
-end
+end

data/lib/cassandra_backup/importer.rb

@@ -1,17 +1,37 @@
-require 'cassandra_backup/backup_process'
-
 module CassandraBackup
-  class Importer < BackupProcess
+  class Importer
+    def self.run_command(command)
+      new(command.config).run
+    end
+
+    attr_reader :config
+    def initialize(config)
+      @config = config
+    end
+
     def run
-      parser.parse(input_io) do |column_family, key, attributes|
-        p "inserting"
-        connection.insert column_family, key, attributes
+      each_full_statement(config.input) do |cql|
+        execute cql
       end
     end
 
-    private
-      def parser
-        @parser ||= Yajl::Parser.new
+    def execute(cql)
+      config.connection.execute(cql)
+    end
+
+    def each_full_statement(io)
+      current_cql = ''
+
+      io.each_line do |line|
+        next unless line =~ /[^[:space:]]/
+
+        current_cql << line.rstrip
+
+        if current_cql =~ /;$/
+          yield(current_cql)
+          current_cql = ''
+        end
       end
+    end
   end
 end

data/test/cassandra_backup/command_test.rb

@@ -0,0 +1,33 @@
+require 'helper'
+require 'shellwords'
+
+class CassandraBackup::CommandTest < MiniTest::Spec
+  def test_keyspace
+    assert_equal 'zombies', command('zombies').options[:keyspace]
+  end
+
+  def test_single_value_options
+    [:where, :version].each do |option|
+      assert_nil command('zombies').options[option]
+      assert_equal 'foo', command("zombies --#{option} foo").options[option]
+    end
+  end
+
+  def test_array_options
+    [:servers, :tables].each do |option|
+      assert_nil command('zombies').options[option]
+      assert_equal ['foo', 'bar'], command("zombies --#{option} \"foo bar\"").options[option]
+      assert_equal ['foo', 'bar'], command("zombies --#{option} foo,bar").options[option]
+      assert_equal ['foo', 'bar'], command("zombies --#{option} foo --#{option} bar").options[option]
+    end
+  end
+
+  def test_config
+    assert_kind_of CassandraBackup::Config, command('zombies').config
+  end
+
+  private
+    def command(line)
+      CassandraBackup::Command.new Shellwords.shellsplit(line)
+    end
+end

data/test/cassandra_backup/config_test.rb

@@ -0,0 +1,29 @@
+require 'helper'
+
+class CassandraBackup::ConfigTest < MiniTest::Spec
+  def test_servers
+    assert_equal ['127.0.0.1:9160'], config.servers
+    assert_equal ['foo'], config(servers: ['foo']).servers
+  end
+
+  def test_input
+    assert_equal STDIN, config.input
+    refute_equal STDIN, config(input: StringIO.new).input
+  end
+
+  def test_output
+    assert_equal STDOUT, config.output
+    refute_equal STDOUT, config(output: StringIO.new).output
+  end
+
+  def test_require_version
+    # expects(:require).with('cassandra-cql/1.0').once
+    # CassandraBackup::Config.new(version: '1.0')
+  end
+
+  private
+    def config(options = {})
+      CassandraBackup::Config.new(options)
+    end
+
+end

data/test/cassandra_backup/dumper_test.rb

@@ -0,0 +1,43 @@
+require 'helper'
+
+class CassandraBackup::DumperTest < MiniTest::Spec
+  def test_table_names
+    assert_equal ['presidents', 'senators'], create_dumper.table_names
+    assert_equal ['presidents'], create_dumper(tables: ['presidents']).table_names
+  end
+
+  def test_dump_table_schema
+    output = StringIO.new
+    dumper = create_dumper output: output
+
+    dumper.dump_table_schema('senators')
+
+    output.rewind
+    lines = output.readlines
+    assert_match /CREATE TABLE senators/, lines.first
+    assert_equal "\n", lines.last
+    assert_equal({"senators" => "id"}, dumper.primary_keys)
+  end
+
+  def test_dump_table_rows
+    output = StringIO.new
+    dumper = create_dumper output: output
+    dumper.execute "TRUNCATE senators"
+    dumper.execute "INSERT INTO senators ('id', 'name', 'state') VALUES ('abc', 'K''la', 'WA')"
+    dumper.execute "INSERT INTO senators ('id', 'name', 'state') VALUES ('xyz', 'Bob', 'NY')"
+
+    dumper.dump_table_rows('senators')
+
+    output.rewind
+    lines = output.readlines
+
+    assert_equal 2, lines.size
+    assert lines.include?("INSERT INTO senators ('id','name','state') VALUES ('abc','K''la','WA');\n")
+    assert lines.include?("INSERT INTO senators ('id','name','state') VALUES ('xyz','Bob','NY');\n")
+  end  
+
+  private
+    def create_dumper(options = {})
+      CassandraBackup::Dumper.new CassandraBackup::Config.new(options.merge(keyspace: 'cassandra_dump_test'))
+    end
+end

data/test/cassandra_backup/importer_test.rb

@@ -0,0 +1,40 @@
+require 'helper'
+
+class CassandraBackup::ImporterTest < MiniTest::Spec
+  def test_run
+    input = StringIO.new("INSERT INTO presidents (id, name) VALUES ('1st', 'George Washington');")
+    importer = create_importer input: input
+    importer.execute "TRUNCATE presidents"
+
+    importer.run
+
+    assert_equal 1, importer.execute("select * from presidents").rows
+  end
+
+  def test_each_full_statement
+    input = StringIO.new(
+      "INSERT INTO presidents (id, name) VALUES\n" +
+      " ('1st', 'George Washington');\n" +
+      " \n" +
+      "UPDATE presidents SET name='Zombie' WHERE id='1st';\n" +
+      "\n"
+    )
+    stmts = []
+
+    create_importer.each_full_statement(input) do |cql|
+      stmts << cql
+    end
+
+    expected = [
+      "INSERT INTO presidents (id, name) VALUES ('1st', 'George Washington');",
+      "UPDATE presidents SET name='Zombie' WHERE id='1st';"
+    ]
+    assert_equal expected, stmts
+  end
+
+  private
+
+    def create_importer(options = {})
+      CassandraBackup::Importer.new CassandraBackup::Config.new(options.merge(keyspace: 'cassandra_dump_test'))
+    end
+end

data/test/helper.rb

@@ -0,0 +1,6 @@
+require 'bundler/setup'
+require 'minitest/autorun'
+Bundler.require
+
+require 'cassandra_backup'
+require 'setup/connection'

data/test/setup/connection.rb

@@ -0,0 +1,11 @@
+CassandraBackup::Config.new.tap do |config|
+  begin
+    config.connection.execute("DROP KEYSPACE cassandra_dump_test")
+  rescue Exception => e
+  end
+  config.connection.execute("CREATE KEYSPACE cassandra_dump_test WITH strategy_class='org.apache.cassandra.locator.SimpleStrategy' AND strategy_options:replication_factor=1")
+  config.connection.execute("USE cassandra_dump_test")
+  config.connection.execute("CREATE TABLE presidents (id varchar PRIMARY KEY)")
+  config.connection.execute("CREATE TABLE senators (id varchar PRIMARY KEY, state varchar)")
+  config.connection.execute("CREATE INDEX senators_state_index ON senators(state)")  
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cassandra_backup
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.1.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,26 +9,10 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-06-19 00:00:00.000000000 Z
+date: 2012-11-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: mcmire-cassandra
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: 0.12.3
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: 0.12.3
-- !ruby/object:Gem::Dependency
-  name: yajl-ruby
+  name: cassandra-cql
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
@@ -43,22 +27,6 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
 description: Cassandra Backup
 email: gems@gotime.com
 executables:
@@ -71,15 +39,22 @@ files:
 - .gitignore
 - Gemfile
 - README.rdoc
+- Rakefile
 - bin/cassandra_dump
 - bin/cassandra_import
 - cassandra_backup.gemspec
 - lib/cassandra_backup.rb
-- lib/cassandra_backup/backup_process.rb
 - lib/cassandra_backup/command.rb
+- lib/cassandra_backup/config.rb
+- lib/cassandra_backup/cqlsh.rb
 - lib/cassandra_backup/dumper.rb
 - lib/cassandra_backup/importer.rb
-- test/test_helper.rb
+- test/cassandra_backup/command_test.rb
+- test/cassandra_backup/config_test.rb
+- test/cassandra_backup/dumper_test.rb
+- test/cassandra_backup/importer_test.rb
+- test/helper.rb
+- test/setup/connection.rb
 homepage: http://github.com/data-axle/cassandra_backup
 licenses: []
 post_install_message: 

data/lib/cassandra_backup/backup_process.rb

@@ -1,30 +0,0 @@
-require 'cassandra_backup/command'
-
-module CassandraBackup
-  class BackupProcess
-    def self.run!
-      new(CassandraBackup::Command.new(ARGV)).run
-    end
-
-    attr_reader :command
-    def initialize(command)
-      @command = command
-    end
-
-    def run
-      raise "inheriting class must implement"
-    end
-
-    def connection
-      @connection ||= Cassandra.new(command.keyspace, command.servers)
-    end
-
-    def input_io
-      $stdin
-    end
-
-    def output_io
-      $stdout
-    end
-  end
-end

data/test/test_helper.rb

@@ -1,2 +0,0 @@
-# require 'bundler/setup'
-# Bundler.require