cassback 0.2.2 → 0.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4e7a97a9c8109931fcc60bc344d4bbd21e77b52d
-  data.tar.gz: ae8b20c65edb06200c39cff9d6807c30bce641b9
+  metadata.gz: 76766f06f7d636939af96dc78a6af157c652a303
+  data.tar.gz: 4c850cec7d6ca903b5750df42c84575fc7e01137
 SHA512:
-  metadata.gz: c1f13b8a68a8f8c1ab87f6128824acf796c41ae2ed3c1adc6844c264cd5a3a7c74bfd9aad7f301e529dd080b6328dc09f642c18db57df15ad5c22f51d7f440a8
-  data.tar.gz: 3e125579865dd92ee08f1bff96aaf8991ccc3066e51d5ca3ffc121531af1b0f2b90c9b3bfa812d303fcef1e77154e7e8922debc4b08d984185996ef206f1f511
+  metadata.gz: 093b50c2986c8baf54169530db3d7f5c76da98428cc95e16fcceb03467783b0ac2891112ea4b8a5182460c1c4614b1d3fc22090ceed8d010f728177868714384
+  data.tar.gz: 009f9557521432e5d580163e5b6ca82073f1fbecb70652b0166f46104ac2867f7f393475d35910e49ca83b0b0058e6e7ad8ac32ed2d1c05da176abc64734276f

data/bin/cassback

@@ -63,6 +63,8 @@ options = {
   },
   'restore'   => {
     'destination' => 'cassandra',
+    'keyspace' => 'ALL',
+    'table' => 'ALL',
   },
 
   'cleanup'   => {
@@ -113,9 +115,15 @@ parser = OptionParser.new do |opts|
   opts.on('-d', '--date DATE', 'snapshot date, like YYYY_MM_DD') do |v|
     options['date'] = v
   end
-  opts.on('-t', '--destination DIR', 'local destination path for restore (default is cassandra)') do |v|
+  opts.on('-o', '--destination DIR', 'local destination/output path for restore (default is cassandra)') do |v|
     command_line_config['restore']['destination'] = v
   end
+  opts.on('-k', '--keyspace KEYSPACE', 'keyspace that will be restored (by default all are restored)') do |v|
+    command_line_config['restore']['keyspace'] = v
+  end
+  opts.on('-t', '--table TABLE', 'table that will be restored (by default all are restored)') do |v|
+    command_line_config['restore']['table'] = v
+  end
 
   opts.separator ''
   opts.separator 'Hadoop (WebHDFS):'
@@ -194,7 +202,8 @@ begin
   # Restore a snapshot
   elsif action == 'restore'
     raise('No date given') unless options.include? 'date'
-    bck.restore_snapshot(options['node'], options['date'], options['restore']['destination'])
+    bck.restore_snapshot(options['node'], options['date'], options['restore']['destination'],
+                         keyspace: options['restore']['keyspace'], table: options['restore']['table'])
 
   # List snapshots
   elsif action == 'list'

data/lib/backuptool.rb

@@ -243,7 +243,7 @@ class BackupTool
   #   - +node+ -> node where the snapshot comes from
   #   - +date+ -> snapshot date
   #   - +destination+ -> local directory where to restore
-  def restore_snapshot(node, date, destination)
+  def restore_snapshot(node, date, destination, keyspace: 'ALL', table: 'ALL')
     # Search the snapshot matching node and date
     snapshots = search_snapshots(node: node, date: date)
 
@@ -254,10 +254,20 @@ class BackupTool
     else
       snapshot = snapshots[0]
       @logger.info("Restoring snapshot #{snapshot}")
-      @logger.info("#{snapshot.metadata.length} files to restore")
+      @logger.info("Snapshot has #{snapshot.metadata.length} files")
 
-      # For each file in metadata
-      snapshot.metadata.each do |file|
+      files_to_be_restored = snapshot.metadata.select { |item|
+        filename = File.basename(item)
+        matches_keyspace = keyspace == 'ALL' || (filename.include? keyspace)
+        matches_table =  table == 'ALL' || (filename.include? table)
+        matches_keyspace && matches_table
+      }
+
+      @logger.info("Found #{files_to_be_restored.length} to be restored that match
+              keyspace #{keyspace} and table #{table}")
+
+      # For each file in the list
+      files_to_be_restored.each do |file|
         @logger.info("Restoring file #{file}")
         local = destination + '/' + file
         remote = @hadoop.base_dir + '/' + snapshot.cluster + '/' + snapshot.node + '/' + file

data/lib/cassback/version.rb

@@ -1,3 +1,3 @@
 module Cassback
-  VERSION = '0.2.2'.freeze
+  VERSION = '0.2.3'.freeze
 end

data/test/cassandra_stub.rb

@@ -12,14 +12,17 @@ class CassandraStub
     @data_path = 'test/cassandra' + '/' + cluster_name + '/' + node_name + '/'
     FileUtils.mkdir_p(@data_path)
 
-    # create some fake sstables
     @metadata = Set.new
+    end
+  end
+
+  public
+  def add_fake_files(file_indexes, keyspace, table)
     file_indexes.each do |index|
-      file_name = "SSTable-#{index}-Data.db"
+      file_name = "#{keyspace}-#{table}#{index}-Data.db"
       file_path = @data_path + '/' + file_name
       File.open(file_path, 'w') { |file| file.write('This is a test file that simulates an SSTable') }
       @metadata.add(file_name)
-    end
   end
 
   def new_snapshot

data/test/test_backuptool.rb

@@ -6,9 +6,12 @@ require_relative '../lib/backuptool'
 require_relative 'hadoop_stub'
 require_relative 'cassandra_stub'
 
-class TestSimpleNumber < Test::Unit::TestCase
+class TestBackupTool < Test::Unit::TestCase
   def test_new_snapshot
     hadoop = HadoopStub.new('test/hadoop')
+
+    clean_test_data(hadoop)
+
     create_new_snapshot(hadoop, 'node1', '2016_04_22', [1, 2])
 
     remote_files = hadoop.list_files('test/hadoop')
@@ -26,12 +29,14 @@ class TestSimpleNumber < Test::Unit::TestCase
     assert(metadata_content.include?('SSTable-2-Data.db'))
 
     # cleanup
-    hadoop.delete('test/hadoop')
-    hadoop.delete('test/cassandra')
+    clean_test_data(hadoop)
   end
 
   def test_two_snapshots
     hadoop = HadoopStub.new('test/hadoop')
+
+    clean_test_data(hadoop)
+
     create_new_snapshot(hadoop, 'node1', '2016_04_22', [1, 2])
     create_new_snapshot(hadoop, 'node1', '2016_04_23', [2, 3, 4])
 
@@ -60,12 +65,14 @@ class TestSimpleNumber < Test::Unit::TestCase
     assert(metadata_content.include?('SSTable-4-Data.db'))
 
     # cleanup
-    hadoop.delete('test/hadoop')
-    hadoop.delete('test/cassandra')
+    clean_test_data(hadoop)
   end
 
   def test_restore
     hadoop = HadoopStub.new('test/hadoop')
+
+    clean_test_data(hadoop)
+
     backup_tool = create_new_snapshot(hadoop, 'node1', '2016_04_22', [1, 2])
 
     # restore a newly created snapshot
@@ -78,13 +85,36 @@ class TestSimpleNumber < Test::Unit::TestCase
     assert_equal('test/restore/SSTable-2-Data.db', restored_files[1])
 
     # cleanup
-    hadoop.delete('test/hadoop')
-    hadoop.delete('test/restore')
-    hadoop.delete('test/cassandra')
+    clean_test_data(hadoop)
+  end
+
+  def test_restore_with_filtering
+    hadoop = HadoopStub.new('test/hadoop')
+    clean_test_data(hadoop)
+
+    keyspace = 'profile'
+    table = 'hash_status'
+
+    backup_tool = create_new_snapshot_2(hadoop, 'node1', '2016_04_22', [1, 2], keyspace, table)
+
+    # restore a newly created snapshot
+    backup_tool.restore_snapshot('node1', '2016_04_22', 'test/restore', keyspace: keyspace, table: table)
+
+    restored_files = hadoop.list_files('test/restore')
+    # two files were restored
+    assert_equal(2, restored_files.size)
+    assert_equal('test/restore/profile-hash_status1-Data.db', restored_files[0])
+    assert_equal('test/restore/profile-hash_status2-Data.db', restored_files[1])
+
+    # cleanup
+    clean_test_data(hadoop)
   end
 
   def test_delete
     hadoop = HadoopStub.new('test/hadoop')
+
+    clean_test_data(hadoop)
+
     backup_tool = create_new_snapshot(hadoop, 'node1', '2016_04_22', [1, 2])
 
     # delete a newly created snapshot
@@ -93,11 +123,14 @@ class TestSimpleNumber < Test::Unit::TestCase
     remote_files = hadoop.list_files('test/hadoop')
     assert_equal(0, remote_files.size)
 
-    hadoop.delete('test/cassandra')
+    clean_test_data(hadoop)
   end
 
   def test_backup_flag
     hadoop = HadoopStub.new('test/hadoop')
+
+    clean_test_data(hadoop)
+
     backup_tool = create_new_snapshot(hadoop, 'node1', '2016_04_22', [1, 2])
 
     backup_tool.create_backup_flag('2016_04_22')
@@ -108,12 +141,14 @@ class TestSimpleNumber < Test::Unit::TestCase
     assert_equal('test/hadoop/cass_snap_metadata/cluster1/BACKUP_COMPLETED_2016_04_22', remote_files[0])
 
     # cleanup
-    hadoop.delete('test/hadoop')
-    hadoop.delete('test/cassandra')
+    clean_test_data(hadoop)
   end
 
   def test_get_backup_flag
     hadoop = HadoopStub.new('test/hadoop')
+
+    clean_test_data(hadoop)
+
     backup_tool = create_new_snapshot(hadoop, 'node1', '2016_04_22', [1, 2])
 
     backup_tool.create_backup_flag('2016_04_22')
@@ -126,12 +161,14 @@ class TestSimpleNumber < Test::Unit::TestCase
     assert_equal('BACKUP_COMPLETED_2016_04_22', flags[0].file)
 
     # cleanup
-    hadoop.delete('test/hadoop')
-    hadoop.delete('test/cassandra')
+    clean_test_data(hadoop)
   end
 
   def test_cleanup
     hadoop = HadoopStub.new('test/hadoop')
+
+    clean_test_data(hadoop)
+
     retention_days = 30
 
     date_31_days_back = (Date.today - 31).strftime('%Y_%m_%d')
@@ -164,17 +201,43 @@ class TestSimpleNumber < Test::Unit::TestCase
     assert_equal("BACKUP_COMPLETED_#{date_30_days_back}", backup_flags[0].file)
 
     # cleanup
-    hadoop.delete('test/hadoop')
-    hadoop.delete('test/cassandra')
+    clean_test_data(hadoop)
   end
 
   def create_new_snapshot(hadoop, node, date, file_indexes)
     logger = Logger.new(STDOUT)
     cassandra = CassandraStub.new('cluster1', node, date, file_indexes)
+
+    # Add some fake files
+    cassandra.add_fake_files(file_indexes, 'SSTable', '')
+
     backup_tool = BackupTool.new(cassandra, hadoop, logger)
+    backup_tool.new_snapshot
+
+    backup_tool
+  end
+
+  def create_new_snapshot_2(hadoop, node, date, file_indexes, keyspace, table)
+    logger = Logger.new(STDOUT)
+
+    cassandra = CassandraStub.new('cluster1', node, date, file_indexes)
+
+    # Add some files with keyspace and table
+    cassandra.add_fake_files(file_indexes, keyspace, table)
+    # Add some files not matching keyspace and table
+    cassandra.add_fake_files(file_indexes, 'test', 'test')
+
 
+    backup_tool = BackupTool.new(cassandra, hadoop, logger)
     backup_tool.new_snapshot
 
     backup_tool
   end
+
+  def clean_test_data(hadoop)
+    # cleanup
+    hadoop.delete('test/hadoop')
+    hadoop.delete('test/restore')
+    hadoop.delete('test/cassandra')
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cassback
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.2.3
 platform: ruby
 authors:
 - Vincent Van Hollebeke
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-09-21 00:00:00.000000000 Z
+date: 2016-10-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -168,7 +168,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.4.8
 signing_key: 
 specification_version: 4
 summary: Cassandra backup to HDFS.