cassback 0.1.8 → 0.1.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d1b84e227b8ac885872dca30ef87343ef497d6b1
-  data.tar.gz: b2f0923cc561e07b14c3c316ae86aa091bdd829f
+  metadata.gz: 493265549ae69ac85882959fd0751368fcf1e940
+  data.tar.gz: 1f7e6550297e0451c35c62a52b8002883d343e81
 SHA512:
-  metadata.gz: f899bacdc79ae008b9e471cb2d5835dbf31140a68adde2166efed9461f0cbf9ae006f84a43436d68dbaa2781e601e902f012d2e8d6956cd8048bd14bf1722ad0
-  data.tar.gz: 8719a219fb94a1ce7337b388c460add9b3e7ead073b3e15d1d8eec3780a65004ff403f994687c1e48a5c1cdd0c12c86eb351379999df63006aff224c20a23c94
+  metadata.gz: b03e231ff64cf1b6ef5d72c6bf28aef338ecab1f90fac64d75c2a6e811d914edb9df4d83fb959bf2383d22ee0e5b118dc2a02c76e60626a12dfa8b2323b4eb36
+  data.tar.gz: cf5ba0d94bdda1a1cbf490e6d54a8413fa28e000e61b3d4fabe55528ba690390c8e55376a4a82f2d51ef12216052516834ed8e412c169533f6422adc45914583

data/README.md

@@ -2,6 +2,7 @@
 
 Welcome to your Cassback!
 This is a project that aims backup Cassandra SSTables and load them into HDFS for further usage.
+It intended to be used as a command line tool - for now it can be triggered only by commands.
 
 ## Installation
 
@@ -40,6 +41,31 @@ A simple command that you can use for starting a backup is :
 
     cassback -S -C path_to_some_config_file.yml
 
+## Structure of the backup
+
+A backup, that has been pushed from multiple Cassandra nodes to one HDFS location, has the following structure :
+
+**a) Backup files**
+
+    <hadoop_root_folder>/<cluster_name>/<node_name>/<keyspace>/<table>/<snapshot_path>/<backup_file>
+
+
+**b) Backup complete flags** (stored at cluster level in the metadata folder) :
+
+    <hadoop_root_folder>/<cass_snap_metadata>/<cluster_name>/BACKUP_COMPLETE_<date>
+
+
+**c) Metadata files** (stored at node level in metadata folder) :
+
+    <hadoop_root_folder>/cass_snap_metadata/<cluster_name>/<node_name>/cass_snap_<date>
+
+## Incremental or full backups ?
+
+**Backups are done incrementally, but published as full backups** - the tool checks locally which files will have to be uploaded to HDFS and checks
+if those files are already present in HDFS (because Cassandra files are immutable we don't have to risk to have two files
+with same name but different content). However when the metadata file is published it points to all the files that
+compose the backup so it basically publishes it as being a full backup.
+
 ## Configuration
 
 The application has some default configuration defined.
@@ -50,6 +76,8 @@ You can overwrite the default configuration using two meanings :
 2. Using individual configuration properties passed as parameters on the command line.
 The command line parameters have precedence over the configuration file.
 
+An example of configuration file is provided under conf/local.yml.
+
 ## Orchestration
 
 The tool is designed to do snapshots at **node level** (and not at **cluster level**) - basically it has to be installed
@@ -94,13 +122,12 @@ The command for triggering a cleanup is :
     cassback -A -C conf/path_to_some_config_file.yml
 
 # Unit tests
+
 Unit tests can be executed locally by running the following command :
 
     rake test
 
 ## Contributing
 
-For now this is an internal Criteo project, but were aiming for making it open source and publishing to GitHub.
-
-Issue reports and merge requests are welcome on Criteo's GitLab at : https://gitlab.criteois.com/ruby-gems/cassback
+Bug reports and pull requests are welcome on GitHub at : https://github.com/criteo/cassback
 

data/bin/cassback

@@ -49,6 +49,7 @@ command_line_config = {
 options = {
   'cassandra' => {
     'config' => '/etc/cassandra/conf/cassandra.yaml',
+    'disk_threshold' => 75
   },
   'hadoop'    => {
     'hostname'      => 'localhost',
@@ -165,7 +166,7 @@ begin
   retry_interval: options['hadoop']['retryInterval'], read_timeout: options['hadoop']['readTimeout'])
 
   #  Create the Cassandra object
-  cassandra = Cassandra.new(options['cassandra']['config'], logger)
+  cassandra = Cassandra.new(options['cassandra']['config'], options['cassandra']['disk_threshold'], logger)
 
   #  Create the backup object
   bck = BackupTool.new(cassandra, hadoop, logger)

data/lib/cassandra.rb

@@ -5,7 +5,7 @@ require 'yaml'
 class Cassandra
   attr_reader :data_path, :cluster_name, :node_name
 
-  def initialize(config_file, logger)
+  def initialize(config_file, disk_threshold, logger)
     @logger = logger
 
     read_config_file(config_file)
@@ -15,6 +15,8 @@ class Cassandra
     @logger.info("Cassandra cluster name = #{@cluster_name}")
     @logger.info("Cassandra node name = #{@node_name}")
     @logger.info("Cassandra data path = #{@data_path}")
+
+    @disk_threshold = disk_threshold
   end
 
   def read_config_file(config_file)
@@ -45,6 +47,13 @@ class Cassandra
     # First delete the snapshot if it exists.
     nodetool_clearsnapshot(name)
 
+    # Check if we have enough disk space left
+    m = /\ ([0-9]+)%\ /.match(IO.popen("df #{@data_path}").readlines[1])
+    used = Integer(m[1])
+    if used > @disk_threshold
+      raise("Not enough disk space remaining for snapshot (#{used}% used > #{@disk_threshold}% required)")
+    end
+
     # Then trigger it.
     @logger.debug("Starting a new Cassandra snapshot #{name}")
     begin

data/lib/cassback/version.rb

@@ -1,3 +1,3 @@
 module Cassback
-  VERSION = '0.1.8'.freeze
+  VERSION = '0.1.9'.freeze
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cassback
 version: !ruby/object:Gem::Version
-  version: 0.1.8
+  version: 0.1.9
 platform: ruby
 authors:
 - Vincent Van Hollebeke
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-04-28 00:00:00.000000000 Z
+date: 2016-05-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -130,7 +130,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
-- ".rubocop.yml_disabled"
+- ".rubocop.yml"
 - Gemfile
 - LICENSE
 - README.md
@@ -144,15 +144,6 @@ files:
 - lib/cassandra.rb
 - lib/cassback/version.rb
 - lib/hadoop.rb
-- scripts/deploy.sh
-- scripts/manualbackups/ansible.cfg
-- scripts/manualbackups/inventory.txt
-- scripts/manualbackups/play_book.sh
-- scripts/manualbackups/playbooks/backups.yml
-- scripts/manualbackups/roles/planb/files/backup.sh
-- scripts/manualbackups/roles/planb/files/httpfs.sh
-- scripts/manualbackups/roles/planb/files/krb5.conf
-- scripts/manualbackups/roles/planb/tasks/main.yml
 - scripts/pre-push
 - test/cassandra_stub.rb
 - test/hadoop_stub.rb
@@ -177,7 +168,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.2
+rubygems_version: 2.6.3
 signing_key: 
 specification_version: 4
 summary: Cassandra backup to HDFS.

data/scripts/deploy.sh

@@ -1,3 +0,0 @@
-#!/bin/bash
-
-while [ 1 = 1 ]; do inotifywait .;scp -r . cstars01e01-par.storage.criteo.preprod:cassback2;scp -r . cstars01e02-par.storage.criteo.preprod:cassback2;done

data/scripts/manualbackups/ansible.cfg

@@ -1,12 +0,0 @@
-[defaults]
-host_key_checking=false
-record_host_keys=false
-remote_tmp=/tmp/.ansible/tmp
-forks=128
-roles_path=roles
-library=library
-
-[ssh_connection]
-control_path=%(directory)s/%%h-%%r
-pipelining=True
-scp_if_ssh=True

data/scripts/manualbackups/inventory.txt

@@ -1,18 +0,0 @@
-[cstars02-par]
-cstars02e01-par ansible_ssh_host="cstars02e01-par.storage.criteo.prod"
-cstars02e02-par ansible_ssh_host="cstars02e02-par.storage.criteo.prod"
-cstars02e03-par ansible_ssh_host="cstars02e03-par.storage.criteo.prod"
-cstars02e04-par ansible_ssh_host="cstars02e04-par.storage.criteo.prod"
-cstars02e05-par ansible_ssh_host="cstars02e05-par.storage.criteo.prod"
-cstars02e06-par ansible_ssh_host="cstars02e06-par.storage.criteo.prod"
-cstars02e07-par ansible_ssh_host="cstars02e07-par.storage.criteo.prod"
-cstars02e08-par ansible_ssh_host="cstars02e08-par.storage.criteo.prod"
-cstars02e09-par ansible_ssh_host="cstars02e09-par.storage.criteo.prod"
-cstars02e10-par ansible_ssh_host="cstars02e10-par.storage.criteo.prod"
-cstars02e11-par ansible_ssh_host="cstars02e11-par.storage.criteo.prod"
-cstars02e12-par ansible_ssh_host="cstars02e12-par.storage.criteo.prod"
-cstars02e13-par ansible_ssh_host="cstars02e13-par.storage.criteo.prod"
-cstars02e14-par ansible_ssh_host="cstars02e14-par.storage.criteo.prod"
-cstars02e15-par ansible_ssh_host="cstars02e15-par.storage.criteo.prod"
-cstars02e16-par ansible_ssh_host="cstars02e16-par.storage.criteo.prod"
-cstars02e17-par ansible_ssh_host="cstars02e17-par.storage.criteo.prod"

data/scripts/manualbackups/play_book.sh

@@ -1,13 +0,0 @@
-#!/bin/bash
-
-PLAYBOOK=$1
-
-if [ "$PLAYBOOK" = "" ]; then
-	echo "Usage: $0 <playbook> [ansible options]"
-	exit 65
-fi
-
-shift
-ansible-playbook --inventory-file=inventory.txt playbooks/$PLAYBOOK.yml --extra-vars $*
-
-exit $?

data/scripts/manualbackups/playbooks/backups.yml

@@ -1,6 +0,0 @@
----
-
-- gather_facts: no
-  hosts: cstars02-par
-  roles:
-    - role: planb

data/scripts/manualbackups/roles/planb/files/backup.sh

@@ -1,27 +0,0 @@
-#!/bin/bash
-
-kinit v.vanhollebeke@CRITEOIS.LAN -k -t ~/keytab
-
-date=`date +%Y_%m_%d`
-
-nodetool clearsnapshot
-
-snapdir=$(nodetool snapshot| grep directory| awk '{print $NF}')
-echo "Snapshot is $snapdir"
-
-for dir in $(find /var/opt/cassandra/data -type d |grep snapshots/$snapdir); do
-    kok=$(klist -l|grep v.vanhollebeke@CRITEOIS.LAN|grep -v Expired|wc -l)
-    if [ $kok == 0 ]; then
-        echo "Must renew Kerberos ticket"
-        kinit v.vanhollebeke@CRITEOIS.LAN -k -t ~/keytab
-    else
-        echo "Kerberos ticket OK"
-    fi
-    keyspace=`echo $dir|awk -F\/ '{print $6}'`
-    table=`echo $dir|awk -F\/ '{print $7}'`
-    echo "Saving $keyspace $table"
-    ./httpfs.sh /var/opt/cassandra/data/$keyspace/$table/snapshots/$snapdir tmp/cassandrabackups/prod/cstars02/$date/$HOSTNAME/$table
-
-done
-
-echo "FINISHED !!!!"

data/scripts/manualbackups/roles/planb/files/httpfs.sh

@@ -1,27 +0,0 @@
-#!/bin/sh
-
-BASE='http://0.httpfs.hpc.criteo.prod:14000/webhdfs/v1'
-#BASE='http://httpfs.pa4.hpc.criteo.prod:14000'
-
-IN=$1
-OUT=$2
-
-echo "Creating destination directory: $OUT"
-curl --negotiate -u : "$BASE/$OUT?op=MKDIRS&permission=0777" -X PUT -s > /dev/null
-
-for p in $(find $IN -type f)
-do
-    f=$(basename $p)
-    echo "$IN/$f"
-
-    # Create file
-    dest=$(curl --negotiate -u : "$BASE/$OUT/$f?op=CREATE&overwrite=true&permission=0777" -i -X PUT -s | grep Location | tail -n1 | cut -d\  -f2 | tr -d '\r\n')
-    [ $? != 0 ] && echo "ERROR"
-
-    echo "DEST IS ${dest}"
-
-    # Upload file
-    curl --negotiate -u : "$dest" -i -X PUT -T "$IN/$f" -H 'Content-Type: application/octet-stream' > /dev/null
-    [ $? != 0 ] && echo "ERROR"
-
-done

data/scripts/manualbackups/roles/planb/files/krb5.conf

@@ -1,26 +0,0 @@
-[libdefaults]
-  dns_lookup_realm = true
-  dns_lookup_kdc = true
-  ticket_lifetime = 24h
-  renew_lifetime = 7d
-  forwardable = true
-  default_realm = CRITEOIS.LAN
-  udp_preference_limit = 1
-  realm_try_domains = 1
-  permitted_enctypes = aes128-cts-hmac-sha1-96 des3-cbc-sha1 arcfour-hmac
-  default_tkt_enctypes = aes128-cts-hmac-sha1-96 des3-cbc-sha1 arcfour-hmac
-[domain_realm]
-  .hpc.criteo.preprod = HPC.CRITEO.PREPROD
-  .hpc.criteo.prod = AMS.HPC.CRITEO.PROD
-  .pa4.hpc.criteo.prod = PA4.HPC.CRITEO.PROD
-  .as.hpc.criteo.prod = AS.HPC.CRITEO.PROD
-  .na.hpc.criteo.prod = NA.HPC.CRITEO.PROD
-  .cn.hpc.criteo.prod = CN.HPC.CRITEO.PROD
-[capaths]
-  CRITEOIS.LAN = {
-    AMS.HPC.CRITEO.PROD = .
-    PA4.HPC.CRITEO.PROD = AMS.HPC.CRITEO.PROD
-    AS.HPC.CRITEO.PROD = AMS.HPC.CRITEO.PROD
-    NA.HPC.CRITEO.PROD = AMS.HPC.CRITEO.PROD
-    CN.HPC.CRITEO.PROD = AMS.HPC.CRITEO.PROD
-  }

data/scripts/manualbackups/roles/planb/tasks/main.yml

@@ -1,34 +0,0 @@
----
-
-- name: Copy krb5.conf into /etc
-  copy: src=krb5.conf dest=/etc/krb5.conf
-  sudo: yes
-  tags: keytab
-
-- name: Copy my keytab
-  copy: src=keytab dest=~/keytab
-  tags: keytab
-
-- name: Check if keytab works
-  command: kinit $USER@CRITEOIS.LAN -k -t ~/keytab
-  tags: keytab
-
-- name: Copy httpfs.sh script
-  copy: src=httpfs.sh dest=~/httpfs.sh mode=750
-  tags: backup
-
-- name: Copy backup.sh script
-  copy: src=backup.sh dest=~/backup.sh mode=750
-  tags: backup
-
-- name: Start Backup
-  shell: ./backup.sh >logfile 2>&1 chdir=~
-  tags: backup
-
-- name: Clear snapshots
-  shell: sudo nodetool clearsnapshot
-  tags: clear
-
-- name: Verify if snapshots are REALLY deleted
-  shell: "[ $(find /var/opt/cassandra -type d |grep snap|wc -l) == 0 ]"
-  tags: verify