cassandra_migrate 0.0.1

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in cassandra_migrate.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Noah Gibbs
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,84 @@
+# Cassandra Migrate
+
+This gem is designed to allow Cassandra migrations in very roughly the
+style of Rails migrations, but with a few extra features...  And
+without the Rails-style DSL for running the migrations.  CQL is
+basically fine.  But we *do* want Erb!
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'cassandra_migrate'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install cassandra_migrate
+
+You don't have to use cassandra_migrate as part of an app, but you'll
+need a recent (1.9.2+) Ruby with Rubygems.
+
+## Usage
+
+To run migrations from the current directory if you have them:
+
+> cassandra_migrate latest
+
+You can also migrate to a filename or a date string, either of which
+will use the date as the important part -- you'll migrate forward or
+back until the specified migration is the last one performed.  You can
+also roll back a single migration:
+
+> cassandra_migrate rollback
+
+See other options:
+
+> cassandra_migrate --help
+
+## Writing Migrations
+
+You'll need a directory for Cassandra migrations.  Every migration
+should have a filename of the form:
+
+20131023000000_create_keyspace_argus_up.cql.erb
+
+The first fourteen digits are the date in YYYYMMDD format, followed
+by six digits of your choice -- base them on time, or just make sure
+they don't conflict.  You can't have two migrations with exactly the
+same fourteen-digit time code!
+
+Then you can use a freeform description, like "create_keyspace_argus"
+above.  Then an action, like "up" or "down" (later, scripts as well),
+and one or more extensions to tell Cassandra Migrate how to use the
+file.  Most commonly you'll want .cql or .cql.erb as the extension.
+Such files will be run through Cassandra, optionally after Erubis
+processing.
+
+Here's an example that uses Erb, in this case to set the replication
+factor of the keyspace via an environment variable:
+
+~~~
+# 20131024001100_create_keyspace_cryptic_up.cql.erb
+CREATE KEYSPACE "cryptic" WITH REPLICATION =
+  { 'class' : 'SimpleStrategy', 'replication_factor' : <%= ENV['CASS_REPLICATION'] || 1 %> };
+~~~
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+## Future Directions
+
+A cassandra_migrate_rails gem could add generators for simple
+Cassandra migrations from Rails.
+
+We could add Ruby, bash or other scripts/executables to be run before
+and after migrations, or as the migration itself.

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/cassandra_migrate

@@ -0,0 +1,42 @@
+#!/usr/bin/env ruby
+# Copyright (C) 2013 OL2, inc.  See LICENSE.txt for details.
+
+require "trollop"
+require "cassandra_migrate"
+
+OPTS = Trollop::options do
+  banner <<-BANNER
+Migrate Cassandra to latest revision, based on your directory of migrations.
+
+Usage: cassandra_migrate [options] [target_migration_date_or_file|latest|rollback]
+  BANNER
+  opt :migration_dir, "Directory of Cassandra migrations",            :type => String,     :default => "cassandra_migrations"
+  opt :host,          "Cassandra host",                               :type => String,     :default => "localhost"
+  opt :port,          "Cassandra port",                               :type => Integer,    :default => 9042
+  opt :dry_run,       "Dry run, print Cassandra commands"
+end
+
+Trollop::die "Must specify no more than one argument: date, filename, latest or rollback!" if ARGV.size > 1
+
+arg = ARGV.size > 0 ? ARGV[0] : "latest"
+
+migrate = CassandraMigrate.new
+
+# Pass arguments along to the migrate object
+[:host, :port, :migration_dir].each do |arg|
+  migrate.send "#{arg}=", OPTS[arg]
+end
+
+options = {}
+options[:dry_run] = OPTS[:dry_run]
+
+case arg
+when "latest"
+  migrate.to_latest options
+when "rollback"
+  migrate.rollback options
+when proc { arg =~ /^(\d{14})/ }
+  migrate.to_target $1, options
+else
+  raise "Didn't recognize argument as a 14-digit date string, a migration pathname, 'latest' or 'rollback': #{arg.inspect}!"
+end

data/cassandra_migrate.gemspec

@@ -0,0 +1,26 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'cassandra_migrate/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "cassandra_migrate"
+  spec.version       = CassandraMigrate::VERSION
+  spec.authors       = ["Noah Gibbs"]
+  spec.email         = ["noah.gibbs@onlive.com"]
+  spec.description   = %q{Migrations for Cassandra in CQL and Erb.}
+  spec.summary       = %q{Migrations for Cassandra in CQL and Erb.}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_runtime_dependency "trollop"
+  spec.add_runtime_dependency "cql-rb"
+  spec.add_runtime_dependency "erubis"
+end

data/lib/cassandra_migrate.rb

@@ -0,0 +1,220 @@
+# Copyright (C) 2013 OL2, inc.  See LICENSE.txt for details.
+
+require "cassandra_migrate/version"
+
+require "cql"
+require "erubis"
+
+require "digest/sha1"
+
+class CassandraMigrate
+  attr_accessor :host
+  attr_accessor :port
+  attr_accessor :migration_dir
+
+  private
+
+  def cql_client
+    return @cassandra_client if @cassandra_client
+
+    STDERR.puts "Connecting to Cassandra: #{host.inspect} / #{port.inspect}"
+    @cassandra_client = Cql::Client.connect(hosts: [host].flatten, port: port, consistency: :quorum)
+
+    @cassandra_client
+  end
+
+  def execute_cql(cql, options = {})
+    if options[:dry_run]
+      puts "Dry run, execute: #{cql}"
+      return
+    end
+
+    last_result = nil
+    # Can only execute single chunks at once
+    cql.split(";").map(&:strip).select {|s| s != ""}.each do |statement|
+      # Prep-then-execute so that a syntax error will be detectable as such
+      last_result = cql_client.execute statement
+      puts "Executing CQL: #{statement}"
+    end
+
+    last_result
+  end
+
+  def migrations_in_dir(refresh = false)
+    return @migrations_in_dir if @migrations_in_dir && !refresh
+
+    @migrations_in_dir = {}
+    Dir[File.join migration_dir, "*"].each do |file|
+      unless File.basename(file) =~ /^(\d{14})_/
+        puts "No match: #{file.inspect}"
+        next
+      end
+
+      unless /^(?<date_str>\d{14})_(?<desc>[^.]+)_(?<action>[^_.]+)(?<extensions>\..*)$/ =~ File.basename(file)
+        puts "No match with regexp: #{file.inspect}"
+        next
+      end
+
+      @migrations_in_dir[date_str] ||= {}
+      migration = @migrations_in_dir[date_str]
+      migration[:actions] ||= {}
+
+      if migration[:desc] && migration[:desc] != desc
+        raise "Only one migration name per date string!  #{desc.inspect} != #{migration[:desc].inspect}"
+      end
+      migration[:desc] = desc
+
+      migration[:actions][action.to_sym] = {
+        file: file
+      }
+    end
+
+    raise "No migrations in directory #{migration_dir.inspect}!  Did you mean to specify a migration directory?" if @migrations_in_dir.empty?
+
+    @migrations_in_dir
+  end
+
+  def ensure_schema_keyspace_exists(options = {})
+    ks = execute_cql "SELECT keyspace_name FROM system.schema_keyspaces WHERE keyspace_name = 'schema';"
+
+    if ks.empty?
+      raise "No schema keyspace in a dry run!" if options[:dry_run]
+
+      peers = execute_cql "SELECT peer FROM system.peers;"
+
+      @replication = [3, peers.to_a.size + 1].min
+      execute_cql <<-MIGRATION, options
+        CREATE KEYSPACE "schema" WITH REPLICATION =
+          { 'class' : 'SimpleStrategy', 'replication_factor' : #{@replication} };
+      MIGRATION
+    end
+
+    cf = execute_cql "SELECT columnfamily_name FROM system.schema_columnfamilies WHERE columnfamily_name = 'migrations' AND keyspace_name = 'schema';"
+    if cf.empty?
+      raise "No migration table in a dry run!" if options[:dry_run]
+
+      execute_cql <<-MIGRATION
+        CREATE TABLE "schema"."migrations" (
+          "date_string" varchar,
+          "up_filename" varchar,
+          "sha1" varchar,
+        PRIMARY KEY ("date_string", "up_filename"));
+      MIGRATION
+    end
+  end
+
+  def migrations_completed(refresh = false, options = {})
+    return @migrations_completed if @migrations_completed && !refresh
+
+    ensure_schema_keyspace_exists(options)
+
+    @migrations_completed = {}
+
+    migrations = execute_cql 'SELECT * FROM "schema"."migrations";'
+    migrations.each do |migration|
+      @migrations_completed[migration["date_string"]] = migration.to_hash
+    end
+
+    @migrations_completed
+  end
+
+  def sha1(path)
+    Digest::SHA1.hexdigest File.read path
+  end
+
+  def execute_migration_file(path, options)
+    ensure_schema_keyspace_exists(options)
+
+    STDERR.puts "Executing migration file: #{path.inspect}"
+
+    components = File.basename(path).split(".")
+    components.shift   # Take just the extensions
+
+    content = File.read path
+
+    while components.size > 1
+      ext = components.pop
+
+      if ext == "erb" || ext == "erubis"
+        eruby = Erubis::Eruby.new content
+        content = eruby.result :replication => @replication
+      else
+        raise "Unknown intermediate extension in path #{path.inspect}: #{ext.inspect}!"
+      end
+    end
+
+    final_type = components.first
+    if ["cql", "cqlsh"].include?(final_type)
+      execute_cql content, options
+    elsif ["erb", "erubis"].include?(final_type)
+      raise "Can't use erb as the final extension in path #{path.inspect}!"
+    else
+      raise "Unknown extension #{final_type.inspect} in path #{path.inspect}!"
+    end
+  end
+
+  public
+
+  def up(date_str, options = {})
+    raise "Can't apply migration #{date_str} that already happened!" if migrations_completed(false,options)[date_str]
+    raise "Can't apply migration #{date_str} that has no migration files!" unless migrations_in_dir[date_str]
+    raise "Can't apply migration #{date_str} with no up migration!" unless migrations_in_dir[date_str][:actions][:up]
+
+    up_filename = migrations_in_dir[date_str][:actions][:up][:file]
+    execute_migration_file up_filename, options
+    execute_cql "INSERT INTO \"schema\".\"migrations\" (date_string, up_filename, sha1) VALUES ('#{date_str}', '#{up_filename}', '#{sha1 up_filename}')", options
+  end
+
+  def down(date_str, options = {})
+    raise "Can't reverse migration #{date_str} that didn't happen!" unless migrations_completed(false,options)[date_str]
+    raise "Can't reverse migration #{date_str} that has no migration files!" unless migrations_in_dir[date_str]
+    raise "Can't reverse migration #{date_str} with no down migration!" unless migrations_in_dir[date_str][:actions][:down]
+
+    execute_migration_file migrations_in_dir[date_str][:actions][:down][:file], options
+    execute_cql "DELETE FROM \"schema\".\"migrations\" WHERE date_string = '#{date_str}';", options
+  end
+
+  def up_to(date_str, options = {})
+    uncompleted_dates = migrations_in_dir.keys - migrations_completed(false,options).keys
+
+    STDERR.puts "Uncompleted: #{uncompleted_dates.inspect}"
+    migrations_to_run = uncompleted_dates.select { |d| d <= date_str }
+
+    STDERR.puts "Run #{migrations_to_run.size} migrations, update to #{date_str}."
+    migrations_to_run.each { |m| up(m, options) }
+  end
+
+  def down_to(date_str, options = {})
+    migrations_to_run = migrations_completed(false,options).keys.select { |d| d > date_str }
+
+    STDERR.puts "Run #{migrations_to_run.size} migrations, roll back to #{date_str}."
+    migrations_to_run.each { |m| down(m, options) }
+  end
+
+  def current_latest(options = {})
+    migrations_completed(false,options).keys.max
+  end
+
+  def latest_in_directory
+    migrations_in_dir.keys.max
+  end
+
+  def to_latest(options = {})
+    latest = latest_in_directory
+    raise "No latest migration!" unless latest
+    up_to latest, options
+  end
+
+  def to_target(date_str, options = {})
+    if date_str < current_latest(options)
+      down_to date_str, options
+    else
+      up_to date_str, options
+    end
+  end
+
+  def rollback(options = {})
+    down(current_latest(options), options)
+  end
+
+end

data/lib/cassandra_migrate/version.rb

@@ -0,0 +1,5 @@
+# Copyright (C) 2013 OL2, inc.  See LICENSE.txt for details.
+
+class CassandraMigrate
+  VERSION = "0.0.1"
+end

metadata

@@ -0,0 +1,137 @@
+--- !ruby/object:Gem::Specification
+name: cassandra_migrate
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Noah Gibbs
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-12-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: trollop
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: cql-rb
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: erubis
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Migrations for Cassandra in CQL and Erb.
+email:
+- noah.gibbs@onlive.com
+executables:
+- cassandra_migrate
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/cassandra_migrate
+- cassandra_migrate.gemspec
+- lib/cassandra_migrate.rb
+- lib/cassandra_migrate/version.rb
+homepage: ''
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.25
+signing_key: 
+specification_version: 3
+summary: Migrations for Cassandra in CQL and Erb.
+test_files: []
+has_rdoc: