datastax_rails 1.0.5

data/lib/datastax_rails/tasks/column_family.rb

@@ -0,0 +1,162 @@
+require 'digest/sha1'
+
+module DatastaxRails
+  module Tasks
+    class SchemaMigration
+      def self.column_family
+        'schema_migrations'
+      end
+    end
+    
+    class ColumnFamily
+      COMPARATOR_TYPES = [:blob, :ascii, :text, :varint, :bigint, :uuid, :timestamp, :boolean, :float, :doublt, :decimal]
+
+      COLUMN_TYPES = [:standard, :super]
+
+      def initialize(keyspace)
+        @keyspace = keyspace
+      end
+
+      def exists?(name)
+        connection.schema.column_family_names.include?(name)
+      end
+
+      def create(name, &block)
+        cql = DatastaxRails::Cql::CreateColumnFamily.new(name.to_s)
+        cql.comparator_type = 'text'
+        cql.column_type = 'Standard'
+
+        block.call cf if block
+
+        connection.execute_cql_query(cql.to_cql)
+      end
+
+      def drop(name)
+        connection.execute_cql_query(DatastaxRails::Cql::DropColumnFamily.new(name.to_s).to_cql)
+      end
+
+      def rename(old_name, new_name)
+        raise NotImplementedError, "Renaming of column families is not currently supported"
+      end
+
+      def clear(name)
+        connection.execute_cql_query(DatastaxRails::Cql::Truncate.new(name.to_s).to_cql)
+      end
+      
+      def generate_solr_schema(model)
+        @fields = []
+        @copy_fields = []
+        @fulltext_fields = []
+        model.attribute_definitions.values.each do |attr|
+          coder = attr.coder
+          if coder.options[:solr_type] && (coder.options[:indexed] || coder.options[:stored])
+          @fields.push({ :name => attr.name,
+                         :type => coder.options[:solr_type].to_s,
+                         :indexed => coder.options[:indexed].to_s,
+                         :stored => coder.options[:stored].to_s,
+                         :multi_valued => coder.options[:multi_valued].to_s })
+          end
+          if coder.options[:sortable] && coder.options[:tokenized]
+            @fields.push({ :name => "sort_" + attr.name,
+                           :type => "string",
+                           :indexed => true,
+                           :stored => false,
+                           :multi_valued => false })
+            @copy_fields.push({ :source => attr.name, :dest => "sort_" + attr.name })
+          end
+          if coder.options[:fulltext]
+            @fulltext_fields << attr.name
+          end
+        end
+        # Sort the fields so that no matter what order the attributes are arranged into the
+        # same schema file gets generated
+        @fields.sort! {|a,b| a[:name] <=> b[:name]}
+        @copy_fields.sort! {|a,b| a[:source] <=> b[:source]}
+        @fulltext_fields.sort!
+        
+        return ERB.new(File.read(File.join(File.dirname(__FILE__),"..","..","..","config","schema.xml.erb"))).result(binding)
+      end
+      
+      def upload_solr_schemas(column_family = :all)
+        # Ensure schema migrations CF exists
+        unless connection.schema.column_families['schema_migrations']
+          connection.execute_cql_query(DatastaxRails::Cql::CreateColumnFamily.new('schema_migrations').key_type(:text).to_cql)
+        end
+        
+        solrconfig = File.read(File.join(File.dirname(__FILE__),"..","..","..","config","solrconfig.xml"))
+        stopwords = File.read(File.join(File.dirname(__FILE__),"..","..","..","config","stopwords.txt"))
+        solrconfig_digest = Digest::SHA1.hexdigest(solrconfig)
+        stopwords_digest = Digest::SHA1.hexdigest(stopwords)
+        
+        models_to_upload = []
+        
+        if column_family == :all
+          # Ensure all models are loaded
+          Dir[Rails.root.join("app","models",'*.rb').to_s].each do |file| 
+            require File.basename(file, File.extname(file))
+          end
+          
+          models_to_upload += DatastaxRails::Base.models
+        else
+          models_to_upload << column_family.constantize
+        end
+        
+        puts "models: #{models_to_upload.collect(&:to_s).join(",")}"
+        
+        models_to_upload.each do |model|
+          schema = generate_solr_schema(model)
+          schema_digest = Digest::SHA1.hexdigest(schema)
+          
+          results = DatastaxRails::Cql::Select.new(SchemaMigration, ['*']).conditions(:key => model.column_family).execute
+          sm_digests = CassandraCQL::Result.new(results).fetch.to_hash
+          
+          solr_url = "#{DatastaxRails::Base.config[:solr][:url]}/resource/#{DatastaxRails::Base.config[:keyspace]}.#{model.column_family}"
+          uri = URI.parse(solr_url)
+          Net::HTTP.start(uri.host, uri.port) do |http|
+            if solrconfig_digest != sm_digests['solrconfig'] 
+              puts "Posting Solr Config file to '#{uri.path}/solrconfig.xml'"
+              http.post(uri.path+"/solrconfig.xml", solrconfig)
+              DatastaxRails::Cql::Update.new(SchemaMigration, model.column_family).columns(:solrconfig => solrconfig_digest).execute
+            end
+            if stopwords_digest != sm_digests['stopwords']
+              puts "Posting Solr Stopwords file to '#{uri.path}/stopwords.txt'"
+              http.post(uri.path+"/stopwords.txt", stopwords)
+              DatastaxRails::Cql::Update.new(SchemaMigration, model.column_family).columns(:stopwords => stopwords_digest).execute
+            end
+            if schema_digest != sm_digests['digest']
+              puts "Posting Solr Schema file to '#{uri.path}/schema.xml'"
+              http.post(uri.path+"/schema.xml", schema)
+              DatastaxRails::Cql::Update.new(SchemaMigration, model.column_family).columns(:digest => schema_digest).execute
+            end
+          end
+          
+        end
+      end
+
+      private
+
+      def connection
+        DatastaxRails::Base.connection
+      end
+
+      def post_process_column_family(cf)
+        comp_type = cf.comparator_type
+        if comp_type && COMPARATOR_TYPES.has_key?(comp_type)
+          cf.comparator_type = COMPARATOR_TYPES[comp_type]
+        end
+
+        comp_type = cf.subcomparator_type
+        if comp_type && COMPARATOR_TYPES.has_key?(comp_type)
+          cf.subcomparator_type = COMPARATOR_TYPES[comp_type]
+        end
+
+        col_type = cf.column_type
+        if col_type && COLUMN_TYPES.has_key?(col_type)
+          cf.column_type = COLUMN_TYPES[col_type]
+        end
+
+        cf
+      end
+    end
+  end
+end

data/lib/datastax_rails/tasks/ds.rake

@@ -0,0 +1,63 @@
+namespace :ds do
+  task :configure => :environment do
+    @configs = YAML.load_file(Rails.root.join("config", "datastax.yml"))
+    @config = @configs[Rails.env || 'development']
+  end
+
+  desc 'Create the keyspace in config/cassandra.yml for the current environment'
+  task :create do
+    @configs = YAML.load_file(Rails.root.join("config", "datastax.yml"))
+    @config = @configs[Rails.env || 'development']
+    ret = DatastaxRails::Tasks::Keyspace.create @config['keyspace'], @config
+    puts "Created keyspace: #{@config['keyspace']}" if ret
+  end
+
+  namespace :create do
+    desc 'Create keyspaces in config/cassandra.yml for all environments'
+    task :all => :configure do
+      created = []
+      @configs.values.each do |config|
+        DatastaxRails::Tasks::Keyspace.create config['keyspace'], config
+        created << config['keyspace']
+      end
+      puts "Created keyspaces: #{created.join(', ')}"
+    end
+  end
+
+  desc 'Drop keyspace in config/cassandra.yml for the current environment'
+  task :drop => :configure do
+    DatastaxRails::Tasks::Keyspace.drop @config['keyspace']
+    puts "Dropped keyspace: #{@config['keyspace']}"
+  end
+
+  namespace :drop do
+    desc 'Drop keyspaces in config/cassandra.yml for all environments'
+    task :all => :configure do
+      dropped = []
+      @configs.values.each do |config|
+        DatastaxRails::Tasks::Keyspace.drop config['keyspace']
+        dropped << config['keyspace']
+      end
+      puts "Dropped keyspaces: #{dropped.join(', ')}"
+    end
+  end
+  
+  task :schema => :configure do
+    cf = DatastaxRails::Tasks::ColumnFamily.new(@config['keyspace'])
+    cf.upload_solr_schemas
+  end
+
+  desc 'Migrate the keyspace (options: VERSION=x)'
+  task :migrate => :configure do
+    version = ( ENV['VERSION'] ? ENV['VERSION'].to_i : nil )
+    DatastaxRails::Schema::Migrator.migrate DatastaxRails::Schema::Migrator.migrations_path, version
+    schema_dump
+  end
+
+  desc 'Load the seed data from ks/seeds.rb'
+  task :seed => :environment do
+    seed_file = Rails.root.join("ks","seeds.rb")
+    load(seed_file) if seed_file.exist?
+  end
+end
+

data/lib/datastax_rails/tasks/keyspace.rb

@@ -0,0 +1,57 @@
+module DatastaxRails
+  module Tasks
+    class Keyspace
+      class << self
+        def exists?(name)
+          connection.keyspaces.collect(&:name).include? name.to_s
+        end
+  
+        def create(name, options = {})
+          opts = { :name => name.to_s,
+                   :strategy_class => 'org.apache.cassandra.locator.NetworkTopologyStrategy'}.merge(options)
+  
+          if(exists?(name.to_s))
+            puts "Keyspace #{name.to_s} already exists"
+            return false
+          else
+            cql = DatastaxRails::Cql::CreateKeyspace.new(opts.delete(:name))
+            cql.strategy_class(opts.delete(:strategy_class))
+            strategy_options = opts.delete('strategy_options')
+            cql.strategy_options(strategy_options.symbolize_keys)
+            puts cql.to_cql
+            connection.execute_cql_query(cql.to_cql)
+            return true
+          end
+        end
+  
+        def drop(name)
+          connection.execute_cql_query(DatastaxRails::Cql::DropKeyspace.new(name.to_s).to_cql)
+        end
+  
+        def set(name)
+          connection.execute_cql_query(DatastaxRails::Cql::UseKeyspace.new(name.to_s).to_cql)
+        end
+  
+        def get
+          connection.keyspace
+        end
+  
+        def clear
+          return puts 'Cannot clear system keyspace' if connection.keyspace == 'system'
+  
+          connection.clear_keyspace!
+        end
+  
+        private
+  
+          def connection
+            unless @connection
+              config = YAML.load_file(Rails.root.join("config", "datastax.yml"))[Rails.env]
+              @connection = CassandraCQL::Database.new(config["servers"], :keyspace => 'system')
+            end
+            @connection
+          end
+      end
+    end
+  end
+end

data/lib/datastax_rails/timestamps.rb

@@ -0,0 +1,19 @@
+module DatastaxRails
+  module Timestamps
+    extend ActiveSupport::Concern
+
+    included do
+      # attribute :created_at, :type => :time#_with_zone
+      # attribute :updated_at, :type => :time#_with_zone
+
+      before_create do #|r|
+        self.created_at ||= Time.current
+        self.updated_at ||= Time.current
+      end
+
+      before_update :if => :changed? do #|r|
+        self.updated_at = Time.current
+      end
+    end
+  end
+end

data/lib/datastax_rails/type.rb

@@ -0,0 +1,16 @@
+module DatastaxRails
+  class Type
+    cattr_accessor :attribute_types
+    self.attribute_types = {}.with_indifferent_access
+
+    class << self
+      def register(name, coder)
+        attribute_types[name] = coder
+      end
+
+      def get_coder(name)
+        attribute_types[name]
+      end
+    end
+  end
+end

data/lib/datastax_rails/types/array_type.rb

@@ -0,0 +1,77 @@
+module DatastaxRails
+  module Types
+    class ArrayType < BaseType
+      DEFAULTS = {:solr_type => 'array', :indexed => true, :stored => true, :multi_valued => false, :sortable => false, :tokenized => true, :fulltext => false}
+      class DirtyArray < Array
+        attr_accessor :record, :name, :options
+        def initialize(record, name, array, options)
+          @record   = record
+          @name     = name.to_s
+          @options  = options
+
+          super(array)
+          setify!
+        end
+
+        def <<(obj)
+          modifying do
+            super
+            setify!
+          end
+        end
+
+        def delete(obj)
+          modifying do
+            super
+          end
+        end
+
+        private
+          def setify!
+            if options[:unique]
+              compact!
+              uniq!
+              begin sort! rescue ArgumentError end
+            end
+          end
+
+          def modifying
+            unless record.changed_attributes.include?(name)
+              original = dup
+            end
+
+            result = yield
+
+            if !record.changed_attributes.key?(name) && original != self
+              record.changed_attributes[name] = original
+            end
+
+            record.send("#{name}=", self)
+
+            result
+          end
+      end
+
+      def default
+        []
+      end
+
+      def encode(array)
+        raise ArgumentError.new("#{self} requires an Array") unless array.kind_of?(Array)
+        ar = Array(array)
+        ar.uniq! if options[:unique]
+        ar.join("&&&&")
+      end
+
+      def decode(str)
+        return [] if str.blank?
+        
+        str.split(/&&&&/)
+      end
+
+      def wrap(record, name, value)
+        DirtyArray.new(record, name, value, options)
+      end
+    end
+  end
+end

data/lib/datastax_rails/types/base_type.rb

@@ -0,0 +1,26 @@
+module DatastaxRails
+  module Types
+    class BaseType
+      attr_accessor :options
+      def initialize(options = {})
+        @options = self.class::DEFAULTS.merge(options)
+      end
+
+      def default
+        options[:default].dup if options[:default]
+      end
+
+      def encode(value)
+        value.to_s
+      end
+
+      def decode(str)
+        str
+      end
+
+      def wrap(record, name, value)
+        value
+      end
+    end
+  end
+end

data/lib/datastax_rails/types/binary_type.rb

@@ -0,0 +1,15 @@
+module DatastaxRails
+  module Types
+    class BinaryType < BaseType
+      DEFAULTS = {:solr_type => false, :indexed => false, :stored => false, :multi_valued => false, :sortable => false, :tokenized => false, :fulltext => false}
+      def encode(str)
+        raise ArgumentError.new("#{self} requires a String") unless str.kind_of?(String)
+        str.dup
+      end
+
+      def wrap(record, name, value)
+        (value.frozen? ? value.dup : value)
+      end
+    end
+  end
+end

data/lib/datastax_rails/types/boolean_type.rb

@@ -0,0 +1,22 @@
+module DatastaxRails
+  module Types
+    class BooleanType < BaseType
+      DEFAULTS = {:solr_type => 'boolean', :indexed => true, :stored => true, :multi_valued => false, :sortable => true, :tokenized => false, :fulltext => false}
+      TRUE_VALS = [true, 'true', '1', 'Y']
+      FALSE_VALS = [false, 'false', '0', '', 'N', nil]
+      VALID_VALS = TRUE_VALS + FALSE_VALS
+      
+      def encode(bool)
+        unless VALID_VALS.include?(bool)
+          raise ArgumentError.new("#{self} requires a boolean")
+        end
+        TRUE_VALS.include?(bool) ? '1' : '0'
+      end
+
+      def decode(str)
+        raise ArgumentError.new("Cannot convert #{str} into a boolean") unless VALID_VALS.include?(str)
+        TRUE_VALS.include?(str)
+      end
+    end
+  end
+end

data/lib/datastax_rails/types/date_type.rb

@@ -0,0 +1,17 @@
+module DatastaxRails
+  module Types
+    class DateType < BaseType
+      DEFAULTS = {:solr_type => 'date', :indexed => true, :stored => true, :multi_valued => false, :sortable => true, :tokenized => false, :fulltext => false}
+      FORMAT = '%Y-%m-%dT%H:%M:%SZ'
+
+      def encode(value)
+        raise ArgumentError.new("#{self} requires a Date") unless value.to_date
+        value.to_date.strftime(FORMAT)
+      end
+
+      def decode(str)
+        Date.parse(str) rescue nil
+      end
+    end
+  end
+end

data/lib/datastax_rails/types/float_type.rb

@@ -0,0 +1,18 @@
+module DatastaxRails
+  module Types
+    class FloatType < BaseType
+      DEFAULTS = {:solr_type => 'float', :indexed => true, :stored => true, :multi_valued => false, :sortable => true, :tokenized => false, :fulltext => false}
+      REGEX = /\A[-+]?(\d+(\.\d+)?|\.\d+)\Z/
+      def encode(float)
+        raise ArgumentError.new("#{self} requires a Float") unless float.kind_of?(Float)
+        float.to_s
+      end
+
+      def decode(str)
+        return nil if str.empty?
+        return nil unless str.kind_of?(String) && str.match(REGEX)
+        str.to_f
+      end
+    end
+  end
+end

data/lib/datastax_rails/types/integer_type.rb

@@ -0,0 +1,18 @@
+module DatastaxRails
+  module Types
+    class IntegerType < BaseType
+      DEFAULTS = {:solr_type => 'int', :indexed => true, :stored => true, :multi_valued => false, :sortable => true, :tokenized => false, :fulltext => false}
+      REGEX = /\A[-+]?\d+\Z/
+      def encode(int)
+        raise ArgumentError.new("#{self} requires an Integer. You passed #{int.inspect}") unless int.kind_of?(Integer)
+        int.to_s
+      end
+
+      def decode(str)
+        return nil if str.empty?
+        return nil unless str.kind_of?(String) && str.match(REGEX)
+        str.to_i
+      end
+    end
+  end
+end

data/lib/datastax_rails/types/string_type.rb

@@ -0,0 +1,16 @@
+module DatastaxRails
+  module Types
+    class StringType < BaseType
+      DEFAULTS = {:solr_type => 'string', :indexed => true, :stored => true, :multi_valued => false, :sortable => true, :tokenized => false, :fulltext => true}
+      def encode(str)
+        raise ArgumentError.new("#{self} requires a String") unless str.kind_of?(String)
+        str.dup
+      end
+
+      def wrap(record, name, value)
+        txt = (value.frozen? ? value.dup : value)
+        txt.respond_to?(:force_encoding) ? txt.force_encoding('UTF-8') : txt
+      end
+    end
+  end
+end

data/lib/datastax_rails/types/text_type.rb

@@ -0,0 +1,16 @@
+module DatastaxRails
+  module Types
+    class TextType < BaseType
+      DEFAULTS = {:solr_type => 'text', :indexed => true, :stored => true, :multi_valued => false, :sortable => false, :tokenized => true, :fulltext => true}
+      def encode(str)
+        raise ArgumentError.new("#{self} requires a String") unless str.kind_of?(String)
+        str.dup
+      end
+
+      def wrap(record, name, value)
+        txt = (value.frozen? ? value.dup : value)
+        txt.respond_to?(:force_encoding) ? txt.force_encoding('UTF-8') : txt
+      end
+    end
+  end
+end

data/lib/datastax_rails/types/time_type.rb

@@ -0,0 +1,17 @@
+module DatastaxRails
+  module Types
+    class TimeType < BaseType
+      DEFAULTS = {:solr_type => 'date', :indexed => true, :stored => true, :multi_valued => false, :sortable => true, :tokenized => false, :fulltext => false}
+      FORMAT = "%Y-%m-%dT%H:%M:%SZ"
+
+      def encode(time)
+        raise ArgumentError.new("#{self} requires a Time") unless time.kind_of?(Time)
+        time.strftime(FORMAT)
+      end
+
+      def decode(str)
+        Time.parse(str) rescue nil
+      end
+    end
+  end
+end

data/lib/datastax_rails/types.rb

@@ -0,0 +1,9 @@
+DatastaxRails::Type.register(:array,          DatastaxRails::Types::ArrayType)
+DatastaxRails::Type.register(:binary,         DatastaxRails::Types::BinaryType)
+DatastaxRails::Type.register(:boolean,        DatastaxRails::Types::BooleanType)
+DatastaxRails::Type.register(:date,           DatastaxRails::Types::DateType)
+DatastaxRails::Type.register(:float,          DatastaxRails::Types::FloatType)
+DatastaxRails::Type.register(:integer,        DatastaxRails::Types::IntegerType)
+DatastaxRails::Type.register(:time,           DatastaxRails::Types::TimeType)
+DatastaxRails::Type.register(:string,         DatastaxRails::Types::StringType)
+DatastaxRails::Type.register(:text,           DatastaxRails::Types::TextType)