datastax_rails 1.0.16.3 → 1.0.17.1

data/lib/datastax_rails/tasks/column_family.rb

@@ -44,11 +44,11 @@ module DatastaxRails
         model.attribute_definitions.values.each do |attr|
           coder = attr.coder
           if coder.options[:solr_type] && (coder.options[:indexed] || coder.options[:stored])
-          @fields.push({ :name => attr.name,
-                         :type => coder.options[:solr_type].to_s,
-                         :indexed => coder.options[:indexed].to_s,
-                         :stored => coder.options[:stored].to_s,
-                         :multi_valued => coder.options[:multi_valued].to_s })
+            @fields.push({ :name => attr.name,
+                           :type => coder.options[:solr_type].to_s,
+                           :indexed => coder.options[:indexed].to_s,
+                           :stored => coder.options[:stored].to_s,
+                           :multi_valued => coder.options[:multi_valued].to_s })
           end
           if coder.options[:sortable] && coder.options[:tokenized]
             @fields.push({ :name => "sort_" + attr.name,
@@ -76,7 +76,7 @@ module DatastaxRails
         column_family ||= :all
         # Ensure schema migrations CF exists
         unless connection.schema.column_families['schema_migrations']
-          connection.execute_cql_query(DatastaxRails::Cql::CreateColumnFamily.new('schema_migrations').key_type(:text).to_cql)
+          connection.execute_cql_query(DatastaxRails::Cql::CreateColumnFamily.new('schema_migrations').key_type(:text).columns(:digest => :text, :solrconfig => :text, :stopwords => :text).to_cql)
         end
         
         solrconfig = File.read(File.join(File.dirname(__FILE__),"..","..","..","config","solrconfig.xml"))
@@ -100,61 +100,89 @@ module DatastaxRails
         puts "models: #{models_to_upload.collect(&:to_s).join(",")}"
         
         models_to_upload.each do |model|
-          schema = generate_solr_schema(model)
-          schema_digest = Digest::SHA1.hexdigest(schema)
-          
-          results = DatastaxRails::Cql::Select.new(SchemaMigration, ['*']).conditions(:key => model.column_family).execute
-          sm_digests = CassandraCQL::Result.new(results).fetch.to_hash
-          
-          solr_url = "#{DatastaxRails::Base.solr_base_url}/resource/#{DatastaxRails::Base.config[:keyspace]}.#{model.column_family}"
-          uri = URI.parse(solr_url)
-          http = Net::HTTP.new(uri.host, uri.port)
-          if uri.scheme == 'https'
-            http.use_ssl = true
-            http.cert = OpenSSL::X509::Certificate.new(Rails.root.join("config","pow.crt").read)
-            http.key = OpenSSL::PKey::RSA.new(Rails.root.join("config","pow.key").read)
-            http.ca_path = Rails.root.join("config","sade_ca.crt").to_s
-            http.verify_mode = OpenSSL::SSL::VERIFY_NONE
-          end
-          http.read_timeout = 300
-          if force || solrconfig_digest != sm_digests['solrconfig']
-            loop do 
-              puts "Posting Solr Config file to '#{solr_url}/solrconfig.xml'"
-              http.post(uri.path+"/solrconfig.xml", solrconfig)
-              if Rails.env.production?
-                sleep(5)
-                resp = http.get(uri.path+"/solrconfig.xml")
-                continue unless resp.message == 'OK'                  
+          if model.payload_model?
+            next if model == DatastaxRails::PayloadModel
+            unless connection.schema.column_families[model.column_family.to_s]
+              puts "Creating payload model #{model.column_family}"
+              columns = {:chunk => :int, :payload => :text}
+              cql = DatastaxRails::Cql::CreateColumnFamily.new(model.column_family).key_name(:digest).key_columns("digest\", \"chunk").key_type(:text).columns(columns).with("COMPACT STORAGE").to_cql
+              puts cql
+              connection.execute_cql_query(cql)
+            end
+          else
+            unless connection.schema.column_families[model.column_family.to_s]
+              puts "Creating normal model #{model.column_family}"
+              cql = DatastaxRails::Cql::CreateColumnFamily.new(model.column_family).key_type(:text).columns(:updated_at => :text, :created_at => :text).to_cql
+              puts cql
+              connection.execute_cql_query(cql)
+            end
+            schema = generate_solr_schema(model)
+            schema_digest = Digest::SHA1.hexdigest(schema)
+            
+            results = DatastaxRails::Cql::Select.new(SchemaMigration, ['*']).conditions(:KEY => model.column_family).execute
+            sm_digests = CassandraCQL::Result.new(results).fetch.try(:to_hash) || {}
+            
+            solr_url = "#{DatastaxRails::Base.solr_base_url}/resource/#{DatastaxRails::Base.config[:keyspace]}.#{model.column_family}"
+            uri = URI.parse(solr_url)
+            http = Net::HTTP.new(uri.host, uri.port)
+            if uri.scheme == 'https'
+              http.use_ssl = true
+              http.cert = OpenSSL::X509::Certificate.new(Rails.root.join("config","datastax_rails.crt").read)
+              http.key = OpenSSL::PKey::RSA.new(Rails.root.join("config","datastax_rails.key").read)
+              http.ca_path = Rails.root.join("config","sade_ca.crt").to_s
+              http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+            end
+            http.read_timeout = 300
+            if force || solrconfig_digest != sm_digests['solrconfig']
+              loop do 
+                puts "Posting Solr Config file to '#{solr_url}/solrconfig.xml'"
+                http.post(uri.path+"/solrconfig.xml", solrconfig)
+                if Rails.env.production?
+                  sleep(5)
+                  resp = http.get(uri.path+"/solrconfig.xml")
+                  continue unless resp.message == 'OK'                  
+                end
+                break
               end
-              break
+              DatastaxRails::Cql::Update.new(SchemaMigration, model.column_family).columns(:solrconfig => solrconfig_digest).execute
             end
-            DatastaxRails::Cql::Update.new(SchemaMigration, model.column_family).columns(:solrconfig => solrconfig_digest).execute
-          end
-          if force || stopwords_digest != sm_digests['stopwords']
-            loop do
-              puts "Posting Solr Stopwords file to '#{solr_url}/stopwords.txt'"
-              http.post(uri.path+"/stopwords.txt", stopwords)
-              if Rails.env.production?
-                sleep(5)
-                resp = http.get(uri.path+"/stopwords.txt")
-                continue unless resp.message == 'OK'
+            if force || stopwords_digest != sm_digests['stopwords']
+              loop do
+                puts "Posting Solr Stopwords file to '#{solr_url}/stopwords.txt'"
+                http.post(uri.path+"/stopwords.txt", stopwords)
+                if Rails.env.production?
+                  sleep(5)
+                  resp = http.get(uri.path+"/stopwords.txt")
+                  continue unless resp.message == 'OK'
+                end
+                break
               end
-              break
+              DatastaxRails::Cql::Update.new(SchemaMigration, model.column_family).columns(:stopwords => stopwords_digest).execute
             end
-            DatastaxRails::Cql::Update.new(SchemaMigration, model.column_family).columns(:stopwords => stopwords_digest).execute
-          end
-          if force || schema_digest != sm_digests['digest']
-            loop do
-              puts "Posting Solr Schema file to '#{solr_url}/schema.xml'"
-              http.post(uri.path+"/schema.xml", schema)
-              if Rails.env.production?
-                sleep(5)
-                resp = http.get(uri.path+"/schema.xml")
-                continue unless resp.message == 'OK'
+            if force || schema_digest != sm_digests['digest']
+              loop do
+                puts "Posting Solr Schema file to '#{solr_url}/schema.xml'"
+                http.post(uri.path+"/schema.xml", schema)
+                if Rails.env.production?
+                  sleep(5)
+                  resp = http.get(uri.path+"/schema.xml")
+                  continue unless resp.message == 'OK'
+                end
+                break
+              end
+              DatastaxRails::Cql::Update.new(SchemaMigration, model.column_family).columns(:digest => schema_digest).execute
+            end
+            
+            # Check for unindexed columns
+            model.attribute_definitions.each do |attribute, definition|
+              if !connection.schema.column_families[model.column_family.to_s].columns.has_key?(attribute.to_s)# && 
+                 #!definition.coder.options[:stored] && 
+                 #!definition.coder.options[:indexed]
+                 
+                puts "Adding column '#{attribute}' to '#{model.column_family}'"
+                DatastaxRails::Cql::AlterColumnFamily.new(model.column_family).add(attribute => :text).execute
               end
-              break
             end
-            DatastaxRails::Cql::Update.new(SchemaMigration, model.column_family).columns(:digest => schema_digest).execute
           end
         end
       end

data/lib/datastax_rails/timestamps.rb

@@ -7,12 +7,18 @@ module DatastaxRails
       # attribute :updated_at, :type => :time#_with_zone
 
       before_create do #|r|
-        self.created_at ||= Time.current
-        self.updated_at ||= Time.current
+        if self.respond_to?(:created_at=)
+          self.created_at ||= Time.current
+        end
+        if self.respond_to?(:updated_at=)
+          self.updated_at ||= Time.current
+        end
       end
 
       before_update :if => :changed? do #|r|
-        self.updated_at = Time.current
+        if self.respond_to?(:updated_at=)
+          self.updated_at = Time.current
+        end
       end
     end
   end

data/lib/datastax_rails/types/array_type.rb

@@ -7,7 +7,7 @@ module DatastaxRails
     #
     # That would give you all the posts that have Technology somewhere in the tags array. 
     class ArrayType < BaseType
-      DEFAULTS = {:solr_type => 'array', :indexed => true, :stored => true, :multi_valued => false, :sortable => false, :tokenized => true, :fulltext => false}
+      DEFAULTS = {:solr_type => 'array', :indexed => true, :stored => true, :multi_valued => false, :sortable => false, :tokenized => true, :fulltext => true}
       
       # An extension to normal arrays that allow for tracking of dirty values.  This is
       # used by ActiveModel's change tracking framework.

data/lib/datastax_rails/types/binary_type.rb

@@ -4,27 +4,11 @@ module DatastaxRails
       DEFAULTS = {:solr_type => false, :indexed => false, :stored => false, :multi_valued => false, :sortable => false, :tokenized => false, :fulltext => false}
       def encode(str)
         raise ArgumentError.new("#{self} requires a String") unless str.kind_of?(String)
-        io = StringIO.new(Base64.encode64(str))
-        #io = StringIO.new(str)
-        chunks = []
-        while chunk = io.read(1.megabyte)
-          chunks << chunk
-        end
-        chunks
+        Base64.encode64(str)
       end
       
-      def decode(arr)
-        if(arr.is_a?(Array))
-          io = StringIO.new("","w+")
-          arr.each do |chunk|
-            io.write(chunk)
-          end
-          io.rewind
-          Base64.decode64(io.read)
-          #io.read
-        else
-          arr
-        end
+      def decode(str)
+        Base64.decode64(str)
       end
 
       def wrap(record, name, value)

data/lib/datastax_rails/version.rb

@@ -1,4 +1,4 @@
 module DatastaxRails
   # The current version of the gem
-  VERSION = "1.0.16.3"
+  VERSION = "1.0.17.1"
 end

data/lib/datastax_rails.rb

@@ -18,6 +18,7 @@ module DatastaxRails
   autoload :GroupedCollection
   autoload :Identity
   autoload :Migrations
+  autoload :PayloadModel
   autoload :Persistence
   autoload :Reflection
   autoload :Relation
@@ -77,7 +78,12 @@ require "thrift"
 module Thrift
   class BinaryProtocol
     def write_string(str)
-      write_i32(str.bytesize)
+      if(str.respond_to?(:bytesize))
+        size = str.bytesize
+      else
+        size = str.size
+      end
+      write_i32(size)
       trans.write(str)
     end
   end

data/spec/datastax_rails/base_spec.rb

@@ -14,8 +14,6 @@ describe DatastaxRails::Base do
   end
   
   it "should raise RecordNotFound when finding a bogus ID" do
-    pending "Datastax Enterprise 2.2 should fix this" do
-      lambda { Person.find("xyzzy") }.should raise_exception(DatastaxRails::RecordNotFound)
-    end
+    lambda { Person.find("xyzzy") }.should raise_exception(DatastaxRails::RecordNotFound)
   end
 end

data/spec/datastax_rails/cql/select_spec.rb

@@ -7,8 +7,8 @@ describe DatastaxRails::Cql::Select do
   
   it "should generate valid CQL" do
     cql = DatastaxRails::Cql::Select.new(@model_class, ["*"])
-    cql.using(DatastaxRails::Cql::Consistency::QUORUM).conditions(:key => '12345').limit(1)
-    cql.to_cql.should == "SELECT * FROM users USING CONSISTENCY QUORUM WHERE key = '12345' LIMIT 1"
+    cql.using(DatastaxRails::Cql::Consistency::QUORUM).conditions(:KEY => '12345').limit(1)
+    cql.to_cql.should == "SELECT * FROM users USING CONSISTENCY QUORUM WHERE \"KEY\" = '12345' LIMIT 1 "
   end
   
   it_has_behavior "default_consistency"

data/spec/datastax_rails/cql/update_spec.rb

@@ -8,7 +8,7 @@ describe DatastaxRails::Cql::Update do
   it "should generate valid CQL" do
     cql = DatastaxRails::Cql::Update.new(@model_class, "12345")
     cql.using(DatastaxRails::Cql::Consistency::QUORUM).columns(:name => 'John', :age => '23')
-    cql.to_cql.should match(/update users using consistency QUORUM SET (name = 'John', age = '23'|age = '23', name = 'John') WHERE KEY IN \('12345'\)/)
+    cql.to_cql.should match(/update users using consistency QUORUM SET ("name" = 'John', "age" = '23'|"age" = '23', "name" = 'John') WHERE "KEY" IN \('12345'\)/)
   end
   
   it_has_behavior "default_consistency"

data/spec/datastax_rails/persistence_spec.rb

@@ -4,7 +4,6 @@ describe "DatastaxRails::Base" do
   describe "persistence" do
     describe "#create" do
       it "should persist at the given consistency level" do
-        DatastaxRails::Base.connection.stub(:execute_cql_query)
         DatastaxRails::Base.connection.should_receive(:execute_cql_query).with(/USING CONSISTENCY LOCAL_QUORUM/i).and_return(true)
         Person.create({:name => 'Steven'},{:consistency => 'LOCAL_QUORUM'})
       end
@@ -12,7 +11,6 @@ describe "DatastaxRails::Base" do
     
     describe "#save" do
       it "should persist at the given consistency level" do
-        DatastaxRails::Base.connection.stub(:execute_cql_query)
         DatastaxRails::Base.connection.should_receive(:execute_cql_query).with(/USING CONSISTENCY LOCAL_QUORUM/i).and_return(true)
         p=Person.new(:name => 'Steven')
         p.save(:consistency => 'LOCAL_QUORUM')
@@ -22,10 +20,32 @@ describe "DatastaxRails::Base" do
     describe "#remove" do
       it "should remove at the given consistency level" do
         p=Person.create(:name => 'Steven')
-        DatastaxRails::Base.connection.stub(:execute_cql_query)
         DatastaxRails::Base.connection.should_receive(:execute_cql_query).with(/USING CONSISTENCY LOCAL_QUORUM/i).and_return(true)
         p.destroy(:consistency => :local_quorum)
       end
     end
+    
+    describe "#store_file" do
+      it "should store a file" do
+        file = "abcd"*1.megabyte
+        CarPayload.create(:digest => 'limo', :payload => file)
+        CarPayload.find('limo').payload.should == file
+      end
+      
+      it "should store really large files" do
+        file = IO.read("/dev/urandom", 25.megabyte)
+        CarPayload.create(:digest => 'limo', :payload => file)
+        CarPayload.find('limo').payload.should == file
+      end
+      
+      it "should successfully overwrite a larger file with a smaller one" do
+        file = "abcd"*1.megabyte
+        car = CarPayload.create(:digest => 'limo', :payload => file)
+        smallfile = "e"*1.kilobyte
+        car.payload = smallfile
+        car.save
+        CarPayload.find('limo').payload.should == smallfile
+      end
+    end
   end
 end

data/spec/dummy/config/datastax.yml

@@ -6,12 +6,8 @@ development:
   connection_options:
     timeout: 10
   solr:
-    port: 8984
+    port: 8983
     path: /solr
-    ssl:
-      use_ssl: true
-      cert: config/datastax_rails.crt
-      key: config/datastax_rails.key
     
 test:
   servers: ["127.0.0.1:9160"]
@@ -19,11 +15,7 @@ test:
   strategy_class: "org.apache.cassandra.locator.SimpleStrategy"
   strategy_options: {"replication_factor": "1"}
   connection_options:
-    timeout: 10
+    timeout: 30
   solr:
-    port: 8984
+    port: 8983
     path: /solr
-    ssl:
-      use_ssl: true
-      cert: config/datastax_rails.crt
-      key: config/datastax_rails.key