spiderfw 0.5.19 → 0.6.0

data/lib/spiderfw/model/storage/db/connectors/jdbc_oracle.rb

@@ -126,8 +126,6 @@ module Spider; module Model; module Storage; module Db; module Connectors
             case type.name
             when 'Spider::DataTypes::Decimal', 'BigDecimal'
                 java_bigdecimal(value)
-            when 'Spider::DataTypes::Bool'
-                value ? 1 : 0
             when 'Date', 'DateTime'
                 java_date(value)
             when 'Time'
@@ -185,6 +183,8 @@ module Spider; module Model; module Storage; module Db; module Connectors
             method = case type
             when :INTEGER, :SMALLINT, :TINYING, :BIGINTEGER, :NUMBER, :NUMERIC
                 :getInt
+            when :LONG
+                :getLong
             when :FLOAT, :REAL
                 :getFloat
             when :DECIMAL

data/lib/spiderfw/model/storage/db/connectors/oci8.rb

@@ -66,9 +66,8 @@ module Spider; module Model; module Storage; module Db; module Connectors
         
         def prepare_value(type, value)
             value = super
-            if (type < Spider::Model::BaseModel)
-                type = type.primary_keys[0].type
-            end
+            type = type.primary_keys[0].type if type < Spider::Model::BaseModel
+            type = Fixnum if type == Spider::DataTypes::PK
             return Oracle::OracleNilValue.new(Spider::Model.ruby_type(type)) if (value == nil)
             case type.name
             when 'Spider::DataTypes::Binary'
@@ -84,7 +83,7 @@ module Spider; module Model; module Storage; module Db; module Connectors
                     debug_vars = bind_vars.map{|var| var = var.to_s; var && var.length > 50 ? var[0..50]+"...(#{var.length-50} chars more)" : var}
                 end
                 curr[:last_executed] = [sql, bind_vars]
-                if (Spider.conf.get('storage.db.replace_debug_vars'))
+                if Spider.conf.get('storage.db.replace_debug_vars')
                     debug("oci8 #{connection} executing: "+sql.gsub(/:(\d+)/){
                         i = $1.to_i
                         v = bind_vars[i-1]

data/lib/spiderfw/model/storage/db/db_storage.rb

@@ -1,6 +1,5 @@
 require 'spiderfw/model/storage/base_storage'
 require 'spiderfw/model/mappers/db_mapper'
-require 'spiderfw/model/storage/db/db_connection_pool'
 require 'iconv'
 
 module Spider; module Model; module Storage; module Db
@@ -31,60 +30,19 @@ module Spider; module Model; module Storage; module Db
             attr_reader :type_synonyms
             # Type conversions which do not lose data. See also #safe_schema_conversion?
             attr_reader :safe_conversions
-            # An Hash of DB capabilities. The default is 
-            # {:autoincrement => false, :sequences => true, :transactions => true}
-            # (The BaseStorage class provides file sequences in case the subclass does not support them.)
-            attr_reader :capabilities
 
-            # Returns a new connection. Must be implemented by the subclasses; args are implementation specific.
-            def new_connection(*args)
-                raise "Unimplemented"
-            end
-            
-            def max_connections
-                nil
-            end
-            
-            def connection_pools
-                @pools ||= {}
-            end
-            
-            def get_connection(*args)
-                @pools ||= {}
-                @pools[args] ||= DbConnectionPool.new(args, self)
-                @pools[args].get_connection
-            end
-            
-            # Frees a connection, relasing it to the pool
-            def release_connection(conn, conn_params)
-                return unless conn
-                return unless @pools && @pools[conn_params]
-                @pools[conn_params].release(conn)
-            end
-            
-            # Removes a connection from the pool.
-            def remove_connection(conn, conn_params)
-                return unless conn
-                return unless @pools && @pools[conn_params]
-                @pools[conn_params].remove(conn)
+
+            def storage_type
+                :db
             end
             
-            def disconnect(conn)
-                raise "Virtual"
-            end
-                
-            # Checks whether a connection is still alive. Must be implemented by subclasses.
-            def connection_alive?(conn)
-                raise "Virtual"
+            def inherited(subclass)
+                subclass.instance_variable_set("@reserved_keywords", @reserved_keywords)
+                subclass.instance_variable_set("@type_synonyms", @type_synonyms)
+                subclass.instance_variable_set("@safe_conversions", @safe_conversions)
+                super
             end
-            
-             def inherited(subclass)
-                 subclass.instance_variable_set("@reserved_keywords", @reserved_keywords)
-                 subclass.instance_variable_set("@type_synonyms", @type_synonyms)
-                 subclass.instance_variable_set("@safe_conversions", @safe_conversions)
-                 subclass.instance_variable_set("@capabilities", @capabilities)
-             end
-            
+
         end
         
         def query_start
@@ -100,70 +58,11 @@ module Spider; module Model; module Storage; module Db
             Spider.logger.info("Db query (#{@instance_name}) done in #{diff}ms")
         end
         
-        def curr
-            Thread.current[:db_storages] ||= {}
-            Thread.current[:db_storages][@connection_params] ||= {
-                :transaction_nesting => 0, :savepoints => []
-            }
-        end
-        
-        def connection_pool
-            self.class.connection_pools[@connection_params]
-        end
-        
         # The constructor takes the connection URL, which will be parsed into connection params.
         def initialize(url)
             super
         end
         
-        # Instantiates a new connection with current connection params.
-        def connect
-            return self.class.get_connection(*@connection_params)
-            #Spider::Logger.debug("#{self.class.name} in thread #{Thread.current} acquired connection #{@conn}")
-        end
-        
-        # True if currently connected.
-        def connected?
-            curr[:conn] != nil
-        end
-
-        
-        # Returns the current connection, or creates a new one.
-        # If a block is given, will release the connection after yielding.
-        def connection
-            # is_connected = connected?
-            #             Spider.logger.debug("#{self} already connected with conn #{@conn}") if is_connected
-            #             connect unless is_connected
-            curr[:conn] = connect
-            if block_given?
-                yield curr[:conn]
-                release # unless is_connected
-                return true
-            else
-                #debugger unless @conn
-                return curr[:conn]
-            end
-        end
-        
-        def self.connection_attributes
-            @connection_attributes ||= {}
-        end
-        
-        def connection_attributes
-            self.class.connection_attributes[connection] ||= {}
-        end
-        
-        # Releases the current connection to the pool.
-        def release
-            # The subclass should check if the connection is alive, and if it is not call remove_connection instead
-            c = curr[:conn]
-            #Spider.logger.debug("#{self} in thread #{Thread.current} releasing #{curr[:conn]}")
-            curr[:conn] = nil
-            self.class.release_connection(c, @connection_params)
-            #Spider.logger.debug("#{self} in thread #{Thread.current} released #{curr[:conn]}")
-            return nil
-            #@conn = nil
-        end
         
         # Returns the default mapper for the storage.
         # If the storage subclass contains a MapperExtension module, it will be mixed-in with the mapper.
@@ -175,106 +74,6 @@ module Spider; module Model; module Storage; module Db
             return mapper
         end
         
-        # True if given named capability is supported by the DB.
-        def supports?(capability)
-            self.class.capabilities[capability]
-        end
-        
-        def supports_transactions?
-            return self.class.capabilities[:transactions]
-        end
-        
-        def transactions_enabled?
-            @configuration['enable_transactions'] && supports_transactions?
-        end
-        
-        def start_transaction
-            return unless transactions_enabled?
-            return savepoint("point#{curr[:savepoints].length}") if in_transaction?
-            curr[:transaction_nesting] += 1
-            Spider.logger.debug("#{self.class.name} starting transaction for connection #{connection.object_id}")
-            do_start_transaction
-            return true
-        end
-        
-        # May be implemented by subclasses.
-        def do_start_transaction
-           raise StorageException, "The current storage does not support transactions" 
-        end
-        
-        def in_transaction
-            if in_transaction?
-                curr[:transaction_nesting] += 1
-                return true
-            else
-                start_transaction
-                return false
-            end
-        end
-        
-        def in_transaction?
-            return false
-        end
-        
-        def commit
-            return false unless transactions_enabled?
-            raise StorageException, "Commit without a transaction" unless in_transaction?
-            return curr[:savepoints].pop unless curr[:savepoints].empty?
-            commit!
-        end
-        
-        def commit_or_continue
-            return false unless transactions_enabled?
-            raise StorageException, "Commit without a transaction" unless in_transaction?
-            if curr[:transaction_nesting] == 1
-                commit
-                return true
-            else
-                curr[:transaction_nesting] -= 1
-            end
-        end
-        
-        def commit!
-            Spider.logger.debug("#{self.class.name} commit connection #{curr[:conn].object_id}")
-            curr[:transaction_nesting] = 0
-            do_commit
-            release
-        end
-        
-        def do_commit
-            raise StorageException, "The current storage does not support transactions" 
-        end
-        
-        def rollback
-            raise "Can't rollback in a nested transaction" if curr[:transaction_nesting] > 1
-            return rollback_savepoint(curr[:savepoints].last) unless curr[:savepoints].empty?
-            rollback!
-        end
-        
-        def rollback!
-            curr[:transaction_nesting] = 0
-            Spider.logger.debug("#{self.class.name} rollback")
-            do_rollback
-            curr[:savepoints] = []
-            release
-        end
-        
-        def do_rollback
-            raise StorageException, "The current storage does not support transactions" 
-        end
-        
-        def savepoint(name)
-            curr[:savepoints] << name
-        end
-        
-        def rollback_savepoint(name=nil)
-            if name
-                curr[:savepoints] = curr[:savepoints][0,(curr[:savepoints].index(name))]
-                name
-            else
-                curr[:savepoints].pop
-            end
-        end
         
         def lock(table, mode=:exclusive)
             lockmode = case(mode)
@@ -340,6 +139,9 @@ module Spider; module Model; module Storage; module Db
         def column_attributes(type, attributes)
             db_attributes = {}
             case type.name
+            when 'Spider::DataTypes::PK'
+                db_attributes[:autoincrement] = true if supports?(:autoincrement)
+                db_attributes[:length] = 11
             when 'String', 'Spider::DataTypes::Text'
                 db_attributes[:length] = attributes[:length] if (attributes[:length])
             when 'Float'
@@ -379,6 +181,8 @@ module Spider; module Model; module Storage; module Db
                 return "SUBSTR(#{fields.join(', ')}, #{arguments})"
             when :subtract
                 return "(#{fields[0]} - #{fields[1]})"
+            when :rownum
+                return "ROWNUM()"
             end
             raise NotImplementedError, "#{self.class} does not support function #{func.func_name}"
         end
@@ -387,15 +191,6 @@ module Spider; module Model; module Storage; module Db
         #   Preparing values                                             #
         ##################################################################
         
-        # Prepares a value for saving.
-        def value_for_save(type, value, save_mode)
-            return prepare_value(type, value)
-        end
-        
-        # Prepares a value that will be used in a condition.
-        def value_for_condition(type, value)
-            return prepare_value(type, value)
-        end
         
         # Converts a value loaded from the DB to return it to the mapper.
         def value_to_mapper(type, value)
@@ -669,6 +464,10 @@ module Spider; module Model; module Storage; module Db
             return [sql, bind_vars]
         end
         
+        def sql_truncate(table)
+            "TRUNCATE #{table}"
+        end
+        
         # Returns an array of SQL statements for a create structured description.
         def sql_create_table(create)
             name = create[:table]

data/lib/spiderfw/model/storage/document/adapters/mongodb.rb

@@ -0,0 +1,307 @@
+require 'spiderfw/model/storage/document/document_storage'
+require 'mongo'
+
+module Spider; module Model; module Storage; module Document
+    
+    class Mongodb < DocumentStorage
+        CONDITION_OPS = {
+            '>' => '$gt', '<' => '$lt', '>=' => '$gte', '<=' => '$lte', 'not' => '$e', '<>' => '$ne'
+        }
+        
+        def self.parse_url(url)
+            # doc:mongodb://<username:password>@<host>:<port>/<database>
+            if (url =~ /.+:\/\/(?:(.+):(.+)@)?(.+)?\/(.+)/)
+                user = $1
+                pass = $2
+                location = $3
+                db_name = $4
+            else
+                raise ArgumentError, "Mongodb url '#{url}' is invalid"
+            end
+            location =~ /(.+)(?::(\d+))/
+            host = $1
+            port = $2
+            return [host, user, pass, db_name, port]
+        end
+        
+        def parse_url(url)
+            @host, @user, @pass, @db_name, @port = self.class.parse_url(url)
+            @connection_params = [@host, @user, @pass, @db_name, @port]
+        end
+        
+        def self.new_connection(host=nil, user=nil, passwd=nil, db=nil, port=nil)
+            conn = ::Mongo::Connection.new(host, port)
+            conn.authenticate(user, passwd) if user
+            return conn
+        end
+        
+        def self.disconnect(conn)
+            conn.close
+        end
+        
+        def self.connection_alive?(conn)
+            conn.connected?
+        end
+        
+        def connect
+            conn = super
+            curr[:db] = conn.db(@connection_params[3])
+            conn
+        end
+        
+        def release
+            curr[:db] = nil
+            super
+        end
+                
+        def db
+            connection
+            curr[:db]
+        end
+        
+        def collection(coll)
+            db.collection(coll)
+        end
+        
+        def insert(collection, doc)
+            Spider.logger.debug("Mongodb insert #{collection}:")
+            Spider.logger.debug(doc)
+            begin
+                collection(collection).insert(doc)
+            ensure
+                release
+            end
+        end
+        
+        def update(collection, selector, vals)
+            Spider.logger.debug("Mongodb update #{collection}, #{selector.inspect}:")
+            Spider.logger.debug(vals)
+            begin
+                collection(collection).update(selector, {'$set' => vals})
+            ensure
+                release
+            end
+        end
+        
+        def delete(collection, doc)
+            begin
+                collection(collection).remove({"_id" => doc["_id"]}, doc)
+            ensure
+                release
+            end
+        end
+        
+        def prepare_value(type, value)
+            return value if value.nil?
+            case type.name
+            when 'Time'
+                value = value.utc
+            when 'DateTime'
+                value = value.to_local_time.utc.to_time
+            when 'Date'
+                value = value.to_gm_time.utc
+            when 'BigDecimal'
+                # FIXME: should multiply on value.attributes[:scale] and store an integer, converting it back on load
+                value = value.to_f
+            end
+            value 
+        end
+        
+        def value_to_mapper(type, value)
+            if value.class == Time
+                if type.name == 'Date'
+                    value = value.to_date
+                elsif type.name == 'DateTime'
+                    value = value.to_datetime
+                end
+                return value
+            elsif type.name == 'Fixnum'
+                return value.to_i
+            end
+            return super
+        end
+        
+        def find(collection, condition, request, options)
+            options[:fields] = request
+            Spider.logger.debug("Mongodb find #{collection}, #{condition.inspect}")
+            begin
+                res = collection(collection).find(condition, options).to_a
+            ensure
+                release
+            end
+            res = res.map{ |row| keys_to_symbols(row) }
+            res.extend(StorageResult)
+            res.total_rows = res.length # FIXME
+            res
+        end
+                
+        def keys_to_symbols(h)
+            sh = {}
+            h.each do |k, v|
+                v = if v.is_a?(Hash)
+                    keys_to_symbols(v)
+                elsif v.is_a?(Array)
+                    v.map{ |row| keys_to_symbols(row) }
+                else
+                    v
+                end
+                #name = k == "_id" ? :id : k.to_sym
+                name = k.to_sym
+                sh[name] = v
+            end
+            sh
+        end
+        
+        def drop_db!
+            conn = connection
+            begin
+                conn.drop_database(@db_name)
+            ensure
+                release
+            end
+        end
+        
+        def generate_pk
+            ::BSON::ObjectId.new
+        end
+        
+        # Mapper extension
+         
+         module MapperExtension
+        
+             def prepare_condition(condition)
+                 cond_hash = {}
+                 pks = []
+                 condition.each_with_comparison do |k, v, comp|
+                     element = model.elements[k.to_sym]
+                     name = nil
+                     if element.primary_key?
+                         if @model.primary_keys.length == 1
+                             name = '_id'
+                         else
+                             raise "Mongo conditions for multiple primary keys are not supported yet"
+                         end
+                     else
+                         name = element.name.to_s
+                     end
+                     #next unless model.mapper.mapped?(element)
+                     if element.model?
+                         if element.attributes[:embedded]
+                             # TODO
+                         else
+                             pks = []
+                             element.model.primary_keys.each do |ek|
+                                 kv = v[ek.name]
+                                 raise "Document mapper can't join #{element.name}: condition #{condition}" unless kv
+                                 pks << element.mapper.map_condition_value(ek.type, kv)
+                             end
+                             hash_v = element.model.keys_string(pks)
+                             hash_v = {Mongodb::CONDITION_OPS[comp] => hash_v} unless comp == '='
+                             cond_hash[name] = hash_v
+                         end
+                     else
+                         hash_v = map_condition_value(element.type, v)
+                         comp ||= '='
+                         if comp != '='
+                             if comp == 'like' || comp == 'ilike'
+                                 options = comp == 'ilike' ? Regexp::IGNORECASE : nil
+                                 parts = hash_v.split('%')
+                                 parts << "" if hash_v[-1].chr == '%'
+                                 re_str = parts.map{ |p| p.blank? ? '' : Regexp.quote(p)}.join('.+') 
+                                 hash_v = Regexp.new(re_str, options)
+                             else
+                                 hash_v = {Mongodb::CONDITION_OPS[comp] => hash_v}
+                             end
+                         end
+                         cond_hash[name] = hash_v
+                     end
+                 end
+                 or_conds = []
+                 or_conds << cond_hash if condition.conjunction == :or && condition.subconditions.length > 0
+                 condition.subconditions.each do |sub|
+                     sub_res = self.prepare_condition(sub)
+                     if condition.conjunction == :and
+                         cond_hash.merge!(sub_res)
+                     elsif condition.conjunction == :or
+                         or_conds << sub_res
+                     end
+                 end
+                 unless or_conds.empty?
+                     cond_hash = { '$or' => or_conds }
+                 end
+                 cond_hash
+             end
+             
+             def prepare_request(request)
+                 h = {}
+                 request.each do |k, v|
+                     element = @model.elements[k.to_sym]
+                     next unless element
+                     name = element.primary_key? ? '_id' : element.name.to_s
+                     if v.is_a?(Spider::Model::Request)
+                         sub = prepare_request(v)
+                         sub.each do |sk, sv|
+                             h["#{name}.#{sk}"] = sv
+                         end
+                     else
+                         h[name] = 1
+                     end
+                 end
+                 h
+             end
+             
+             def fetch(query)
+                 condition = prepare_condition(query.condition)
+                 request = prepare_request(query.request)
+                 options = {}
+                 options[:limit] = query.limit if query.limit
+                 options[:skip] = query.offset if query.offset
+                 if query.order
+                     order_array = []
+                     query.order.each do |order|
+                         order_element, direction = order
+                         dir_str = direction == :desc ? 'descending' : 'ascending'
+                         order_array << [order_element.to_s, dir_str]
+                     end
+                     options[:sort] = order_array unless order_array.empty?
+                 end
+                 res = storage.find(@model.name, condition, request, options)
+                 r = res.map{ |row| postprocess_result(row) }
+                 r.extend(Storage::StorageResult) # FIXME: avoid doing this twice
+                 r.total_rows = res.total_rows
+                 r
+             end
+             
+             def postprocess_result(h, model=nil)
+                 model ||= @model
+                 sh = {}
+                 h.each do |k, v|
+                     if k == :_id
+                         pks = model.split_keys_string(v)
+                         model.primary_keys.each_with_index do |pk, i|
+                             pkval = pks[i]
+                             pkval = storage_value_to_mapper(pk.type, pkval)
+                             sh[pk.name] = pks[i]
+                         end
+                     else
+                         el = model.elements[k]
+                         v = if v.is_a?(Hash)
+                             postprocess_result(v, el.model)
+                         elsif v.is_a?(Array)
+                             v.map{ |row| postprocess_result(row, el.model) }
+                         else
+                             v
+                         end
+                         sh[k] = v
+                     end
+                 end
+                 sh
+             end
+             
+
+             
+         end
+        
+    end
+    
+end; end; end; end

data/lib/spiderfw/model/storage/document/document.rb

@@ -0,0 +1,7 @@
+module Spider; module Model; module Storage
+    module Document
+    end
+    
+    Document.autoload(:Mongodb, 'spiderfw/model/storage/document/adapters/mongodb')
+    
+end; end; end

data/lib/spiderfw/model/storage/document/document_storage.rb

@@ -0,0 +1,34 @@
+require 'spiderfw/model/storage/base_storage'
+require 'spiderfw/model/mappers/document_mapper'
+
+module Spider; module Model; module Storage; module Document
+    
+    class DocumentStorage < Storage::BaseStorage
+        
+        @capabilities = {
+            :embedding => true,
+            :transactions => false
+        }
+        
+        def self.storage_type
+            :document
+        end
+        
+        # Returns the default mapper for the storage.
+        # If the storage subclass contains a MapperExtension module, it will be mixed-in with the mapper.
+        def get_mapper(model)
+            mapper = Spider::Model::Mappers::DocumentMapper.new(model, self)
+            if (self.class.const_defined?(:MapperExtension))
+                mapper.extend(self.class.const_get(:MapperExtension))
+            end
+            return mapper
+        end
+        
+        def generate_pk
+            Spider::DataTypes::UUID.auto_value
+        end
+        
+    end
+    
+    
+end; end; end; end