spiderfw 0.5.19 → 0.6.0

data/lib/spiderfw/model/model.rb

@@ -8,7 +8,7 @@ module Spider
         
         @base_types = [
             String, Spider::DataTypes::Text, Fixnum, Float, BigDecimal, Date, DateTime, Time,
-            Spider::DataTypes::Bool
+            Spider::DataTypes::Bool, Spider::DataTypes::PK
         ]
         
         # Base types are:
@@ -50,7 +50,7 @@ module Spider
             return klass if base_types.include?(klass)
             return klass if klass <= Spider::Model::BaseModel
             return t if t = map_types[klass]
-            return klass.maps_to if (klass.subclass_of?(Spider::DataType) && klass.maps_to)
+            return klass.maps_to if klass.subclass_of?(Spider::DataType) && klass.maps_to
             return klass.superclass if klass.superclass
             return nil
         end
@@ -126,6 +126,10 @@ module Spider
             self.unit_of_work = uow
         end
         
+        def self.unit_of_work_running?
+            self.unit_of_work && self.unit_of_work.running?
+        end
+        
         def self.no_identity_mapper(&proc)
             im = self.identity_mapper
             self.identity_mapper = nil
@@ -136,7 +140,7 @@ module Spider
         def self.no_context(&proc)
             uow = self.unit_of_work
             self.unit_of_work = nil
-            im = self.identity_mapper
+            im = self.identity_mapper            
             self.identity_mapper = nil
             yield
             self.identity_mapper = im
@@ -212,7 +216,7 @@ module Spider
         end
         
         # Load YAML data
-        def self.load_fixtures(file)
+        def self.load_fixtures(file, truncate=false)
             if (file =~ /\.([^\.]+)$/)
                 extension = $1
             else
@@ -229,8 +233,20 @@ module Spider
             data.each do |step|
                 step.each do |mod_name, mod_data|
                     mod = const_get_full(mod_name)
+                    mod.mapper.truncate! if truncate
                     mod_data.each do |row|
-                        obj = mod.new(row)
+                        h = {}
+                        row.each do |k, v|
+                            if v.is_a?(String)
+                                if v[0..1] == '@@'
+                                    v = v[1..-1]
+                                elsif v[0].chr == '@'
+                                    v = eval(v[1..-1].to_s)
+                                end
+                            end
+                            h[k] = v
+                        end
+                        obj = mod.new(h)
                         obj.insert
                     end
                 end
@@ -268,9 +284,11 @@ module Spider
         class TypeError < ArgumentError
         end
         
-        def self.sort(models)
-
-            sorter = Sorter.new(models)
+        def self.sort(models, options={})
+            options = {
+                :association_dependencies => true
+            }.merge(options)
+            sorter = Sorter.new(models, options)
             sorter.sort
         end
         
@@ -279,11 +297,12 @@ module Spider
         class Sorter
             include TSort
             
-            def initialize(models)
+            def initialize(models, options={})
                 @model_tasks = {}
                 @processed_deps = {}
                 @processed = {}
                 @models = models
+                @options = options
                 @models.each{ |m| collect_dependencies(m) }
             end
             
@@ -303,6 +322,14 @@ module Spider
                      @model_tasks[model.superclass] ||= SortTask.new(model.superclass)
                      @model_tasks[model] << @model_tasks[model.superclass]
                  end
+                 if @options[:association_dependencies]
+                     model.elements.each do |name, element|
+                         if element.model? && !element.attributes[:added_reverse] && @models.include?(element.type)
+                             @model_tasks[element.type] ||= SortTask.new(element.type)
+                             @model_tasks[model] << @model_tasks[element.type]
+                         end
+                     end
+                 end
              end
 
 

data/lib/spiderfw/model/query_funcs.rb

@@ -122,6 +122,9 @@ module Spider; module QueryFuncs
 
     class CurrentDate < ZeroArityFunction
     end
+    
+    class RowNum < ZeroArityFunction
+    end
 
     class Length < UnaryFunction
     end

data/lib/spiderfw/model/query_set.rb

@@ -28,7 +28,7 @@ module Spider; module Model
         # Total number of objects present in the Storage for the Query
         attr_accessor :total_rows
         # (bool) Wether the QuerySet has been loaded
-        attr_accessor :loaded
+        attr_reader :loaded
         # (Fixnum) How many objects to load at a time. If nil, all the objects returned by the Query 
         # will be loaded.
         attr_accessor :fetch_window
@@ -50,6 +50,7 @@ module Spider; module Model
         attr_accessor :loadable # :nodoc: TODO: remove?
         # Don't put this queryset's objects into the IdentityMapper
         attr_accessor :_no_identity_mapper
+        attr_accessor :modified
         
         # Instantiates a non-autoloading queryset
         def self.static(model, query_or_val=nil)
@@ -172,6 +173,7 @@ module Spider; module Model
             end
             index_object(obj)
             @raw_data[@objects.length-1] = raw if raw
+            @modified = true
         end
 
             
@@ -221,6 +223,14 @@ module Spider; module Model
             @loaded_elements.merge!(f_loaded)
         end
         
+        def modified?
+            return true if @modified
+            @objects.each do |obj|
+                return true if obj.modified?
+            end
+            return false
+        end
+        
         # Returns the last object.
         def last
             load unless (@loaded || !autoload?) && loaded?(total_rows-1)
@@ -232,6 +242,10 @@ module Spider; module Model
             @objects.delete_at(index)
         end
         
+        def delete(obj)
+            @objects.delete(obj)
+        end
+        
         # Returns a new QuerySet containing objects from both this and the other.
         def +(other)
             qs = self.clone
@@ -456,6 +470,7 @@ module Spider; module Model
             return load_next if @fetch_window && !@query.offset
             mapper.find(@query.clone, self)
             @loaded = true
+            @modified = false
             return self
         end
         
@@ -500,6 +515,11 @@ module Spider; module Model
             return false
         end
         
+        def loaded=(val)
+            @loaded = val
+            @modified = false if @loaded
+        end
+        
         def loadable?
             @loadable
         end

data/lib/spiderfw/model/storage/base_storage.rb

@@ -1,3 +1,5 @@
+require 'spiderfw/model/storage/connection_pool'
+
 module Spider; module Model; module Storage
     
     class BaseStorage
@@ -5,14 +7,82 @@ module Spider; module Model; module Storage
         attr_reader :url
         attr_accessor :instance_name
         
-        def self.sequence_sync
-            @sequence_sync ||= ::Sync.new
-        end
+        @capabilities = {
+            
+        }
         
-        def self.base_types
-            Model.base_types
+        class << self
+            # An Hash of storage capabilities. The default for db storages is 
+            # {:autoincrement => false, :sequences => true, :transactions => true}
+            # (The BaseStorage class provides file sequences in case the subclass does not support them.)
+            attr_reader :capabilities
+            
+            def storage_type
+                :none
+            end
+            
+            def sequence_sync
+                @sequence_sync ||= ::Sync.new
+            end
+
+            def base_types
+                Model.base_types
+            end
+            
+            # True if given named capability is supported by the Storage.
+            def supports?(capability)
+                @capabilities[capability]
+            end
+            
+            # Returns a new connection. Must be implemented by the subclasses; args are implementation specific.
+            def new_connection(*args)
+                raise "Unimplemented"
+            end
+
+            def max_connections
+                nil
+            end
+
+            def connection_pools
+                @pools ||= {}
+            end
+
+            def get_connection(*args)
+                @pools ||= {}
+                @pools[args] ||= ConnectionPool.new(args, self)
+                @pools[args].get_connection
+            end
+
+            # Frees a connection, relasing it to the pool
+            def release_connection(conn, conn_params)
+                return unless conn
+                return unless @pools && @pools[conn_params]
+                @pools[conn_params].release(conn)
+            end
+
+            # Removes a connection from the pool.
+            def remove_connection(conn, conn_params)
+                return unless conn
+                return unless @pools && @pools[conn_params]
+                @pools[conn_params].remove(conn)
+            end
+
+            def disconnect(conn)
+                raise "Virtual"
+            end
+
+            # Checks whether a connection is still alive. Must be implemented by subclasses.
+            def connection_alive?(conn)
+                raise "Virtual"
+            end
+
+            def inherited(subclass)
+                subclass.instance_variable_set("@capabilities", @capabilities)
+            end
+            
         end
         
+        
         def initialize(url)
             @url = url
             @configuration = {}
@@ -31,6 +101,89 @@ module Spider; module Model; module Storage
             raise StorageException, "Unimplemented"
         end
         
+        def supports?(capability)
+            self.class.supports?(capability)
+        end
+        
+        def curr
+            var = nil
+            if Spider.conf.get('storage.shared_connection')
+                $STORAGES ||= {}
+                var = $STORAGES
+            else
+                var = Thread.current
+            end
+            var[:storages] ||= {}
+            var[:storages][self.class.storage_type] ||= {}
+            var[:storages][self.class.storage_type][@connection_params] ||= {
+                :transaction_nesting => 0, :savepoints => []
+            }
+        end
+        
+        def connection_pool
+            self.class.connection_pools[@connection_params]
+        end
+        
+        # Instantiates a new connection with current connection params.
+        def connect
+            return self.class.get_connection(*@connection_params)
+            #Spider::Logger.debug("#{self.class.name} in thread #{Thread.current} acquired connection #{@conn}")
+        end
+        
+        # True if currently connected.
+        def connected?
+            curr[:conn] != nil
+        end
+
+        
+        # Returns the current connection, or creates a new one.
+        # If a block is given, will release the connection after yielding.
+        def connection
+            curr[:conn] = connect
+            if block_given?
+                yield curr[:conn]
+                release # unless is_connected
+                return true
+            else
+                return curr[:conn]
+            end
+        end
+        
+        def self.connection_attributes
+            @connection_attributes ||= {}
+        end
+        
+        def connection_attributes
+            self.class.connection_attributes[connection] ||= {}
+        end
+        
+        # Releases the current connection to the pool.
+        def release
+            # The subclass should check if the connection is alive, and if it is not call remove_connection instead
+            c = curr[:conn]
+            #Spider.logger.debug("#{self} in thread #{Thread.current} releasing #{curr[:conn]}")
+            curr[:conn] = nil
+            self.class.release_connection(c, @connection_params)
+            #Spider.logger.debug("#{self} in thread #{Thread.current} released #{curr[:conn]}")
+            return nil
+            #@conn = nil
+        end
+        
+        # Prepares a value for saving.
+        def value_for_save(type, value, save_mode)
+            return prepare_value(type, value)
+        end
+        
+        # Prepares a value that will be used in a condition.
+        def value_for_condition(type, value)
+            return prepare_value(type, value)
+        end
+        
+        def value_to_mapper(type, value)
+            value
+        end
+        
+        
         def prepare_value(type, value)
             return value
         end
@@ -41,6 +194,106 @@ module Spider; module Model; module Storage
             return true
         end
         
+        
+        def supports_transactions?
+            return self.class.supports?(:transactions)
+        end
+        
+        def transactions_enabled?
+            @configuration['enable_transactions'] && supports_transactions?
+        end
+        
+        def start_transaction
+            return unless transactions_enabled?
+            curr[:transaction_nesting] += 1
+            return savepoint("point#{curr[:savepoints].length}") if in_transaction?
+        
+            Spider.logger.debug("#{self.class.name} starting transaction for connection #{connection.object_id}")
+            do_start_transaction
+            return true
+        end
+        
+        # May be implemented by subclasses.
+        def do_start_transaction
+           raise StorageException, "The current storage does not support transactions" 
+        end
+        
+        def in_transaction
+            if in_transaction?
+                curr[:transaction_nesting] += 1
+                return true
+            else
+                start_transaction
+                return false
+            end
+        end
+        
+        def in_transaction?
+            return false
+        end
+        
+        
+        def commit
+            return false unless transactions_enabled?
+            raise StorageException, "Commit without a transaction" unless in_transaction?
+            return curr[:savepoints].pop unless curr[:savepoints].empty?
+            commit!
+        end
+        
+        def commit_or_continue
+            return false unless transactions_enabled?
+            raise StorageException, "Commit without a transaction" unless in_transaction?
+            if curr[:transaction_nesting] == 1
+                commit
+                curr[:transaction_nesting] = 0
+                return true
+            else
+                curr[:transaction_nesting] -= 1
+            end
+        end
+        
+        def commit!
+            Spider.logger.debug("#{self.class.name} commit connection #{curr[:conn].object_id}")
+            curr[:transaction_nesting] = 0
+            do_commit
+            release
+        end
+        
+        def do_commit
+            raise StorageException, "The current storage does not support transactions" 
+        end
+        
+        def rollback
+            raise "Can't rollback in a nested transaction" if curr[:transaction_nesting] > 1
+            return rollback_savepoint(curr[:savepoints].last) unless curr[:savepoints].empty?
+            rollback!
+        end
+        
+        def rollback!
+            curr[:transaction_nesting] = 0
+            Spider.logger.debug("#{self.class.name} rollback")
+            do_rollback
+            curr[:savepoints] = []
+            release
+        end
+        
+        def do_rollback
+            raise StorageException, "The current storage does not support transactions" 
+        end
+        
+        def savepoint(name)
+            curr[:savepoints] << name
+        end
+        
+        def rollback_savepoint(name=nil)
+            if name
+                curr[:savepoints] = curr[:savepoints][0,(curr[:savepoints].index(name))]
+                name
+            else
+                curr[:savepoints].pop
+            end
+        end
+        
         # Utility methods
         
         def sequence_file_path(name)
@@ -99,7 +352,7 @@ module Spider; module Model; module Storage
             self.class.sequence_sync.lock(::Sync::UN)
             return seq
         end
-            
+        
         
     end
     

data/lib/spiderfw/model/storage/{db/db_connection_pool.rb → connection_pool.rb}

@@ -1,8 +1,8 @@
 require 'monitor'
 
-module Spider; module Model; module Storage; module Db
+module Spider; module Model; module Storage
 
-    class DbConnectionPool
+    class ConnectionPool
         attr_reader :max_size
         attr_accessor :timeout, :retry
 
@@ -11,21 +11,13 @@ module Spider; module Model; module Storage; module Db
             @provider = provider
             @connection_mutex = Monitor.new
             @queue = @connection_mutex.new_cond
-            @max_size = Spider.conf.get('storage.db.pool.size')
+            @max_size = Spider.conf.get('storage.pool.size')
             @max_size = provider.max_connections if provider.max_connections && provider.max_connections < @max_size
-            @timeout = Spider.conf.get('storage.db.pool.timeout')
-            @retry = Spider.conf.get('storage.db.pool.retry')
+            @timeout = Spider.conf.get('storage.pool.timeout')
+            @retry = Spider.conf.get('storage.pool.retry')
             @connections = []
             @free_connections = []
             @thread_connections = {}
-            # if Spider.runmode == 'devel'
-            #     Thread.new do
-            #         loop do
-            #             Spider.logger.debug("DB Pool: #{@connections.length} connections, #{@free_connections.length} free")
-            #             sleep(10)
-            #         end
-            #     end
-            # end
         end
         
         def size
@@ -35,18 +27,27 @@ module Spider; module Model; module Storage; module Db
         def free_size
             @free_connections.length
         end
+        
+        def storage_type
+            @provider.storage_type
+        end
 
         def get_connection
-            Thread.current[:db_connections] ||= {}
+            if Spider.conf.get('storage.shared_connection')
+                @shared_conn ||= _checkout
+                return @shared_conn
+            end
+            Thread.current[:storage_connections] ||= {}
+            Thread.current[:storage_connections][storage_type] ||= {}
             @connection_mutex.synchronize do
                 #Spider.logger.debug("DB Pool (#{Thread.current}): trying to get connection")
-                if conn = Thread.current[:db_connections][@connection_params]
+                if conn = Thread.current[:storage_connections][storage_type][@connection_params]
                     #Spider.logger.debug("DB Pool (#{Thread.current}): returning thread connection #{conn}")
                     @free_connections.delete(conn)
                     conn
                 else
                     conn = _checkout
-                    Thread.current[:db_connections][@connection_params] = conn
+                    Thread.current[:storage_connections][storage_type][@connection_params] = conn
                     @thread_connections[Thread.current.object_id] = [conn, Time.now]
                     conn
                 end
@@ -60,10 +61,13 @@ module Spider; module Model; module Storage; module Db
         end
         
         def release(conn)
+            if Spider.conf.get('storage.shared_connection')
+                return
+            end
             @connection_mutex.synchronize do
                 #Spider.logger.debug("DB Pool (#{Thread.current}): releasing #{conn}")
                 @free_connections << conn
-                Thread.current[:db_connections].delete(@connection_params)
+                Thread.current[:storage_connections][storage_type].delete(@connection_params)
                 @thread_connections.delete(Thread.current.object_id)
                 @queue.signal
             end
@@ -102,7 +106,7 @@ module Spider; module Model; module Storage; module Db
                             create_new_connection if @free_connections.empty? && @connections.length < @max_size
                             if @free_connections.empty?
                                 Spider.logger.error "#{Thread.current} GOT TIRED WAITING, #{@queue.count_waiters} IN QUEUE"
-                                raise StorageException, "Unable to get a db connection in #{@timeout} seconds" if @timeout
+                                raise StorageException, "Unable to get a #{storage_type} connection in #{@timeout} seconds" if @timeout
                             end
                         end
                     end
@@ -111,7 +115,7 @@ module Spider; module Model; module Storage; module Db
                 end
                 conn = @free_connections.pop
                 while conn && !@provider.connection_alive?(conn)
-                    Spider.logger.warn("DB Pool (#{Thread.current}): connection #{conn} dead")
+                    Spider.logger.warn("Storage #{storage_type} Pool (#{Thread.current}): connection #{conn} dead")
                     remove_connection(conn)
                     conn = nil
                     conn = @free_connections.pop unless @free_connections.empty?
@@ -158,7 +162,7 @@ module Spider; module Model; module Storage; module Db
         
         def create_new_connection
             conn = @provider.new_connection(*@connection_params)
-            Spider.logger.debug("DB Pool (#{Thread.current}): creating new connection #{conn} (#{@connections.length} already in pool)")
+            Spider.logger.debug("Storage #{storage_type } Pool (#{Thread.current}): creating new connection #{conn} (#{@connections.length} already in pool)")
             @connections << conn
             @free_connections << conn
         end
@@ -168,4 +172,4 @@ module Spider; module Model; module Storage; module Db
     end
 
 
-end; end; end; end
+end; end; end

data/lib/spiderfw/model/storage/db/adapters/mysql.rb

@@ -84,12 +84,17 @@ module Spider; module Model; module Storage; module Db
             conn.close
         end
         
+        def configure(conf)
+            super
+            @configuration['default_engine'] ||= Spider.conf.get('db.mysql.default_engine')
+        end
+        
         def release
             begin
                 #Spider::Logger.debug("MYSQL #{self.object_id} in thread #{Thread.current} releasing connection #{@conn}")
-                @conn.autocommit(true) if @conn
+                @conn.autocommit(true) if @conn && !Spider.conf.get('storage.db.shared_connection')
                 super
-            rescue exc
+            rescue => exc
                 Spider::Logger.error("MYSQL #{self.object_id} in thread #{Thread.current} exception #{exc.message} while trying to release connection #{@conn}")
                 self.class.remove_connection(@conn, @connection_params)
                 @conn = nil
@@ -151,7 +156,7 @@ module Spider; module Model; module Storage; module Db
         end
         
         def do_rollback
-            curr[:conn].rollback
+            curr[:conn].rollback if curr[:conn]
             curr[:in_transaction] = false
         end
         
@@ -168,10 +173,10 @@ module Spider; module Model; module Storage; module Db
                 curr[:last_executed] = [sql, bind_vars]
                 if (Spider.conf.get('storage.db.replace_debug_vars'))
                     cnt = -1
-                    debug("mysql executing: "+sql.gsub('?'){ debug_vars[cnt+=1] })
+                    debug("mysql #{curr[:conn]} executing: "+sql.gsub('?'){ debug_vars[cnt+=1] })
                 else
                     debug_vars_str = debug_vars ? debug_vars.join(', ') : ''
-                    debug("mysql executing:\n#{sql}\n[#{debug_vars_str}]")
+                    debug("mysql #{curr[:conn]} executing:\n#{sql}\n[#{debug_vars_str}]")
                 end
                 query_start
                 stmt = connection.prepare(sql)
@@ -300,6 +305,12 @@ module Spider; module Model; module Storage; module Db
              return sql
          end
          
+         def sql_create_table(create)
+             sqls = super
+             sqls[0] += " ENGINE=#{@configuration['default_engine']}" if @configuration['default_engine']
+             sqls
+         end
+         
          def function(func)
              return super unless func.func_name == :concat
              fields = func.elements.map{ |func_el|
@@ -426,6 +437,15 @@ module Spider; module Model; module Storage; module Db
              return db_attributes
          end
          
+         def function(func)
+             case func.func_name
+             when :rownum
+                 "if(@rn, @rn:=@rn+1, @rn:=1)-1"
+             else
+                 super
+             end
+         end
+         
          def schema_field_text_equal?(current, field)
              # FIXME
              return true

data/lib/spiderfw/model/storage/db/adapters/oracle.rb

@@ -359,7 +359,7 @@ module Spider; module Model; module Storage; module Db
          end
          
          def column_name(name)
-             super.upcase
+             shorten_identifier(super, 30).upcase
          end
          
          def sequence_name(name)