spiderfw 0.5.13 → 0.5.14

data/lib/spiderfw/model/storage/db/db_storage.rb

@@ -15,7 +15,8 @@ module Spider; module Model; module Storage; module Db
         @safe_conversions = {
             'TEXT' => ['LONGTEXT'],
             'INT' => ['TEXT', 'LONGTEXT', 'REAL'],
-            'REAL' => ['TEXT']
+            'REAL' => ['TEXT'],
+            'DATETIME' => ['DATE', 'TIME']
         }
         @capabilities = {
             :autoincrement => false,
@@ -86,6 +87,18 @@ module Spider; module Model; module Storage; module Db
             
         end
         
+        def query_start
+            curr[:query_start] = Time.now
+        end
+        
+        def query_finished
+            now = Time.now
+            diff = now - curr[:query_start]
+            diff = 0 if diff < 0 # ??? 
+            diff = diff*1000
+            Spider.logger.info("Db query (#{@instance_name}) done in #{diff}ms")
+        end
+        
         def curr
             Thread.current[:db_storages] ||= {}
             Thread.current[:db_storages][@connection_params] ||= {
@@ -313,7 +326,7 @@ module Spider; module Model; module Storage; module Db
                 'REAL'
             when 'BigDecimal', 'Spider::DataTypes::Decimal'
                 'DECIMAL'
-            when 'Date', 'DateTime'
+            when 'Date', 'DateTime', 'Time'
                 'DATE'
             when 'Spider::DataTypes::Binary'
                 'BLOB'
@@ -411,7 +424,7 @@ module Spider; module Model; module Storage; module Db
                     end
                 end
             when 'BigDecimal'
-                value = value.to_f
+                value = value.to_f if value
             end
             return value
         end
@@ -542,6 +555,9 @@ module Spider; module Model; module Storage; module Db
                 else
                     val = bound_vars ? '?' : value
                     sql = "#{key} #{comp} #{val}"
+                    if comp == '<>'
+                        sql = "(#{sql} or #{key} IS NULL)"
+                    end
                 end
             end
             return sql
@@ -570,7 +586,11 @@ module Spider; module Model; module Storage; module Db
                 to_t = join[:as] || join[:to]
                 sql_on = join[:keys].map{ |from_f, to_f|
                     to_field = to_f.is_a?(FieldExpression) ? to_f.expression : "#{to_t}.#{to_f.name}"
-                    "#{from_f} = #{to_field}"
+                    if from_f.is_a?(FieldExpression)
+                        "#{to_field} = #{from_f.expression}"
+                    else
+                        "#{from_f} = #{to_field}"
+                    end
                 }.join(' AND ')
                 if (join[:condition])
                     condition_sql, condition_values = sql_condition({:condition => join[:condition]})
@@ -616,7 +636,7 @@ module Spider; module Model; module Storage; module Db
         def sql_update(update)
             curr[:last_query_type] = :update
             values = []
-            tables = update[:table]
+            tables = update[:table].to_s
             if (update[:joins] && update[:joins][update[:table]])
                 join_str, join_values = sql_tables_join(update, update[:table])
                 tables += " "+join_str

data/lib/spiderfw/model/unit_of_work.rb

@@ -7,52 +7,133 @@ module Spider; module Model
         
         def initialize(&proc)
             @objects = {}
+            @actions = {}
+            @to_delete = {}
+            @new_objects = []
             if (proc)
-                Thread.current[:unit_of_work] = self
+                start
                 yield self
-                Thread.current[:unit_of_work] = nil
+                stop
             end
         end
         
-        def run() #(&proc)
+        def start
+            Spider.current[:unit_of_work] = self
+        end
+        
+        def stop
+            Spider.current[:unit_of_work] = nil
+        end
+        
+        def run #(&proc)
             #proc.call
             @tasks = {}
             @processed_tasks = {}
+            while objs = new_objects
+                objs.each do |obj|
+                    @actions[obj.object_id].each do |action, params|
+                        if action == :save
+                            next unless obj.mapper && obj.mapper.class.write?
+                            next unless obj.modified?
+                            obj.save_mode do
+                                obj.before_save
+                            end
+                        elsif action == :delete
+                            obj.before_delete
+                        end
+                    end
+                end
+            end
+            @running = true
             @objects.each do |obj_id, obj|
-                next unless obj.mapper && obj.mapper.class.write?
-                task = Spider::Model::MapperTask.new(obj, :save)
-                @tasks[task] ||= task
-                find_dependencies(task)
+                @actions[obj_id].each do |action, params|
+                    if action == :save
+                        next unless obj.mapper && obj.mapper.class.write?
+                        next unless obj.modified?
+                    end
+                    task = Spider::Model::MapperTask.new(obj, action, params)
+                    @tasks[task] = task
+                    find_dependencies(task)
+                end
             end
             tasks = tsort()
-            tasks.each{ |task| p task}
+            
+            Spider.logger.debug("Tasks:")
+            tasks.each do |task| 
+                Spider.logger.debug "-- #{task.action} on #{task.object.class} #{task.object.primary_keys}"
+            end
+            
             tasks.each do |task|
-                Spider::Logger.debug("Executing task #{task.inspect}")
+                #Spider::Logger.debug("Executing task #{task.inspect}")
                 task.execute()
             end
+            @objects = {}
+            @new_objects = []
+            @running = false
+        end
+        
+        def running?
+            @running
         end
         
+        alias :commit :run
+        
         def find_dependencies(model_task)
             return if (@processed_tasks[model_task])
             @processed_tasks[model_task] = true
-            dependencies = model_task.object.mapper.get_dependencies(model_task.object, model_task.action)
+            dependencies = model_task.object.mapper.get_dependencies(model_task.object, model_task.action).uniq
             dependencies.each do |dep|
+                had0 = @tasks[dep[0]]
                 @tasks[dep[0]] ||= dep[0]
+                had1 = @tasks[dep[1]]
                 @tasks[dep[1]] ||= dep[1]
                 @tasks[dep[0]] << @tasks[dep[1]]
-                find_dependencies(dep[1])
+                find_dependencies(dep[0]) unless had0
+                find_dependencies(dep[1]) unless had1
             end
         end
                 
         
-        def add(obj)
+        def add(obj, action = :save, params = {})
+            raise "Objects can't be added to the UnitOfWork while it is running" if @running
             if (obj.class == QuerySet)
                 obj.each do |item|
-                    @objects[item.object_id] = item
+                    add(item, action, params)
                 end
-            else
-                @objects[obj.object_id] = obj
+                return
+            end
+            curr = @actions[obj.object_id]
+            if curr && (curr_act = curr.select{ |c| c[0] == action }).length > 0
+                curr_act.each{ |c| c[1] = params}
+                return
+            end
+            if action == :delete # FIXME: abstract
+                @actions[obj.object_id] = []
+            end
+            @actions[obj.object_id] ||= []
+            @actions[obj.object_id] << [action, params]
+            @objects[obj.object_id] = obj
+            @new_objects << obj unless curr
+            traverse(obj, action, params) if action == :save
+        end
+        
+        def traverse(obj, action, params)
+            obj.class.elements_array.each do |el|
+                next unless obj.element_has_value?(el)
+                next unless el.model?
+                add(obj.get(el), action, params)
             end
+            
+        end
+        
+        def to_delete(obj)
+            
+        end
+        
+        def new_objects
+            objects = @new_objects.clone
+            @new_objects = []
+            objects.length > 0 ? objects : nil
         end
         
         

data/lib/spiderfw/requires.rb

@@ -1,6 +1,7 @@
 require 'spiderfw/i18n/gettext'
 require 'spiderfw/i18n/i18n'
 require 'spiderfw/exceptions'
+require 'spiderfw/tag/tag'
 require 'spiderfw/utils/monkey/kernel'
 require 'spiderfw/utils/monkey/module'
 require 'spiderfw/utils/monkey/class'
@@ -11,14 +12,21 @@ require 'spiderfw/utils/monkey/numeric'
 require 'spiderfw/utils/monkey/exception'
 require 'spiderfw/utils/monkey/nil_class'
 require 'spiderfw/utils/monkey/object'
+require 'spiderfw/utils/json'
 require 'spiderfw/utils/multi_level_hash'
 require 'spiderfw/utils/thread_out'
 require 'spiderfw/utils/inflector'
 require 'spiderfw/utils/periodic_runner'
 require 'spiderfw/utils/fork'
+require 'spiderfw/utils/events/event_source'
+require 'spiderfw/utils/profiling'
+require 'spiderfw/utils/ordered_hash'
+require 'spiderfw/utils/memory'
+require 'spiderfw/utils/secure_random'
 require 'spiderfw/config/configuration'
 require 'spiderfw/config/configurable'
 require 'spiderfw/model/model'
 require 'spiderfw/home'
+require 'spiderfw/site'
 require 'spiderfw/resource'
-
+require 'rational'

data/lib/spiderfw/setup/app_manager.rb

@@ -0,0 +1,81 @@
+require 'tempfile'
+require 'fileutils'
+
+module Spider
+
+    module AppManager
+
+        def self.install(specs, home_path, options)
+            options[:use_git] = true unless options[:use_git] == false
+            specs = [specs] unless specs.is_a?(Array)
+            pre_setup(specs, options)
+            specs.each do |spec|
+                if spec.git_repo && options[:use_git]
+                    git_install(spec, home_path)
+                else
+                    pack_install(spec, home_path)
+                end
+            end
+        end
+
+        def self.git_install(spec, home_path)
+            require 'grit'
+            repo = Grit::Repo.new(home_path)
+            puts _("Fetching %s from %s") % [spec.app_id, spec.git_repo]
+            `#{Grit::Git.git_binary} submodule add #{spec.git_repo} apps/#{spec.id}`
+            repo.git.submodule({}, "init")
+            repo.git.submodule({}, "update")
+            repo.add('.gitmodules', "apps/#{spec.id}")
+            repo.commit_index(_("Added app %s") % spec.id) 
+        end
+
+        def self.pack_install(spec, home_path)
+            require 'rubygems/package'
+            client = AppServerClient.new(spec.app_server)
+            print _("Fetching %s from server... ") % spec.app_id
+            tmp_path = client.fetch_app(spec.app_id)
+            puts _("Fetched.")
+            dest = File.join(home_path, "apps/#{spec.app_id}")
+            FileUtils.mkdir_p(dest)
+            open tmp_path, Gem.binary_mode do |io|
+                Gem::Package::TarReader.new(io) do |reader|
+                    reader.each do |entry|
+                        dest_path = File.join(dest, entry.full_name)
+                        if entry.directory?
+                            FileUtils.mkdir(dest_path)
+                        elsif entry.file?
+                            File.open(dest_path, 'w') do |f|
+                                f << entry.read
+                            end
+                        end
+                    end
+                end
+            end
+
+        end
+        
+        def self.pre_setup(specs, options={})
+            require 'rubygems'
+            require 'rubygems/command.rb'
+            require 'rubygems/dependency_installer.rb'
+            unless options[:no_gems]
+                gems = specs.map{ |s| s.gems }
+                # unless options[:no_optional_gems]
+                #     gems += specs.map{ |s| s.gems_optional }
+                # end
+                gems = gems.flatten.uniq
+                gems.reject!{ |g| Gem.available?(g) }
+                unless gems.empty?
+                    puts _("Installing the following needed gems:")
+                    puts gems.inspect
+                    inst = Gem::DependencyInstaller.new
+                    gems.each do |g|
+                        inst.install g
+                    end
+                end
+            end
+        end
+
+    end
+
+end

data/lib/spiderfw/setup/app_server_client.rb

@@ -0,0 +1,59 @@
+require 'net/http'
+require 'uri'
+require 'json'
+require 'open-uri'
+require 'spiderfw/app'
+
+module Spider
+
+    class AppServerClient
+        
+        def initialize(url)
+            @url = url
+        end
+        
+        def specs
+            load_specs unless @specs
+            @specs
+        end
+        
+        def http_get(url)
+            uri = URI.parse(url)
+            proxy = uri.find_proxy
+            klass = proxy ? Net::HTTP::Proxy(proxy.host, proxy.port) : Net::HTTP
+            result = klass.get_response(uri)
+            raise "#{result.code} #{result.message} #{uri}" if result.is_a?(Net::HTTPClientError)
+            result.body
+        end
+        
+        def load_specs
+            result = http_get(@url+'/list.json')
+            list = JSON.parse(result)
+            @specs = list.map{ |app| Spider::App::AppSpec.parse_hash(app) }
+        end
+        
+        def get_specs(app_ids)
+            app_ids = [app_ids] unless app_ids.is_a?(Array)
+            result = http_get(@url+"/list/#{app_ids.join('+')}.json")
+            JSON.parse(result).map{ |app| Spider::App::AppSpec.parse_hash(app) }
+        end
+        
+        def get_deps(app_ids, options={})
+            app_ids = [app_ids] unless app_ids.is_a?(Array)
+            url = "#{@url}/deps/#{app_ids.join('+')}.json"
+            url += "?no_optional=true" if options[:no_optional]
+            result = http_get(url)
+            JSON.parse(result).map{ |app| Spider::App::AppSpec.parse_hash(app) }
+        end
+        
+        def fetch_app(app_id)
+            tmp = Tempfile.open("spider-app-archive")
+            res = http_get(@url+"/pack/#{app_id}")
+            tmp << res
+            tmp.path
+        end
+        
+
+    end 
+
+end

data/lib/spiderfw/setup/setup_task.rb

@@ -2,29 +2,33 @@ module Spider
     
     class SetupTask
         attr_reader :path, :version
+        attr_accessor :up, :down, :app
         
         class <<self
             def tasks
                 @tasks ||= {}
             end
             
-            def inherited(subclass)
-                @last_class = subclass
-            end
+            # def inherited(subclass)
+            #     @last_class = subclass
+            # end
             
             def load(path)
                 crit = Thread.critical
                 Thread.critical = true unless crit
                 Kernel.load(path)
-                obj = @last_class.new(path)
+                obj = self.new(path)
+                obj.up = Setup.up
+                obj.down = Setup.down
+                #obj = @last_class.new(path)
 #                Kernel.send(:remove_const, @last_class.name)
-                @last_class = nil
+                #@last_class = nil
                 Thread.critical = crit
                 return obj
             end
         end
         
-        def initialize(path)
+        def initialize(path, up=nil, down=nil)
             @path = path
             name = File.basename(path, '.rb')
             if (name =~ /^((?:\d\.?){1,3})/)
@@ -38,7 +42,65 @@ module Spider
             @version = Gem::Version.new(version)
             @desc = desc
         end
+        
+        def do_up
+            intance_eval(&@up)
+            sync_schema unless @no_sync || @sync_done
+        end
+        
+        def do_down
+            instance_eval(&@down)
+        end
+        
+        def no_sync_schema
+            @no_sync = true
+        end
+        
+        def sync_schema(*models)
+            if models[-1].is_a?(Hash)
+                options = models.pop
+            else
+                options = {}
+            end
+            if models.empty?
+                models = @app.models.reject{ |m| !(m < Spider::Model::Managed) }
+            end
+            Spider::Model.sync_schema(
+                model, options[:force], 
+                :drop_fields => options[:drop_fields], 
+                :update_sequences => options[:update_sequences], 
+                :no_foreign_key_constraints => options[:no_foreign_key_constraints]
+            )
+            @sync_done = true
+        end
+        
+        def sync_schema!(*models)
+            if models[-1].is_a?(Hash)
+                options = models.pop
+            else
+                options = {}
+            end
+            options[:force] = true
+            args = models + [options]
+            sync_schema(*args)
+        end
+        
 
     end
     
+    module Setup
+        
+        
+        def self.up(&proc)
+            @up = proc if proc
+            @up
+        end
+        
+        def self.down(&proc)
+            @down = proc if proc
+            @down
+        end
+        
+    end
+    
 end