spiderfw 0.6.20 → 0.6.21

data/lib/spiderfw/http/server.rb

@@ -43,7 +43,12 @@ module Spider; module HTTP
             
             start = lambda{
                 $SPIDER_WEB_SERVER = true
-                require 'spiderfw'
+                begin
+                    require 'spiderfw/init'
+                rescue Exception => exc
+                    Spider.logger.error(exc)
+                    return
+                end
                 require 'spiderfw/controller/http_controller'
                 
                 port ||= Spider.conf.get('webserver.port')
@@ -54,11 +59,15 @@ module Spider; module HTTP
                 server = Spider::HTTP.const_get(servers[server_name]).new
                 ssl_server = nil
                 Spider.startup
-                if Spider.conf.get('devel.trace.extended')
-                    require 'ruby-debug'
-                    require 'spiderfw/utils/monkey/debugger'
-                    Debugger.start
-                    Debugger.post_mortem
+                begin
+                    if Spider.conf.get('devel.trace.extended')
+                        require 'ruby-debug'
+                        require 'spiderfw/utils/monkey/debugger'
+                        Debugger.start
+                        Debugger.post_mortem
+                    end
+                rescue Exception => exc
+                    Spider.logger.warn "Unable to start debugger"
                 end
                 
                 thread = Thread.new do
@@ -79,7 +88,7 @@ module Spider; module HTTP
                     end
                 end
                 do_shutdown = lambda{
-                    Debugger.post_mortem = false
+                    Debugger.post_mortem = false if defined?(Debugger)
                     # debugger
                     server.shutdown
                     ssl_server.shutdown if ssl_server
@@ -95,7 +104,7 @@ module Spider; module HTTP
                 ssl_thread.join if ssl_thread
             }
             if options[:daemonize]
-                require 'spiderfw'
+                require 'spiderfw/init'
                 require 'spiderfw/utils/fork'
                 pid_file = File.join(Spider.paths[:var], 'run/server.pid')
                 process_name = (options[:daemonize] == true) ? 'spider-server' : options[:daemonize]
@@ -117,9 +126,11 @@ module Spider; module HTTP
                 if Spider.conf.get('webserver.respawn_on_change')
                     Spider.start_loggers
                     begin
-                        begin
+                        gemfile = File.join(Spider.paths[:root], 'Gemfile')
+                        gemfile_lock = File.join(Spider.paths[:root], 'Gemfile.lock')
+                        if File.file?(gemfile) && File.file?(gemfile_lock)
+                            require 'bundler'
                             Bundler.require :default, Spider.runmode.to_sym
-                        rescue
                         end
                         spawner = Spawner.new({'spawn' => start})
                         spawner.run('spawn')
@@ -132,7 +143,11 @@ module Spider; module HTTP
                 unless spawner_started
                     Spider.main_process_startup
                     Spider.startup
-                    start.call 
+                    begin
+                        start.call 
+                    rescue Exception => exc
+                        Spider.logger.error(exc)
+                    end
                 end
             end
         end

data/lib/spiderfw/i18n/cldr.rb

@@ -40,8 +40,8 @@ module Spider; module I18n
             # FIXME: handle more efficiently
             d = ['sun', 'mon', 'tue', 'wed', 'thu', 'fri', 'sat']
             obj_d = d[object.wday]
-            days = @cldr.calendar.days[options[:calendar].to_sym]
-            months = @cldr.calendar.months[options[:calendar].to_sym]
+            days = @cldr.calendar.days[options[:calendar].to_sym][:format] || @cldr.calendar.days[options[:calendar].to_sym]
+            months = @cldr.calendar.months[options[:calendar].to_sym][:format] || @cldr.calendar.months[options[:calendar].to_sym]
             replacements = [
                 [/y{1,4}/, '%Y'], # year  don't use two digits year, they cause confusion [/y{1,2}/, '%y']
                 [/M{5}/, months[:narrow][object.month.to_s]], [/M{4}/, months[:wide][object.month.to_s]], #month
@@ -115,9 +115,9 @@ module Spider; module I18n
             end
         end
         
-        def day_names(format = :wide, calendar = self.default_calendar)
+        def day_names(format = :wide, context = :stand_alone, calendar = self.default_calendar)
             begin
-                days = @cldr.calendar.days[calendar][format]
+                days = @cldr.calendar.days[calendar][format] || @cldr.calendar.days[calendar][context][format]
                 return [days['sun'], days['mon'], days['tue'], days['wed'], days['thu'], days['fri'], days['sat']]
             rescue NoMethodError
                 raise ArgumentError, "Calendar #{calendar} not found" unless @cldr.days[calendar]
@@ -126,10 +126,12 @@ module Spider; module I18n
             
         end
         
-        def month_names(format = :wide, calendar = self.default_calendar)
+        def month_names(format = :wide, context = :stand_alone, calendar = self.default_calendar)
             months = []
             begin
-                @cldr.calendar.months[calendar][format].each do |k, v|
+                # CLDR 1 doesn't have context, CLDR 2 does
+                months = @cldr.calendar.months[calendar][format] || @cldr.calendar.months[calendar][context][format]
+                months.each do |k, v|
                     months[k.to_i] = v
                 end
             rescue NoMethodError

data/lib/spiderfw/i18n/javascript_parser.rb

@@ -1,5 +1,6 @@
 require 'gettext/tools'
 require 'spiderfw/templates/blocks/text'
+require 'json'
 
 module Spider; module I18n
 

data/lib/spiderfw/init.rb

@@ -0,0 +1,2 @@
+require 'spiderfw/spider'
+Spider.init

data/lib/spiderfw/model/base_model.rb

@@ -957,6 +957,11 @@ module Spider; module Model
         def self.elements
             @elements
         end
+
+        # An array of non-integrated elements
+        def self.own_elements
+            elements_array.reject{ |el| el.integrated? }
+        end
         
         # An array of the model's Elements.
         def self.elements_array
@@ -1251,8 +1256,8 @@ module Spider; module Model
             self.load(values) || self.create(values)
         end
         
-        def self.get(values)
-            return self.new(values) unless Spider::Model.identity_mapper
+        def self.get(values, static=false)
+            return static ? self.static(values) : self.new(values) unless Spider::Model.identity_mapper
             values = [values] unless values.is_a?(Hash) || values.is_a?(Array)
             if values.is_a?(Array)
                 vals = {}
@@ -1262,11 +1267,16 @@ module Spider; module Model
                 values = vals
             end
             curr = Spider::Model.identity_mapper.get(self, values)
+            curr.autoload = false if static
             return curr if curr
-            obj = self.new(values)
+            obj = static ? self.static(values) : self.new(values)
             Spider::Model.identity_mapper.put(obj)
             obj
         end
+
+        def self.get_static(values)
+            self.get(values, true)
+        end
         
         def set_values(values)
             if (values.is_a? Hash)

data/lib/spiderfw/model/condition.rb

@@ -144,6 +144,10 @@ module Spider; module Model
                 sub.all_each_with_comparison{ |k, v, c| yield k, v, c }
             end
         end
+
+        def primary_keys_only?(model)
+            self.select{ |key, value| !model.elements[key] || !model.elements[key].primary_key? }.empty?
+        end
         
         # Returns the result of merging the condition with another one (does not modify the original condition).
         def +(condition)

data/lib/spiderfw/model/mappers/db_mapper.rb

@@ -595,8 +595,7 @@ module Spider; module Model; module Mappers
                             el_join_info[jk[k.to_s.length+1..-1]] = jv
                         end
                     end
-                    if (v && model.mapper.have_references?(element.name) && v.select{ |key, value| 
-                        !element.model.elements[key] || !element.model.elements[key].primary_key? }.empty?)
+                    if v && model.mapper.have_references?(element.name) && v.primary_keys_only?(element.model)
                         # 1/n <-> 1 with only primary keys
                         element_cond = {:conj => 'AND', :values => [], :is_having => is_having}
                         v.each_with_comparison do |el_k, el_v, el_comp|
@@ -781,20 +780,23 @@ module Spider; module Model; module Mappers
             joins = []
             el = nil
 #            Spider::Logger.debug("GETTING DEEP JOIN TO #{dotted_element} (#{@model})")
+            cnt = 0
             parts.each do |part|
+                cnt += 1
                 el = current_model.elements[part]
                 raise "Can't find element #{part} in model #{current_model}" unless el
-                if (el.integrated?)
+                next if have_references?(el) && cnt == parts.length
+                if el.integrated?
                     joins << current_model.mapper.get_join(el.integrated_from)
                     current_model = el.integrated_from.type
                     el = current_model.elements[el.integrated_from_element]
                 end
-                if (el.model? && can_join?(el))
+                if el.model? && can_join?(el)
                     joins << current_model.mapper.get_join(el)
                     current_model = el.model
                 end
             end
-            while (el.integrated?)
+            while el.integrated? && !have_references?(el)
                 joins << current_model.mapper.get_join(el.integrated_from)
 #                joins << current_model.integrated_from.mapper.get_join(el.integrated_from_element)
                 current_model = el.integrated_from.type
@@ -845,9 +847,8 @@ module Spider; module Model; module Mappers
                     fields << [field, direction]
                 else
                     el_joins, el_model, el = get_deep_join(order_element)
-                    if (el.model?)
-                        # FIXME: integrated elements
-                        if el.model.storage != storage
+                    if el.model?
+                        if el_model.mapper.have_references?(el) || el.model.storage != storage
                             el.model.primary_keys.each do |pk|
                                 fields << [el_model.mapper.schema.foreign_key_field(el.name, pk.name), direction]
                             end

data/lib/spiderfw/model/mappers/mapper.rb

@@ -20,7 +20,6 @@ module Spider; module Model
         def initialize(model, storage)
             @model = model
             @storage = storage
-            @raw_data = {}
             @options = {}
             @no_map_elements = {}
             @sequences = []
@@ -416,7 +415,10 @@ module Spider; module Model
         
         # Deletes an object, or objects according to a condition.
         # Will not delete with null condition (i.e. all objects) unless force is true
-        def delete(obj_or_condition, force=false)
+        # Options can be:
+        # :keep_single_reverse: don't delete associations that have a single reverse.
+        # Useful when an object will be re-inserted with the same keys.
+        def delete(obj_or_condition, force=false, options={})
             
             def prepare_delete_condition(obj)
                 condition = Condition.and
@@ -459,6 +461,7 @@ module Spider; module Model
                     end
                     vals << obj_vals
                     assocs.each do |el|
+                        next if el.has_single_reverse? && options[:keep_single_reverse]
                         delete_element_associations(curr_obj, el)
                     end
                 end
@@ -580,7 +583,6 @@ module Spider; module Model
                     @model.primary_keys.each{ |key| query.request[key] = true}
                     expand_request(query.request, set) unless options[:no_expand_request] || !query.request.expandable?
                     query = prepare_query(query, query_set)
-                    query.request.total_rows = true unless query.request.total_rows == false
                     result = fetch(query)
                     if !result || result.empty?
                         set.each_current do |obj|
@@ -600,7 +602,6 @@ module Spider; module Model
                         next unless obj
                         merged_obj = merge_object(set, obj, query.request)
                         merged[merged_obj.object_id] = true
-                        @raw_data[obj.object_id] = row
                     end
                     query.request.keys.each do |k, v|
                         set.element_loaded(k) if have_references?(k)
@@ -932,8 +933,7 @@ module Spider; module Model
                     next if k.is_a?(Spider::QueryFuncs::Function)
                     next unless element = model.elements[k]
                     changed_v = false
-                    if element.type < Spider::DataType && !v.is_a?(element.type)
-                        condition.delete(k)
+                    if element.type < Spider::DataType && !v.is_a?(element.type) && element.type.force_wrap?
                         begin
                             v = element.type.from_value(v)
                             changed_v = true
@@ -944,8 +944,10 @@ module Spider; module Model
                         v = DateTime.parse(v)
                         changed_v = true
                     elsif element.model? && v.is_a?(Spider::Model::Condition)
-                        v = element.mapper.preprocess_condition(v)
-                        changed_v = true
+                        unless v.primary_keys_only?(element.model)
+                            v = element.mapper.preprocess_condition(v)
+                            changed_v = true
+                        end
                     end
                     if element.integrated?
                         condition.delete(k)
@@ -953,15 +955,21 @@ module Spider; module Model
                         integrated_from_element = element.integrated_from_element
                         sub = condition.get_deep_obj
                         sub.set(integrated_from_element, c, v)
-                        condition[integrated_from.name] = integrated_from.model.mapper.preprocess_condition(sub) 
+                        unless sub.primary_keys_only?(integrated_from.model)
+                            sub = integrated_from.model.mapper.preprocess_condition(sub) 
+                        end
+                        condition[integrated_from.name] = sub
                     elsif element.junction? && !v.is_a?(BaseModel) && !v.is_a?(Hash) && !v.nil? # conditions on junction id don't make sense
                         condition.delete(k)
-                        condition.set("#{k}.#{element.attributes[:junction_their_element]}", c, v)
+                        sub = condition.get_deep_obj
+                        sub.set(element.attributes[:junction_their_element], c, v)
+                        condition[k] = element.model.mapper.preprocess_condition(sub)
                     elsif changed_v
                         condition.delete(k)
                         condition.set(k, c, v)
                     end
                 end
+                condition
             end
             
             basic_preprocess(condition)

data/lib/spiderfw/model/migrations/drop_table.rb

@@ -0,0 +1,20 @@
+module Spider; module Migrations
+
+    class DropTable < IrreversibleMigration
+
+        def initialize(model, options={})
+            @model = model
+            @options = options
+        end
+
+        def run
+            table = @options[:table_name]
+            if !table
+                table = @model.mapper.schema.table.name
+            end
+            @model.mapper.storage.drop_table(table)
+        end
+
+    end
+
+end; end

data/lib/spiderfw/model/migrations/replace.rb

@@ -3,28 +3,33 @@ module Spider; module Migrations
     class Replace < Migration
         
         def initialize(model, element, values)
-            @model = model
+            @models = model
+            @models = [@models] unless @models.is_a?(Enumerable)
             @element = element
             @values = values
         end
         
         def run
-            field = @model.mapper.schema.field(@model.get_element(@element).name)
-            table = @model.mapper.schema.table
-            raise "Table #{table} does not have a field #{field}" unless field
-            @values.each do |from, to|
-                save = {
-                    :table => table, 
-                    :values => {field => to},
-                    :condition => {:values => [[field, '=', from]]}
-                }
-                sql, bind_vars = @model.storage.sql_update(save)
-                return @model.storage.execute(sql, *bind_vars)                
+            @models.each do |model|
+                field = model.mapper.schema.field(model.get_element(@element).name)
+                table = model.mapper.schema.table
+                raise "Table #{table} does not have a field #{field}" unless field
+                @values.each do |from, to|
+                    save = {
+                        :table => table, 
+                        :values => {field => to},
+                        :condition => {:values => [[field, '=', from]]}
+                    }
+                    sql, bind_vars = model.storage.sql_update(save)
+                    model.storage.execute(sql, *bind_vars)                
+                end
             end
         end
         
         def undo
-            Replace.new(@model, @element, @values.invert).run
+            @models.each do |model|
+                Replace.new(model, @element, @values.invert).run
+            end
         end
         
     end

data/lib/spiderfw/model/migrations.rb

@@ -2,6 +2,7 @@ require 'spiderfw/model/migrations/migration'
 require 'spiderfw/model/migrations/irreversible_migration'
 require 'spiderfw/model/migrations/replace'
 require 'spiderfw/model/migrations/drop_element'
+require 'spiderfw/model/migrations/drop_table'
 
 module Spider
     
@@ -14,6 +15,10 @@ module Spider
         def self.drop_element!(model, element, options={})
         	Spider::Migrations::DropElement.new(model, element, options={})
         end
+
+        def self.drop_table!(model, options={})
+            Spider::Migrations::DropTable.new(model, element, options={})
+        end
         
         
     end

data/lib/spiderfw/model/mixins/tree.rb

@@ -165,7 +165,7 @@ module Spider; module Model
                        cnt += 1
                        break if sub == self
                    end
-                   cnt = nil if cnt = 0
+                   cnt = nil if cnt == 0
                    cnt
                end
 

data/lib/spiderfw/model/model.rb

@@ -230,6 +230,7 @@ module Spider
             end
              # Ruby 1.9: steps are not needed with ordered hashes
             data = [data] unless data.is_a?(Array)
+            loaded = []
             data.each do |step|
                 step.each do |mod_name, mod_data|
                     mod = const_get_full(mod_name)
@@ -248,9 +249,11 @@ module Spider
                         end
                         obj = mod.new(h)
                         obj.insert
+                        loaded << obj
                     end
                 end
             end
+            loaded
         end
         
         # Generic Model error.

data/lib/spiderfw/model/storage/db/adapters/oracle.rb

@@ -67,7 +67,7 @@ module Spider; module Model; module Storage; module Db
              return nil unless curr[:last_executed]
              q = curr[:last_query].clone
              unless (q[:offset] || q[:limit])
-                 return curr[:last_result] ? curr[:last_result].length : nil
+                 return curr[:last_result_length] ? curr[:last_result_length] : nil
              end
              q.delete(:offset); q.delete(:limit)
              q[:query_type] = :count

data/lib/spiderfw/model/storage/db/adapters/sqlite.rb

@@ -121,7 +121,7 @@ module Spider; module Model; module Storage; module Db
              return nil unless curr[:last_query]
              q = curr[:last_query]
              unless (q[:offset] || q[:limit])
-                 return curr[:last_result] ? curr[:last_result].length : nil
+                 return curr[:last_result_length] ? curr[:last_result_length] : nil
              end
              q[:offset] = q[:limit] = nil
              q[:keys] = ["COUNT(*) AS N"]

data/lib/spiderfw/model/storage/db/connectors/jdbc_oracle.rb

@@ -257,8 +257,8 @@ module Spider; module Model; module Storage; module Db; module Connectors
                 end
                 if (res)
                     unless block_given?
+                        curr[:last_result_length] = result.length
                         result.extend(StorageResult)
-                        curr[:last_result] = result
                         return result
                     end
                 else

data/lib/spiderfw/model/storage/db/connectors/oci8.rb

@@ -125,8 +125,8 @@ module Spider; module Model; module Storage; module Db; module Connectors
                 end
                 if (have_result)
                     unless block_given?
+                        curr[:last_result_length] = result.length
                         result.extend(StorageResult)
-                        curr[:last_result] = result
                         return result
                     end
                 else

data/lib/spiderfw/model/storage/db/db_schema.rb

@@ -267,7 +267,10 @@ module Spider; module Model; module Storage; module Db
         attr_reader :expression
         
         def initialize(table, name, type, attributes={})
-            super
+            @table = table
+            @name = name.to_s
+            @type = type
+            @attributes = attributes
             @expression = attributes[:expression]
         end
         

data/lib/spiderfw/model/storage/db/dialects/no_total_rows.rb

@@ -6,7 +6,7 @@ module Spider; module Model; module Storage; module Db; module Dialects
             return nil unless @last_executed
             q = @last_query.clone
             unless (q[:offset] || q[:limit])
-                return @last_result ? @last_result.length : nil
+                return @last_result_length ? @last_result_length : nil
             end
             q.delete(:offset); q.delete(:limit); q[:order]= []
             q[:query_type] = :count

data/lib/spiderfw/setup/app_manager.rb

@@ -4,6 +4,10 @@ require 'fileutils'
 module Spider
 
     class AppManager
+
+        def initialize(options)
+            @options = options
+        end
         
         def self.installed?(app)
             require 'spiderfw/home'
@@ -269,7 +273,8 @@ module Spider
              require 'rubygems/package'
              client = AppServerClient.new(spec.app_server)
              print _("Fetching %s from server... ") % spec.app_id
-             tmp_path = client.fetch_app(spec.app_id)
+             options[:branch] ||= 'master'
+             tmp_path = client.fetch_app(spec.app_id, options[:branch])
              Spider.output _("Fetched.")
              dest = File.join(@home_path, "apps/#{spec.app_id}")
              FileUtils.mkdir_p(dest)
@@ -307,8 +312,9 @@ module Spider
             app_path = File.join(@home_path, "apps", spec.id)
             app_repo = Git.open(app_path)
             Spider.output _("Updating %s from %s") % [spec.app_id, spec.git_repo]
+            options[:branch] ||= 'master'
             Dir.chdir(app_path) do
-                app_repo.branch('master').checkout
+                app_repo.branch(options[:branch]).checkout
             end
             response = err = nil
             Dir.chdir(app_path) do
@@ -321,7 +327,7 @@ module Spider
             end
             Dir.chdir(app_path) do
                 app_repo.reset('HEAD', :hard => true)
-                app_repo.branch('master').checkout
+                app_repo.branch(options[:branch]).checkout
             end
             
             home_repo.add("apps/#{spec.id}")
@@ -401,11 +407,13 @@ module Spider
                 end
             end
             done_tasks = []
+
             
             tasks.each do |task|
                 Spider.output _("Running setup task #{path+'/'+task}...")
                 t = Spider::SetupTask.load("#{path}/#{task}")
                 t.app = app
+                raise "Can't run interactive task" if t.interactive? && !@options[:interactive]
                 begin
                     done_tasks << t
                     t.do_sync

data/lib/spiderfw/setup/app_server_client.rb

@@ -12,8 +12,8 @@ module Spider
             @url = url
         end
         
-        def specs
-            load_specs unless @specs
+        def specs(branch=nil)
+            load_specs(branch) unless @specs
             @specs
         end
         
@@ -34,29 +34,35 @@ module Spider
         end
         
         def load_specs
-            result = http_get(@url+'/list.json')
+            url = @url+'/list.json'
+            result = http_get(url)
             list = JSON.parse(result)
             @specs = list.map{ |app| Spider::App::AppSpec.parse_hash(app) }
         end
         
         def get_specs(app_ids)
             app_ids = [app_ids] unless app_ids.is_a?(Array)
-            result = http_get(@url+"/list/#{app_ids.join('+')}.json")
+            url = @url+"/list/#{app_ids.join('+')}.json"
+            result = http_get(url)
             JSON.parse(result).map{ |app| Spider::App::AppSpec.parse_hash(app) }
         end
         
         def get_deps(app_ids, options={})
             app_ids = [app_ids] unless app_ids.is_a?(Array)
             url = "#{@url}/deps/#{app_ids.join('+')}.json"
-            url += "?no_optional=true" if options[:no_optional]
+            params = []
+            params << 'no_optional=true' if options[:no_optional]
+            url += '?'+params.join('&') unless params.empty?
             result = http_get(url)
             JSON.parse(result).map{ |app| Spider::App::AppSpec.parse_hash(app) }
         end
         
-        def fetch_app(app_id)
+        def fetch_app(app_id, branch=nil)
             tmp = Tempfile.new("spider-app-archive")
             tmp.binmode
-            res = http_get(@url+"/pack/#{app_id}")
+            url = @url+"/pack/#{app_id}"
+            url += "?branch=#{branch}" if branch
+            res = http_get(url)
             tmp << res
             tmp.flush
             tmp.path