spiderfw 0.6.21 → 0.6.22

data/lib/spiderfw/model/storage/db/db_storage.rb

@@ -22,6 +22,7 @@ module Spider; module Model; module Storage; module Db
             :sequences => true,
             :transactions => true
         }
+        @fixed_length_types = ['TIME', 'DATE', 'DATETIME']
 
         class << self
             # An Array of keywords that can not be used in schema names.
@@ -30,6 +31,8 @@ module Spider; module Model; module Storage; module Db
             attr_reader :type_synonyms
             # Type conversions which do not lose data. See also #safe_schema_conversion?
             attr_reader :safe_conversions
+            # Types for which we can safely ignore length in conversions
+            attr_reader :fixed_length_types
 
 
             def storage_type
@@ -40,6 +43,7 @@ module Spider; module Model; module Storage; module Db
                 subclass.instance_variable_set("@reserved_keywords", @reserved_keywords)
                 subclass.instance_variable_set("@type_synonyms", @type_synonyms)
                 subclass.instance_variable_set("@safe_conversions", @safe_conversions)
+                subclass.instance_variable_set("@fixed_length_types", @fixed_length_types)
                 super
             end
 
@@ -249,17 +253,17 @@ module Spider; module Model; module Storage; module Db
             sql = "SELECT #{sql_keys(query)} FROM #{tables_sql} "
             bind_vars += tables_values
             where, vals = sql_condition(query)
-            bind_vars += vals
+            bind_vars += vals if vals
             sql += "WHERE #{where} " if where && !where.empty?
             having, having_vals = sql_condition(query, true)
-            unless having.blank?
-                group_fields = (
+            unless having.blank? && query[:group_by].blank?
+                group_fields = query[:group_by] || (
                     query[:keys].select{ |k| !k.is_a?(FieldExpression)
                 } + collect_having_fields(query[:condition])).flatten.uniq
                 group_keys = sql_keys(group_fields)
                 sql += "GROUP BY #{group_keys} "
-                sql += "HAVING #{having} "
-                bind_vars += having_vals
+                sql += "HAVING #{having} " unless having.blank?
+                bind_vars += having_vals if having_vals
             end
             order = sql_order(query)
             sql += "ORDER BY #{order} " if order && !order.empty?
@@ -275,7 +279,7 @@ module Spider; module Model; module Storage; module Db
         # Returns the SQL for select keys.
         def sql_keys(query)
             query = {:keys => query} unless query.is_a?(Hash)
-            query[:keys].join(',')
+            query[:keys].join(', ')
         end
         
         # Returns an array containing the 'FROM' part of an SQL query (including joins),
@@ -427,8 +431,9 @@ module Spider; module Model; module Storage; module Db
             return query[:order].map{|o| 
                 repl = replacements[o[0].to_s]
                 ofield = repl ? repl : o[0]
+                ofield = ofield.name if ofield.is_a?(FieldExpression)
                 "#{ofield} #{o[1]}"
-            }.join(' ,')
+            }.join(', ')
         end
         
         # Returns the LIMIT and OFFSET SQL.
@@ -448,7 +453,7 @@ module Spider; module Model; module Storage; module Db
         end
         
         # Returns SQL and values for an update statement.
-        def sql_update(update)
+        def sql_update(update, allow_all=false)
             curr[:last_query_type] = :update
             values = []
             tables = update[:table].to_s
@@ -462,7 +467,8 @@ module Spider; module Model; module Storage; module Db
             sql += sql_update_values(update)
             where, bind_vars = sql_condition(update)
             values += bind_vars
-            sql += " WHERE #{where}"
+            raise "Update without conditions" if where.blank? && !allow_all
+            sql += " WHERE #{where}" unless where.blank?
             return [sql, values]
         end
         
@@ -524,8 +530,10 @@ module Spider; module Model; module Storage; module Db
                 sqls += sql_alter_field(table_name, field[:name], field[:type], field[:attributes])
             end
             if (alter_attributes[:primary_keys] && !alter_attributes[:primary_keys].empty?)
-                sqls << sql_drop_primary_key(table_name) if (current[:primary_keys] && !current[:primary_keys].empty? && current[:primary_keys] != alter_attributes[:primary_keys])
-                sqls << sql_create_primary_key(table_name, alter_attributes[:primary_keys])
+                sqls.unshift sql_drop_primary_key(table_name) if (current[:primary_keys] && !current[:primary_keys].empty? && current[:primary_keys] != alter_attributes[:primary_keys])
+                # unshift avoids problems with a field being created with primary key before the drop
+                sql_pk = sql_create_primary_key(table_name, alter_attributes[:primary_keys])
+                sqls << sql_pk if sql_pk
             end
             if (alter_attributes[:foreign_key_constraints])
                 cur_fkc = current && current[:foreign_key_constraints] ? current[:foreign_key_constraints] : []
@@ -567,6 +575,12 @@ module Spider; module Model; module Storage; module Db
             sqls = sql_drop_field(table_name, field_name)
             sqls.each{ |sql| execute(sql) }
         end
+
+        def change_field(table_name, field_name, new_field_name, type, attributes)
+            sqls = sql_change_field(table_name, field_name, new_field_name, type, attributes)
+            sqls.each{ |sql| execute(sql) }
+
+        end
         
         # Drops a table from the DB.
         def drop_table(table_name)
@@ -622,6 +636,10 @@ module Spider; module Model; module Storage; module Db
         def sql_drop_table(table_name)
             ["DROP TABLE #{table_name}"]
         end
+
+        def sql_change_field(table_name, field, new_field, type, attributes)
+            ["ALTER TABLE #{table_name} CHANGE #{field} #{sql_table_field(new_field, type, attributes)}"]
+        end
         
         # Checks if a DB field is equal to a schema field.
         def schema_field_equal?(current, field)
@@ -630,9 +648,11 @@ module Spider; module Model; module Storage; module Db
                 (self.class.type_synonyms && self.class.type_synonyms[current[:type]] && self.class.type_synonyms[current[:type]].include?(field[:type]))
             try_method = :"schema_field_#{field[:type].downcase}_equal?"
             return send(try_method, current, field) if (respond_to?(try_method))
-            current[:length] ||= 0; attributes[:length] ||= 0; current[:precision] ||= 0; attributes[:precision] ||= 0
-            return false unless current[:length] == attributes[:length]
-            return false unless current[:precision] == attributes[:precision]
+            unless self.class.fixed_length_types.include?(field[:type])
+                current[:length] ||= 0; attributes[:length] ||= 0; current[:precision] ||= 0; attributes[:precision] ||= 0
+                return false unless current[:length] == attributes[:length]
+                return false unless current[:precision] == attributes[:precision]
+            end
             return true
         end
 
@@ -640,7 +660,7 @@ module Spider; module Model; module Storage; module Db
         # Checks if the conversion from a current DB field to a schema field is safe, i.e. can 
         # be done without loss of data.
         def safe_schema_conversion?(current, field)
-            attributes = field[:attributes]
+            attributes = field[:attributes].clone
             safe = self.class.safe_conversions
             if (current[:type] != field[:type])
                 if safe[current[:type]] && safe[current[:type]].include?(field[:type])
@@ -649,9 +669,11 @@ module Spider; module Model; module Storage; module Db
                     return false
                 end
             end
-            return true if ((!current[:length] || current[:length] == 0) \
+            cur = current
+            return true if self.class.fixed_length_types.include?(current[:type])
+            return true if ((!cur[:length] || cur[:length] == 0) \
                             || (attributes[:length] && current[:length] <= attributes[:length])) && \
-                           ((!current[:precision] || current[:precision] == 0) \
+                           ((!cur[:precision] || cur[:precision] == 0) \
                            || (attributes[:precision] && current[:precision] <= attributes[:precision]))
             return false
         end

data/lib/spiderfw/setup/app_manager.rb

@@ -5,7 +5,7 @@ module Spider
 
     class AppManager
 
-        def initialize(options)
+        def initialize(options={})
             @options = options
         end
         
@@ -47,6 +47,9 @@ module Spider
             return res if apps.empty?
             require 'spiderfw/setup/app_server_client'
             client = Spider::AppServerClient.new(url)
+            if options[:branch]
+                apps = apps.map{ |a| "#{a}@#{options[:branch]}"}
+            end
             if options[:no_deps]
                 specs = client.get_specs(apps)
             else
@@ -215,6 +218,13 @@ module Spider
                     end
                 end
             end
+            if @options[:interactive]
+                puts "\n\n"
+                @done_tasks.each do |app, tasks|
+                    next unless tasks
+                    task.print_release_notes 
+                end
+            end
             if options[:clear_cache]
                 Spider.output _("Clearing cache...")
                 Spider::Template.cache.clear!
@@ -251,7 +261,10 @@ module Spider
                  Spider.output _("%s already installed, skipping") % spec.id
                  return
              end
-             repo = Git.open(@home_path)
+             repo = nil
+             if ::File.directory?(File.join(@home_path, '.git'))
+                repo = Git.open(@home_path)
+             end
              repo_url = spec.git_repo_rw || spec.git_repo
              Spider.output _("Fetching %s from %s") % [spec.app_id, repo_url]
 
@@ -260,13 +273,19 @@ module Spider
              end
 
              ENV['GIT_WORK_TREE'] = nil
-             Dir.chdir(@home_path) do
-                 `git submodule add #{repo_url} apps/#{spec.id}`
-                 `git submodule init`
-                 `git submodule update`
-             end
-             repo.add(['.gitmodules', "apps/#{spec.id}"])
-             repo.commit(_("Added app %s") % spec.id) 
+             if repo
+                 Dir.chdir(@home_path) do   
+                     `git submodule add #{repo_url} apps/#{spec.id}`
+                     `git submodule init`
+                     `git submodule update`
+                     repo.add(['.gitmodules', "apps/#{spec.id}"])
+                     repo.commit(_("Added app %s") % spec.id) 
+                 end
+            else
+                Dir.chdir(File.join(@home_path, 'apps')) do
+                    Git.clone(repo_url, spec.id) 
+                end
+            end
          end
 
          def pack_install(spec, options={})
@@ -308,33 +327,45 @@ module Spider
         
         def git_update(spec, options={})
             require 'git'
-            home_repo = Git.open(@home_path)
+            home_repo = nil
+            if ::File.directory?(File.join(@home_path, '.git'))
+                home_repo = Git.open(@home_path)
+            end
             app_path = File.join(@home_path, "apps", spec.id)
             app_repo = Git.open(app_path)
             Spider.output _("Updating %s from %s") % [spec.app_id, spec.git_repo]
             options[:branch] ||= 'master'
             Dir.chdir(app_path) do
-                app_repo.branch(options[:branch]).checkout
-            end
-            response = err = nil
-            Dir.chdir(app_path) do
-                response = `git --git-dir='#{app_path}/.git' pull origin master`
-            end
-            require 'ruby-debug'
-            if response =~ /Aborting/
-                Spider.output err, :ERROR
-                raise "Unable to update"
-            end
-            Dir.chdir(app_path) do
-                app_repo.reset('HEAD', :hard => true)
-                app_repo.branch(options[:branch]).checkout
+                app_repo.fetch
+                if app_repo.is_branch?(options[:branch])
+                    app_repo.checkout(options[:branch])
+                else 
+                    app_repo.checkout("origin/#{options[:branch]}", :new_branch => options[:branch])
+                end
+                app_repo.merge("origin/#{options[:branch]}")
             end
+            # response = err = nil
+            # Dir.chdir(app_path) do
+            #     response = `git --git-dir='#{app_path}/.git' pull origin #{options[:branch]}`
+            # end
+            #  app_repo.branch(options[:branch]).checkout
+            # require 'ruby-debug'
+            # if response =~ /Aborting/
+            #     Spider.output err, :ERROR
+            #     raise "Unable to update"
+            # end
+            # Dir.chdir(app_path) do
+            #     app_repo.reset('HEAD', :hard => true)
+            #     app_repo.branch(options[:branch]).checkout
+            # end
             
-            home_repo.add("apps/#{spec.id}")
-            begin
-                home_repo.commit(_("Updated app %s") % spec.id) 
-            rescue => exc
-                raise unless exc.message =~ /no changes added to commit/
+            if home_repo
+                home_repo.add("apps/#{spec.id}")
+                begin
+                    home_repo.commit(_("Updated app %s") % spec.id) 
+                rescue => exc
+                    raise unless exc.message =~ /no changes added to commit/
+                end
             end
         end
         
@@ -388,7 +419,7 @@ module Spider
             if version
                 tasks = ["#{@version}.rb"]
             else
-                tasks = Dir.entries(path).reject{ |p| p[0].chr == '.'}.sort{ |a, b| 
+                tasks = Dir.entries(path).reject{ |p| !File.file?(File.join(path, p)) || p[0].chr == '.'}.sort{ |a, b| 
                     va = Gem::Version.new(File.basename(a, '.rb'))
                     vb = Gem::Version.new(File.basename(b, '.rb'))
                     va <=> vb
@@ -421,7 +452,14 @@ module Spider
                     Spider.output _("Setup task done")
                 rescue => exc
                     Spider.output exc, :ERROR
-                    done_tasks.reverse.each{ |dt| dt.do_down } # FIXME: rescue and log errors in down
+                    done_tasks.reverse.each do |dt| 
+                        begin
+                            dt.do_down 
+                        rescue => exc
+                            Spider.output("Unable to do down on #{dt}!", :ERROR)
+                            Spider.output(exc, :ERROR)
+                        end
+                    end
                     raise
                 end
             end

data/lib/spiderfw/setup/setup_task.rb

@@ -4,7 +4,7 @@ module Spider
     
     class SetupTask
         attr_reader :path, :version
-        attr_accessor :before, :up, :down, :cleanup, :sync_models, :app
+        attr_accessor :before, :up, :down, :cleanup, :sync_models, :app, :interactive
         
         class <<self
             def tasks
@@ -25,10 +25,12 @@ module Spider
                 obj.down = Setup.down
                 obj.cleanup = Setup.cleanup
                 obj.sync_models = Setup.sync_models
+                obj.interactive = Setup.interactive?
                 #obj = @last_class.new(path)
 #                Kernel.send(:remove_const, @last_class.name)
                 #@last_class = nil
                 Thread.critical = crit
+                obj.do_before
                 return obj
             end
         end
@@ -47,8 +49,21 @@ module Spider
             @version = Gem::Version.new(version)
             @desc = desc
         end
+
+        def dir
+            File.join(::File.dirname(@path), @version.to_s)
+        end
+
+        def do_before
+            return unless @before
+            Spider::Model::Managed.no_set_dates = true
+            instance_eval(&@before)
+            #sync_schema unless @no_sync || @sync_done
+            Spider::Model::Managed.no_set_dates = false
+        end
         
         def do_up
+            return unless @up
             Spider::Model::Managed.no_set_dates = true
             instance_eval(&@up)
             #sync_schema unless @no_sync || @sync_done
@@ -56,8 +71,9 @@ module Spider
         end
         
         def do_down
+            return unless @down
             Spider::Model::Managed.no_set_dates = true
-            instance_eval(&@down) if @down
+            instance_eval(&@down)
             Spider::Model::Managed.no_set_dates = false
         end
 
@@ -66,12 +82,25 @@ module Spider
         end
 
         def do_sync
-            return unless @sync_models
+            sync_schema
+        end
+
+        def sync_schema(*models)
+            had_models = models
+            models = @sync_models if models.blank?
+            return unless models
+            @sync_options = models.pop if models.last.is_a?(Hash)
+            models = models.map{ |m| 
+                m <= Spider::App ? m.models.reject{ |n| n <= Spider::Migrations::PreviousModel } : m 
+            }.flatten
+            @sync_options ||= {}
             options = {
-                :no_foreign_key_constraints => true
-            }
-            @sync_models.each do |m|
-                m.mapper.sync_schema(false, options)
+                :no_foreign_key_constraints => true,
+                :force => false
+            }.merge(@sync_options)
+            models.each do |m|
+                next unless m.mapper.respond_to?(:sync_schema)
+                m.mapper.sync_schema(options[:force], options)
             end
         end
 
@@ -89,7 +118,7 @@ module Spider
             good = false
 
             while !good
-                print "#{msg} #{y}/#{n}]: "
+                print "#{msg} [#{y}/#{n}]: "
                 res = $stdin.gets.strip
                 
                 good = true
@@ -109,6 +138,17 @@ module Spider
 
 
         end
+
+        def warn(msg)
+            puts msg
+            print "\n"+_("Press any key to continue ")
+            $stdin.getch
+            print "\n"
+        end
+
+        def interactive?
+            @interactive
+        end
         
         # def no_sync_schema
         #     @no_sync = true
@@ -142,6 +182,32 @@ module Spider
         #     args = models + [options]
         #     sync_schema(*args)
         # end
+
+        def setup_previous_templates(templates)
+            msg = ""
+            templates.each do |t|
+                from = File.join(self.dir, t[:prev])
+                to = File.join(Spider.paths[:root], 'views', t[:dest])
+                unless File.file?(to)
+                    FileUtils.mkdir_p(File.dirname(to))
+                    FileUtils.cp(from, to)
+                    msg += _(t[:msg]) % to
+                    msg += "\n"
+                end
+            end
+            warn(msg) unless msg.blank?
+        end
+
+        def print_release_notes
+            readme = File.join(self.dir, "README.#{Spider.locale.language}")
+            unless File.file?(readme)
+                readme = File.join(self.dir, "README")
+            end
+            if File.file?(readme)
+                puts File.read(readme)
+            end
+
+        end
         
 
     end
@@ -154,7 +220,9 @@ module Spider
 
         # TODO: pass options
         def self.sync_schema(*models)
+            options = models.pop if models.last.is_a?(Hash)
             @sync_models = models
+            @sync_options = options || {}
         end
 
         def self.sync_models

data/lib/spiderfw/spider.rb

@@ -1,6 +1,7 @@
 require 'spiderfw/env'
 
 require 'rubygems'
+require 'backports'
 require 'find'
 require 'fileutils'
 require 'pathname'
@@ -151,6 +152,7 @@ module Spider
             FileUtils.mkdir_p(Spider.paths[:tmp])
             FileUtils.mkdir_p(Spider.paths[:var])
             FileUtils.mkdir_p(File.join(Spider.paths[:var], 'memory'))
+            FileUtils.mkdir_p(File.join(Spider.paths[:var], 'data'))
             
         end
 
@@ -183,7 +185,7 @@ module Spider
         end
         
         def main_process_startup
-            if Object.const_defined?(:FSSM)
+            if defined?(FSSM)
                 monitor = FSSM::Monitor.new
 
                 monitor.path(Spider.paths[:tmp], 'restart.txt') do
@@ -192,12 +194,30 @@ module Spider
 
                 end
 
+                if Spider.conf.get('template.cache.use_fssm')
+                    monitor.path(Spider.paths[:root]) do
+                        glob '**/*.shtml'
+                        create { |base, relative| FileUtils.rm_rf(File.join(Spider.paths[:var], 'cache', 'templates')) }
+                        update { |base, relative| FileUtils.rm_rf(File.join(Spider.paths[:var], 'cache', 'templates')) }                                    
+                    end
+                    monitor.path($SPIDER_PATH) do
+                        glob '**/*.shtml'
+                        create { |base, relative| FileUtils.rm_rf(File.join(Spider.paths[:var], 'cache', 'templates')) }
+                        update { |base, relative| FileUtils.rm_rf(File.join(Spider.paths[:var], 'cache', 'templates')) }                                    
+                    end
+                    FileUtils.rm_rf(File.join(Spider.paths[:var], 'cache', 'templates'))
+                end
+
                 @fssm_thread = Thread.new do
                     monitor.run
                 end
                 Spider.output("Monitoring restart.txt")
+
             else
                 Spider.output("FSSM not installed, unable to monitor restart.txt")
+                if Spider.conf.get('template.cache.use_fssm')
+                    raise "Unable to use FSSM for monitoring templates; use template.cache.disable instead"
+                end
             end
             trap('TERM'){ Spider.main_process_shutdown; exit }
             trap('INT'){ Spider.main_process_shutdown; exit }

data/lib/spiderfw/templates/blocks/text.rb

@@ -3,7 +3,7 @@ require 'strscan'
 
 module Spider; module TemplateBlocks
     ExpressionOutputRegexp = /\{?\{\s([^\s].*?)\s\}\}?/
-    GettextRegexp = /_\(([^\)]+)?\)(\s%\s([^\s,]+(?:,\s*\S+\s*)?))?/
+    GettextRegexp = /([snp][snp]?)?_\(([^\)]+)?\)(\s%\s([^\s,]+(?:,\s*\S+\s*)?))?/
     ERBRegexp = /(<%(.+)?%>)/
     
     class Text < Block
@@ -21,9 +21,9 @@ module Spider; module TemplateBlocks
                 when :expr
                     c += "$out << #{vars_to_scene(val)}\n"
                 when :gettext
-                    c += "$out << _('#{escape_text(val[0])}')"
-                    if val[1]
-                        c += " % [#{vars_to_scene(val[1])}]" 
+                    c += "$out << #{val[:func]}_('#{escape_text(val[:val])}')"
+                    if val[:vars]
+                        c += " % [#{vars_to_scene(val[:vars])}]" 
                     end
                     c += "\n"
                 when :erb

data/lib/spiderfw/templates/blocks/text_domain.rb

@@ -0,0 +1,25 @@
+require 'spiderfw/templates/template_blocks'
+
+module Spider; module TemplateBlocks
+    
+    class TextDomain < Block
+        
+        def compile(options={})
+            init = ""
+            td = @el.get_attribute('tpl:text-domain')
+            c = "Spider::GetText.in_domain('#{td}') do\n"
+            @el.remove_attribute('tpl:text-domain')
+            content = Spider::TemplateBlocks.parse_element(@el, @allowed_blocks, @template).compile(options)
+            init += content.init_code
+            content.run_code.each_line do |line|
+                c += '  '+line
+            end
+            c += "end\n"
+            return CompiledBlock.new(init, c)
+        end
+        
+        
+    end
+    
+    
+end; end

data/lib/spiderfw/templates/blocks/widget.rb

@@ -37,7 +37,6 @@ module Spider; module TemplateBlocks
             @el.each_child do |ch|
                 html += ch.to_html
             end
-            html = "<sp:widget-content>#{html}</sp:widget-content>" unless html.empty?
             runtime_content, overrides = klass.parse_content_xml(html)
 
             template = nil
@@ -73,6 +72,7 @@ module Spider; module TemplateBlocks
             html.gsub!("'", "\\\\'")
             
             init_params = self.class.attributes_to_init_params(@el.attributes.to_hash)
+            runtime_content.gsub!("'", "\\\\'") if runtime_content
             
             init += "add_widget('#{id}', #{klass}.new(@request, @response), {#{init_params.join(', ')}}, '#{runtime_content}', #{t_param})\n"
             c = "yield :\"#{id}\"\n"