xmigra 1.5.1 → 1.6.0

data/lib/xmigra/declarative_support.rb

@@ -0,0 +1,158 @@
+module XMigra
+  module DeclarativeSupport
+    class SpecificationError < Error; end
+    
+    class StructureReader
+      EXTENSION_PREFIX = 'X-'
+      
+      def initialize(data, keypath=[])
+        @keypath = keypath
+        @data = data
+        @children = []
+        @used_keys = Set.new
+      end
+      
+      def [](key)
+        result, result_key = item_and_keypath(key)
+        
+        case result
+        when Hash
+          result = StructureReader.new(result, result_key).tap do |r|
+            r.parent = self
+          end
+        when Array
+          raise "Invalid to fetch an array via [] -- use array_fetch"
+        end
+        
+        @used_keys << key if @data.kind_of? Hash
+        return result
+      end
+      
+      def fetch(key, *args)
+        if args.length > 1
+          raise ArgumentError, "fetch takes 1 or 2 arguments"
+        end
+        
+        if @data.has_key?(key)
+          return self[key]
+        elsif args.length == 1
+          return args[0]
+        else
+          raise KeyError, "#{key.inspect} not present"
+        end
+      end
+      
+      def array_fetch(key, key_finder_proc)
+        result, result_key = item_and_keypath(key)
+        unless result.kind_of? Array
+          raise ::TypeError, "Expected key for array"
+        end
+        
+        @used_keys << key if @data.kind_of? Hash
+        return StructureReader.new(result, result_key).tap do |r|
+          r.parent = self
+          r.key_finder_proc = key_finder_proc
+        end
+      end
+      
+      def raw_item(key)
+        @used_keys << key if @data.kind_of? Hash
+        return @data[key]
+      end
+      
+      def each
+        to_enum(:each) unless block_given?
+        
+        if @data.kind_of? Hash
+          @data.each_key {|k| yield k, self[k]}
+        else
+          (0...@data.length).each {|i| yield self[i]}
+        end
+      end
+      include Enumerable
+      
+      def kind_of?(klass)
+        return super(klass) || @data.kind_of?(klass)
+      end
+      
+      def hash
+        @data.hash
+      end
+      
+      def eql?(other)
+        @data.eql?(other)
+      end
+      
+      def keys
+        @data.keys
+      end
+      
+      def values
+        @data.values
+      end
+      
+      def uniq
+        collect {|o| o}.uniq
+      end
+      
+      def length
+        @data.length
+      end
+      
+      def join(sep=$,)
+        @data.join(sep)
+      end
+      
+      def each_extension(&blk)
+        return to_enum(:each_extension) if blk.nil?
+        
+        if @data.kind_of? Hash
+          @data.each_pair do |k, val|
+            next unless k.kind_of?(String) and k.start_with?(EXTENSION_PREFIX)
+            blk.call((@keypath + [k]).join('.'), val)
+          end
+        end
+        
+        children.each do |child|
+          child.each_extension(&blk)
+        end
+      end
+      
+      def each_unused_standard_key(&blk)
+        return to_enum(:each_unused_standard_key) if blk.nil?
+        
+        if @data.kind_of? Hash
+          @data.each_key do |k|
+            next if @used_keys.include?(k)
+            next if k.kind_of?(String) && k.start_with?(EXTENSION_PREFIX)
+            blk.call(@keypath + [k]).join('.')
+          end
+        end
+        children.each {|child| child.each_unused_standard_key(&blk)}
+      end
+      
+      protected
+      attr_accessor :key_finder_proc
+      attr_reader :parent, :children
+      
+      def parent=(new_val)
+        @parent.children.delete(self) if @parent
+        @parent = new_val
+        @parent.children << self if @parent
+        new_val
+      end
+      
+      def item_and_keypath(key)
+        item = @data[key]
+        subkey = begin
+          if @key_finder_proc
+            @keypath + [@key_finder_proc.call(item)]
+          else
+            @keypath + [key]
+          end
+        end
+        return item, subkey
+      end
+    end
+  end
+end

data/lib/xmigra/impdecl_migration_adder.rb

@@ -0,0 +1,249 @@
+require 'xmigra/declarative_migration'
+require 'xmigra/new_migration_adder'
+
+module XMigra
+  class ImpdeclMigrationAdder < NewMigrationAdder
+    class NoChangesError < Error; end
+    
+    @support_types = {}
+    def self.register_support_type(tag, klass)
+      if @support_types.has_key? tag
+        raise Error, "#{@support_types[tag]} already registered to handle #{tag}"
+      end
+      @support_types[tag] = klass
+      if block_given?
+        begin
+          yield
+        ensure
+          @support_types.delete(tag)
+        end
+      end
+    end
+    
+    def self.support_type(tag)
+      @support_types[tag]
+    end
+    
+    def self.each_support_type(&blk)
+      @support_types.each_pair(&blk)
+    end
+    
+    module SupportedDatabaseObject
+      module ClassMethods
+        def for_declarative_tagged(tag)
+          XMigra::ImpdeclMigrationAdder.register_support_type(tag, self)
+        end
+      end
+      
+      def self.included(mod)
+        mod.extend(ClassMethods)
+      end
+      
+      # Classes including this Module should define:
+      #     #creation_sql
+      #     #sql_to_effect_from(old_state)
+      #     #destruction_sql
+      #
+      # and expect to receive as arguments to their constructor the name of
+      # the object and the Ruby-ized data present at the top level of the
+      # declarative file.
+    end
+    
+    def initialize(path)
+      super(path)
+      @migrations = MigrationChain.new(
+        self.path.join(STRUCTURE_SUBDIR),
+        :db_specifics=>@db_specifics,
+        :vcs_specifics=>@vcs_specifics,
+      )
+    end
+    
+    attr_accessor :strict
+    
+    def add_migration_implementing_changes(file_path, options={})
+      file_path = Pathname(file_path)
+      prev_impl = @migrations.latest_declarative_implementations[file_path]
+      decl_stat = prev_impl.declarative_status
+      
+      # Declarative doesn't make any sense without version control
+      unless VersionControlSupportModules.find {|m| self.kind_of? m}
+        raise Error, "#{self.path} is not under version control (required for declarative)"
+      end
+      
+      # Check if an implementation is needed/allowed
+      if bad_rel = {
+          :equal=>"the same revision as",
+          :older=>"an older revision than",
+      }[decl_stat]
+        raise NoChangesError, "#{file_path} changed in #{bad_rel} the latest implementing migration #{prev_impl.file_path}"
+      end
+      
+      # This should require the same user to generate a migration on the same
+      # day starting from the same committed version working on the same
+      # branch to cause a collision of migration file names:
+      file_hash = begin
+        file_base = begin
+          [
+            SchemaUpdater.new(path).branch_identifier,
+            vcs_latest_revision(file_path),
+          ].join("\x00")
+        rescue VersionControlError
+          ''
+        end
+        XMigra.secure_digest(
+          [(ENV['USER'] || ENV['USERNAME']).to_s, file_base.to_s].join("\x00"),
+          :encoding=>:base32
+        )[0,12]
+      end
+      summary = "#{file_path.basename('.yaml')}-#{file_hash}.decl"
+      
+      add_migration_options = {
+        :file_path=>file_path,
+      }
+      
+      # Figure out the goal of the change to the declarative
+      fail_options = []
+      case decl_stat
+      when :unimplemented
+        fail_options << :renounce
+        add_migration_options[:goal] = options[:adopt] ? 'adoption' : 'creation'
+      when :newer
+        fail_options.concat [:adopt, :renounce]
+        add_migration_options[:goal] = 'revision'
+      when :missing
+        fail_options << :adopt
+        add_migration_options[:goal] = options[:renounce] ? 'renunciation' : 'destruction'
+      end
+      
+      if opt = fail_options.find {|o| options[o]}
+        raise Program::ArgumentError, "--#{opt} flag is invalid when declarative file is #{decl_stat}"
+      end
+      
+      # gsub gets rid of trailing whitespace on a line (which would force double-quote syntax)
+      add_migration_options[:delta] = prev_impl.delta(file_path).gsub(/\s+$/, '').extend(LiteralYamlStyle)
+      unless options[:adopt] || options[:renounce]
+        begin
+          if suggested_sql = build_suggested_sql(decl_stat, file_path, prev_impl)
+            add_migration_options[:sql] = suggested_sql
+            add_migration_options[:sql_suggested] = true
+          end
+        rescue DeclarativeSupport::SpecificationError
+          add_migration_options[:spec_error] = $!.to_s
+        end
+      end
+      
+      add_migration(summary, add_migration_options)
+    end
+    
+    def migration_data(head_info, options)
+      target_object = options[:file_path].basename('.yaml')
+      goal = options[:goal].to_sym
+      super(head_info, options).tap do |data|
+        # The "changes" key is not used by declarative implementation
+        #migrations -- the "of object" (TARGET_KEY) is used instead
+        data.delete(Migration::CHANGES)
+        
+        data[DeclarativeMigration::GOAL_KEY] = options[:goal].to_s
+        data[DeclarativeMigration::TARGET_KEY] = target_object.to_s
+        data[DeclarativeMigration::DECLARATION_VERSION_KEY] = begin
+          if [:renunciation, :destruction].include?(goal)
+            'DELETED'
+          else
+            XMigra.secure_digest(options[:file_path].read)
+          end
+        end
+        data['delta'] = options[:delta]
+        options[:spec_error].tap do |message|
+          data['specification error'] = message if message
+        end
+        
+        # Reorder "sql" key to here (unless adopting or renouncing, then
+        # remove "sql" completely)
+        provided_sql = data.delete('sql')
+        unless [:adoption, :renunciation].include? goal
+          data['sql'] = provided_sql
+          data[DeclarativeMigration::QUALIFICATION_KEY] = begin
+            if options[:sql_suggested]
+              'suggested command sequence'
+            else
+              'unimplemented'
+            end
+          end 
+        end
+        
+        # Reorder "description" key to here with 
+        data.delete('description')
+        data['description'] = "Declarative #{goal} of #{target_object}"
+      end
+    end
+    
+    def build_suggested_sql(decl_stat, file_path, prev_impl)
+      d = SupportedObjectDeserializer.new(
+        file_path.basename('.yaml').to_s,
+        @db_specifics
+      )
+      case decl_stat
+      when :unimplemented
+        initial_state = YAML.parse_file(file_path)
+        initial_state = d.deserialize(initial_state.children[0])
+        
+        if initial_state.kind_of?(SupportedDatabaseObject)
+          initial_state.creation_sql
+        end
+      when :newer
+        old_state = YAML.parse(
+          vcs_contents(file_path, :revision=>prev_impl.vcs_latest_revision),
+          file_path
+        )
+        old_state = d.deserialize(old_state.children[0])
+        new_state = YAML.parse_file(file_path)
+        new_state = d.deserialize(new_state.children[0])
+        
+        if new_state.kind_of?(SupportedDatabaseObject) && old_state.class == new_state.class
+          new_state.sql_to_effect_from old_state
+        end
+      when :missing
+        penultimate_state = YAML.parse(
+          vcs_contents(file_path, :revision=>prev_impl.vcs_latest_revision),
+          file_path
+        )
+        penultimate_state = d.deserialize(penultimate_state.children[0])
+        
+        if penultimate_state.kind_of?(SupportedDatabaseObject)
+          penultimate_state.destruction_sql
+        end
+      end
+    rescue DeclarativeSupport::SpecificationError
+      raise
+    rescue StandardError
+      raise if strict
+      nil
+    end
+    
+    class SupportedObjectDeserializer
+      def initialize(object_name, db_specifics)
+        @object_name = object_name
+        @db_specifics = db_specifics
+      end
+      
+      attr_reader :object_name, :db_specifics
+      
+      def deserialize(yaml_node)
+        data = yaml_node.to_ruby
+        if klass = ImpdeclMigrationAdder.support_type(yaml_node.tag)
+          klass.new(@object_name, data).extend(@db_specifics)
+        else
+          if data.respond_to? :name=
+            data.name = @object_name
+          elsif data.kind_of? Hash
+            data['name'] = @object_name
+          end
+          data
+        end
+      end
+    end
+  end
+end
+
+require 'xmigra/declarative_support'
+require 'xmigra/declarative_support/table'

data/lib/xmigra/migration.rb

@@ -12,22 +12,29 @@ module XMigra
       @id = info['id'].dup.freeze
       _follows = info[FOLLOWS]
       @follows = (_follows.dup.freeze unless _follows == EMPTY_DB)
-      @sql = info["sql"].dup.freeze
+      @sql = info.has_key?('sql') ? info["sql"].dup.freeze : nil
       @description = info["description"].dup.freeze
       @changes = (info[CHANGES] || []).dup.freeze
       @changes.each {|c| c.freeze}
+      @all_info = Marshal.load(Marshal.dump(info))
     end
     
     attr_reader :id, :follows, :description, :changes
     attr_accessor :file_path
     
     def schema_dir
-      Pathname(file_path).dirname.join('..')
+      @schema_dir ||= begin
+        result = Pathname(file_path).dirname
+        while result.basename.to_s != SchemaManipulator::STRUCTURE_SUBDIR
+          result = result.dirname
+        end
+        result.join('..')
+      end
     end
     
     def sql
       if Plugin.active
-        @sql.dup.tap do |result|
+        (@sql || "").dup.tap do |result|
           Plugin.active.amend_source_sql(result)
         end
       else

data/lib/xmigra/migration_chain.rb

@@ -1,4 +1,5 @@
 
+require 'xmigra/declarative_migration'
 require 'xmigra/migration'
 
 module XMigra
@@ -7,13 +8,16 @@ module XMigra
     LATEST_CHANGE = 'latest change'
     MIGRATION_FILE_PATTERN = /^\d{4}-\d\d-\d\d.*\.yaml$/i
     
+    include DeclarativeMigration::ChainSupport
+    
     def initialize(path, options={})
       super()
+      @path = Pathname(path)
       
       db_specifics = options[:db_specifics]
       vcs_specifics = options[:vcs_specifics]
       
-      head_info = YAML.load_file(File.join(path, HEAD_FILE))
+      head_info = yaml_of_file(File.join(path, HEAD_FILE)) || {}
       file = head_info[LATEST_CHANGE]
       prev_file = HEAD_FILE
       files_loaded = []
@@ -21,18 +25,16 @@ module XMigra
       until file.nil?
         file = XMigra.yaml_path(file)
         fpath = File.join(path, file)
-        break unless File.file?(fpath)
-        begin
-          mig_info = YAML.load_file(fpath)
-        rescue
-          raise XMigra::Error, "Error loading/parsing #{fpath}"
-        end
+        break if (mig_info = yaml_of_file(fpath)).nil?
         files_loaded << file
         mig_info["id"] = Migration::id_from_filename(file)
         migration = Migration.new(mig_info)
         migration.file_path = File.expand_path(fpath)
         migration.extend(db_specifics) if db_specifics
         migration.extend(vcs_specifics) if vcs_specifics
+        if migration.file_path.end_with? ".decl.yaml"
+          migration.extend(DeclarativeMigration)
+        end
         unshift(migration)
         prev_file = file
         file = migration.follows
@@ -50,14 +52,26 @@ module XMigra
       @other_migrations.freeze
     end
     
+    attr_reader :path
+    
     # Test if the chain reaches back to the empty database
     def complete?
-      length > 0 && self[0].follows.nil?
+      length == 0 || self[0].follows.nil?
     end
     
     # Test if the chain encompasses all migration-like filenames in the path
     def includes_all?
       @other_migrations.empty?
     end
+    
+    protected
+    def yaml_of_file(fpath)
+      return nil unless File.file?(fpath)
+      begin
+        return YAML.load_file(fpath)
+      rescue
+        raise XMigra::Error, "Error loading/parsing #{fpath}"
+      end
+    end
   end
 end

data/lib/xmigra/migration_conflict.rb

@@ -63,6 +63,33 @@ module XMigra
       if @after_fix
         @after_fix.call
       end
+      
+      fix_merged_declarative_relations!
+    end
+    
+    def fix_merged_declarative_relations!
+      # Update the latest implementing migration for any declarative
+      # file that is modified in the working copy
+      each_decohered_implementing_migration(
+        &method(:fix_decohered_implmeneting_migration!)
+      )
+    end
+    
+    def each_decohered_implementing_migration
+      tool = SchemaUpdater.new(@path.join(".."))
+      latest_impls = tool.migrations.latest_declarative_implementations
+      latest_impls.each_pair do |decl_file, migration|
+        next unless tool.vcs_file_modified?(decl_file)
+        yield migration.file_path, tool.vcs_latest_revision(migration.file_path)
+      end
+    end
+    
+    def fix_decohered_implmeneting_migration!(file_path, last_commit)
+      migration_info = YAML.load_file(file_path)
+      migration_info['pre-unbranch'] = last_commit
+      file_path.open('w') do |f|
+        $xmigra_yamler.dump(migration_info, f)
+      end
     end
   end
 end

data/lib/xmigra/new_access_artifact_adder.rb

@@ -0,0 +1,44 @@
+
+require 'xmigra/schema_manipulator'
+
+module XMigra
+  class NewAccessArtifactAdder < SchemaManipulator
+    class UnsupportedArtifactType < XMigra::Error
+      def initialize(artifact_type, system_name)
+        super("#{system_name} does not support #{artifact_type} artifacts")
+        @artifact_type = artifact_type
+        @system_name = system_name
+      end
+      
+      attr_reader :artifact_type, :system_name
+    end
+    
+    def initialize(path)
+      super(path)
+    end
+    
+    def add_artifact(type, name)
+      access_dir = @path.join(ACCESS_SUBDIR)
+      FileUtils.mkdir_p(access_dir) unless access_dir.exist?
+      
+      new_fpath = access_dir.join(name + '.yaml')
+      raise(XMigra::Error, "Access object \"#{new_fpath.basename}\" already exists") if new_fpath.exist?
+      
+      template_method = begin
+        method("#{type}_definition_template_sql".to_sym)
+      rescue NameError
+        proc {''}
+      end
+      new_data = {
+        'define'=>type.to_s,
+        'sql'=>template_method.call.dup.extend(LiteralYamlStyle),
+      }
+      
+      File.open(new_fpath, "w") do |f|
+        $xmigra_yamler.dump(new_data, f)
+      end
+      
+      return new_fpath
+    end
+  end
+end

data/lib/xmigra/new_index_adder.rb

@@ -0,0 +1,33 @@
+
+require 'xmigra/schema_manipulator'
+
+module XMigra
+  class NewIndexAdder < SchemaManipulator
+    def initialize(path)
+      super(path)
+    end
+    
+    def add_index(name)
+      indexes_dir = @path.join(INDEXES_SUBDIR)
+      FileUtils.mkdir_p(indexes_dir) unless indexes_dir.exist?
+      
+      new_fpath = indexes_dir.join(name + '.yaml')
+      raise(XMigra::Error, "Index \"#{new_fpath.basename}\" already exists") if new_fpath.exist?
+      
+      index_creation_template = begin
+        index_template_sql
+      rescue NameError
+        ''
+      end
+      new_data = {
+        'sql'=>index_creation_template.dup.extend(LiteralYamlStyle),
+      }
+      
+      File.open(new_fpath, "w") do |f|
+        $xmigra_yamler.dump(new_data, f)
+      end
+      
+      return new_fpath
+    end
+  end
+end

data/lib/xmigra/new_migration_adder.rb

@@ -40,12 +40,7 @@ module XMigra
       )
       raise(XMigra::Error, "Migration file\"#{new_fpath.basename}\" already exists") if new_fpath.exist?
       
-      new_data = {
-        Migration::FOLLOWS=>head_info.fetch(MigrationChain::LATEST_CHANGE, Migration::EMPTY_DB),
-        'sql'=>options.fetch(:sql, "<<<<< INSERT SQL HERE >>>>>\n").dup.extend(LiteralYamlStyle),
-        'description'=>options.fetch(:description, "<<<<< DESCRIPTION OF MIGRATION >>>>>").dup.extend(FoldedYamlStyle),
-        Migration::CHANGES=>options.fetch(:changes, ["<<<<< WHAT THIS MIGRATION CHANGES >>>>>"]),
-      }
+      new_data = migration_data(head_info, options)
       
       # Write the head file first, in case a lock is required
       old_head_info = head_info.dup
@@ -85,6 +80,15 @@ module XMigra
       return new_fpath
     end
     
+    def migration_data(head_info, options={})
+      {}.tap do |data|
+        data[Migration::FOLLOWS] = head_info.fetch(MigrationChain::LATEST_CHANGE, Migration::EMPTY_DB)
+        data['sql'] = options.fetch(:sql, "<<<<< INSERT SQL HERE >>>>>\n").dup.extend(LiteralYamlStyle)
+        data['description'] = options.fetch(:description, "<<<<< DESCRIPTION OF MIGRATION >>>>>").dup.extend(FoldedYamlStyle)
+        data[Migration::CHANGES] = options.fetch(:changes, ["<<<<< WHAT THIS MIGRATION CHANGES >>>>>"])
+      end
+    end
+    
     # Called when the chain of migrations in the production/master branch is
     # extended with a new migration.
     #

data/lib/xmigra/permission_script_writer.rb

@@ -17,11 +17,11 @@ module XMigra
     
     def ddl_block_separator; "\n"; end
     
-    def permissions_sql
-      intro_comment = @db_info.fetch('script comment', '') + "\n\n"
-      
-      intro_comment + in_ddl_transaction do
-        [
+    def permissions_sql(options = {})
+      "".tap do |result|
+        result << @db_info.fetch('script comment', '') + "\n\n"
+        
+        transaction_sql = [
           # Check for blatantly incorrect application of script, e.g. running
           # on master or template database.
           check_execution_environment_sql,
@@ -39,6 +39,12 @@ module XMigra
         ].flatten.compact.join(ddl_block_separator).tap do |result|
           Plugin.active.amend_composed_sql(result) if Plugin.active
         end
+        
+        if options.fetch(:transactional, true)
+          result << in_ddl_transaction {transaction_sql}
+        else
+          result << transaction_sql
+        end
       end
     end