pg_migrate 0.0.1

data/.gitignore

@@ -0,0 +1,34 @@
+*.gem
+*.rbc
+.bundle
+.config
+coverage
+InstalledFiles
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+
+# YARD artifacts
+.yardoc
+_yardoc
+doc/
+
+# Intellij artifacts
+.idea
+*.iml
+
+# RVM
+.rvmrc
+
+# VIM
+*~
+
+# rspec output directory
+target
+
+# local ruby symbolic link to fool IDE
+ruby

data/.gitmodules

@@ -0,0 +1,3 @@
+[submodule "lib/pg_migrate/templates"]
+	path = lib/pg_migrate/templates
+	url = git@github.com:sethcall/pg_migrate_templates.git

data/Gemfile

@@ -0,0 +1,10 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in pg_migrate.gemspec
+gemspec
+
+group :development do
+  gem 'rspec', '2.11.0'
+  #gem 'files', :git => 'git@github.com:sethcall/files.git'
+  gem 'files', :path => '/Users/seth/workspace/files'
+end

data/Gemfile.lock

@@ -0,0 +1,38 @@
+PATH
+  remote: .
+  specs:
+    pg_migrate (0.0.1)
+      logging (= 1.7.2)
+      pg (= 0.14.0)
+      thor (= 0.15.4)
+
+PATH
+  remote: /Users/seth/workspace/files
+  specs:
+    files (0.2.1)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    diff-lcs (1.1.3)
+    little-plugger (1.1.3)
+    logging (1.7.2)
+      little-plugger (>= 1.1.3)
+    pg (0.14.0)
+    rspec (2.11.0)
+      rspec-core (~> 2.11.0)
+      rspec-expectations (~> 2.11.0)
+      rspec-mocks (~> 2.11.0)
+    rspec-core (2.11.0)
+    rspec-expectations (2.11.1)
+      diff-lcs (~> 1.1.3)
+    rspec-mocks (2.11.0)
+    thor (0.15.4)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  files!
+  pg_migrate!
+  rspec (= 2.11.0)

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2012 TODO: Write your name
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,35 @@
+# PgMigrate
+
+The ruby implementation of pg_migrate.  
+
+With this gem, there are two primary features:
+* Manifest Builder - process a user's pg_migrate manifest into a stable set of SQL migration scripts
+* Migrator - migrate a database, using a built pg_migrate manifest
+
+## Installation
+
+**NOTE: this gem is not yet published**
+
+Add this line to your application's Gemfile:
+
+    gem 'pg_migrate'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install pg_migrate
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,2 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"

data/bin/pg_migrate

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+require 'pg_migrate'
+
+include PgMigrate
+
+CommandLine.start

data/lib/pg_migrate/builder.rb

@@ -0,0 +1,158 @@
+require 'pathname'
+require 'fileutils'
+require 'find'
+require 'erb'
+
+
+module PgMigrate
+  # takes a unprocessed manifest directory, and adds before/after headers to each file
+  class Builder
+
+
+    attr_accessor :manifest_reader, :sql_reader
+
+
+    def initialize(manifest_reader, sql_reader)
+      @log = Logging.logger[self]
+      @manifest_reader = manifest_reader
+      @sql_reader = sql_reader
+      @template_dir = File.join(File.dirname(__FILE__), 'templates')
+    end
+
+
+    # input_dir is root path, contains file 'manifest' and 'migrations'
+    # output_dir will have a manifest and migrations folder, but processed
+    # force will create the output dir if needed, and *delete an existing directory* if it's in the way
+    def build(input_dir, output_dir, options={:force=>true})
+      input_dir = File.expand_path(input_dir)
+      output_dir = File.expand_path(output_dir)
+                                                      
+      if input_dir == output_dir
+        raise 'input_dir can not be same as output_dir: #{input_dir}'
+      end
+
+      @log.debug "building migration directory #{input_dir} and placing result at: #{output_dir}"
+
+      output = Pathname.new(output_dir)
+      if !output.exist?
+        if !options[:force]
+          raise "Output directory '#{output_dir}' does not exist.  Create it or specify create_output_dir=true"
+        else
+          output.mkpath
+        end
+      else
+        # verify that it's is a directory
+        if !output.directory?
+          raise "output_dir #{output_dir} is a file; not a directory."
+        else
+          @log.debug("deleting & recreating existing output_dir #{output_dir}")
+          output.rmtree
+          output.mkpath
+        end
+      end
+
+      # manifest always goes over mostly as-is,
+      # just with a comment added at top indicating our version
+
+      input_manifest = File.join(input_dir, MANIFEST_FILENAME)
+      output_manifest = File.join(output_dir, MANIFEST_FILENAME)
+
+      File.open(output_manifest, 'w') do |fout|
+        fout.puts "#{BUILDER_VERSION_HEADER}pg_migrate_ruby-#{PgMigrate::VERSION}"
+        IO.readlines(input_manifest).each do |input|
+          fout.puts input
+        end
+      end
+
+      # in order array of manifest declarations
+      loaded_manifest = @manifest_reader.load_input_manifest(input_dir)
+      # hashed on migration name hash of manifest
+
+      loaded_manifest_hash = @manifest_reader.hash_loaded_manifest(loaded_manifest)
+      @manifest_reader.validate_migration_paths(input_dir, loaded_manifest)
+
+      build_up(input_dir, output_dir, loaded_manifest_hash)
+    end
+
+    def build_up(input_dir, output_dir, loaded_manifest_hash)
+      migrations_input = File.join(input_dir, UP_DIRNAME)
+      migrations_output = File.join(output_dir, UP_DIRNAME)
+
+      # iterate through files in input migrations path, wrapping files with transactions and other required bits
+
+      Find.find(migrations_input) do |path|
+        if path == ".."
+          Find.prune
+        else
+          @log.debug "building #{path}"
+
+          # create relative bit
+          relative_path = path[migrations_input.length..-1]
+
+          # create the filename correct for the input directory, for this file
+          migration_in_path = path
+
+          # create the filename correct for the output directory, for this file
+          migration_out_path = File.join(migrations_output, relative_path)
+
+          process_and_copy_up(migration_in_path, migration_out_path, relative_path, loaded_manifest_hash)
+        end
+      end
+
+      create_bootstrap_script(migrations_output)
+    end
+
+    # creates the 'pg_migrations table'
+    def create_bootstrap_script(migration_out_path)
+      run_template("bootstrap.erb", binding, File.join(migration_out_path, BOOTSTRAP_FILENAME))
+    end
+
+    def create_wrapped_up_migration(migration_in_filepath, migration_out_filepath, migration_def)
+    builder_version="pg_migrate_ruby-#{PgMigrate::VERSION}"
+      migration_content = nil
+      File.open(migration_in_filepath, 'r') {|reader| migration_content = reader.read }
+      run_template("up.erb", binding, File.join(migration_out_filepath))
+    end
+
+    # given an input template and binding, writes to an output file
+    def run_template(template, binding, output_filepath)
+      bootstrap_template = nil
+      File.open(File.join(@template_dir, template), 'r') do |reader|
+        bootstrap_template   = reader.read
+      end
+
+
+      template = ERB.new(bootstrap_template, 0, "%<>")
+      content = template.result(binding)
+      File.open(output_filepath, 'w') do |writer|
+        writer.syswrite(content)
+      end
+    end
+
+    def process_and_copy_up(migration_in_path, migration_out_path, relative_path, loaded_manifest_hash)
+
+      if FileTest.directory?(migration_in_path)
+        # copy over directories
+        # find relative-to-migrations dir this path
+        FileUtils.mkdir(migration_out_path)
+      else
+        if migration_in_path.end_with?('.sql')
+          # if a .sql file, then copy & process
+
+          # create the the 'key' version of this name, which is basically the filepath
+          # of the .sql file relative without the leading '/' directory
+          manifest_name = relative_path[1..-1]
+
+          @log.debug("retrieving manifest definition for #{manifest_name}")
+
+          migration_def = loaded_manifest_hash[manifest_name]
+
+          create_wrapped_up_migration(migration_in_path, migration_out_path, migration_def)
+        else
+          # if not a .sql file, just copy it over
+          FileUtils.cp(migration_in_path, migration_out_path)
+        end
+      end
+    end
+  end
+end

data/lib/pg_migrate/command_line.rb

@@ -0,0 +1,78 @@
+module PgMigrate
+  class CommandLine < Thor
+
+    desc "up", "migrates the database forwards, applying migrations found in the source directory"
+    method_option :source, :aliases => "-s", :default => '.', :lazy_default => '.', :banner => 'input directory', :desc => "a pg_migrate built manifest. Should contain your processed manifest and up|down|test folders"
+    method_option :connopts, :aliases => "-o", :type => :hash, :required => true, :banner => "connection options", :desc => "database connection options used by gem 'pg': dbname|host|hostaddr|port|user|password|connection_timeout|options|sslmode|krbsrvname|gsslib|service"
+    method_option :verbose, :aliases => "-v", :type => :boolean, :default => false, :banner => "verbose", :desc=> "set to raise verbosity"
+
+    def up
+      bootstrap_logger(options[:verbose])
+
+      manifest_reader = ManifestReader.new
+      sql_reader = SqlReader.new
+
+      connopts = options[:connopts]
+      if !connopts[:port].nil?
+        connopts[:port] = connopts[:port].to_i
+      end
+
+      migrator = Migrator.new(manifest_reader, sql_reader, connopts)
+
+      begin
+        migrator.migrate(options[:source])
+      rescue Exception => e
+        if !options[:verbose]
+          # catch common exceptions and make pretty on command-line
+          if !e.message.index("ManifestReader: code=unloadable_manifest").nil?
+            puts "Unable to load manifest in source directory '#{options[:source]}' .  Check -s|--source option and run again."
+            exit 1
+          else
+            raise e
+          end
+        else
+          raise e
+        end
+      end
+
+
+    end
+
+    desc "down", "not implemented"
+
+    def down
+      bootstrap_logger(options[:verbose])
+
+      raise 'Not implemented'
+    end
+
+    desc "build", "processes a pg_migrate source directory and places the result in the specified output directory"
+    method_option :source, :aliases => "-s", :default => '.', :lazy_default => '.', :banner => 'input directory', :desc => "the input directory containing a manifest file and up|down|test folders"
+    method_option :out, :aliases => "-o", :required => true, :banner => "output directory", :desc => "where the processed migrations will be placed"
+    method_option :force, :aliases => "-f", :default => false, :type => :boolean, :banner => "overwrite out", :desc => "if specified, the out directory will be created before processing occurs, replacing any existing directory"
+    method_option :verbose, :aliases => "-v", :type => :boolean, :default => false, :banner => "verbose", :desc=> "set to raise verbosity"
+
+    def build
+
+      bootstrap_logger(options[:verbose])
+
+      manifest_reader = ManifestReader.new
+      sql_reader = SqlReader.new
+      builder = Builder.new(manifest_reader, sql_reader)
+      builder.build(options[:source], options[:out], :force => options[:force])
+    end
+
+    no_tasks do
+      def bootstrap_logger(verbose)
+        # bootstrap logger
+        if verbose
+          Logging.logger.root.level = :debug
+        else
+          Logging.logger.root.level = :info
+        end
+
+        Logging.logger.root.appenders = Logging.appenders.stdout
+      end
+    end
+  end
+end

data/lib/pg_migrate/config_parser.rb

@@ -0,0 +1,49 @@
+require 'yaml'
+
+module PgMigrate
+
+  class ConfigParser
+
+    def self.rails(path, environment)
+
+      config = {}
+
+      rails_config = YAML.load_file(path)
+
+      if !rails_config.has_key?(environment)
+        raise "no environment #{environment} found in rails config file: #{path}"
+      end
+
+      rails_config = rails_config[environment]
+
+      # populate from rails YAML to PG
+
+      # required parameters 1st
+      if !rails_config.has_key?("database")
+        raise "no database key found in #{path} with environment #{environment}"
+      end
+
+      config[:dbname] = rails_config["database"]
+
+      if rails_config.has_key?("host")
+        config[:host] = rails_config["host"]
+      end
+
+      if rails_config.has_key?("port")
+        config[:port] = rails_config["port"]
+      end
+
+      if rails_config.has_key?("username")
+        config[:user] = rails_config["username"]
+      end
+
+      if rails_config.has_key?("password")
+        config[:password] = rails_config["password"]
+      end
+
+      return config
+
+    end
+
+  end
+end

data/lib/pg_migrate/manifest_reader.rb

@@ -0,0 +1,103 @@
+require 'pathname'
+
+module PgMigrate
+  class ManifestReader
+
+    def initialize
+      @log = Logging.logger[self]
+    end
+
+    # returns array of migration paths
+    def load_input_manifest(manifest_path)
+      manifest, version = load_manifest(manifest_path, false)
+      return manifest
+    end
+
+    # returns [array of migration paths, version]
+    def load_output_manifest(manifest_path)
+      return load_manifest(manifest_path, true)
+    end
+
+
+    # verify that the migration files exist
+    def validate_migration_paths(manifest_path, manifest)
+      # each item in the manifest should be a valid file
+      manifest.each do |item|
+        item_path = build_migration_path(manifest_path, item.name)
+        if !Pathname.new(item_path).exist?
+          raise "manifest reference #{item.name} does not exist at path #{item_path}"
+        end
+      end
+    end
+
+    # construct a migration file path location based on the manifest basedir and the name of the migration
+    def build_migration_path(manifest_path, migration_name)
+      File.join(manifest_path, UP_DIRNAME, "#{migration_name}")
+    end
+
+    def hash_loaded_manifest(loaded_manifest)
+      hash = {}
+      loaded_manifest.each do |manifest|
+        hash[manifest.name] = manifest
+      end
+      return hash
+    end
+
+    # read in the manifest, saving each migration declaration in order as they are found
+    private
+    def load_manifest(manifest_path, is_output)
+
+      manifest = []
+      version = nil
+
+      manifest_filepath = File.join(manifest_path, MANIFEST_FILENAME)
+
+      @log.debug "loading manifest from #{manifest_path}"
+
+      if !FileTest::exist?(manifest_filepath)
+        raise "ManifestReader: code=unloadable_manifest: manifest not found at #{manifest_path}"
+      end
+
+      # there should be a file called 'manifest' at this location
+      manifest_lines = IO.readlines(manifest_filepath)
+
+      ordinal = 0
+      manifest_lines.each_with_index do |line, index|
+        # ignore comments
+        migration_name = line.strip
+
+        @log.debug "processing line:#{index} #{line}"
+
+        # output files must have a version header as 1st line o file
+        if is_output
+          if index == 0
+            # the first line must be the version comment. if not, error out.
+            if migration_name.index(BUILDER_VERSION_HEADER) == 0 && migration_name.length > BUILDER_VERSION_HEADER.length
+              version = migration_name[BUILDER_VERSION_HEADER.length..-1]
+              @log.debug "manifest has builder_version #{version}"
+            else
+              raise "manifest invalid: missing/malformed version.  expecting '# pg_migrate-VERSION' to begin first line '#{line}' of manifest file: '#{manifest_path}'"
+            end
+          end
+        end
+
+        if migration_name.empty? or migration_name.start_with?('#')
+          # ignored!
+        else
+          @log.debug "adding manifest #{migration_name} with ordinal #{ordinal}"
+          manifest.push(Migration.new(migration_name, ordinal, build_migration_path(manifest_path, migration_name)))
+          ordinal += 1
+        end
+
+        # the logic above wouldn't get upset with an empty manifest
+        if is_output
+          if version.nil?
+            raise "manifest invalid: empty"
+          end
+        end
+      end
+      return manifest, version
+    end
+
+  end
+end

data/lib/pg_migrate/migration.rb

@@ -0,0 +1,12 @@
+module PgMigrate
+  class Migration
+    attr_accessor :name, :ordinal, :md5, :created, :production, :filepath
+
+    def initialize(name, ordinal, filepath)
+      @name = name
+      @ordinal = ordinal
+      @filepath = filepath
+    end
+    
+  end
+end

data/lib/pg_migrate/migrator.rb

@@ -0,0 +1,101 @@
+module PgMigrate
+
+  class Migrator
+
+    attr_accessor :conn, :connection_hash, :manifest_path, :manifest, :manifest_reader, :sql_reader
+
+    # options = gem 'pg' connection_hash options, or connstring => dbname=test port=5432, or pgconn => PG::Connection object
+    def initialize(manifest_reader, sql_reader, options = {})
+      @log = Logging.logger[self]
+      @connection_hash = options
+      @manifest = nil
+      @builder_version = nil
+      @manifest_reader = manifest_reader
+      @sql_reader = sql_reader
+    end
+
+    # 'migrate' attempt to migrate your database based on the contents of your built manifest
+    # The manifest_path argument should point to your manifest
+    # manifest_path = the directory containing your 'manifest' file, 'up' directory, 'down' directory, 'test' directory
+    # this method will throw an exception if anything goes wrong (such as bad SQL in the migrations themselves)
+
+    def migrate(manifest_path)
+      @manifest_path = manifest_path
+
+      if !@connection_hash[:pgconn].nil?
+        @conn = @connection_hash[:pgconn]
+      elsif !@connection_hash[:connstring].nil?
+        @conn = PG::Connection.open(@connection_hash[:connstring])
+      else
+        @conn = PG::Connection.open(@connection_hash)
+      end
+
+      # this is used to record the version of the 'migrator' in the pg_migrate table
+      @conn.exec("SET application_name = 'pg_migrate_ruby-#{PgMigrate::VERSION}'")
+
+      # load the manifest, and version of the builder that made it
+      process_manifest()
+
+      # execute the migrations
+      run_migrations()
+    end
+
+
+    # load the manifest's migration declarations, and validate that each migration points to a real file
+    def process_manifest
+      @manifest, @builder_version = @manifest_reader.load_output_manifest(@manifest_path)
+      @manifest_reader.validate_migration_paths(@manifest_path, @manifest)
+    end
+
+    # run all necessary migrations
+    def run_migrations
+
+      # run bootstrap before user migrations to prepare database
+      run_bootstrap
+
+      # loop through the manifest, executing migrations in turn
+      manifest.each_with_index do |migration, index|
+        execute_migration(migration.name, migration.filepath)
+      end
+
+    end
+
+    # executes the bootstrap method
+    def run_bootstrap
+      bootstrap = File.join(@manifest_path, UP_DIRNAME, BOOTSTRAP_FILENAME)
+      execute_migration('bootstrap.sql', bootstrap)
+    end
+
+    # execute a single migration by loading it's statements from file, and then executing each
+    def execute_migration(name, filepath)
+      @log.debug "executing migration #{filepath}"
+
+      statements = @sql_reader.load_migration(filepath)
+      if statements.length == 0
+        raise 'no statements found in migration #{migration_path}'
+      end
+      run_migration(name, statements)
+    end
+
+    # execute all the statements of a single migration
+    def run_migration(name, statements)
+
+      begin
+        statements.each do |statement|
+          conn.exec(statement).clear
+        end
+      rescue Exception => e
+        # we make a special allowance for one exception; it just means this migration
+        # has already occured, and we should just treat it like a continue
+        if e.message.index('pg_migrate: code=migration_exists').nil?
+          conn.exec("ROLLBACK")
+          raise e
+        else
+          conn.exec("ROLLBACK")
+          @log.info "migration #{name} already run"
+        end
+      end
+    end
+  end
+end
+