psql-cm 0.0.5 → 0.0.6

data/History.md

@@ -1,3 +1,15 @@
+# 0.0.6 - 2012-04-20
+
+Merged {base.sql, cm.sql} into a single sql-path/{database}/{schema}.sql file.
+
+Base is now the first change in the {schema}.sql CM file.
+
+Made Configuration Managment table name configurable via cli using -c|--cm-table=
+
+Made debug message output more structured and clear.
+
+Created {drop,create,setup,dump,restore} rake tasks for qa-ing.
+
 # 0.0.5 - 2012-04-18
 
 Added git repository feature to sql-path for dump action, a commit per run of

data/README.md

@@ -15,8 +15,7 @@ combined with complimentary process and adherence.
 
 ## What psql-cm is not
 
-psql-cm is not intended on being a solution whatsoever for data backup.
-
+psql-cm is not intended on being a solution for data backup.
 For backup of data instead use the
 [pg\_dump](http://www.postgresql.org/docs/current/static/app-pgdump.html)
 command line utility for backing up data in addition to a
@@ -61,12 +60,16 @@ database cluster.
 
     $ psql-cm --databases psqlcm_test --uri "postgres://127.0.0.1:5432" restore
 
+## Change
+
+TODO: Document how to commit a change.
+
 ## Command line parameters
 
 --databases argument may take multiple database targets, to do this pass them
 in ',' separated format, no spaces. Specifically the format is,
 
-    $ psql-cm --databases psqlcm_test,psqlcm_test2,... ...
+    $ psql-cm --databases adatabase,anotherdatabase,... ...
 
 --uri has the format,
 
@@ -119,16 +122,57 @@ Next we'll dump the schema to sql/ within our working directory
 
     $ psql-cm --databases psqlcm_test --uri "postgres://127.0.0.1:5432" dump
 
-At this point we have the base schema for the psqlcm\_test database recorded and
-we can test to see that this is true by droping the database and then running
-the psql-cm restore action.
+At this point we have the base schema for the psqlcm\_test database recorded to
+the filesystem. You can see the filesystem structure and contents with
+a find command on \*nix:
+
+    $ find sql/psqlcm_test
+    sql/psqlcm_test
+    sql/psqlcm_test/public
+    sql/psqlcm_test/public/base.sql
+    sql/psqlcm_test/public/cm.sql
+    sql/psqlcm_test/schema_one
+    sql/psqlcm_test/schema_one/base.sql
+    sql/psqlcm_test/schema_one/cm.sql
+    sql/psqlcm_test/schema_two
+    sql/psqlcm_test/schema_two/base.sql
+    sql/psqlcm_test/schema_two/cm.sql
+
+We can now do a restore restore by droping the database and then running the
+psql-cm restore action.
 
     $ dropdb psqlcm_test
     $ psql-cm --databases psqlcm_test --uri "postgres://127.0.0.1:5432" restore
 
-Once again use yoru favorite client tool and verify that the schema is inded
-what it was after setup was run.
+Once again useing a client tool and verify that the schema is inded what it was
+after setup was run.
+
+NOTE: one caveat is that psql-cm does not handle ROLEs and USERs so these will
+have to be accounted for after doing a restore.
+
+## Debugging
+
+Debugging output can be enabled by exporting DEBUG=true in the environment
+before calling the psql-cm command:
+
+    $ export debug=true
+
+## Development
+
+To play around inside of a running psql-cm Ruby environment use the console:
+
+    rake console    # Development console, builds installs then runs console
+
+The 'Walkthrough' from above is encoded as rake tasks, each step can be
+seen including all debugging output by running:
 
-Note that one caveat is that psql-cm does not handle ROLEs and USERs so these
-will have to be accounted for after doing a restore.
+    rake clean      # Remove the sql/ directory in the current working directory.
+    rake create     # Create the development database psqlcm_development, including two schemas.
+    rake debug      # Enable debugging using environment variable DEBUG
+    rake drop       # Drop the development database psqlcm_development
+    rake dump       # Remove sql/ from CWD and then run the psql-cm dump action on psqlcm_development
+    rake build      # Build the psql-cm gem.
+    rake install    # Build then install the psql-cm gem.
+    rake restore    # Create psqlcm_development, run psql-cm actions {setup, dump, restore} in order.
+    rake setup      # Create psqlcm_development and run psql-cm setup on it
 

data/lib/psql-cm/base.rb

@@ -20,12 +20,13 @@ module PSQLCM
     def databases
       @databases = db.
         exec("SELECT datname as name FROM pg_database WHERE datname !~ 'template*|postgres';").
-        map {|row| row["name"]}
+        map {|row| row['name']}
 
-      unless config.databases.to_a.empty? # filter out databases not specified.
+      if config.databases.to_a.empty?
+        halt! 'A list of databases must be given:\n  --databases={database_one}[,{database_two}[,...]]'
+      else # filter out databases not specified.
         @databases.select!{ |name| config.databases.include?(name) }
       end
-
       debug "databases> #{@databases}"
       @databases
     end
@@ -33,7 +34,10 @@ module PSQLCM
     def schemas(name = 'postgres')
       @schemas = db(name).
         exec("SELECT nspname AS name FROM pg_namespace WHERE nspname !~ '^pg_.*|information_schema';").
-        map{|row| row["name"]}
+        map{|row| row['name']}
+
+      # Filter out schemas not specified, if specified.
+      @schemas.select!{ |name| config.schemas.include?(name) } if config.schemas
       debug "schemas> #{@schemas}"
       @schemas
     end
@@ -42,149 +46,88 @@ module PSQLCM
       return @tree if @tree
       @tree = {}
       databases.each do |dbname|
-        @tree[dbname] = {}
-        schemas(dbname).each do |schema|
-          @tree[dbname][schema] = ['base.sql', 'cm.sql']
-        end
+        @tree[dbname] = schemas(dbname)
       end
-      debug "tree> tree: #{@tree}"
+      debug "tree> #{@tree}"
       @tree
     end
 
-    def dump!
-      unless config.sql_path
-        $stdout.puts "Warning: --sql-path was not set, defaulting to $PWD/sql."
-        config.sql_path = "#{ENV["PWD"]}/sql"
+    def run!(action = config.action)
+      case action
+      when 'console'
+        require 'psql-cm/cli'
+        ::PSQLCM::Console.run!
+      when 'dump'
+        dump!
+      when 'restore'
+        restore!
+      when 'setup'
+        setup!
+      when 'submit'
+        submit!
+      else
+        halt! "An action must be given! {setup, dump, restore}" if action.nil?
+        halt! "Action '#{action}' is not handled."
       end
+    end
 
-      debug "dump> sql_path: #{config.sql_path}"
-      FileUtils.mkdir(config.sql_path) unless File.directory?(config.sql_path)
-      Dir.chdir(config.sql_path) do
-        tree.each_pair do |database, schema_hash|
-          debug "dump> database: #{database}"
-
-          File.directory?(File.join(config.sql_path,database)) or
-            FileUtils.mkdir(File.join(config.sql_path,database))
-
-          schema_hash.each_pair do |schema, files|
-          debug "dump> schema: #{schema}"
-            File.directory?(File.join(config.sql_path,database,schema)) or
-              FileUtils.mkdir(File.join(config.sql_path,database,schema))
-
-            base_file = File.join(config.sql_path,database,schema,'base.sql')
-            cm_file = File.join(config.sql_path,database,schema,'cm.sql')
-
-            FileUtils.touch(base_file)
-            FileUtils.touch(cm_file)
+    def sh(command)
+      debug "$ #{command}"
+      %x[#{command} 2>&1 | awk '! /NOTICE/']
+    end
 
-            command = "pg_dump #{db(database).psql_args} --schema-only --no-owner --schema=#{schema} "
-            if File.size(base_file) > 0
-              command += "--file=#{cm_file}  --table=psql_cm "
-            else
-              command += "--file=#{base_file} --exclude-table=psql_cm "
-            end
-            command += "#{database}"
-            sh 'dump', command
-          end
-        end
+    def uri
+      return config.uri unless config.uri.to_s.empty?
+      $stdout.puts "NOTICE: uri is not set, defaulting to postgres://127.0.0.1:5432 (format: postgres://{user}:{password}@{host}:{port}/{database}\nwhere user, password, port and database are optional)"
+      config.uri = "postgres://127.0.0.1:5432"
+    end
 
-        sh 'git', "git init; git add ." unless File.exists?('.git') && File.directory?('.git')
+    private
 
-        sh 'git', "git commit -a -m 'PostgreSQL Change Management (psql-cm).\nDatabases: #{databases}\nTree: #{tree}'"
-      end
-    end # def dump!
-
-    def setup!
-      # Create psql_cm tables for each schema on the target db.
-      debug "setup> Setting up pg_psql_cm table in each target schema."
-      tree.each_pair do |database, schema_hash|
-        schema_hash.keys.each do |schema|
-          sql = <<-SQL
-            SET search_path = #{schema}, public;
-            CREATE TABLE IF NOT EXISTS #{schema}.pg_psql_cm
-            (
-              id bigserial NOT NULL PRIMARY KEY ,
-              is_base boolean NOT NULL,
-              created_at timestamp with time zone DEFAULT now(),
-              implementer text NOT NULL,
-              content text NOT NULL
-            );
-          SQL
-          debug "setup:#{database}:#{schema}> sql\n#{sql}"
-          db(database).exec sql
-        end
+    def ensure_database_exists(database)
+      begin
+        db('postgres').exec("CREATE DATABASE #{database};")
+        debug "create> #{database}"
+      rescue => error
+        raise error unless error.message =~ /already exists/
       end
     end
 
-    def restore!
-      unless config.sql_path
-        $stdout.puts "Warning: --sql-path was not set, defaulting to $PWD/sql."
-        config.sql_path = "#{ENV["PWD"]}/sql"
-      end
-
-      debug "restore> sql_path: #{config.sql_path}"
-      File.directory?(config.sql_path) or
-        halt! "Cannot restore from a path that does not exist: #{config.sql_path}"
-
-      Dir.chdir(config.sql_path) do
-
-        Dir['*'].each do |database|
-          next unless File.directory? database
-
-          Dir.chdir(database) do
-            sh "restore", "createdb #{db(database).psql_args} #{database}"
-
-            debug "restore:#{database}>"
-            Dir['*'].each do |schema|
-              next unless File.directory? schema
-
-              base_file = File.join(config.sql_path,database,schema,'base.sql')
-              cm_file = File.join(config.sql_path,database,schema,'cm.sql')
-
-              debug "restore:#{database}:#{schema}> #{base_file}"
-              sh 'restore', "psql #{db(database).psql_args} #{database} < #{base_file}"
-
-              next if File.size(cm_file) == 0
-
-              debug "restore:#{database}:#{schema}> #{cm_file}"
-              sh 'restore', "psql #{db(database).psql_args} #{database} < #{cm_file}"
-            end
-          end
-        end
+    def ensure_schema_exists(database,schema)
+      begin
+        db(database).exec("CREATE SCHEMA #{schema};")
+        debug "create> #{database}.#{schema}"
+      rescue => error
+        raise error unless error.message =~ /already exists/
       end
-    end # def restore!
-
-    def change!
-      puts "TODO: allow change string and/or file to be specified and add to the
-      specified database scema control table"
     end
 
-    def run!(action = config.action, parent_id = config.parent_id)
-      case action
-      when "console"
-        ::PSQLCM.debug "Starting Console"
-        require 'psql-cm/cli'
-        ::PSQLCM::Console.run!
-      when "dump"
-        dump!
-      when "restore"
-        restore!
-      when "setup"
-        setup!
-      when "change"
-        change!
-      else
-        halt! "Action '#{action}' is not handled."
-      end
+    def ensure_cm_table_exists(database,schema)
+      sql = <<-SQL
+        SET search_path = #{schema};
+        SET client_min_messages = warning;
+        CREATE TABLE IF NOT EXISTS #{schema}.#{config.cm_table}
+        (
+          id bigserial NOT NULL PRIMARY KEY ,
+          is_base boolean NOT NULL,
+          created_at timestamp with time zone DEFAULT now(),
+          implementer text NOT NULL,
+          content text NOT NULL
+        );
+      SQL
+      debug "create> #{database}.#{schema}.#{config.cm_table}"
+      db(database).exec(sql)
     end
 
-    def sh(tag, command)
-      debug "sh:#{tag}> #{command}"
-      %x[#{command}]
+    def sql_path
+      return config.sql_path unless config.sql_path.to_s.empty?
+      $stdout.puts "NOTICE: sql_path is not set, defaulting to #{ENV["PWD"]}/sql"
+      config.sql_path = "#{ENV["PWD"]}/sql"
     end
 
   end # class << self
 end
 
 ::PSQLCM.config.debug = !!ENV['DEBUG']
+::PSQLCM.config.cm_table = 'pg_psql_cm' # Default --cm-table name.
 

data/lib/psql-cm/cli.rb

@@ -6,36 +6,40 @@ module PSQLCM
     class << self
       def parse!(arguments)
         ::OptionParser.new do |options|
-          options.banner = "Usage: psql-cm [options]"
-          options.separator ""
-          options.separator "Specific options:"
+          options.banner = 'Usage: psql-cm [options]'
+          options.separator ''
+          options.separator 'Specific options:'
 
-          options.on("-s", "--sql-path PATH", "Path to dump SQL cm files into.") do |path|
+          options.on('-s', '--sql-path PATH', 'Path to dump SQL cm files into.') do |path|
             ::PSQLCM.config.sql_path = path
           end
 
-          options.on("-a", "--databases NAMES", "A comma separated list of databases to cm.") do |names|
+          options.on('-a', '--databases=NAMES', 'A comma separated list of databases to cm.') do |names|
             ::PSQLCM.config.databases = names.split(',')
           end
 
-          options.on("-u", "--uri URI", "Path to the sink database connection file.") do |uri|
+          options.on('-u', '--uri=URI', 'Path to the sink database connection file.') do |uri|
             ::PSQLCM.config.uri = uri
           end
 
-          options.on("-D", "--[no-]debug", "Output debugging information.") do |debug|
+          options.on('-D', '--[no-]debug', 'Output debugging information.') do |debug|
             ::PSQLCM.config.debug = debug.nil? ? false : true
           end
 
-          options.on("-v", "--[no-]verbose", "Output verbosley.") do |verbose|
+          options.on('-v', '--[no-]verbose', 'Output verbosley.') do |verbose|
             ::PSQLCM.config.verbose = verbose
           end
 
-          options.on_tail("-h", "--help", "Print help and exit.") do
+          options.on('-c', '--cm-table=TABLE') do |cm_table|
+            ::PSQLCM.config.cm_table = cm_table
+          end
+
+          options.on_tail('-h', '--help', 'Print help and exit.') do
             puts options
             exit 0
           end
 
-          options.on_tail("--version", "Print version and exit.") do
+          options.on_tail('--version', 'Print version and exit.') do
             require 'psql-cm/version'
             puts ::PSQLCM::Version
             exit 0

data/lib/psql-cm/database.rb

@@ -16,7 +16,7 @@ module PSQLCM
 
     def db
       unless @config[:dbname] == 'postgres'
-        ::PSQLCM.sh 'createdb', "createdb #{psql_args} #{@config[:dbname]}"
+        ::PSQLCM.sh "createdb #{psql_args} #{@config[:dbname]}"
       end
 
       @db || connect!
@@ -69,9 +69,12 @@ module PSQLCM
       @db[name] = Connection.new(:dbname => name)
     end
 
-    # "postgres://{user}:{password}@{host}:{port}/{database}"
     def configure!
-      uri = URI.parse(::PSQLCM.config.uri)
+      begin
+        uri = URI.parse(::PSQLCM.uri)
+      rescue => error
+        halt! "PostgreSQL URI was incorrectly specified, format is:\n  --uri=postgres://{user}:{password}@{host}:{port}/{database}\nwhere user, password, port and database are optional."
+      end
 
       query = uri.query.to_s.split('&')
 

data/lib/psql-cm/dump.rb

@@ -0,0 +1,30 @@
+module PSQLCM
+  class << self
+    def dump!
+      debug "dump> sql_path: #{sql_path}"
+      FileUtils.mkdir(sql_path) unless File.directory?(sql_path)
+      Dir.chdir(sql_path) do
+        debug "tree> #{tree}"
+        tree.each_pair do |database, database_schemas|
+          debug "dump> database: #{database}"
+          FileUtils.mkdir_p(File.join(sql_path,database))
+          database_schemas.each do |schema|
+            debug "dump> schema: #{schema}"
+            FileUtils.mkdir_p(File.join(sql_path,database))
+
+            cm_file = File.join(sql_path,database,"#{schema}.sql")
+
+            sh %W[ pg_dump #{db(database).psql_args}
+              --no-owner --schema=#{schema} --file=#{cm_file} --table=#{config.cm_table}
+              #{database} ].join(' ')
+          end
+        end
+
+        sh "git init; git add ." unless File.exists?('.git') && File.directory?('.git')
+
+        sh "git commit -a -m 'PostgreSQL Change Management (psql-cm).\nDatabases: #{databases.join(', ')}\nTree: #{tree}'"
+      end
+    end # def dump!
+
+  end
+end

data/lib/psql-cm/restore.rb

@@ -0,0 +1,49 @@
+module PSQLCM
+  class << self
+    def restore!
+      File.directory?(sql_path) or
+        halt! "Cannot restore from sql-path (#{sql_path}), it does not exist!"
+
+      Dir.chdir(sql_path) do
+
+        Dir['*'].each do |database|
+          next unless File.directory? database
+
+          Dir.chdir(database) do
+            ensure_database_exists(database)
+
+            debug "restore> #{database}"
+            Dir['*.sql'].each do |cm_file|
+              next if File.size(cm_file) == 0
+
+              schema = cm_file.sub(".sql",'')
+              ensure_schema_exists(database,schema)
+
+              debug "restore> #{database}:#{schema} < #{cm_file}"
+              sh "psql #{db(database).psql_args} #{database} < #{cm_file}"
+
+              ensure_cm_table_exists(database,schema)
+              row = db(database).exec("SELECT content from #{schema}.#{config.cm_table}
+                                      WHERE is_base IS true ORDER BY created_at
+                                      DESC LIMIT 1;")
+              Tempfile.open('base.sql') do |temp_file|
+                temp_file.write(row)
+                sh "psql #{db(database).psql_args} #{database} < #{temp_file.path}"
+              end
+
+              sql = "SELECT content from #{schema}.#{config.cm_table} where is_base IS false ORDER BY created_at ASC;"
+              db(database).exec(sql).each do |row|
+                debug "restoring cm row: #{row}"
+                Tempfile.open('base.sql') do |temp_file|
+                  temp_file.write(row)
+                  sh "psql #{db(database).psql_args} #{database} < #{temp_file.path}"
+                end
+              end
+            end
+          end
+        end
+      end
+    end # def restore!
+
+  end
+end

data/lib/psql-cm/setup.rb

@@ -0,0 +1,27 @@
+module PSQLCM
+  class << self
+    def setup!
+      tree.each_pair do |database, schemas|
+        ensure_database_exists(database)
+        schemas(database).each do |schema|
+          ensure_schema_exists(database,schema)
+          ensure_cm_table_exists(database,schema)
+
+          Tempfile.open('base.sql') do |temp_file|
+            sh " pg_dump #{db(database).psql_args} --schema-only --no-owner --schema=#{schema} --file=#{temp_file.path} #{database}"
+
+            contents = %x{cat #{temp_file.path}}
+            implementer = %Q|#{%x[git config user.name].strip} <#{%x[git config user.email].strip}>|
+            db(database).exec(
+              "INSERT INTO #{schema}.#{config.cm_table} (is_base, implementer, content) VALUES (true, $1, $2);",
+              [implementer, contents]
+            )
+          end
+
+        end
+      end
+
+      dump!
+    end
+  end
+end

data/lib/psql-cm/submit.rb

@@ -0,0 +1,14 @@
+module PSQLCM
+  class << self
+    def submit!
+      puts "TODO: allow change string and/or file to be specified and add to the
+      specified database scema control table"
+
+      # TOOD: get values x,y,z
+      sql = "INSERT INTO #{schema}.#{config.cm_table} (is_base,implementer,content) VALUES (x,y,z)"
+      debug "submit:#{database}:#{schema}> sql\n#{sql}"
+      db(database).exec sql
+    end # def submit!
+  end
+end
+

data/lib/psql-cm/version.rb

@@ -1,4 +1,4 @@
 module PSQLCM
-  Version = '0.0.5'
+  Version = '0.0.6'
 end
 

data/lib/psql-cm.rb

@@ -3,7 +3,12 @@ require 'fileutils'
 require 'ostruct'
 require 'uri'
 require 'delegate'
+require 'tempfile'
 
 require_relative 'psql-cm/base'
 require_relative 'psql-cm/database'
+require_relative 'psql-cm/setup'
+require_relative 'psql-cm/dump'
+require_relative 'psql-cm/restore'
+require_relative 'psql-cm/submit'
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: psql-cm
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-04-19 00:00:00.000000000 Z
+date: 2012-04-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
@@ -39,6 +39,10 @@ files:
 - lib/psql-cm/base.rb
 - lib/psql-cm/cli.rb
 - lib/psql-cm/database.rb
+- lib/psql-cm/dump.rb
+- lib/psql-cm/restore.rb
+- lib/psql-cm/setup.rb
+- lib/psql-cm/submit.rb
 - lib/psql-cm/version.rb
 - lib/psql-cm.rb
 - LICENCE