db_suit_rails 0.4.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: ec869f044990a140af694c70bef1002b5e3e8e9cde39af3dde4a889f57a4706a
+  data.tar.gz: cce91a8c112295d0993a063f2b1dbc5f9927d02b0cd87ab65fa946df8feaca1c
+SHA512:
+  metadata.gz: 75a00ce98daf73a16f248f90703de44976796c694a86ef24227661da06cb6a8c5849b07291fdd310c048e571a7cef7b14e18376fae700a89d27a05612232d7a2
+  data.tar.gz: e5e11f5f1bb6a03a36a566487c38ab988145ec8712a93bdee491150685e2e1edcccc1182663d95ad670f54592131a23071480c08c6bd088b3b4ed9f410f9eb6e

data/.gitignore

@@ -0,0 +1,11 @@
+*.[oa]
+*.so
+*~
+*.nogem
+*nogem.*
+*.bak
+*.org
+*.orig
+*.gem
+*.pyc
+*.yardoc

data/ChangeLog

@@ -0,0 +1,41 @@
+-----
+(Version: 0.4.1)
+2018-03-31  Masa Sakano
+  * Updated the document.
+
+-----
+(Version: 0.4.1)
+2018-03-31  Masa Sakano
+  * Updated the document.
+
+-----
+(Version: 0.4)
+2018-03-26  Masa Sakano
+  * Added new command-line options (--[no-]withsql --mappingsql=FILE.SQL --mapping-tbl-prefix=PREFIX --mapping-tbl-suffix-from=SUFFIX --mapping-tbl-suffix-to=SUFFIX) so a normalised SQL-mapping file is output.
+  * Now it can deal with the table names in the form of "public.talbe_name"
+  * lib/db_suit_rails/sql_skelton.rb
+    * SqlSkelton::DefMappingFilename changed into DefMappingCsvname .
+    * New constant DefMappingSql is defined.
+    * new() and run() accept an additional option "mappingsql".
+    * new class method: self.colname4id()
+  * lib/db_suit_rails/sql_skelton/fkey.rb
+    * now requires "active_support/all" and "db_suit_rails/sql_skelton/utils" and includes the latter.
+  * lib/db_suit_rails/sql_skelton/utils.rb
+    * New private method: remove_sqlprefix()
+  * test/test_sql_skelton.rb : updated with a few new tests.
+
+-----
+(Version: 0.3)
+2018-03-13  Masa Sakano
+  * Fixed a serious bug in bin/copy_nline where some lines after a small number of data are omitted.  Added --version option in bin/copy_nline and bin/mk_sqlskelton .
+
+-----
+(Version: 0.2)
+2018-03-13  Masa Sakano
+  * Added an executable bin/copy_nline . Added a new module SqlSkelton::Utils (lib/db_suit_rails/sql_skelton/utils.rb).  Added routines to check reserved words for table/column names in col_index.rb and tbl_index.rb and implemented related tests.
+
+-----
+(Version: 0.1)
+2018-03-10  Masa Sakano
+
+ * Initial preliminary commit.

data/Makefile

@@ -0,0 +1,22 @@
+ALL	= 
+
+objs	= 
+
+.SUFFIXES:	.so .o .c .f
+
+#.o.so:
+#	${LD} ${LFLAGS} -o $@ $< ${LINK_LIB}
+
+all: ${ALL}
+
+
+.PHONY: clean test doc
+clean:
+	$(RM) bin/*~
+
+test:
+	rake test
+
+doc:
+	yard doc
+

data/README.en.rdoc

@@ -0,0 +1,118 @@
+
+= db_suit_rails - Database conversion to suit Ruby-on-Rails
+
+== Summary
+
+Script to create a SQL file to inject a table that follows the
+convention of Rails.
+
+== Install
+
+This script requires {Ruby}[http://www.ruby-lang.org] Version 2.0
+or above, and Rails library (called gem) of version 5.0 or above.
+The standard way to install Rails gem is from the command-line
+   Prompt% gem install rails
+
+Put the main-script file +mk_sqlskelton+ in any of your command-line search
+paths and give it the executable permission.
+
+You may need to modify the first line (Shebang line) of the script to suit your
+environment (it should be unnecessary for Linux and MacOS), or run it
+explicitly with your Ruby command as
+   Prompt% /YOUR/ENV/ruby /YOUR/INSTALLED/mk_sqlskelton
+
+Prior to it, make sure the RUBYLIB environment
+variable contains the library directory to this gem, which is
+   /THIS/GEM/LIBRARY/PATH/db_suit_rails/lib
+
+== Simple Examples
+
+All you need to do is to run
+   mk_sqlskelton [OPTIONS] OriginalSQL.sql
+
+You may start with
+   mk_sqlskelton --help
+to see the available options, which are described below in detail.
+
+== Description
+
+This script
+
+* pluralizes all the table names, 
+* changes table names containing `*_[0-9]*` into `*_n[0-9]*` ,
+* changes table names `id` into `id_cli` ,
+* changes table names containing `*_id` into `*_id_cli` ,
+* removes all the PRIMARY KEY constraints (n.b., those individual columns are not UNIQUE, either, any more),
+* creates the new PRIMARY KEY named 'id' in each table,
+* sets sets of composite PRIMARY KEY as equivalent UNIQUE keys,
+* changes the reference settings, reflecting all these changes,
+* removes all the triggers.
+
+The script is far from perfect, but is useful enough for practical uses.
+
+For example, the script assumes a certain format for the input SQL
+file.  Crucially, CREATE TABLE statements should have newlines at
+usual locations, and other statements should not have newlines inside
+a single sentence.
+
+It tries to handle references.  However, there are many forms to
+specify references in PostgreSQL, and this script handles not all of them.
+
+Also, its support for composite unique keys and their references are
+also limited.  If you encounter an error like
+
+   there is no unique constraint matching given keys for referenced table "...."
+
+then chances are, unique constrants are not set appropriately.
+
+SEQUENCEs are preserved, though probably they should not be.
+
+Essentially, not all the unique keys in the current table may be
+applicable in the generated table for Rails, depending on the
+specification of the table!  For example, if a column references the
+primary key in another table, would it be still valid,
+after the primary key is changed into a normal column, that is the
+ex-primary-key should really be unique after the migration?  If a
+new table combines entries from multiple tables, the column that used
+to be unique in a single table man not be unique any more.  It
+entirely depends on what you wnat in the newly generated database and tables.
+
+== Command-line options
+
+  USAGE: mk_sqlskelton [options] OriginalSQL.sql
+
+-e::     Regular expression for the tables to convert
+-o, --output::       Output SQL file.  In default, OriginalSQL_ror.sql
+-i, --index::       Index SQL file to list old and new.  In default, ./index.sql
+-n, --[no-]dryrun::       Dryrun (Def: false). It still attempts to access the files, and then you can tell which files exist and which do not.
+
+
+== Examples
+
+
+== Developer's note
+
+
+=== Algorithm
+
+* The components for the output SQL filename and components (table and
+  column names) are represented by a hash in the current
+  imprementation as in Ver.0.3, but it had better be by a class.
+
+=== Tests
+
+Ruby codes under the directory <tt>test/</tt> are the test scripts.
+You can run them from the top directory as <tt>ruby test/test_****.rb</tt>
+or simply run <tt>make test</tt>.
+
+
+== Known bugs
+
+None.
+
+
+== Copyright
+
+Author::  Masa Sakano < info a_t wisebabel dot com >
+Versions:: The versions of this package follow Semantic Versioning (2.0.0) http://semver.org/
+

data/Rakefile

@@ -0,0 +1,9 @@
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+end
+
+desc "Run tests"
+task :default => :test
+

data/bin/copy_nline

@@ -0,0 +1,141 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+
+require 'optparse'
+require 'open3'
+
+BANNER = <<"__EOF__"
+USAGE: #{File.basename($0)} [--nlines=N] [--pg-dump-command=COM] -- [options] [DBNAME] > STDOUT
+__EOF__
+
+
+# Initialising the hash for the command-line options.
+OPTS = {
+  :nlines => 5,
+  :pg_dump_command => 'pg_dump',
+  :read_file => nil,
+  :version => nil,
+  # :debug => false,
+}
+
+# Print the version number and exit
+#
+# If the global constant VERSION is defiend, it is returned.
+# Else, this assumes the library to load follows the proper Ruby/Gem style,
+# where the library is found in the path /.../gems/GEM_NAME-1.2.3/lib/
+#
+# @param libfile [String] the library file, e.g., 'my_gem_name/lib_to_load', where the file is used as in require 'my_gem_name/lib_to_load'
+# @return [String,NilClass] eg., '(Gem) did_you_mean-1.2.0'
+def get_version_in_gem(libfile)
+  defined?(VERSION) && (return VERSION)
+
+  libfile_base = libfile.sub(/^(\/)?(.+)(\.rb)?$/, '\2')
+  $LOAD_PATH.each do |dir|
+    if Dir.glob(dir+"/#{libfile_base}{,.rb}").size > 1
+      retstr = dir.sub(/\/lib\/?$/, '').sub(/.*\//, '')
+      if (%r@/gems/$@ =~ $&)
+        retstr.sub!(/^/, '(Gem) ')
+      else
+        retstr = sprintf("(Library under directory): %s", dir)
+      end
+      return retstr
+    end
+  end
+
+  ## Perhaps not in Gem
+  nil
+end
+
+
+# Print the version number and exit
+#
+# @param see #get_version_in_gem
+# @return [NilClass]
+def print_version_and_exit(libfile)
+  version = get_version_in_gem(libfile)
+
+  if version
+    puts version
+    exit
+  end
+
+  warn "ERROR: No version is found.  Perhaps the command is not in the properly installed Gem style (or currently in development)?"
+  exit 1
+end
+
+
+# VERSION = (get_version_in_gem('db_suit_rails/sql_skelton') || false)
+
+# Function to handle the command-line arguments.
+#
+# ARGV will be modified, and the constant variable OPTS is set.
+#
+# @return [Hash]  Optional-argument hash.
+#
+def handle_argv
+  # opt = OptionParser.new(BANNER, version: get_version_in_gem('db_suit_rails/sql_skelton')) 
+  opt = OptionParser.new(BANNER)
+
+  opt.on(  '--nlines=NUM', sprintf("Number of lines to be included (Def: %s)", OPTS[:nlines]), Integer) { |v| OPTS[:nlines]=v }
+  opt.on(  '--pg-dump-command=COM', sprintf("pg_dump command name (Def: %s)", OPTS[:pg_dump_command])) { |v| OPTS[:pg_dump_command]=v }
+  opt.on(  '--read-file=FILENAME', sprintf("Specify the filename for direct input from a file (Def: '').  If you specify this, you probablly do not want to specify any other options.", OPTS[:read_file])) { |v| OPTS[:read_file]=v }
+  # opt.on(  '--[no-]debug', "Debug (Def: false)", TrueClass) {|v| OPTS[:debug] = v}
+  opt.on(  '--version', "Display the version and exits.", TrueClass) {|v| OPTS[:version] = v}  # Consider opts.on_tail
+  opt.separator ""        # Way to control a help message.
+  opt.separator "Note:"
+  opt.separator " NUM lines (or rows) of data at most per table are included."
+  opt.separator ' Please make sure to include "--" always!'
+  opt.separator ' "[options]" above are any options for pg_dump'
+  opt.separator " Do not include --schema-only as it does not make sense!"
+  opt.separator " Do not include --(*-)inserts as this script would not handle it."
+  opt.separator " Useful options include '-t|-T table' '--disable-triggers', '--no-owner'"
+  opt.parse!(ARGV)
+
+  if OPTS[:version]
+    print_version_and_exit('db_suit_rails/sql_skelton')
+    # print_version_and_exit('did_you_mean')  # for testing
+  end
+
+  OPTS
+end
+
+
+################################################
+# MAIN
+################################################
+
+$stdout.sync=true
+$stderr.sync=true
+
+# Handle the command-line options => OPTS
+handle_argv()
+
+st_argv = ARGV.join(' ')
+com = if OPTS[:read_file]
+        sprintf("cat %s %s", st_argv, OPTS[:read_file])
+      else
+        OPTS[:pg_dump_command]+' '+st_argv
+      end
+
+counter = -1;
+Open3.popen2(com){|ioIn,ioStdOut,wt|
+  ioStdOut.each_line do |ea_line|
+    next if counter >= OPTS[:nlines] && /^\\\./ !~ ea_line
+      
+    print ea_line
+
+    if counter >= OPTS[:nlines] || (/^\\\./ !~ ea_line && counter >= 0)
+      counter = -1;
+    elsif counter >= 0
+      counter += 1;
+    elsif /^COPY / =~ ea_line
+      counter = 0
+    end
+  end
+  # p wt.value.exitstatus
+}
+
+exit
+
+__END__
+

data/bin/mk_sqlskelton

@@ -0,0 +1,158 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+
+require 'optparse'
+require 'db_suit_rails/sql_skelton'
+
+BANNER = <<"__EOF__"
+USAGE: #{File.basename($0)} [options] [INFILE.sql] < STDIN
+  INFILE (or STDIN) is an output of pg_dump (-s, -t|-T)
+__EOF__
+
+# Initialising the hash for the command-line options.
+OPTS = {
+  :outfile  => 'rails_db.sql',
+  :mappingcsv => SqlSkelton::DefMappingCsvname,  # 'mapping.csv'
+  :mappingsql => {:filename => nil},
+  :withsql => true,
+  :mapping_tbl_prefix => '',
+  :mapping_col_suffix_from => SqlSkelton::DefMappingSql[:colsuffix4from],
+  :mapping_col_suffix_to   => SqlSkelton::DefMappingSql[:colsuffix4to],
+  :delete_primary => true,
+  :delete_sequence => false,  ###### Investigate...
+  :delete_trigger => true,
+  # :clobber => true,     # Default
+  # :chatter => 3,        # Default
+  :dryrun => false,
+  :debug => false,
+}
+
+# Print the version number and exit
+#
+# If the global constant VERSION is defiend, it is returned.
+# Else, this assumes the library to load follows the proper Ruby/Gem style,
+# where the library is found in the path /.../gems/GEM_NAME-1.2.3/lib/
+#
+# @param libfile [String] the library file, e.g., 'my_gem_name/lib_to_load', where the file is used as in require 'my_gem_name/lib_to_load'
+# @return [String,NilClass] eg., '(Gem) did_you_mean-1.2.0'
+def get_version_in_gem(libfile)
+  defined?(VERSION) && (return VERSION)
+
+  libfile_base = libfile.sub(/^(\/)?(.+)(\.rb)?$/, '\2')
+  $LOAD_PATH.each do |dir|
+    if Dir.glob(dir+"/#{libfile_base}{,.rb}").size > 1
+      retstr = dir.sub(/\/lib\/?$/, '').sub(/.*\//, '')
+      if (%r@/gems/$@ =~ $&)
+        retstr.sub!(/^/, '(Gem) ')
+      else
+        retstr = sprintf("(Library under directory): %s", dir)
+      end
+      return retstr
+    end
+  end
+
+  ## Perhaps not in Gem
+  nil
+end
+
+
+# Print the version number and exit
+#
+# @param see #get_version_in_gem
+# @return [NilClass]
+def print_version_and_exit(libfile)
+  version = get_version_in_gem(libfile)
+
+  if version
+    puts version
+    exit
+  end
+
+  warn "ERROR: No version is found.  Perhaps the command is not in the properly installed Gem style (or currently in development)?"
+  exit 1
+end
+
+
+# Function to handle the command-line arguments.
+#
+# ARGV will be modified, and the constant variable OPTS is set.
+#
+# @return [Hash]  Optional-argument hash.
+#
+def handle_argv
+  opt = OptionParser.new(BANNER)
+  opt.on('-o OUTFILE', '--outfile=FILE.SQL', sprintf("Output SQL filename (Def: %s)", OPTS[:outfile]), String) { |v| OPTS[:outfile]=v }
+  opt.on('-m MAPPINGCSV', '--mappingfile=FILE.CSV', sprintf("Output mapping CSV filename (Def: %s)", OPTS[:mappingcsv])) { |v| OPTS[:mappingcsv]=v }
+  opt.on(  '--[no-]withsql', sprintf("If the SQL file is created (Def: %s)", OPTS[:withsql].inspect), TrueClass) {|v| OPTS[:withsql] = v}
+  opt.on(  '--mappingsql=FILE.SQL', sprintf("Output mapping SQL filename (Def: mapping.sql)")) { |v| OPTS[:mappingsql][:filename]=v }
+  opt.on(  '--mapping-tbl-prefix=PREFIX', sprintf("Mapping SQL-tablename prefix (Def: %s)", OPTS[:mapping_tbl_prefix].inspect)) { |v| OPTS[:mapping_tbl_prefix]=v }
+  opt.on(  '--mapping-tbl-suffix-from=SUFFIX', sprintf("Mapping SQL-column-name suffix for 'from' (Def: %s)", OPTS[:mapping_col_suffix_from].inspect)) { |v| OPTS[:mapping_col_suffix_from]=v }
+  opt.on(  '--mapping-tbl-suffix-to=SUFFIX',   sprintf("Mapping SQL-column-name suffix for 'to' (Def: %s)",   OPTS[:mapping_col_suffix_to].inspect)  ) { |v| OPTS[:mapping_col_suffix_to]=v }
+  opt.on(  '--[no-]delete-primary', "Delete primary keys (Def: #{OPTS[:delete_primary]})", TrueClass) {|v| OPTS[:delete_primary] = v}
+  # opt.on(  '--[no-]delete-sequence', "Delete sequences (Def: #{OPTS[:delete_sequence]})", TrueClass) {|v| OPTS[:delete_sequence] = v}
+  # opt.on(  '--[no-]delete-trigger', "Delete triggers (Def: #{OPTS[:delete_trigger]})", TrueClass) {|v| OPTS[:delete_trigger] = v}
+  # opt.on(  '--[no-]clobber', sprintf("Overwrites the existing files if true (Default: %s)", OPTS[:clobber].inspect.capitalize), TrueClass) { |v| OPTS[:clobber] = v}
+  # opt.on(  '--chatter=INT', Integer, sprintf("Chatter level (0(silent) to 5(chatty), Default: %d)", OPTS[:chatter])) { |v| OPTS[:chatter] = v}
+  opt.on('-n', '--[no-]dryrun', "Dryrun (Def: #{OPTS[:dryrun]})", TrueClass) {|v| OPTS[:dryrun] = v}
+  opt.on(  '--[no-]debug', "Debug (Def: false)", TrueClass) {|v| OPTS[:debug] = v}
+  opt.on(  '--version', "Display the version and exits.", TrueClass) {|v| OPTS[:version] = v}  # Consider opts.on_tail
+  opt.separator ""        # Way to control a help message.
+  opt.separator "Note:"
+  opt.separator " Mapping-CSV-file has the columns of  Old_Table, New_Table, Old_Column, New_Column."
+  opt.separator " Output files are overwritten."
+  opt.separator " In running psql after this script, --single-transaction option is highly recommended."
+
+  opt.parse!(ARGV)
+
+  if OPTS[:version]
+    print_version_and_exit('db_suit_rails/sql_skelton')
+    # print_version_and_exit('did_you_mean')  # for testing
+  end
+
+  if OPTS[:withsql]
+    ## See {SqlSkelton#new} for the specification of the Hash
+    
+    # Sets a default if --mappingsql is not specified explicitly.
+    OPTS[:mappingsql][:filename] ||= OPTS[:mappingcsv].sub(/\.csv$/i, '') + '.sql'  # => (Default)"mapping.sql"
+
+    OPTS[:mappingsql][:tblname] = {
+      :tbl => OPTS[:mapping_tbl_prefix] + SqlSkelton::DefMappingSql[:tblname4tbl],
+      :col => OPTS[:mapping_tbl_prefix] + SqlSkelton::DefMappingSql[:tblname4col],
+    }
+    OPTS[:mappingsql][:colname] = {
+      :tbl => {
+        :from => SqlSkelton::DefMappingSql[:colprefix4tbl] + OPTS[:mapping_col_suffix_from],
+        :to   => SqlSkelton::DefMappingSql[:colprefix4tbl] + OPTS[:mapping_col_suffix_to],
+      },
+      :col => {
+        :from => SqlSkelton::DefMappingSql[:colprefix4col] + OPTS[:mapping_col_suffix_from],
+        :to   => SqlSkelton::DefMappingSql[:colprefix4col] + OPTS[:mapping_col_suffix_to],
+      },
+    }
+  end
+
+  if 'stdout' == OPTS[:outfile].downcase
+    OPTS[:outfile] = $stdout
+  end
+
+  OPTS
+end
+
+
+################################################
+# MAIN
+################################################
+
+$stdout.sync=true
+$stderr.sync=true
+
+# Handle the command-line options => OPTS
+handle_argv()
+
+ss = SqlSkelton.new(ARGF, OPTS[:outfile], mappingcsv: OPTS[:mappingcsv], mappingsql: OPTS[:mappingsql])
+ss.run(dryrun: OPTS[:dryrun], delete_primary: OPTS[:delete_primary], delete_sequence: OPTS[:delete_sequence], delete_trigger: OPTS[:delete_trigger])
+
+exit
+
+__END__
+