blackwinter-flattendb 0.0.4

data/ChangeLog

@@ -0,0 +1,5 @@
+= Revision history for flattendb
+
+== x.y.z [yyyy-mm-dd]
+
+* ...

data/README

@@ -0,0 +1,42 @@
+= flattendb - Flatten relational databases
+
+== VERSION
+
+This documentation refers to flattendb version 0.0.4
+
+
+== DESCRIPTION
+
+TODO: well, the description... ;-)
+
+
+== LINKS
+
+<b></b>
+Documentation::     <http://prometheus.rubyforge.org/flattendb>
+Source code (old):: <http://prometheus.rubyforge.org/svn/scratch/flattendb>
+Source code::       <http://github.com/blackwinter/flattendb>
+Rubyforge project:: <http://rubyforge.org/projects/prometheus>
+
+
+== AUTHORS
+
+* Jens Wille <mailto:jens.wille@uni-koeln.de>
+
+
+== LICENSE AND COPYRIGHT
+
+Copyright (C) 2007 University of Cologne,
+Albertus-Magnus-Platz, 50932 Cologne, Germany
+
+flattendb is free software: you can redistribute it and/or modify it under the
+terms of the GNU General Public License as published by the Free Software
+Foundation, either version 3 of the License, or (at your option) any later
+version.
+
+flattendb is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along with
+flattendb. If not, see <http://www.gnu.org/licenses/>.

data/Rakefile

@@ -0,0 +1,21 @@
+begin
+  require 'hen'
+rescue LoadError
+  abort "Please install the 'hen' gem first."
+end
+
+require 'lib/flattendb/version'
+
+Hen.lay! {{
+  :rubyforge => {
+    :package => 'flattendb'
+  },
+
+  :gem => {
+    :version      => FlattenDB::VERSION,
+    :summary      => 'Flatten relational databases.',
+    :files        => FileList['lib/**/*.rb', 'bin/*'].to_a,
+    :extra_files  => FileList['[A-Z]*', 'example/*'].to_a,
+    :dependencies => %w[highline libxml-ruby builder ruby-nuggets]
+  }
+}}

data/bin/flattendb

@@ -0,0 +1,195 @@
+#! /usr/bin/ruby
+
+#--
+###############################################################################
+#                                                                             #
+# flattendb -- Flatten relational databases                                   #
+#                                                                             #
+# Copyright (C) 2007 University of Cologne,                                   #
+#                    Albertus-Magnus-Platz,                                   #
+#                    50932 Cologne, Germany                                   #
+#                                                                             #
+# Authors:                                                                    #
+#     Jens Wille <jens.wille@uni-koeln.de>                                    #
+#                                                                             #
+# flattendb is free software; you can redistribute it and/or modify it under  #
+# the terms of the GNU General Public License as published by the Free        #
+# Software Foundation; either version 3 of the License, or (at your option)   #
+# any later version.                                                          #
+#                                                                             #
+# flattendb is distributed in the hope that it will be useful, but WITHOUT    #
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or       #
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for    #
+# more details.                                                               #
+#                                                                             #
+# You should have received a copy of the GNU General Public License along     #
+# with flattendb. If not, see <http://www.gnu.org/licenses/>.                 #
+#                                                                             #
+###############################################################################
+#++
+
+require 'optparse'
+
+require 'rubygems'
+require 'highline/import'
+require 'nuggets/array/flatten_once'
+
+$: << File.join(File.dirname(__FILE__), '..', 'lib')
+
+require 'flattendb'
+require 'flattendb/cli'
+
+include FlattenDB::CLI
+
+USAGE = "Usage: #{$0} [-h|--help] [options]"
+abort USAGE if ARGV.empty?
+
+$global_options = {
+  :type    => :mysql,
+  :infiles => [],
+  :outfile => nil,
+  :confile => nil,
+  :keep    => false,
+  :mysql => {
+    :intype => :xml
+  },
+  :mdb => {
+    :intype => :mdb
+  }
+}
+
+$opts_by_type = {
+  :mysql => {
+    :title => 'MySQL',
+    :opts  => lambda { |opts|
+      opts.on('-x', '--mysql-xml', "Input file is of type XML [This is the default]") {
+        $global_options[:mysql][:intype] = :xml
+      }
+      opts.on('-s', '--sql', "Input file is of type SQL") {
+        $global_options[:mysql][:intype] = :sql
+      }
+    }
+  },
+  :mdb => {
+    :title => 'MS Access',
+    :opts  => lambda { |opts|
+      opts.on('-m', '--mdb', "Input file is of type MDB [This is the default]") {
+        $global_options[:mdb][:intype] = :mdb
+      }
+      opts.on('-y', '--mdb-xml', "Input file is of type XML") {
+        $global_options[:mdb][:intype] = :xml
+      }
+    }
+  }
+}
+
+def type_options(type, opts, with_heading = true)
+  if type == :all
+    $opts_by_type.keys.sort_by { |t| t.to_s }.each { |t|
+      type_options(t, opts, with_heading)
+    }
+  else
+    opt = $opts_by_type[type.to_sym]
+
+    if with_heading
+      opts.separator ''
+      opts.separator " - [#{type}] #{opt[:title]}"
+    end
+
+    opt[:opts][opts]
+  end
+end
+
+OptionParser.new { |opts|
+  opts.banner = USAGE
+
+  opts.separator ''
+  opts.separator 'Options:'
+
+  if $type
+    opts.separator "   [-t, --type]                      TYPE OVERRIDE IN EFFECT (#{$type})"
+  else
+    opts.on('-t', '--type TYPE', "Type of database at hand [Default: #{$global_options[:type]}]") { |t|
+      $global_options[:type] = t.downcase.to_sym
+    }
+  end
+
+  opts.separator ''
+
+  opts.on('-i', '--input-file FILE', "Input file; depending on the database type, this option", "may be given multiple times [REQUIRED]") { |f|
+    $global_options[:infiles] << f
+  }
+
+  opts.on('-o', '--output-file FILE', "Output file (flat XML) [REQUIRED]") { |f|
+    $global_options[:outfile] = f
+  }
+
+  opts.on('-c', '--config-file FILE', "Configuration file (YAML) [REQUIRED]") { |f|
+    $global_options[:confile] = f
+  }
+
+  opts.separator ''
+
+  opts.on('-k', '--keep', "Keep any intermediate files (generated XML dumps, etc.)") {
+    $global_options[:keep] = true
+  }
+
+  opts.separator ''
+  opts.separator 'Database-specific Options:'
+
+  if $type
+    type_options($type, opts, false)
+  else
+    type_options(:all, opts)
+  end
+
+  opts.separator ''
+  opts.separator 'Generic options:'
+
+  opts.on('-h', '--help', 'Print this help message and exit') {
+    abort opts.to_s
+  }
+
+  opts.on('--version', 'Print program version and exit') {
+    abort "#{File.basename($0)} v#{FlattenDB::VERSION}"
+  }
+}.parse!
+
+$type  ||= $global_options[:type]
+$options = $global_options[$type] || {}
+
+# Load corresponding module
+begin
+  require "flattendb/types/#{$type}"
+rescue LoadError
+  abort "Database type '#{$type}' is not supported at the moment."
+end
+
+abort "No output file specified" unless $global_options[:outfile]
+
+if confile = $global_options[:confile]
+  abort "Configuration file not found: #{confile}" unless File.readable?(confile)
+else
+  abort "No configuration file specified"
+end
+
+$global_options[:infiles].each { |infile|
+  abort "Input file not found: #{infile}" unless File.readable?(infile)
+}
+
+$infiles = $global_options[:infiles]
+$outfile = $global_options[:outfile]
+$confile = $global_options[:confile]
+
+# Load type-specific script
+begin
+  load File.join(File.dirname(__FILE__), "flattendb.#{$type}")
+rescue LoadError
+  # silently ignore
+end
+
+FlattenDB[$type].new($infile || $infiles, $outfile, $confile).flatten!.to_xml
+
+unless $global_options[:keep]
+  File.delete($dump_file) if $dump_file
+end

data/bin/flattendb.mdb

@@ -0,0 +1,55 @@
+#! /usr/bin/ruby
+
+# vi:ft=ruby
+
+#--
+###############################################################################
+#                                                                             #
+# flattendb -- Flatten relational databases                                   #
+#                                                                             #
+# Copyright (C) 2007 University of Cologne,                                   #
+#                    Albertus-Magnus-Platz,                                   #
+#                    50932 Cologne, Germany                                   #
+#                                                                             #
+# Authors:                                                                    #
+#     Jens Wille <jens.wille@uni-koeln.de>                                    #
+#                                                                             #
+# flattendb is free software; you can redistribute it and/or modify it under  #
+# the terms of the GNU General Public License as published by the Free        #
+# Software Foundation; either version 3 of the License, or (at your option)   #
+# any later version.                                                          #
+#                                                                             #
+# flattendb is distributed in the hope that it will be useful, but WITHOUT    #
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or       #
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for    #
+# more details.                                                               #
+#                                                                             #
+# You should have received a copy of the GNU General Public License along     #
+# with flattendb. If not, see <http://www.gnu.org/licenses/>.                 #
+#                                                                             #
+###############################################################################
+#++
+
+unless $type
+  $type = :mdb
+  load File.join(File.dirname(__FILE__), 'flattendb')
+else
+  case $options[:intype]
+    when :xml
+      $infiles.each { |infile|
+        unless IO.read(infile, 6) == '<?xml '
+          abort "Input file doesn't seem to be a valid XML file, XML declaration missing: #{infile}"
+        end
+      }
+    when :mdb
+      tables_cmd = 'mdb-tables'
+      export_cmd = 'mdb-export'
+
+      require_commands(tables_cmd, export_cmd, :pkg => 'mdbtools')
+      require_libraries('fastercsv')
+
+      $infiles.map! { |infile|
+        $dump_file
+      }
+  end
+end

data/bin/flattendb.mysql

@@ -0,0 +1,91 @@
+#! /usr/bin/ruby
+
+# vi:ft=ruby
+
+#--
+###############################################################################
+#                                                                             #
+# flattendb -- Flatten relational databases                                   #
+#                                                                             #
+# Copyright (C) 2007 University of Cologne,                                   #
+#                    Albertus-Magnus-Platz,                                   #
+#                    50932 Cologne, Germany                                   #
+#                                                                             #
+# Authors:                                                                    #
+#     Jens Wille <jens.wille@uni-koeln.de>                                    #
+#                                                                             #
+# flattendb is free software; you can redistribute it and/or modify it under  #
+# the terms of the GNU General Public License as published by the Free        #
+# Software Foundation; either version 3 of the License, or (at your option)   #
+# any later version.                                                          #
+#                                                                             #
+# flattendb is distributed in the hope that it will be useful, but WITHOUT    #
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or       #
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for    #
+# more details.                                                               #
+#                                                                             #
+# You should have received a copy of the GNU General Public License along     #
+# with flattendb. If not, see <http://www.gnu.org/licenses/>.                 #
+#                                                                             #
+###############################################################################
+#++
+
+unless $type
+  $type = :mysql
+  load File.join(File.dirname(__FILE__), 'flattendb')
+else
+  $infile = $infiles.first
+
+  case $options[:intype]
+    when :xml
+      unless IO.read($infile, 6) == '<?xml '
+        abort "Input file doesn't seem to be a valid XML file, XML declaration missing"
+      end
+    when :sql
+      $dump_file = $infile.sub(/\.(?:sql|dump)$/i, '') << '.xml'
+      abort "Dump file and output file are the same: #{$dump_file} = #{$outfile}" \
+        if File.expand_path($dump_file) == File.expand_path($outfile)
+
+      mysql_cmd = 'mysql'
+      dump_cmd  = 'mysqldump'
+
+      require_commands(mysql_cmd, dump_cmd, :pkg => 'a suitable MySQL client')
+      require_libraries('mysql')
+
+      mysql_user = ask('Please enter the MySQL user name: ')              \
+        { |q| q.default = ENV['USER'] }
+      mysql_pass = ask("Please enter the MySQL password for that user: ") \
+        { |q| q.echo    = false       }
+
+      # according to <http://www.adamspiers.org/computing/mysqldiff/#how> MySQL
+      # default permissions allow anyone to create databases beginning with the
+      # prefix 'test_'
+      temp_db   = 'flattendb_temp_%d_%d' % [Time.now, $$]
+      temp_user = 'flattendb_user'
+      temp_pass = 'flattendb_pass'
+
+      mysql_args = "--one-database -u#{temp_user} -p#{temp_pass} #{temp_db} < #{$infile}"
+      dump_args  = "--xml -u#{temp_user} -p#{temp_pass} #{temp_db} > #{$dump_file}"
+
+      begin
+        mysql = Mysql.real_connect('localhost', mysql_user, mysql_pass)
+
+        mysql.query("CREATE DATABASE #{temp_db}")
+        mysql.query("GRANT ALL ON #{temp_db}.* TO '#{temp_user}'@'localhost' IDENTIFIED BY '#{temp_pass}'")
+
+        system("#{mysql_cmd} #{mysql_args} && #{dump_cmd} #{dump_args}")
+      rescue Mysql::Error => err
+        abort "ERROR #{err.errno} (#{err.sqlstate}): #{err.error}"
+      ensure
+        if mysql
+          mysql.query("REVOKE ALL ON #{temp_db}.* FROM '#{temp_user}'@'localhost'") \
+            if mysql_user == 'root'
+
+          mysql.query("DROP DATABASE IF EXISTS #{temp_db}")
+          mysql.close
+        end
+      end
+
+      $infile = $dump_file
+  end
+end

data/example/mysql-sample.flat.xml

@@ -0,0 +1,120 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ndr>
+  <row>
+    <Bla>12</Bla>
+    <Blub></Blub>
+    <ObjID>1</ObjID>
+    <attr1>
+      <ObjID>1</ObjID>
+      <Val>3</Val>
+      <attr1ID>1</attr1ID>
+    </attr1>
+    <attr2>
+      <ObjID>1</ObjID>
+      <Val>4</Val>
+      <attr2ID>3</attr2ID>
+    </attr2>
+    <barobject>
+      <Bar>1002</Bar>
+      <ObjID>1</ObjID>
+      <attr4>
+        <Val>0</Val>
+        <attr4ID>2</attr4ID>
+      </attr4>
+      <attr4ID>2</attr4ID>
+    </barobject>
+    <fooobject>
+      <Foo>112</Foo>
+      <ObjID>1</ObjID>
+      <attr1>
+        <ObjID>1</ObjID>
+        <Val>3</Val>
+        <attr1ID>1</attr1ID>
+      </attr1>
+      <attr1ID>1</attr1ID>
+      <attr3>
+        <Val>8</Val>
+        <attr3ID>2</attr3ID>
+      </attr3>
+      <attr3ID>2</attr3ID>
+    </fooobject>
+  </row>
+  <row>
+    <Bla>12</Bla>
+    <Blub>hi</Blub>
+    <ObjID>2</ObjID>
+    <attr1>
+      <ObjID>2</ObjID>
+      <Val>2</Val>
+      <attr1ID>2</attr1ID>
+    </attr1>
+    <attr2>
+      <ObjID>2</ObjID>
+      <Val>5</Val>
+      <attr2ID>2</attr2ID>
+    </attr2>
+    <barobject>
+      <Bar>1200</Bar>
+      <ObjID>2</ObjID>
+      <attr4>
+        <Val>0</Val>
+        <attr4ID>1</attr4ID>
+      </attr4>
+      <attr4ID>1</attr4ID>
+    </barobject>
+    <fooobject>
+      <Foo>122</Foo>
+      <ObjID>2</ObjID>
+      <attr1>
+        <ObjID>2</ObjID>
+        <Val>2</Val>
+        <attr1ID>2</attr1ID>
+      </attr1>
+      <attr1ID>2</attr1ID>
+      <attr3>
+        <Val>0</Val>
+        <attr3ID>1</attr3ID>
+      </attr3>
+      <attr3ID>1</attr3ID>
+    </fooobject>
+  </row>
+  <row>
+    <Bla>1</Bla>
+    <Blub>ho</Blub>
+    <ObjID>3</ObjID>
+    <attr1>
+      <ObjID>3</ObjID>
+      <Val>1</Val>
+      <attr1ID>3</attr1ID>
+    </attr1>
+    <attr2>
+      <ObjID>3</ObjID>
+      <Val>6</Val>
+      <attr2ID>1</attr2ID>
+    </attr2>
+    <barobject>
+      <Bar>1000</Bar>
+      <ObjID>3</ObjID>
+      <attr4>
+        <Val>9</Val>
+        <attr4ID>3</attr4ID>
+      </attr4>
+      <attr4ID>3</attr4ID>
+    </barobject>
+    <fooobject>
+      <Foo>111</Foo>
+      <ObjID>3</ObjID>
+      <attr1>
+        <ObjID>3</ObjID>
+        <Val>1</Val>
+        <attr1ID>3</attr1ID>
+      </attr1>
+      <attr1ID>3</attr1ID>
+      <attr3>
+        <Val>7</Val>
+        <attr3ID>3</attr3ID>
+      </attr3>
+      <attr3ID>3</attr3ID>
+    </fooobject>
+  </row>
+</ndr>