flattendb 0.0.8 → 0.1.0

data/README

@@ -2,7 +2,7 @@
 
 == VERSION
 
-This documentation refers to flattendb version 0.0.8
+This documentation refers to flattendb version 0.1.0
 
 
 == DESCRIPTION

data/Rakefile

@@ -14,7 +14,7 @@ begin
       :summary      => %q{Flatten relational databases.},
       :author       => %q{Jens Wille},
       :email        => %q{jens.wille@uni-koeln.de},
-      :dependencies => %w[highline libxml-ruby builder ruby-nuggets]
+      :dependencies => %w[libxml-ruby builder ruby-nuggets] << ['athena', '>= 0.1.5']
     }
   }}
 rescue LoadError => err

data/bin/flattendb

@@ -28,168 +28,10 @@
 ###############################################################################
 #++
 
-require 'optparse'
-
-require 'rubygems'
-require 'highline/import'
-require 'nuggets/array/flatten_once'
-
-$: << File.join(File.dirname(__FILE__), '..', 'lib')
-
-require 'flattendb'
 require 'flattendb/cli'
 
-include FlattenDB::CLI
-
-USAGE = "Usage: #{$0} [-h|--help] [options]"
-abort USAGE if ARGV.empty?
-
-$global_options = {
-  :type    => :mysql,
-  :infiles => [],
-  :outfile => nil,
-  :confile => nil,
-  :keep    => false,
-  :mysql => {
-    :intype => :xml
-  },
-  :mdb => {
-    :intype => :mdb
-  }
-}
-
-$opts_by_type = {
-  :mysql => {
-    :title => 'MySQL',
-    :opts  => lambda { |opts|
-      opts.on('-x', '--mysql-xml', "Input file is of type XML [This is the default]") {
-        $global_options[:mysql][:intype] = :xml
-      }
-      opts.on('-s', '--sql', "Input file is of type SQL") {
-        $global_options[:mysql][:intype] = :sql
-      }
-    }
-  },
-  :mdb => {
-    :title => 'MS Access',
-    :opts  => lambda { |opts|
-      opts.on('-m', '--mdb', "Input file is of type MDB [This is the default]") {
-        $global_options[:mdb][:intype] = :mdb
-      }
-      opts.on('-y', '--mdb-xml', "Input file is of type XML") {
-        $global_options[:mdb][:intype] = :xml
-      }
-    }
-  }
-}
-
-def type_options(type, opts, with_heading = true)
-  if type == :all
-    $opts_by_type.keys.sort_by { |t| t.to_s }.each { |t|
-      type_options(t, opts, with_heading)
-    }
-  else
-    opt = $opts_by_type[type.to_sym]
-
-    if with_heading
-      opts.separator ''
-      opts.separator " - [#{type}] #{opt[:title]}"
-    end
-
-    opt[:opts][opts]
-  end
-end
-
-OptionParser.new { |opts|
-  opts.banner = USAGE
-
-  opts.separator ''
-  opts.separator 'Options:'
-
-  if $type
-    opts.separator "   [-t, --type]                      TYPE OVERRIDE IN EFFECT (#{$type})"
-  else
-    opts.on('-t', '--type TYPE', "Type of database at hand [Default: #{$global_options[:type]}]") { |t|
-      $global_options[:type] = t.downcase.to_sym
-    }
-  end
-
-  opts.separator ''
-
-  opts.on('-i', '--input-file FILE', "Input file; depending on the database type, this option", "may be given multiple times [REQUIRED]") { |f|
-    $global_options[:infiles] << f
-  }
-
-  opts.on('-o', '--output-file FILE', "Output file (flat XML) [REQUIRED]") { |f|
-    $global_options[:outfile] = f
-  }
-
-  opts.on('-c', '--config-file FILE', "Configuration file (YAML) [REQUIRED]") { |f|
-    $global_options[:confile] = f
-  }
-
-  opts.separator ''
-
-  opts.on('-k', '--keep', "Keep any intermediate files (generated XML dumps, etc.)") {
-    $global_options[:keep] = true
-  }
-
-  opts.separator ''
-  opts.separator 'Database-specific Options:'
-
-  if $type
-    type_options($type, opts, false)
-  else
-    type_options(:all, opts)
-  end
-
-  opts.separator ''
-  opts.separator 'Generic options:'
-
-  opts.on('-h', '--help', 'Print this help message and exit') {
-    abort opts.to_s
-  }
-
-  opts.on('--version', 'Print program version and exit') {
-    abort "#{File.basename($0)} v#{FlattenDB::VERSION}"
-  }
-}.parse!
-
-$type  ||= $global_options[:type]
-$options = $global_options[$type] || {}
-
-# Load corresponding module
-begin
-  require "flattendb/types/#{$type}"
-rescue LoadError
-  abort "Database type '#{$type}' is not supported at the moment."
-end
-
-abort "No output file specified" unless $global_options[:outfile]
-
-if confile = $global_options[:confile]
-  abort "Configuration file not found: #{confile}" unless File.readable?(confile)
-else
-  abort "No configuration file specified"
-end
-
-$global_options[:infiles].each { |infile|
-  abort "Input file not found: #{infile}" unless File.readable?(infile)
-}
-
-$infiles = $global_options[:infiles]
-$outfile = $global_options[:outfile]
-$confile = $global_options[:confile]
-
-# Load type-specific script
 begin
-  load File.join(File.dirname(__FILE__), "flattendb.#{$type}")
-rescue LoadError
-  # silently ignore
-end
-
-FlattenDB[$type].new($infile || $infiles, $outfile, $confile).flatten!.to_xml
-
-unless $global_options[:keep]
-  File.delete($dump_file) if $dump_file
+  FlattenDB::CLI.execute($flattendb_type, ARGV, STDIN, STDOUT, STDERR)
+rescue ArgumentError => err
+  abort err
 end

data/bin/flattendb.mdb

@@ -1,55 +1,7 @@
 #! /usr/bin/ruby
 
-#--
-###############################################################################
-#                                                                             #
-# flattendb -- Flatten relational databases                                   #
-#                                                                             #
-# Copyright (C) 2007-2011 University of Cologne,                              #
-#                         Albertus-Magnus-Platz,                              #
-#                         50923 Cologne, Germany                              #
-#                                                                             #
-# Authors:                                                                    #
-#     Jens Wille <jens.wille@uni-koeln.de>                                    #
-#                                                                             #
-# flattendb is free software; you can redistribute it and/or modify it under  #
-# the terms of the GNU Affero General Public License as published by the Free #
-# Software Foundation; either version 3 of the License, or (at your option)   #
-# any later version.                                                          #
-#                                                                             #
-# flattendb is distributed in the hope that it will be useful, but WITHOUT    #
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or       #
-# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License #
-# for more details.                                                           #
-#                                                                             #
-# You should have received a copy of the GNU Affero General Public License    #
-# along with flattendb. If not, see <http://www.gnu.org/licenses/>.           #
-#                                                                             #
-###############################################################################
-#++
-
-unless $type
-  $type = :mdb
-  load File.join(File.dirname(__FILE__), 'flattendb')
-else
-  case $options[:intype]
-    when :xml
-      $infiles.each { |infile|
-        unless IO.read(infile, 6) == '<?xml '
-          abort "Input file doesn't seem to be a valid XML file, XML declaration missing: #{infile}"
-        end
-      }
-    when :mdb
-      tables_cmd = 'mdb-tables'
-      export_cmd = 'mdb-export'
-
-      require_commands(tables_cmd, export_cmd, :pkg => 'mdbtools')
-      require_libraries('fastercsv')
-
-      $infiles.map! { |infile|
-        $dump_file
-      }
-  end
-end
+ext = File.extname(__FILE__)
+$flattendb_type = ext.sub(/\A\./, '')
+load __FILE__.sub(/#{Regexp.escape(ext)}\z/, '')
 
 # vim:ft=ruby

data/bin/flattendb.mysql

@@ -1,92 +1,7 @@
 #! /usr/bin/ruby
 
-#--
-###############################################################################
-#                                                                             #
-# flattendb -- Flatten relational databases                                   #
-#                                                                             #
-# Copyright (C) 2007-2011 University of Cologne,                              #
-#                         Albertus-Magnus-Platz,                              #
-#                         50923 Cologne, Germany                              #
-#                                                                             #
-# Authors:                                                                    #
-#     Jens Wille <jens.wille@uni-koeln.de>                                    #
-#                                                                             #
-# flattendb is free software; you can redistribute it and/or modify it under  #
-# the terms of the GNU Affero General Public License as published by the Free #
-# Software Foundation; either version 3 of the License, or (at your option)   #
-# any later version.                                                          #
-#                                                                             #
-# flattendb is distributed in the hope that it will be useful, but WITHOUT    #
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or       #
-# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License #
-# for more details.                                                           #
-#                                                                             #
-# You should have received a copy of the GNU Affero General Public License    #
-# along with flattendb. If not, see <http://www.gnu.org/licenses/>.           #
-#                                                                             #
-###############################################################################
-#++
-
-unless $type
-  $type = :mysql
-  load File.join(File.dirname(__FILE__), 'flattendb')
-else
-  $infile = $infiles.first
-
-  case $options[:intype]
-    when :xml
-      unless IO.read($infile, 6) == '<?xml '
-        abort "Input file doesn't seem to be a valid XML file, XML declaration missing"
-      end
-    when :sql
-      $dump_file = $infile.sub(/\.(?:sql|dump)(?:\.gz)?\z/i, '') << '.xml'
-      abort "Dump file and output file are the same: #{$dump_file} = #{$outfile}" \
-        if File.expand_path($dump_file) == File.expand_path($outfile)
-
-      mysql_cmd = 'mysql'
-      dump_cmd  = 'mysqldump'
-
-      require_commands(mysql_cmd, dump_cmd, :pkg => 'a suitable MySQL client')
-      require_libraries('mysql')
-
-      mysql_user = ask('Please enter the MySQL user name: ')              \
-        { |q| q.default = ENV['USER'] }
-      mysql_pass = ask('Please enter the MySQL password for that user: ') \
-        { |q| q.echo    = false       }
-
-      is_root = mysql_user == 'root'
-
-      temp_db   = ENV['FLATTEN_DB']   || 'flattendb_temp'
-      temp_db   = "#{temp_db}_%d_%d" % [Time.now, $$] if is_root
-
-      temp_user = ENV['FLATTEN_USER'] || 'flattendb_user'
-      temp_pass = ENV['FLATTEN_PASS'] || 'flattendb_pass'
-
-      input = $infile =~ /\.gz\z/i ? "<(zcat #{$infile})" : "< #{$infile}"
-      mysql_args = "--one-database -u#{temp_user} -p#{temp_pass} #{temp_db} #{input}"
-      dump_args  = "--xml -u#{temp_user} -p#{temp_pass} #{temp_db} > #{$dump_file}"
-
-      begin
-        mysql = Mysql.real_connect('localhost', mysql_user, mysql_pass)
-
-        mysql.query("CREATE DATABASE #{temp_db}")
-        mysql.query("GRANT ALL ON #{temp_db}.* TO '#{temp_user}'@'localhost' IDENTIFIED BY '#{temp_pass}'") if is_root
-
-        system("#{mysql_cmd} #{mysql_args} && #{dump_cmd} #{dump_args}")
-      rescue Mysql::Error => err
-        abort "ERROR #{err.errno} (#{err.sqlstate}): #{err.error}"
-      ensure
-        if mysql
-          mysql.query("REVOKE ALL ON #{temp_db}.* FROM '#{temp_user}'@'localhost'") if is_root
-          mysql.query("DROP DATABASE IF EXISTS #{temp_db}")
-
-          mysql.close
-        end
-      end
-
-      $infile = $dump_file
-  end
-end
+ext = File.extname(__FILE__)
+$flattendb_type = ext.sub(/\A\./, '')
+load __FILE__.sub(/#{Regexp.escape(ext)}\z/, '')
 
 # vim:ft=ruby

data/example/mysql-sample.flat-sql.xml

@@ -0,0 +1,120 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ndr>
+  <row>
+    <Bla>12</Bla>
+    <Blub></Blub>
+    <ObjID>1</ObjID>
+    <attr1>
+      <ObjID>1</ObjID>
+      <Val>3</Val>
+      <attr1ID>1</attr1ID>
+    </attr1>
+    <attr2>
+      <ObjID>1</ObjID>
+      <Val>4</Val>
+      <attr2ID>3</attr2ID>
+    </attr2>
+    <barobject>
+      <Bar>1002</Bar>
+      <ObjID>1</ObjID>
+      <attr4>
+        <Val>0</Val>
+        <attr4ID>2</attr4ID>
+      </attr4>
+      <attr4ID>2</attr4ID>
+    </barobject>
+    <fooobject>
+      <Foo>112</Foo>
+      <ObjID>1</ObjID>
+      <attr1>
+        <ObjID>1</ObjID>
+        <Val>3</Val>
+        <attr1ID>1</attr1ID>
+      </attr1>
+      <attr1ID>1</attr1ID>
+      <attr3>
+        <Val>8</Val>
+        <attr3ID>2</attr3ID>
+      </attr3>
+      <attr3ID>2</attr3ID>
+    </fooobject>
+  </row>
+  <row>
+    <Bla>12</Bla>
+    <Blub>hi</Blub>
+    <ObjID>2</ObjID>
+    <attr1>
+      <ObjID>2</ObjID>
+      <Val>2</Val>
+      <attr1ID>2</attr1ID>
+    </attr1>
+    <attr2>
+      <ObjID>2</ObjID>
+      <Val>5</Val>
+      <attr2ID>2</attr2ID>
+    </attr2>
+    <barobject>
+      <Bar>1200</Bar>
+      <ObjID>2</ObjID>
+      <attr4>
+        <Val>0</Val>
+        <attr4ID>1</attr4ID>
+      </attr4>
+      <attr4ID>1</attr4ID>
+    </barobject>
+    <fooobject>
+      <Foo>122</Foo>
+      <ObjID>2</ObjID>
+      <attr1>
+        <ObjID>2</ObjID>
+        <Val>2</Val>
+        <attr1ID>2</attr1ID>
+      </attr1>
+      <attr1ID>2</attr1ID>
+      <attr3>
+        <Val>0</Val>
+        <attr3ID>1</attr3ID>
+      </attr3>
+      <attr3ID>1</attr3ID>
+    </fooobject>
+  </row>
+  <row>
+    <Bla>1</Bla>
+    <Blub>ho</Blub>
+    <ObjID>3</ObjID>
+    <attr1>
+      <ObjID>3</ObjID>
+      <Val>1</Val>
+      <attr1ID>3</attr1ID>
+    </attr1>
+    <attr2>
+      <ObjID>3</ObjID>
+      <Val>6</Val>
+      <attr2ID>1</attr2ID>
+    </attr2>
+    <barobject>
+      <Bar>1000</Bar>
+      <ObjID>3</ObjID>
+      <attr4>
+        <Val>9</Val>
+        <attr4ID>3</attr4ID>
+      </attr4>
+      <attr4ID>3</attr4ID>
+    </barobject>
+    <fooobject>
+      <Foo>111</Foo>
+      <ObjID>3</ObjID>
+      <attr1>
+        <ObjID>3</ObjID>
+        <Val>1</Val>
+        <attr1ID>3</attr1ID>
+      </attr1>
+      <attr1ID>3</attr1ID>
+      <attr3>
+        <Val>7</Val>
+        <attr3ID>3</attr3ID>
+      </attr3>
+      <attr3ID>3</attr3ID>
+    </fooobject>
+  </row>
+</ndr>

data/example/mysql-sample.sql

@@ -0,0 +1,137 @@
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+DROP TABLE IF EXISTS `object`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `object` (
+  `ObjID` int(11) NOT NULL,
+  `Bla` int(11) NOT NULL,
+  `Blub` varchar(50) DEFAULT NULL,
+  PRIMARY KEY (`ObjID`)
+) ENGINE=MyISAM DEFAULT CHARSET=utf8;
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+LOCK TABLES `object` WRITE;
+/*!40000 ALTER TABLE `object` DISABLE KEYS */;
+INSERT INTO `object` VALUES (1,12,NULL),(2,12,'hi'),(3,1,'ho');
+/*!40000 ALTER TABLE `object` ENABLE KEYS */;
+UNLOCK TABLES;
+
+DROP TABLE IF EXISTS `fooobject`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `fooobject` (
+  `ObjID` int(11) NOT NULL,
+  `attr1ID` int(11) NOT NULL,
+  `attr3ID` int(11) NOT NULL,
+  `Foo` int(11) NOT NULL,
+  PRIMARY KEY (`ObjID`)
+) ENGINE=MyISAM DEFAULT CHARSET=utf8;
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+LOCK TABLES `fooobject` WRITE;
+/*!40000 ALTER TABLE `fooobject` DISABLE KEYS */;
+INSERT INTO `fooobject` VALUES (1,1,2,112),(2,2,1,122),(3,3,3,111);
+/*!40000 ALTER TABLE `fooobject` ENABLE KEYS */;
+UNLOCK TABLES;
+
+DROP TABLE IF EXISTS `barobject`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `barobject` (
+  `ObjID` int(11) NOT NULL,
+  `attr4ID` int(11) NOT NULL,
+  `Bar` int(11) NOT NULL,
+  PRIMARY KEY (`ObjID`)
+) ENGINE=MyISAM DEFAULT CHARSET=utf8;
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+LOCK TABLES `barobject` WRITE;
+/*!40000 ALTER TABLE `barobject` DISABLE KEYS */;
+INSERT INTO `barobject` VALUES (1,2,1002),(2,1,1200),(3,3,1000);
+/*!40000 ALTER TABLE `barobject` ENABLE KEYS */;
+UNLOCK TABLES;
+
+DROP TABLE IF EXISTS `attr1`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `attr1` (
+  `ObjID` int(11) NOT NULL,
+  `attr1ID` int(11) NOT NULL,
+  `Val` int(11) NOT NULL,
+  PRIMARY KEY (`attr1ID`)
+) ENGINE=MyISAM DEFAULT CHARSET=utf8;
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+LOCK TABLES `attr1` WRITE;
+/*!40000 ALTER TABLE `attr1` DISABLE KEYS */;
+INSERT INTO `attr1` VALUES (1,1,3),(2,2,2),(3,3,1);
+/*!40000 ALTER TABLE `attr1` ENABLE KEYS */;
+UNLOCK TABLES;
+
+DROP TABLE IF EXISTS `attr2`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `attr2` (
+  `ObjID` int(11) NOT NULL,
+  `attr2ID` int(11) NOT NULL,
+  `Val` int(11) NOT NULL,
+  PRIMARY KEY (`attr2ID`)
+) ENGINE=MyISAM DEFAULT CHARSET=utf8;
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+LOCK TABLES `attr2` WRITE;
+/*!40000 ALTER TABLE `attr2` DISABLE KEYS */;
+INSERT INTO `attr2` VALUES (1,3,4),(2,2,5),(3,1,6);
+/*!40000 ALTER TABLE `attr2` ENABLE KEYS */;
+UNLOCK TABLES;
+
+DROP TABLE IF EXISTS `attr3`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `attr3` (
+  `attr3ID` int(11) NOT NULL,
+  `Val` int(11) NOT NULL,
+  PRIMARY KEY (`attr3ID`)
+) ENGINE=MyISAM DEFAULT CHARSET=utf8;
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+LOCK TABLES `attr3` WRITE;
+/*!40000 ALTER TABLE `attr3` DISABLE KEYS */;
+INSERT INTO `attr3` VALUES (1,0),(2,8),(3,7);
+/*!40000 ALTER TABLE `attr3` ENABLE KEYS */;
+UNLOCK TABLES;
+
+DROP TABLE IF EXISTS `attr4`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `attr4` (
+  `attr4ID` int(11) NOT NULL,
+  `Val` int(11) NOT NULL,
+  PRIMARY KEY (`attr4ID`)
+) ENGINE=MyISAM DEFAULT CHARSET=utf8;
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+LOCK TABLES `attr4` WRITE;
+/*!40000 ALTER TABLE `attr4` DISABLE KEYS */;
+INSERT INTO `attr4` VALUES (1,0),(2,0),(3,9);
+/*!40000 ALTER TABLE `attr4` ENABLE KEYS */;
+UNLOCK TABLES;
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;

data/lib/flattendb/base.rb

@@ -27,9 +27,8 @@
 #++
 
 require 'yaml'
-
-require 'rubygems'
 require 'builder'
+require 'flattendb'
 
 module FlattenDB
 
@@ -49,8 +48,12 @@ module FlattenDB
 
     class << self
 
+      def to_flat!(*args)
+        new(*args).flatten!.to_xml
+      end
+
       def types
-        Base.instance_variable_get :@types
+        Base.instance_variable_get(:@types)
       end
 
       def [](type)
@@ -67,40 +70,13 @@ module FlattenDB
 
     attr_reader :root, :config, :input, :output
 
-    def initialize(infiles, outfile, config)
-      config = case config
-        when Hash
-          config
-        when String
-          # assume file name
-          YAML.load_file(config)
-        else
-          raise ArgumentError, "invalid config argument of type '#{config.class}'"
-      end
+    def initialize(options)
+      config = options.select { |k, _| k.is_a?(String) }
       raise ArgumentError, "can't have more than one primary (root) table" if config.size > 1
 
       (@root, @config), _ = *config  # get "first" (and only) hash element
 
-      @input = [*infiles].map { |infile|
-        case infile
-          when String
-            infile
-          when File
-            infile.path
-          else
-            raise ArgumentError, "invalid infile argument of type '#{infile.class}'"
-        end
-      }
-
-      @output = case outfile
-        when IO
-          outfile
-        when String
-          # assume file name
-          File.open(outfile, 'w')
-        else
-          raise ArgumentError, "invalid outfile argument of type '#{outfile.class}'"
-      end
+      @input, @output = options.values_at(:input, :output)
     end
 
     def flatten!(*args)

data/lib/flattendb/cli.rb

@@ -26,52 +26,212 @@
 ###############################################################################
 #++
 
+require 'optparse'
+require 'yaml'
+require 'zlib'
+require 'flattendb'
+
 module FlattenDB
 
-  module CLI
+  class CLI
 
-    def require_libraries(*libraries)
-      parse_arguments(libraries, :gem).each { |lib, gem|
-        begin
-          require lib
-        rescue LoadError
-          abort_with_msg('Ruby library not found: %s', lib, 'Please install gem %s first', gem)
-        end
+    USAGE = "Usage: #{$0} [-h|--help] [options]"
+
+    DEFAULTS = {
+      :input  => '-',
+      :inputs => [],
+      :output => '-',
+      :config => 'config.yaml'
+    }
+
+    TYPES = {
+      :mysql => {
+        :title => 'MySQL',
+        :opts  => lambda { |opts, options|
+          opts.on('-x', '--xml', 'Input file is of type XML [This is the default]') {
+            options[:type] = :xml
+          }
+          opts.on('-s', '--sql', 'Input file is of type SQL') {
+            options[:type] = :sql
+          }
+        }
+      },
+      :mdb => {
+        :title => 'MS Access',
+        :opts  => lambda { |opts, options|
+          opts.separator("    NOTE: Repeat '-i' for each .mdb file")
+        }
       }
+    }
+
+    def self.execute(type = nil, *args)
+      new(type).execute(*args)
     end
 
-    def require_commands(*commands)
-      parse_arguments(commands, :pkg).each { |cmd, pkg|
-        catch :cmd_found do
-          ENV['PATH'].split(':').each { |path|
-            throw :cmd_found if File.executable?(File.join(path, cmd))
-          }
+    attr_reader :type, :options, :config, :defaults
+    attr_reader :stdin, :stdout, :stderr
 
-          abort_with_msg("Command not found: #{cmd}", "Please install #{pkg} first", pkg || cmd)
-        end
-      }
+    def initialize(type = nil, defaults = DEFAULTS)
+      @defaults = defaults
+
+      reset(type)
+
+      # prevent backtrace on ^C
+      trap(:INT) { exit 130 }
     end
 
-    def abort_with_msg(msg1, arg1, msg2, arg2)
-      msg  = msg1 % arg1
-      msg += " (#{msg2})" % (arg2 || arg1)
+    def execute(arguments = [], *inouterr)
+      reset(type, *inouterr)
+
+      parse_options(arguments, defaults)
 
-      abort msg
+      if type
+        require "flattendb/types/#{type}"
+      else
+        abort 'Database type is required!'
+      end
+
+      options[:input] = if type == :mdb
+        if options[:inputs].empty?
+          if arguments.empty?
+            options[:inputs] << options[:input]
+          else
+            options[:inputs].concat(arguments)
+            arguments.clear
+          end
+        end
+
+        options[:inputs].map! { |file| open_file_or_std(file) }
+      else
+        open_file_or_std(
+          options[:inputs].last || arguments.shift || options[:input]
+        )
+      end
+
+      abort USAGE unless arguments.empty?
+
+      options[:output] = open_file_or_std(options[:output], true)
+
+      FlattenDB[type].to_flat!(options)
+    ensure
+      options[:output].close if options[:output].is_a?(Zlib::GzipWriter)
+    end
+
+    def reset(type = nil, stdin = STDIN, stdout = STDOUT, stderr = STDERR)
+      @stdin, @stdout, @stderr = stdin, stdout, stderr
+      self.type, @options, @config = type, {}, {}
     end
 
     private
 
-    def parse_arguments(arguments, option)
-      options = arguments.last.is_a?(Hash) ? arguments.pop : {}
-      special = options.delete(option)
+    def type=(type)
+      if type
+        @type = type.to_s.downcase.to_sym
+        abort "Database type not supported: #{type}" unless TYPES.has_key?(@type)
+      else
+        @type = nil
+      end
+    end
+
+    def open_file_or_std(file, write = false)
+      if file == '-'
+        write ? stdout : stdin
+      else
+        gz = file =~ /\.gz\z/i
+
+        if write
+          gz ? Zlib::GzipWriter.open(file) : File.open(file, 'w')
+        else
+          abort "No such file: #{file}" unless File.readable?(file)
+          (gz ? Zlib::GzipReader : File).open(file)
+        end
+      end
+    end
+
+    def warn(msg, output = stderr)
+      output.puts(msg)
+    end
+
+    def abort(msg = nil, status = 1, output = stderr)
+      warn(msg, output) if msg
+      exit(status)
+    end
+
+    def parse_options(arguments, defaults)
+      option_parser(defaults).parse!(arguments)
+
+      config_file = options[:config] || defaults[:config]
+      @config = YAML.load_file(config_file) if File.readable?(config_file)
+
+      [config, defaults].each { |hash| hash.each { |key, value| options[key] ||= value } }
+    end
+
+    def option_parser(defaults)
+      sorted_types = TYPES.keys.sort_by { |t| t.to_s }
+
+      OptionParser.new { |opts|
+        opts.banner = USAGE
+
+      if type
+        opts.separator ''
+        opts.separator "TYPE = #{type} (#{TYPES[type][:title]})"
+      end
+
+        opts.separator ''
+        opts.separator 'Options:'
+
+      unless type
+        opts.on('-t', '--type TYPE', 'Type of database [REQUIRED]') { |type|
+          self.type = type
+        }
+
+        opts.separator ''
+      end
+
+        opts.on('-i', '--input FILE', 'Input file(s) [Default: STDIN]') { |input|
+          (options[:inputs] ||= []) << input
+        }
+
+        opts.on('-o', '--output FILE', 'Output file (flat XML) [Default: STDOUT]') { |output|
+          options[:output] = output
+        }
+
+        opts.on('-c', '--config FILE', "Configuration file (YAML) [Default: #{defaults[:config]}#{' (currently not present)' unless File.readable?(defaults[:config])}]") { |config|
+          options[:config] = config
+        }
+
+        opts.separator ''
+        opts.separator 'Database-specific options:'
+
+        type ? type_options(opts) : sorted_types.each { |t| type_options(opts, true, t) }
 
-      arguments.map { |arg|
-        [arg, special]
-      } + options.map { |args, spx|
-        [*args].map { |arg|
-          [arg, spx]
+        opts.separator ''
+        opts.separator 'Generic options:'
+
+        opts.on('-h', '--help', 'Print this help message and exit') {
+          abort opts.to_s
         }
-      }.flatten_once
+
+        opts.on('--version', 'Print program version and exit') {
+          abort "#{File.basename($0)} v#{FlattenDB::VERSION}"
+        }
+
+      unless type
+        opts.separator ''
+        opts.separator "Supported database types: #{sorted_types.map { |t| "#{t} (#{TYPES[t][:title]})" }.join(', ')}."
+      end
+      }
+    end
+
+    def type_options(opts, heading = false, type = type)
+      cfg = TYPES[type]
+
+      if heading
+        opts.separator ''
+        opts.separator " - [#{type}] #{cfg[:title]}"
+      end
+
+      cfg[:opts][opts, options]
     end
 
   end

data/lib/flattendb/types/mdb.rb

@@ -26,16 +26,28 @@
 ###############################################################################
 #++
 
-require 'flattendb/base'
+require 'fastercsv'
+require 'nuggets/file/which'
+require 'flattendb'
 
 module FlattenDB
 
   class MDB < Base
 
-    JOIN_KEY = '@key'
-
-    def initialize(infiles, outfile, config)
+    def initialize(options)
       super
+      parse
+    end
+
+    def parse
+      tables_cmd, export_cmd = 'mdb-tables', 'mdb-export'
+
+      [tables_cmd, export_cmd].each { |cmd|
+        next if File.which(cmd)
+        abort "Command not found: #{cmd}! Please install `mdbtools' first."
+      }
+
+      # ...
     end
 
     def flatten!(options = {}, builder_options = {})

data/lib/flattendb/types/mysql.rb

@@ -27,8 +27,8 @@
 #++
 
 require 'libxml'
-
-require 'flattendb/base'
+require 'athena'
+require 'flattendb'
 
 module FlattenDB
 
@@ -36,22 +36,21 @@ module FlattenDB
 
     JOIN_KEY = '@key'
 
-    attr_reader :document, :database, :name, :tables, :builder
+    attr_reader :type, :name, :tables, :builder
 
-    def initialize(infile, outfile, config)
+    def initialize(options)
       super
 
-      @document = LibXML::XML::Document.file(@input.first)
-      @database = @document.root.find_first('database[@name]')
-      @name     = @database[:name]
-      @tables   = {}
+      @type = options[:type] || :xml
+      @name, @tables = parse
+    end
 
-      parse
+    def parse(tables = {})
+      [send("parse_#{type}", tables) || 'root', tables]
     end
 
     def flatten!(options = {}, builder_options = {})
       flatten_tables!(tables, root, config)
-
       self
     end
 
@@ -76,9 +75,12 @@ module FlattenDB
 
     private
 
-    def parse
+    def parse_xml(tables)
+      document = LibXML::XML::Document.io(input)
+      database = document.root.find_first('database[@name]')
+
       database.find('table_data[@name]').each { |table|
-        rows = []
+        rows = tables[table[:name]] ||= []
 
         table.find('row').each { |row|
           fields = {}
@@ -89,9 +91,44 @@ module FlattenDB
 
           rows << fields
         }
+      }
 
-        tables[table[:name]] = rows
+      database[:name]
+    end
+
+    def parse_sql(tables)
+      columns, table, name = Hash.new { |h, k| h[k] = [] }, nil, nil
+      parser = Athena::Formats::MySQL::SQLParser.new
+
+      input.each { |line|
+        case line
+          when /\AUSE\s+`(.+?)`/i
+            raise 'dump file contains more than one database' if name
+            name = $1
+          when /\ACREATE\s+TABLE\s+`(.+?)`/i
+            table = $1
+          when /\A\s+`(.+?)`/i
+            columns[table] << $1 if table
+          when /\A\).*;\Z/
+            table = nil
+          when /\AINSERT\s+INTO\s+`(.+?)`\s+VALUES\s*(.*);\Z/i
+            _columns = columns[_table = $1]
+            next if _columns.empty?
+
+            parser.parse($2) { |row|
+              fields = {}
+
+              row.each_with_index { |value, index|
+                column = _columns[index] or next
+                fields[column] = value.to_s
+              }
+
+              (tables[_table] ||= []) << fields
+            }
+        end
       }
+
+      name
     end
 
     def flatten_tables!(tables, primary_table, config)
@@ -144,7 +181,7 @@ module FlattenDB
       builder.tag!(table) {
         rows.each { |row|
           row_to_xml('row', row, builder)
-        }
+        } if rows
       }
     end
 

data/lib/flattendb/version.rb

@@ -3,8 +3,8 @@ module FlattenDB
   module Version
 
     MAJOR = 0
-    MINOR = 0
-    TINY  = 8
+    MINOR = 1
+    TINY  = 0
 
     class << self
 

data/lib/flattendb.rb

@@ -27,10 +27,13 @@
 #++
 
 require 'flattendb/version'
-require 'flattendb/base'
 
 module FlattenDB
 
+  autoload :Base,  'flattendb/base'
+  autoload :MDB,   'flattendb/types/mdb'
+  autoload :MySQL, 'flattendb/types/mysql'
+
   extend self
 
   def [](type)

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: flattendb
 version: !ruby/object:Gem::Version 
-  hash: 15
+  hash: 27
   prerelease: 
   segments: 
   - 0
+  - 1
   - 0
-  - 8
-  version: 0.0.8
+  version: 0.1.0
 platform: ruby
 authors: 
 - Jens Wille
@@ -15,10 +15,10 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-07-04 00:00:00 Z
+date: 2011-07-12 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
-  name: highline
+  name: libxml-ruby
   prerelease: false
   requirement: &id001 !ruby/object:Gem::Requirement 
     none: false
@@ -32,7 +32,7 @@ dependencies:
   type: :runtime
   version_requirements: *id001
 - !ruby/object:Gem::Dependency 
-  name: libxml-ruby
+  name: builder
   prerelease: false
   requirement: &id002 !ruby/object:Gem::Requirement 
     none: false
@@ -46,7 +46,7 @@ dependencies:
   type: :runtime
   version_requirements: *id002
 - !ruby/object:Gem::Dependency 
-  name: builder
+  name: ruby-nuggets
   prerelease: false
   requirement: &id003 !ruby/object:Gem::Requirement 
     none: false
@@ -60,17 +60,19 @@ dependencies:
   type: :runtime
   version_requirements: *id003
 - !ruby/object:Gem::Dependency 
-  name: ruby-nuggets
+  name: athena
   prerelease: false
   requirement: &id004 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
+        hash: 17
         segments: 
         - 0
-        version: "0"
+        - 1
+        - 5
+        version: 0.1.5
   type: :runtime
   version_requirements: *id004
 description: Flatten relational databases.
@@ -100,6 +102,7 @@ files:
 - Rakefile
 - COPYING
 - example/mysql-sample.flat.xml
+- example/mysql-sample.flat-sql.xml
 - example/mysql-sample2flat.yaml
 - example/mysql-sample.xml
 - example/mysql-sample.sql
@@ -111,7 +114,7 @@ rdoc_options:
 - --charset
 - UTF-8
 - --title
-- flattendb Application documentation (v0.0.8)
+- flattendb Application documentation (v0.1.0)
 - --main
 - README
 - --line-numbers