athena 0.1.4 → 0.1.5

data/README

@@ -2,7 +2,7 @@
 
 == VERSION
 
-This documentation refers to athena version 0.1.4
+This documentation refers to athena version 0.1.5
 
 
 == DESCRIPTION

data/example/config.yaml

@@ -70,3 +70,12 @@
       - "0370"
       - "0100"
     :string: "ort2sb=%s (HSTZusatz=%s [Untertitstab=%s] / VerfAnsetz=%s)"
+:dump-my:
+  :object:    [ObjID, Bla, Blub]
+  :fooobject: [ObjID, attr1ID, attr3ID, Foo]
+  :barobject: [ObjID, attr4ID, Bar]
+  :attr1:     [ObjID, attr1ID, Val]
+  :attr2:     [ObjID, attr2ID, Val]
+  :attr3:     [attr3ID, Val]
+  :attr4:     [attr4ID, Val]
+:dump-pg:

data/example/dump-my.sql

@@ -0,0 +1,202 @@
+-- MySQL dump 10.13  Distrib 5.1.54, for debian-linux-gnu (x86_64)
+--
+-- Host: localhost    Database: flattendb_temp_1310038141_10671
+-- ------------------------------------------------------
+-- Server version	5.1.54-1ubuntu4
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `attr1`
+--
+
+DROP TABLE IF EXISTS `attr1`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `attr1` (
+  `ObjID` int(11) NOT NULL,
+  `attr1ID` int(11) NOT NULL,
+  `Val` int(11) NOT NULL,
+  PRIMARY KEY (`attr1ID`)
+) ENGINE=MyISAM DEFAULT CHARSET=utf8;
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+--
+-- Dumping data for table `attr1`
+--
+
+LOCK TABLES `attr1` WRITE;
+/*!40000 ALTER TABLE `attr1` DISABLE KEYS */;
+INSERT INTO `attr1` VALUES (1,1,3),(2,2,2),(3,3,1);
+/*!40000 ALTER TABLE `attr1` ENABLE KEYS */;
+UNLOCK TABLES;
+
+--
+-- Table structure for table `attr2`
+--
+
+DROP TABLE IF EXISTS `attr2`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `attr2` (
+  `ObjID` int(11) NOT NULL,
+  `attr2ID` int(11) NOT NULL,
+  `Val` int(11) NOT NULL,
+  PRIMARY KEY (`attr2ID`)
+) ENGINE=MyISAM DEFAULT CHARSET=utf8;
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+--
+-- Dumping data for table `attr2`
+--
+
+LOCK TABLES `attr2` WRITE;
+/*!40000 ALTER TABLE `attr2` DISABLE KEYS */;
+INSERT INTO `attr2` VALUES (1,3,4),(2,2,5),(3,1,6);
+/*!40000 ALTER TABLE `attr2` ENABLE KEYS */;
+UNLOCK TABLES;
+
+--
+-- Table structure for table `attr3`
+--
+
+DROP TABLE IF EXISTS `attr3`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `attr3` (
+  `attr3ID` int(11) NOT NULL,
+  `Val` int(11) NOT NULL,
+  PRIMARY KEY (`attr3ID`)
+) ENGINE=MyISAM DEFAULT CHARSET=utf8;
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+--
+-- Dumping data for table `attr3`
+--
+
+LOCK TABLES `attr3` WRITE;
+/*!40000 ALTER TABLE `attr3` DISABLE KEYS */;
+INSERT INTO `attr3` VALUES (1,0),(2,8),(3,7);
+/*!40000 ALTER TABLE `attr3` ENABLE KEYS */;
+UNLOCK TABLES;
+
+--
+-- Table structure for table `attr4`
+--
+
+DROP TABLE IF EXISTS `attr4`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `attr4` (
+  `attr4ID` int(11) NOT NULL,
+  `Val` int(11) NOT NULL,
+  PRIMARY KEY (`attr4ID`)
+) ENGINE=MyISAM DEFAULT CHARSET=utf8;
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+--
+-- Dumping data for table `attr4`
+--
+
+LOCK TABLES `attr4` WRITE;
+/*!40000 ALTER TABLE `attr4` DISABLE KEYS */;
+INSERT INTO `attr4` VALUES (1,0),(2,0),(3,9);
+/*!40000 ALTER TABLE `attr4` ENABLE KEYS */;
+UNLOCK TABLES;
+
+--
+-- Table structure for table `barobject`
+--
+
+DROP TABLE IF EXISTS `barobject`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `barobject` (
+  `ObjID` int(11) NOT NULL,
+  `attr4ID` int(11) NOT NULL,
+  `Bar` int(11) NOT NULL,
+  PRIMARY KEY (`ObjID`)
+) ENGINE=MyISAM DEFAULT CHARSET=utf8;
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+--
+-- Dumping data for table `barobject`
+--
+
+LOCK TABLES `barobject` WRITE;
+/*!40000 ALTER TABLE `barobject` DISABLE KEYS */;
+INSERT INTO `barobject` VALUES (1,2,1002),(2,1,1200),(3,3,1000);
+/*!40000 ALTER TABLE `barobject` ENABLE KEYS */;
+UNLOCK TABLES;
+
+--
+-- Table structure for table `fooobject`
+--
+
+DROP TABLE IF EXISTS `fooobject`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `fooobject` (
+  `ObjID` int(11) NOT NULL,
+  `attr1ID` int(11) NOT NULL,
+  `attr3ID` int(11) NOT NULL,
+  `Foo` int(11) NOT NULL,
+  PRIMARY KEY (`ObjID`)
+) ENGINE=MyISAM DEFAULT CHARSET=utf8;
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+--
+-- Dumping data for table `fooobject`
+--
+
+LOCK TABLES `fooobject` WRITE;
+/*!40000 ALTER TABLE `fooobject` DISABLE KEYS */;
+INSERT INTO `fooobject` VALUES (1,1,2,112),(2,2,1,122);
+INSERT INTO `fooobject` VALUES (3,3,3,111);
+/*!40000 ALTER TABLE `fooobject` ENABLE KEYS */;
+UNLOCK TABLES;
+
+--
+-- Table structure for table `object`
+--
+
+DROP TABLE IF EXISTS `object`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `object` (
+  `ObjID` int(11) NOT NULL,
+  `Bla` int(11) NOT NULL,
+  `Blub` varchar(50) DEFAULT NULL,
+  PRIMARY KEY (`ObjID`)
+) ENGINE=MyISAM DEFAULT CHARSET=utf8;
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+--
+-- Dumping data for table `object`
+--
+
+LOCK TABLES `object` WRITE;
+/*!40000 ALTER TABLE `object` DISABLE KEYS */;
+INSERT INTO `object` VALUES (1,12,NULL),(2,12,'h),(i'),(30,1,'h\'o');
+/*!40000 ALTER TABLE `object` ENABLE KEYS */;
+UNLOCK TABLES;
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2011-07-07 13:31:05

data/example/dump-pg.sql

@@ -0,0 +1,163 @@
+--
+-- PostgreSQL database cluster dump
+--
+
+SET client_encoding = 'UTF8';
+SET standard_conforming_strings = off;
+SET escape_string_warning = off;
+
+--
+-- PostgreSQL database dump
+--
+
+SET statement_timeout = 0;
+SET client_encoding = 'UTF8';
+SET standard_conforming_strings = off;
+SET check_function_bodies = false;
+SET client_min_messages = warning;
+SET escape_string_warning = off;
+
+SET search_path = public, pg_catalog;
+
+SET default_tablespace = '';
+
+SET default_with_oids = false;
+
+--
+-- Name: attr1; Type: TABLE; Schema: public; Owner: bla; Tablespace: 
+--
+
+CREATE TABLE attr1 (
+  ObjID integer NOT NULL,
+  attr1ID integer NOT NULL,
+  Val integer NOT NULL
+);
+
+--
+-- Data for Name: attr1; Type: TABLE DATA; Schema: public; Owner: bla
+--
+
+COPY attr1 (ObjID, attr1ID, Val) FROM stdin;
+1	1	3
+2	2	2
+3	3	1
+\.
+
+--
+-- Name: attr2; Type: TABLE; Schema: public; Owner: bla; Tablespace: 
+--
+
+CREATE TABLE attr2 (
+  ObjID integer NOT NULL,
+  attr2ID integer NOT NULL,
+  Val integer NOT NULL
+);
+
+--
+-- Data for Name: attr2; Type: TABLE DATA; Schema: public; Owner: bla
+--
+
+COPY attr2 (ObjID, attr2ID, Val) FROM stdin;
+1	3	4
+2	2	5
+3	1	6
+\.
+
+--
+-- Name: attr3; Type: TABLE; Schema: public; Owner: bla; Tablespace: 
+--
+
+CREATE TABLE attr3 (
+  attr3ID integer NOT NULL,
+  Val integer NOT NULL
+);
+
+--
+-- Data for Name: attr3; Type: TABLE DATA; Schema: public; Owner: bla
+--
+
+COPY attr3 (attr3ID, Val) FROM stdin;
+1	0
+2	8
+3	7
+\.
+
+--
+-- Name: attr4; Type: TABLE; Schema: public; Owner: bla; Tablespace: 
+--
+
+CREATE TABLE attr4 (
+  attr4ID integer NOT NULL,
+  Val integer NOT NULL
+);
+
+--
+-- Data for Name: attr4; Type: TABLE DATA; Schema: public; Owner: bla
+--
+
+COPY attr4 (attr4ID, Val) FROM stdin;
+1	0
+2	0
+3	9
+\.
+
+--
+-- Name: barobject; Type: TABLE; Schema: public; Owner: bla; Tablespace: 
+--
+
+CREATE TABLE barobject (
+  ObjID integer NOT NULL,
+  attr4ID integer NOT NULL,
+  Bar integer NOT NULL
+);
+
+--
+-- Data for Name: barobject; Type: TABLE DATA; Schema: public; Owner: bla
+--
+
+COPY barobject (ObjID, attr4ID, Bar) FROM stdin;
+1	2	1002
+2	1	1200
+3	3	1000
+\.
+
+--
+-- Name: fooobject; Type: TABLE; Schema: public; Owner: bla; Tablespace: 
+--
+
+CREATE TABLE fooobject (
+  ObjID integer NOT NULL,
+  attr1ID integer NOT NULL,
+  attr3ID integer NOT NULL,
+  Foo integer NOT NULL
+);
+
+--
+-- Data for Name: fooobject; Type: TABLE DATA; Schema: public; Owner: bla
+--
+
+COPY fooobject (ObjID, attr1ID, attr3ID, Foo) FROM stdin;
+1	1	2	112
+2	2	1	122
+3	3	3	111
+\.
+
+--
+-- Name: object; Type: TABLE; Schema: public; Owner: bla; Tablespace: 
+--
+
+CREATE TABLE object (
+  ObjID integer NOT NULL,
+  Bla integer NOT NULL,
+  Blub character varying(50) DEFAULT NULL
+);
+
+--
+-- Data for Name: object; Type: TABLE DATA; Schema: public; Owner: bla
+--
+
+COPY object (ObjID, Bla, Blub) FROM stdin;
+1	12	NULL
+2	12	h)	(i
+3	1	h'o
+\.

data/lib/athena/formats/ferret.rb

@@ -44,12 +44,12 @@ module Athena
 
     register_format :in do
 
-      attr_reader :record_element, :config, :parser, :match_all_query
+      attr_reader :record_element, :config, :match_all_query
 
       def initialize(parser)
-        config = parser.config.dup
+        @config = parser.config.dup
 
-        case @record_element = config.delete(:__record_element)
+        case @record_element = @config.delete(:__record_element)
           when String
             # fine!
           when nil
@@ -57,9 +57,6 @@ module Athena
           else
             raise IllegalRecordElementError, "illegal record element #{@record_element.inspect}"
         end
-
-        @config = config
-        @parser = parser
       end
 
     end if Object.const_defined?(:Ferret)

data/lib/athena/formats/sisis.rb

@@ -35,12 +35,12 @@ module Athena
 
     register_format :in do
 
-      attr_reader :record_element, :config, :parser
+      attr_reader :record_element, :config
 
       def initialize(parser)
-        config = parser.config.dup
+        @config = parser.config.dup
 
-        case @record_element = config.delete(:__record_element)
+        case @record_element = @config.delete(:__record_element)
           when String
             # fine!
           when nil
@@ -48,9 +48,6 @@ module Athena
           else
             raise IllegalRecordElementError, "illegal record element #{@record_element.inspect}"
         end
-
-        @config = config
-        @parser = parser
       end
 
     end

data/lib/athena/formats/sql.rb

@@ -0,0 +1,249 @@
+#--
+###############################################################################
+#                                                                             #
+# A component of athena, the database file converter.                         #
+#                                                                             #
+# Copyright (C) 2007-2011 University of Cologne,                              #
+#                         Albertus-Magnus-Platz,                              #
+#                         50923 Cologne, Germany                              #
+#                                                                             #
+# Authors:                                                                    #
+#     Jens Wille <jens.wille@uni-koeln.de>                                    #
+#                                                                             #
+# athena is free software; you can redistribute it and/or modify it under the #
+# terms of the GNU Affero General Public License as published by the Free     #
+# Software Foundation; either version 3 of the License, or (at your option)   #
+# any later version.                                                          #
+#                                                                             #
+# athena is distributed in the hope that it will be useful, but WITHOUT ANY   #
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS   #
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for     #
+# more details.                                                               #
+#                                                                             #
+# You should have received a copy of the GNU Affero General Public License    #
+# along with athena. If not, see <http://www.gnu.org/licenses/>.              #
+#                                                                             #
+###############################################################################
+#++
+
+require 'strscan'
+
+module Athena
+  module Formats
+
+  class MYSQL < Base
+
+    register_format :in do
+
+      attr_reader :record_element, :config, :sql_parser
+
+      def initialize(parser)
+        @config = parser.config.dup
+
+        case @record_element = @config.delete(:__record_element)
+          when String, nil
+            # fine!
+          else
+            raise IllegalRecordElementError, "illegal record element #{@record_element.inspect}"
+        end
+
+        @sql_parser = SQLParser.new
+      end
+
+    end
+
+    def parse(source, &block)
+      columns, table, num = Hash.new { |h, k| h[k] = [] }, nil, 0
+
+      source.each { |line|
+        case line = line.chomp
+          when /\ACREATE\s+TABLE\s+`(.+?)`/i
+            table = $1
+          when /\A\s+`(.+?)`/i
+            columns[table] << $1 if table
+          when /\A\).*;\z/
+            table = nil
+          when /\AINSERT\s+INTO\s+`(.+?)`\s+VALUES\s*(.*);\z/i
+            _columns = columns[$1]
+            next if _columns.empty?
+
+            sql_parser.parse($2) { |row|
+              Record.new(nil, block) { |record|
+                row.each_with_index { |value, index|
+                  column = _columns[index] or next
+
+                  if column == record_element
+                    record.instance_variable_set(:@id, value)
+                  end
+
+                  record.update(column, value.to_s, config[column])
+                }
+              }
+
+              num += 1
+            }
+        end
+      }
+
+      num
+    end
+
+    class SQLParser
+
+      AST = Struct.new(:value)
+
+      def self.parse(input)
+        new.parse(input)
+      end
+
+      def parse(input)
+        @input = StringScanner.new(input)
+
+        rows, block_given = [], block_given?
+
+        while result = parse_row
+          row = result.value
+          block_given ? yield(row) : rows << row
+          break unless @input.scan(/,/)
+        end
+
+        @input.scan(/;/)  # optional
+
+        error('Unexpected data') unless @input.eos?
+
+        rows unless block_given
+      end
+
+      private
+
+      def parse_row
+        return unless @input.scan(/\(/)
+
+        row = []
+
+        while result = parse_value
+          row << result.value
+          break unless @input.scan(/,/)
+        end
+
+        error('Unclosed row') unless @input.scan(/\)/)
+
+        AST.new(row)
+      end
+
+      def parse_value
+        parse_string ||
+        parse_number ||
+        parse_keyword
+      end
+
+      def parse_string
+        return unless @input.scan(/'/)
+
+        string = ''
+
+        while contents = parse_string_content || parse_string_escape
+          string << contents.value
+        end
+
+        error('Unclosed string') unless @input.scan(/'/)
+
+        AST.new(string)
+      end
+
+      def parse_string_content
+        if @input.scan(/[^\\']+/)
+          AST.new(@input.matched)
+        end
+      end
+
+      def parse_string_escape
+        if @input.scan(/\\['\\]|''/)
+          AST.new(@input.matched[-1, 1])
+        end
+      end
+
+      def parse_number
+        if @input.scan(/-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?/)
+          AST.new(eval(@input.matched))
+        end
+      end
+
+      def parse_keyword
+        if @input.scan(/null/i)
+          AST.new(nil)
+        end
+      end
+
+      def error(message)
+        if @input.eos?
+          raise "Unexpected end of input (#{message})."
+        else
+          raise "#{message} at #{@input.pos}: #{@input.peek(8).inspect}"
+        end
+      end
+
+    end
+
+  end
+
+  class PGSQL < Base
+
+    register_format :in do
+
+      attr_reader :record_element, :config
+
+      def initialize(parser)
+        @config = parser.config.dup
+
+        case @record_element = @config.delete(:__record_element)
+          when String, nil
+            # fine!
+          else
+            raise IllegalRecordElementError, "illegal record element #{@record_element.inspect}"
+        end
+      end
+
+    end
+
+    def parse(source, &block)
+      columns, table, num = Hash.new { |h, k| h[k] = [] }, nil, 0
+
+      source.each { |line|
+        case line = line.chomp
+          when /\ACOPY\s+(\S+)\s+\((.+?)\)\s+FROM\s+stdin;\z/i
+            columns[table = $1] = $2.split(/\s*,\s*/)
+          when /\A\\\.\z/
+            table = nil
+          else
+            next unless table
+
+            cols = columns[table]
+            next if cols.empty?
+
+            Record.new(nil, block) { |record|
+              line.split(/\t/).each_with_index { |value, index|
+                column = cols[index] or next
+
+                if column == record_element
+                  record.instance_variable_set(:@id, value)
+                end
+
+                record.update(column, value, config[column])
+              }
+            }
+
+            num += 1
+        end
+      }
+
+      num
+    end
+
+  end
+
+  MySQL = MYSQL
+  PgSQL = PGSQL
+
+  end
+end

data/lib/athena/version.rb

@@ -4,7 +4,7 @@ module Athena
 
     MAJOR = 0
     MINOR = 1
-    TINY  = 4
+    TINY  = 5
 
     class << self
 

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: athena
 version: !ruby/object:Gem::Version 
-  hash: 19
+  hash: 17
   prerelease: 
   segments: 
   - 0
   - 1
-  - 4
-  version: 0.1.4
+  - 5
+  version: 0.1.5
 platform: ruby
 authors: 
 - Jens Wille
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-04-29 00:00:00 Z
+date: 2011-07-12 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: builder
@@ -78,6 +78,7 @@ files:
 - lib/athena/formats/lingo.rb
 - lib/athena/formats/ferret.rb
 - lib/athena/formats/dbm.rb
+- lib/athena/formats/sql.rb
 - lib/athena/formats/sisis.rb
 - lib/athena/version.rb
 - lib/athena/parser.rb
@@ -88,7 +89,9 @@ files:
 - ChangeLog
 - Rakefile
 - COPYING
+- example/dump-my.sql
 - example/config.yaml
+- example/dump-pg.sql
 - example/example.xml
 - example/sisis-ex.txt
 homepage: http://prometheus.rubyforge.org/athena
@@ -96,14 +99,14 @@ licenses: []
 
 post_install_message: 
 rdoc_options: 
-- --line-numbers
-- --main
-- README
 - --charset
 - UTF-8
-- --all
 - --title
-- athena Application documentation (v0.1.4)
+- athena Application documentation (v0.1.5)
+- --main
+- README
+- --line-numbers
+- --all
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
@@ -127,7 +130,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: prometheus
-rubygems_version: 1.7.2
+rubygems_version: 1.8.5
 signing_key: 
 specification_version: 3
 summary: Convert database files to various formats.