graph2relational 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ffd5eda800a15b74f1843d5e5e32c9063d8b2646
+  data.tar.gz: a10dc8539499cb64bbcbd88355bfa2a064ce5970
+SHA512:
+  metadata.gz: aa6904f22f311b822dad0cfa1575b155cf1b3e03b58a1316eea3f026e4f69a7fc34984f242044c8ed2ebbf75dee188181a82c0f03d97dbb973a18d6cfb2bb6ea
+  data.tar.gz: 656a7e830a82e14f0fece65483f6efe32b9c82ba8af15f225839c04037623784ba6e9a09b3455fdbd5a97ec98365116837aab9d1c63366c609ebed3acc1c2fb1

data/bin/graph2relational

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+require 'graph2relational'
+
+# parse command line input
+options = G2R::Util::CommandLineParser.new.parse
+
+# create database connections
+neo4j = G2R::Neo4J::Database.new(options[:input])
+rdbms = G2R::RDBMS::Database.new(options[:output], options)
+
+# convert
+rdbms.source = neo4j
+rdbms.convert()

data/lib/graph2relational.rb

@@ -0,0 +1,22 @@
+# external dependencies
+require 'neography'
+require 'sequel'
+require 'optparse'
+require 'set'
+
+#
+$:.unshift File.dirname(__FILE__)
+
+# Utils
+require 'graph2relational/util-commandline-parser'
+
+# Neo4J require
+require 'graph2relational/neo4j-database'
+require 'graph2relational/neo4j-connection'
+
+# RDBMS require
+require 'graph2relational/rdbms-database'
+require 'graph2relational/rdbms-connection'
+require 'graph2relational/rdbms-table'
+require 'graph2relational/rdbms-join-table'
+require 'graph2relational/rdbms-column'

data/lib/graph2relational/neo4j-connection.rb

@@ -0,0 +1,26 @@
+module G2R
+  module Neo4J
+    class Connection
+      def initialize(options)
+        @conn = Neography::Rest.new(options)
+      end
+
+      def query_data(cypher)
+        query(cypher)['data']
+      end
+
+      def query_columns(cypher)
+        columns = Set.new
+        query(cypher)['data'].each do |row|
+          columns += row[0]['data'].keys
+        end
+
+        columns.to_a
+      end
+
+      def query(cypher)        
+        @conn.execute_query(cypher)
+      end
+    end
+  end
+end

data/lib/graph2relational/neo4j-database.rb

@@ -0,0 +1,81 @@
+module G2R
+  module Neo4J
+
+    # The Neo4J database that will have its data converted. It is the source of the conversion.
+    class Database
+
+      def initialize(connection_options)
+        # services
+        @conn = Connection.new(connection_options)
+
+        # data
+        @labels = nil
+        @label_attributes = {}
+        @label_relationships = {}
+        @relationships = nil
+        @relationship_attributes = {}
+      end
+
+      # ==========================================================================
+      # NODES
+      # ==========================================================================
+      def labels
+        # lazy load labels
+        if @labels.nil?
+          rows = @conn.query_data("MATCH (n) UNWIND labels(n) AS label RETURN DISTINCT label ORDER BY label")
+          @labels = rows.map {|row| row[0]}
+        end
+
+        @labels
+      end
+
+      def label_attributes(label)
+        if not @label_attributes.key? label
+          @label_attributes[label] = @conn.query_columns("MATCH (n:#{label}) RETURN n LIMIT 1000")
+        end
+
+        @label_attributes[label]
+      end
+
+      def label_relationships(label)
+        if not @label_relationships.key? label
+          @label_relationships[label] = @conn.query_data("MATCH (n:#{label})-[r]->(m) UNWIND labels(m) AS label WITH label, type(r) as relationship RETURN DISTINCT relationship, label")
+        end
+
+        @label_relationships[label]
+      end
+
+      def label_data(label)
+        return_clause = label_attributes(label).map {|column| "n.#{column}"}.join(", ")
+        @conn.query_data("MATCH (n:#{label}) RETURN id(n), #{return_clause}")
+      end
+
+      # ==========================================================================
+      # RELATIONSIIPS
+      # ==========================================================================
+      def relationships
+        # lazy load relationships
+        if @relationships.nil?
+          rows = @conn.query_data("MATCH (n)-[r]-(m) RETURN DISTINCT type(r) AS relationship ORDER BY relationship")
+          @relationships = rows.map{|row| row[0]}
+        end
+
+        @relationships
+      end
+
+      def relationship_attributes(source, type, target)
+        key = "#{source}_#{type}_#{target}"
+        if not @relationship_attributes.has_key? key
+          @relationship_attributes[key] = @conn.query_columns("MATCH (n:#{source})-[r:#{type}]->(m:#{target}) RETURN r LIMIT 1000")
+        end
+
+        @relationship_attributes[key]
+      end
+
+      def relationship_data(source, type, target)
+        return_clause = relationship_attributes(source, type, target).map {|column| "r.#{column}"}.join(", ")
+        @conn.query_data("MATCH (n:#{source})-[r:#{type}]->(m:#{target}) RETURN id(n), id(m), #{return_clause}")
+      end
+    end
+  end
+end

data/lib/graph2relational/rdbms-column.rb

@@ -0,0 +1,35 @@
+module G2R
+  module RDBMS
+    class Column
+      attr_accessor :name
+
+      def initialize(name)
+        @name = name.downcase.to_sym
+        @primary_key = false
+        @foreign_key = false
+      end
+
+      # Define the column as a primary key
+      def primary_key
+        @primary_key = true
+        self
+      end
+
+      # Define the column as a foreign key
+      def foreign_key
+        @foreign_key = true
+        self
+      end
+
+      # Checks if the column is a primary key
+      def primary_key?
+        @primary_key
+      end
+
+      # Check if the column is a foreign key
+      def foreign_key?
+        @foreign_key
+      end
+    end
+  end
+end

data/lib/graph2relational/rdbms-connection.rb

@@ -0,0 +1,38 @@
+module G2R
+  module RDBMS
+    class Connection
+
+      def initialize(options)
+        # sequel
+        @db = Sequel.connect(options)
+      end
+
+      def create_table(table)
+        @db.create_table! table.name do
+
+          table.columns.each do |column|
+            # primary key
+            if column.primary_key?
+              primary_key column.name
+            # foreign key
+            elsif column.foreign_key?
+              Integer column.name
+            # common column
+            else
+              String column.name
+            end
+          end
+        end
+      end
+
+      def insert_data(table)
+         @db.transaction do
+           table.data.each do |row|
+             @db[table.name.to_sym].insert(row)
+           end
+         end
+      end
+
+    end
+  end
+end

data/lib/graph2relational/rdbms-database.rb

@@ -0,0 +1,146 @@
+module G2R
+  module RDBMS
+
+    # The target RDBMS database that will have the schema and data generated for. It is the target of the conversion.
+    class Database
+
+      # ==========================================================================
+      # INITIALIZATION
+      # ==========================================================================
+      def initialize(connection_options, app_options = {})
+        # services
+        @options = app_options
+        @conn = Connection.new(connection_options)
+
+        # data
+        reset_data()
+      end
+
+      # ==========================================================================
+      # CONVERSION
+      # ==========================================================================
+      def source=(source)
+        @source = source
+      end
+
+      def source
+        @source
+      end
+
+      # Export the schema creation and data import scripts to the specified location
+      def convert
+        # prepare
+        reset_data()
+
+        # create tables
+        puts "Creating base tables"
+        base_tables.each do |table|
+          @conn.create_table(table)
+        end
+
+        puts "Creating relationship tables"
+        relationship_tables.each do |table|
+          @conn.create_table(table)
+        end
+
+        # insert data
+        puts "Inserting base tables data"
+        base_tables.each do |table|
+          @conn.insert_data(table)
+        end
+
+        puts "Inserting relationship tables data"
+        relationship_tables.each do |table|
+          @conn.insert_data(table)
+        end
+      end
+
+      # Reset the saved data to perform a new conversion
+      def reset_data
+        @base_tables = nil
+        @relationship_tables = nil
+      end
+
+      # ==========================================================================
+      # TABLES
+      # ==========================================================================
+      # Get all the base tables that will be generated. They are based on the Neo4J labels.
+      # Columns are based on the Neo4J node attributes.
+      def base_tables
+        # lazy initialization
+        if @base_tables.nil?
+          @base_tables = @source.labels.map do |label|
+
+            # if excluding label, do not transform into base table
+            next if exclude? label
+
+            # generate table
+            table = Table.new(label)
+
+            # generate primary key
+            table.add_columns(Column.new(:id).primary_key)
+
+            # generate columns
+            columns = @source.label_attributes(label).map {|attribute| Column.new(attribute)}
+            table.add_columns(columns)
+
+            # generate data
+            label_data = @source.label_data(label)
+            table.add_data(label_data)
+
+            # generated table
+            table
+          end.compact
+        end
+
+        @base_tables.compact
+      end
+
+      # Get all the relationships tables that will be generated. They are based in the found Neo4J relationships.
+      # Columss are based on the Neo4J relationship attributes.
+      def relationship_tables
+        # lazy initialization
+        if @relationship_tables.nil?
+          @relationship_tables = @source.labels.flat_map do |label|
+
+            # if excluding label, do not transform into relationship table
+            next if exclude? label
+
+            @source.label_relationships(label).map do |relationship|
+              relationship, target_label = relationship
+
+              # if excluding label, do not transform into relationship table
+              next if exclude? target_label
+
+              # gerarate table
+              table_name = "#{label}_#{relationship}_#{target_label}"
+              table = JoinTable.new(table_name)
+
+              # generate keys
+              table.source = label
+              table.target = target_label
+
+              # generate columns
+              columns = @source.relationship_attributes(label, relationship, target_label).map {|attribute| Column.new(attribute)}
+              table.add_columns(columns)
+
+              # generate data
+              relationship_data = @source.relationship_data(label, relationship, target_label)
+              table.add_data(relationship_data)
+
+              # generated table
+              table
+            end
+          end.compact
+        end
+
+        @relationship_tables
+      end
+
+      # Check if a label should be excluded and not be transformed into a table
+      def exclude?(label)
+        @options.has_key? :exclude and @options[:exclude].include? label
+      end
+    end
+  end
+end

data/lib/graph2relational/rdbms-join-table.rb

@@ -0,0 +1,43 @@
+module G2R
+  module RDBMS
+
+    class JoinTable < Table
+
+      # Additionally return the source and target column ids plus all defined columns
+      def columns
+        [source_column, target_column] + super
+      end
+
+      def source_column
+        Column.new(source + "_id").foreign_key
+      end
+
+      def target_column
+        if source != target
+          Column.new(target + "_id").foreign_key
+        else
+          Column.new(target + "_id_2").foreign_key
+        end
+      end
+
+      # ==========================================================================
+      # ACESSORS
+      # ==========================================================================
+      def source=(source)
+        @source = source
+      end
+
+      def source
+        @source
+      end
+
+      def target=(target)
+        @target = target
+      end
+
+      def target
+        @target
+      end
+    end
+  end
+end

data/lib/graph2relational/rdbms-table.rb

@@ -0,0 +1,36 @@
+module G2R
+  module RDBMS
+    class Table
+
+      def initialize(name)
+        @name = name.downcase.to_sym
+        @columns = []
+        @data = []
+      end
+
+      def name
+        @name
+      end
+
+      def add_columns(columns)
+        if columns.class == Array
+          @columns += columns
+        else
+          @columns << columns
+        end
+      end
+
+      def columns
+        @columns
+      end
+
+      def add_data(data)
+        @data += data
+      end
+
+      def data
+        @data
+      end
+    end
+  end
+end

data/lib/graph2relational/util-commandline-parser.rb

@@ -0,0 +1,57 @@
+module G2R
+  module Util
+
+    class CommandLineParser
+
+      def initialize()
+        @valid = true
+      end
+
+      def parse()
+        # default values
+        options = {:input => nil, :output => nil, :exclude => []}
+
+        # display help if no args
+        ARGV << '-h' if ARGV.empty?
+
+        # do parsing
+        OptionParser.new do |opts|
+            opts.banner = "Usage: graph2relational -i <neo4j_connection_url> -o <rdbms_connection_url> [optional_arguments]"
+
+            # input file
+            opts.on("-i CONNECTION", "--input CONNECTION", "Neo4J connection URI from where the schema and data will be geberated") do |input_connection|
+                options[:input] = input_connection
+            end
+
+            # output dir
+            opts.on("-o CONNECTION", "--output CONNECTION", "RDBMS connection URI to where the schema and data will be persisted") do |output_connection|
+              options[:output] = output_connection
+            end
+
+            # exclusion
+            opts.on("-x EXCLUSION", "--exclude EXCLUSION", "Neo4J labels (separated by comma) to exclude from the conversion") do |exclude|
+              options[:exclude] = exclude.split(",").map{|exclude| exclude.strip}
+            end
+
+            # help
+            opts.on_tail("-h", "--help", "Help message") do
+              puts opts
+              exit
+            end
+        end.parse!
+
+        # check mandatory args
+        if options[:input].nil?
+          puts "Missing parameter: specify Neo4J connection URI using -i <connection> parameter"
+          exit
+        end
+        if options[:output].nil?
+          puts "Missing parameter: specify RDBMS connection URI using -o <connection> parameter"
+          exit
+        end
+
+        return options
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,83 @@
+--- !ruby/object:Gem::Specification
+name: graph2relational
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Renato Dinhani Conceição
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-03-25 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: neography
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+- !ruby/object:Gem::Dependency
+  name: sequel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.32'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.32'
+description: Converts a Neo4J graph database to a supported RDBMS database automatically
+  generating tables from its labels, relationships and attributes
+email: renatodinhani@gmail.com
+executables:
+- graph2relational
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/graph2relational
+- lib/graph2relational.rb
+- lib/graph2relational/neo4j-connection.rb
+- lib/graph2relational/neo4j-database.rb
+- lib/graph2relational/rdbms-column.rb
+- lib/graph2relational/rdbms-connection.rb
+- lib/graph2relational/rdbms-database.rb
+- lib/graph2relational/rdbms-join-table.rb
+- lib/graph2relational/rdbms-table.rb
+- lib/graph2relational/util-commandline-parser.rb
+homepage: https://github.com/renatodinhani/graph2relational
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5.1
+signing_key: 
+specification_version: 4
+summary: Converts a Neo4J graph database to a RDBMS database
+test_files: []