neo4j_bolt 0.1.6 → 0.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e754e8e69d3794520280d568a879834a6ceb91bad419385fb68318959f8168c2
-  data.tar.gz: 48ac0bab0ae453aa6d41ce6832271db2798b99f49baaa0307c0795a7d932264d
+  metadata.gz: 1ac7e327b2ece44cd23a4408c81c0a9a41d52faf277c24b56280a15f1b79aeec
+  data.tar.gz: a24414f311a40ff6cb79d18a18497200f79ed7e2a58e070b5e321a4dc31cc757
 SHA512:
-  metadata.gz: dfabf88d8ec3a58298c87ab3a1662bcf43abe83a5bc6d2d948992389704a34c70f2fb5f21877adf7d2e826f39e1da5936764369de3ee1942e4466e6d54ea1acc
-  data.tar.gz: f21d0f51b0a90083a79d5df951b76d8acf5c3bd55386fa3a10039f0ae107115927eea9bbe56ee3d0763b9e42e04f86181b4806f43d8f72f392b82f3538c01e29
+  metadata.gz: f716cd2380decb532dbe6954030e4ead5ea0a1edde1868922b81d2105801d7ed6bb9e5b35e3eb22f79ca9413408582bca6da44a9fa767ad6fa554b929df84436
+  data.tar.gz: 35f82570c1bcf2b56a61a17d9bbf189f2055386efd1c6f83cd3c60313069d26380ce050394665ede60ef9ea4eb27626bf7438b43a4070e361a2b9ed626c51893

data/Gemfile

@@ -5,3 +5,4 @@ gemspec
 
 gem "rake", "~> 12.0"
 gem "rspec", "~> 3.0"
+gem "gli"

data/Gemfile.lock

@@ -1,12 +1,13 @@
 PATH
   remote: .
   specs:
-    neo4j_bolt (0.1.5)
+    neo4j_bolt (0.1.8)
 
 GEM
   remote: https://rubygems.org/
   specs:
     diff-lcs (1.5.0)
+    gli (2.21.0)
     rake (12.3.3)
     rspec (3.11.0)
       rspec-core (~> 3.11.0)
@@ -26,6 +27,7 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
+  gli
   neo4j_bolt!
   rake (~> 12.0)
   rspec (~> 3.0)

data/README.md

@@ -1,6 +1,6 @@
 # Neo4jBolt
 
-A Neo4j/Bolt driver written in pure Ruby. Currently only supporting Neo4j 4.4.
+A Neo4j/Bolt driver written in pure Ruby. Currently only supporting Neo4j 4.4. Caution: This gem is not feature complete, and also the documention is not complete yet.
 
 ## Installation
 

data/bin/neo4j_bolt

@@ -0,0 +1,213 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "neo4j_bolt"
+require "gli"
+
+include Neo4jBolt
+
+class App
+    extend GLI::App
+
+    program_desc 'run various Neo4j housekeeping tasks'
+    version Neo4jBolt::VERSION
+
+    flag [:v, :verbosity], :default_value => 0
+    flag [:h, :host], :default_value => 'localhost:7687'
+
+    pre do |global_options, command, options, args|
+        host = global_options[:host]
+        Neo4jBolt.bolt_host = host.split(':').first
+        Neo4jBolt.bolt_port = host.split(':').last.to_i
+        Neo4jBolt.bolt_verbosity = global_options[:verbosity].to_i
+        true
+    end
+
+    # --------------------------------------------
+
+    desc 'Console'
+    command :console do |c|
+        c.action do
+            require "irb"
+            IRB.start(__FILE__)
+        end
+    end
+
+    # --------------------------------------------
+
+    desc 'Dump database'
+    long_desc 'Dump all nodes and relationships.'
+    command :dump do |c|
+        c.flag [:o, :out_file], :default_value => '/dev/stdout'
+        c.action do |global_options, options|
+            File.open(options[:out_file], 'w') do |f|
+                dump_database(f)
+            end
+        end
+    end
+
+    # --------------------------------------------
+
+    desc 'Load database dump'
+    long_desc 'Load nodes and relationships from a database dump.'
+    command :load do |c|
+        # c.flag [:i, :in_file], :desc => 'input path', :required => true
+        c.switch [:f, :force], :default_value => false, :desc => 'force appending nodes even if the database is not empty'
+        c.action do |global_options, options, args|
+            help_now!('input path is required') if args.empty?
+            path = args.shift
+            File.open(path, 'r') do |f|
+                load_database_dump(f, force_append: options[:force])
+            end
+        end
+    end
+
+    # --------------------------------------------
+
+    desc 'Clear database'
+    long_desc 'Clear all nodes and relationships'
+    command :clear do |c|
+        c.switch [:srsly], :required => true, :negatable => false, :desc => 'Specify --srsly to really clear the database'
+        c.action do |global_options, options, args|
+            if options[:srsly]
+                neo4j_query("MATCH (n) DETACH DELETE n;")
+            else
+                STDERR.puts "Doing nothing unless you specify --srsly."
+            end
+        end
+    end
+
+    # --------------------------------------------
+
+    desc 'Visualize database'
+    long_desc 'Generate a GraphViz-formatted visual representation of the database'
+    command :visualize do |c|
+        c.flag [:o, :out_file], :default_value => '/dev/stdout'
+        c.switch [:p, :properties], :default_value => false, :desc => 'include properties'
+        c.action do |global_options, options|
+            File.open(options[:out_file], 'w') do |f|
+                all_labels = Set.new()
+
+                TR = {'String' => 'string',
+                      'Array' => 'list',
+                      'Hash' => 'hash',
+                      'TrueClass' => 'true',
+                      'FalseClass' => 'false',
+                      'NilClass' => 'null',
+                      'Integer' => 'int',
+                      'Float' => 'float'
+                    }
+
+                neo4j_query("MATCH (n) RETURN DISTINCT labels(n) AS labels") do |entry|
+                    labels = entry['labels']
+                    if labels.size != 1
+                        raise "multiple labels per node not supported yet: #{labels.join(' ')}"
+                    end
+                    all_labels << labels.first
+                end
+
+                all_relationships = Set.new()
+
+                neo4j_query("MATCH (a)-[r]->(b) RETURN DISTINCT labels(a) AS la, type(r) AS t, labels(b) AS lb;") do |entry|
+                    la = entry['la'].first
+                    t = entry['t']
+                    lb = entry['lb'].first
+                    all_relationships << "#{la}/#{t}/#{lb}"
+                end
+
+                properties_for_label = {}
+                counts_for_label = {}
+
+                all_labels.to_a.sort.each do |label|
+                    properties_for_label[label] ||= {}
+                    if options[:properties]
+                        neo4j_query("MATCH (n:#{label}) RETURN n") do |entry|
+                            counts_for_label[label] ||= 0
+                            counts_for_label[label] += 1
+                            node = entry['n']
+                            node.each_pair do |key, value|
+                                properties_for_label[label][key] ||= {:classes => Set.new()}
+                                properties_for_label[label][key][:classes] << value.class
+                            end
+                        end
+                    end
+                end
+
+                all_relationships.each do |s|
+                    properties_for_label[s] ||= {}
+                    parts = s.split('/')
+                    la = parts[0]
+                    type = parts[1]
+                    lb = parts[2]
+                    if options[:properties]
+                        neo4j_query("MATCH (a:#{la})-[r:#{type}]->(b:#{lb}) RETURN r") do |entry|
+                            counts_for_label[s] ||= 0
+                            counts_for_label[s] += 1
+                            rel = entry['r']
+                            rel.each_pair do |key, value|
+                                properties_for_label[s][key] ||= {:classes => Set.new()}
+                                properties_for_label[s][key][:classes] << value.class
+                            end
+                        end
+                    end
+                end
+
+                dot = StringIO.open do |io|
+                    io.puts "digraph {"
+                    io.puts "graph [fontname = Helvetica, fontsize = 10, nodesep = 0.2, ranksep = 0.3];"
+                    io.puts "node [fontname = Helvetica, fontsize = 10, shape = none, margin = 0];"
+                    io.puts "edge [fontname = Helvetica, fontsize = 10, arrowsize = 0.6, color = \"#000000\"];"
+                    io.puts 'rankdir=LR;'
+                    io.puts 'splines=true;'
+                    properties_for_label.keys.sort.each do |lbl|
+                        label = "<<table valign='top' align='left' border='0' cellborder='0' cellspacing='0' cellpadding='4'>"
+                        label += "<tr><td border='1' bgcolor='#fce94f' valign='top' align='left' colspan='2'><b>#{lbl}</b>"
+                        if options[:properties]
+                            label += " <i>(#{counts_for_label[lbl]})</i>"
+                        end
+                        label += "</td></tr>"
+                        properties_for_label[lbl].keys.sort.each do |key|
+                            label += "<tr>"
+                            label += "<td border='1' valign='top' align='left' colspan='1'>#{key}</td>"
+                            label += "<td border='1' valign='top' align='left' colspan='1'>#{properties_for_label[lbl][key][:classes].to_a.map { |x| TR[x.to_s] || x.to_s }.sort.join(' / ')}</td>"
+                            label += "</tr>"
+                        end
+                        label += "</table>>"
+                        io.puts "\"#{lbl}\" [label = #{label}, pencolor = \"#000000\"];"
+                    end
+                    all_relationships.each do |s|
+                        parts = s.split('/')
+                        la = parts[0]
+                        type = parts[1]
+                        lb = parts[2]
+
+                        label = "<<table valign='top' align='left' border='0' cellborder='0' cellspacing='0' cellpadding='4'>"
+                        label += "<tr><td border='1' bgcolor='#d3d7cf' valign='top' align='left' colspan='2'><b>#{type}</b>"
+                        if options[:properties]
+                            label += " <i>(#{counts_for_label[s]})</i>"
+                        end
+                        label += "</td></tr>"
+                        (properties_for_label[s] || {}).keys.sort.each do |key|
+                            label += "<tr>"
+                            label += "<td border='1' valign='top' align='left' colspan='1'>#{key}</td>"
+                            label += "<td border='1' valign='top' align='left' colspan='1'>#{properties_for_label[s][key][:classes].to_a.map { |x| TR[x.to_s] || x.to_s }.sort.join(' / ')}</td>"
+                            label += "</tr>"
+                        end
+                        label += "</table>>"
+                        io.puts "\"#{s}\" [label = #{label}, pencolor = \"#000000\"];"
+
+                        io.puts "\"#{la}\" -> \"#{s}\";"
+                        io.puts "\"#{s}\" -> \"#{lb}\";"
+                    end
+
+                    io.puts "}"
+                    io.string
+                end
+                f.puts dot
+            end
+        end
+    end
+
+end
+
+exit App.run(ARGV)

data/lib/neo4j_bolt/version.rb

@@ -1,3 +1,3 @@
 module Neo4jBolt
-  VERSION = "0.1.6"
+  VERSION = "0.1.8"
 end

data/lib/neo4j_bolt.rb

@@ -5,12 +5,13 @@ require 'yaml'
 
 module Neo4jBolt
     class << self
-        attr_accessor :bolt_host, :bolt_port
+        attr_accessor :bolt_host, :bolt_port, :bolt_verbosity
     end
     self.bolt_host = 'localhost'
     self.bolt_port = 7687
+    self.bolt_verbosity = 0
 
-    NEO4J_DEBUG = 0
+    CONSTRAINT_INDEX_PREFIX = 'neo4j_bolt_'
 
     module ServerState
         DISCONNECTED = 0
@@ -114,7 +115,7 @@ module Neo4jBolt
                     end
                     chunk = @socket.read(length).unpack('C*')
                     @data += chunk
-                    if NEO4J_DEBUG >= 3
+                    if Neo4jBolt.bolt_verbosity >= 3
                         dump()
                     end
                 end
@@ -752,7 +753,7 @@ module Neo4jBolt
         end
 
         def run_query(query, data = {}, &block)
-            if NEO4J_DEBUG >= 1
+            if Neo4jBolt.bolt_verbosity >= 1
                 STDERR.puts query
                 STDERR.puts data.to_json
                 STDERR.puts '-' * 40
@@ -790,7 +791,7 @@ module Neo4jBolt
                                 keys.each.with_index do |key, i|
                                     entry[key] = fix_value(data[:data][i])
                                 end
-                                if NEO4J_DEBUG >= 1
+                                if Neo4jBolt.bolt_verbosity >= 1
                                     STDERR.puts ">>> #{entry.to_json}"
                                     STDERR.puts '-' * 40
                                 end
@@ -830,6 +831,50 @@ module Neo4jBolt
             end
             rows.first
         end
+
+        def setup_constraints_and_indexes(constraints, indexes)
+            wanted_constraints = Set.new()
+            wanted_indexes = Set.new()
+            # STDERR.puts "Setting up constraints and indexes..."
+            constraints.each do |constraint|
+                unless constraint =~ /\w+\/\w+/
+                    raise "Unexpected constraint format: #{constraint}"
+                end
+                constraint_name = "#{CONSTRAINT_INDEX_PREFIX}#{constraint.gsub('/', '_')}"
+                wanted_constraints << constraint_name
+                label = constraint.split('/').first
+                property = constraint.split('/').last
+                query = "CREATE CONSTRAINT #{constraint_name} IF NOT EXISTS FOR (n:#{label}) REQUIRE n.#{property} IS UNIQUE"
+                # STDERR.puts query
+                neo4j_query(query)
+            end
+            indexes.each do |index|
+                unless index =~ /\w+\/\w+/
+                    raise "Unexpected index format: #{index}"
+                end
+                index_name = "#{CONSTRAINT_INDEX_PREFIX}#{index.gsub('/', '_')}"
+                wanted_indexes << index_name
+                label = index.split('/').first
+                property = index.split('/').last
+                query = "CREATE INDEX #{index_name} IF NOT EXISTS FOR (n:#{label}) ON (n.#{property})"
+                # STDERR.puts query
+                neo4j_query(query)
+            end
+            neo4j_query("SHOW ALL CONSTRAINTS").each do |row|
+                next unless row['name'].index(CONSTRAINT_INDEX_PREFIX) == 0
+                next if wanted_constraints.include?(row['name'])
+                query = "DROP CONSTRAINT #{row['name']}"
+                # STDERR.puts query
+                neo4j_query(query)
+            end
+            neo4j_query("SHOW ALL INDEXES").each do |row|
+                next unless row['name'].index(CONSTRAINT_INDEX_PREFIX) == 0
+                next if wanted_indexes.include?(row['name']) || wanted_constraints.include?(row['name'])
+                query = "DROP INDEX #{row['name']}"
+                # STDERR.puts query
+                neo4j_query(query)
+            end
+        end
     end
 
     def transaction(&block)
@@ -864,7 +909,7 @@ module Neo4jBolt
         end
     end
 
-    def dump_database(&block)
+    def dump_database(io)
         tr_id = {}
         id = 0
         neo4j_query("MATCH (n) RETURN n ORDER BY ID(n);") do |row|
@@ -874,7 +919,7 @@ module Neo4jBolt
                 :labels => row['n'].labels,
                 :properties => row['n']
             }
-            yield "n #{node.to_json}"
+            io.puts "n #{node.to_json}"
             id += 1
         end
         neo4j_query("MATCH ()-[r]->() RETURN r;") do |row|
@@ -884,14 +929,99 @@ module Neo4jBolt
                 :type => row['r'].type,
                 :properties => row['r']
             }
-            yield "r #{rel.to_json}"
+            io.puts "r #{rel.to_json}"
         end
     end
 
+    def load_database_dump(io, force_append: false)
+        unless force_append
+            transaction do
+                node_count = neo4j_query_expect_one('MATCH (n) RETURN COUNT(n) as count;')['count']
+                unless node_count == 0
+                    raise "There are nodes in this database, exiting now."
+                end
+            end
+        end
+        n_count = 0
+        r_count = 0
+        node_tr = {}
+        node_batch_by_label = {}
+        relationship_batch_by_type = {}
+        io.each_line do |line|
+            line.strip!
+            next if line.empty?
+            if line[0] == 'n'
+                line = line[2, line.size - 2]
+                node = JSON.parse(line)
+                label_key = node['labels'].sort.join('/')
+                node_batch_by_label[label_key] ||= []
+                node_batch_by_label[label_key] << node
+            elsif line[0] == 'r'
+                line = line[2, line.size - 2]
+                relationship = JSON.parse(line)
+                relationship_batch_by_type[relationship['type']] ||= []
+                relationship_batch_by_type[relationship['type']] << relationship
+            else
+                STDERR.puts "Invalid entry: #{line}"
+                exit(1)
+            end
+        end
+        node_batch_by_label.each_pair do |label_key, batch|
+            while !batch.empty? do
+                slice = []
+                json_size = 0
+                while (!batch.empty?) && json_size < 0x20000 && slice.size < 256
+                    x = batch.shift
+                    slice << x
+                    json_size += x.to_json.size
+                end
+                ids = neo4j_query(<<~END_OF_QUERY, {:properties => slice.map { |x| x['properties']}})
+                    UNWIND $properties AS props
+                    CREATE (n:#{slice.first['labels'].join(':')})
+                    SET n = props
+                    RETURN ID(n) AS id;
+                END_OF_QUERY
+                slice.each.with_index do |node, i|
+                    node_tr[node['id']] = ids[i]['id']
+                end
+                n_count += slice.size
+                STDERR.print "\rLoaded #{n_count} nodes, #{r_count} relationships..."
+            end
+        end
+        relationship_batch_by_type.each_pair do |rel_type, batch|
+            batch.each_slice(256) do |slice|
+                slice.map! do |rel|
+                    rel['from'] = node_tr[rel['from']]
+                    rel['to'] = node_tr[rel['to']]
+                    rel
+                end
+                count = neo4j_query_expect_one(<<~END_OF_QUERY, {:slice => slice})['count_r']
+                    UNWIND $slice AS props
+                    MATCH (from), (to) WHERE ID(from) = props.from AND ID(to) = props.to
+                    CREATE (from)-[r:#{rel_type}]->(to)
+                    SET r = props.properties
+                    RETURN COUNT(r) AS count_r, COUNT(from) AS count_from, COUNT(to) AS count_to;
+                END_OF_QUERY
+                if count != slice.size
+                    raise "Ooops... expected #{slice.size} relationships, got #{count}."
+                end
+                r_count += slice.size
+                STDERR.print "\rLoaded #{n_count} nodes, #{r_count} relationships..."
+            end
+        end
+
+        STDERR.puts
+    end
+
     def cleanup_neo4j
         if @bolt_socket
             @bolt_socket.disconnect()
             @bolt_socket = nil
         end
     end
+
+    def setup_constraints_and_indexes(constraints, indexes)
+        @bolt_socket ||= BoltSocket.new()
+        @bolt_socket.setup_constraints_and_indexes(constraints, indexes)
+    end
 end

data/neo4j_bolt.gemspec

@@ -22,8 +22,8 @@ Gem::Specification.new do |spec|
   spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
     `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   end
-  spec.bindir        = "exe"
-  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.bindir        = "bin"
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
   spec.add_development_dependency "rspec", "~> 3.2"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: neo4j_bolt
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.1.8
 platform: ruby
 authors:
 - Michael Specht
 autorequire: 
-bindir: exe
+bindir: bin
 cert_chain: []
-date: 2022-10-28 00:00:00.000000000 Z
+date: 2022-10-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -27,7 +27,10 @@ dependencies:
 description: 
 email:
 - micha.specht@gmail.com
-executables: []
+executables:
+- console
+- neo4j_bolt
+- setup
 extensions: []
 extra_rdoc_files: []
 files:
@@ -40,6 +43,7 @@ files:
 - README.md
 - Rakefile
 - bin/console
+- bin/neo4j_bolt
 - bin/setup
 - lib/neo4j_bolt.rb
 - lib/neo4j_bolt/version.rb