neoscout 0.1

data/etc/neo4j.yml

@@ -0,0 +1,16 @@
+neostore.nodestore.db.mapped_memory: 400M
+neostore.relationshipstore.db.mapped_memory: 1G
+neostore.propertystore.db.mapped_memory: 4G
+neostore.propertystore.db.index.mapped_memory: 50M
+neostore.propertystore.db.index.keys.mapped_memory: 50M
+neostore.propertystore.db.strings.mapped_memory: 500M
+neostore.propertystore.db.arrays.mapped_memory: 500M
+
+# Keeping the defaults here for reference
+# neostore.nodestore.db.mapped_memory: 25M
+# neostore.relationshipstore.db.mapped_memory: 50M
+# neostore.propertystore.db.mapped_memory: 90M
+# neostore.propertystore.db.index.mapped_memory: 1M
+# neostore.propertystore.db.index.keys.mapped_memory: 1M
+# neostore.propertystore.db.strings.mapped_memory: 130M
+# neostore.propertystore.db.arrays.mapped_memory: 130M

data/lib/neoscout.rb

@@ -0,0 +1,12 @@
+require 'json'
+require 'neo4j'
+require 'set'
+
+require 'neoscout/version'
+
+require 'neoscout/tools'
+require 'neoscout/constraints'
+require 'neoscout/model'
+require 'neoscout/json_schema'
+require 'neoscout/scout'
+require 'neoscout/gdb_neo4j'

data/lib/neoscout/constraints.rb

@@ -0,0 +1,35 @@
+module NeoScout
+
+  module Constraints
+
+    class Constraint
+      def satisfied_by_node?(typer, node) ; satisfied_by?(typer, node) end
+      def satisfied_by_edge?(typer, edge) ; satisfied_by?(typer, edge) end
+
+      def satisfied_by?(typer, obj) ; raise NotImplementedError end
+    end
+
+    class PropConstraint < Constraint
+      attr_reader :name, :opt, :type
+
+      def initialize(args = {})
+        super(args)
+        @name  = args[:name]
+        @opt   = args[:opt]
+        @type  = args[:type]
+
+        raise ArgumentError unless @name.kind_of? String
+        raise ArgumentError unless @name.length > 0
+      end
+
+      def to_s
+        opt_s  = if @opt then " (opt.)" else '' end
+        type_s = if @type then ": #{@type}" else '' end
+
+        "#{@name}#{type_s}#{opt_s}"
+      end
+    end
+
+
+  end
+end

data/lib/neoscout/gdb_neo4j.rb

@@ -0,0 +1,147 @@
+module NeoScout
+
+  module GDB_Neo4j
+
+    module Constraints
+
+      class PropConstraint < NeoScout::Constraints::PropConstraint
+
+        def satisfied_by?(typer, obj)
+          if obj.property?(@name)
+            then satisfies_type?(typer, @type, obj[@name])
+            else self.opt
+          end
+        end
+
+        def satisfies_type?(typer, type, value)
+          if type then typer.valid_value?(type, value) else true end
+        end
+      end
+
+    end
+
+    class Typer < NeoScout::Typer
+
+      attr_accessor :type_attr
+      attr_accessor :nil_type
+      attr_accessor :value_type_table
+      attr_accessor :node_mapper
+      attr_accessor :edge_mapper
+
+      include TyperValueTableMixin
+
+      def initialize
+        @type_attr        = '_classname'
+        @nil_type         = '__NOTYPE__'
+        @node_mapper      = nil
+        @edge_mapper      = nil
+        @value_type_table = {}
+      end
+
+      def node_type(node)
+        props = node.props
+        return node_mapped(props[@type_attr]) if props.has_key? @type_attr
+        @nil_type
+      end
+
+      def edge_type(edge)
+        type = edge.rel_type
+        if type then edge_mapped(type.to_s) else @nil_type end
+      end
+
+      def checked_node_type?(node_type) ; node_type != self.nil_type end
+      def checked_edge_type?(edge_type) ; edge_type != self.nil_type end
+
+      def unknown_node_type?(type) ; type == @nil_type end
+      def unknown_edge_type?(type) ; type == @nil_type end
+
+      protected
+
+      def node_mapped(t)
+        if self.node_mapper then self.node_mapper.call(t) else t end
+      end
+
+      def edge_mapped(t)
+        if self.edge_mapper then self.edge_mapper.call(t) else t end
+      end
+
+    end
+
+    class ElementIterator < NeoScout::ElementIterator
+
+      def iter_nodes(args)
+        if args[:report_progress]
+          then report = args[:report_progress]
+          else report = lambda { |mode, what, num| } end
+        glops = org.neo4j.tooling.GlobalGraphOperations.at(Neo4j.db.graph)
+        iter  = glops.getAllNodes.iterator
+        num   = 0
+        while iter.hasNext do
+          node = iter.next
+          num  = num + 1
+          report.call(:progress, :nodes, num)
+          yield node unless node.getId == 0
+        end
+
+        report.call(:finish, :nodes, num)
+        num
+      end
+
+      def iter_edges(args)
+        if args[:report_progress]
+          then report = args[:report_progress]
+          else report = lambda { |mode, what, num| } end
+        glops = org.neo4j.tooling.GlobalGraphOperations.at(Neo4j.db.graph)
+        iter  = glops.getAllRelationships.iterator
+        num   = 0
+        while iter.hasNext do
+          num  = num + 1
+          report.call(:progress, :edges, num)
+          yield iter.next
+        end
+
+        report.call(:finish, :edges, num)
+        num
+      end
+
+    end
+
+    class Verifier < NeoScout::Verifier
+      def initialize(typer)
+        super()
+        @typer = typer
+      end
+
+      def new_node_prop_constr(args={})
+        Constraints::PropConstraint.new args
+      end
+
+      def new_edge_prop_constr(args={})
+        Constraints::PropConstraint.new args
+      end
+
+      def init_from_json(json)
+        super(json)
+      end
+    end
+
+    class Scout < NeoScout::Scout
+
+      def initialize(args={})
+        args[:typer] = Typer.new unless args[:typer]
+        args[:verifier] = Verifier.new(args[:typer]) unless args[:verifier]
+        args[:iterator] = ElementIterator.new unless args[:iterator]
+        super args
+      end
+
+      def prep_counts(counts)
+        # Ensure __NOTYPE__entries always have a counts array
+        counts.typed_nodes[typer.nil_type]
+        counts.typed_edges[typer.nil_type]
+        counts
+      end
+    end
+
+  end
+
+end

data/lib/neoscout/json_schema.rb

@@ -0,0 +1,136 @@
+module NeoScout
+
+  class Counter
+
+    def to_json
+      { 'num_failed' => num_failed, 'num_total' => num_total }
+    end
+
+  end
+
+
+  # TODO indexes
+  # TODO edges
+  # TODO testing
+
+  class Verifier
+
+    def init_from_json(json)
+      JSON.cd(json, %w(nodes)).each_pair do |type_name, type_json|
+        JSON.cd(type_json, %w(properties)).each_pair do |prop_name, prop_json|
+          prop_constr = new_node_prop_constr name: prop_name, opt: !prop_json['relevant']
+          prop_set    = self.node_props[type_name]
+          prop_set << prop_constr
+        end
+      end
+
+      JSON.cd(json, %w(connections)).each_pair do |type_name, type_json|
+        JSON.cd(type_json, %w(properties)).each_pair do |prop_name, prop_json|
+          prop_constr = new_edge_prop_constr name: prop_name, opt: !prop_json['relevant']
+          prop_set    = self.edge_props[type_name]
+          prop_set << prop_constr
+        end
+
+        sources_json = if type_json.has_key?('sources') then type_json['sources'] else [] end
+        targets_json = if type_json.has_key?('targets') then type_json['targets'] else [] end
+        add_valid_edge_sets type_name, sources_json, targets_json
+      end
+    end
+
+  end
+
+
+  class HashWithDefault
+
+    def to_json
+      self.map_value { |v| v.to_json }
+    end
+  end
+
+  #noinspection RubyTooManyInstanceVariablesInspection
+  class Counts
+
+    def add_to_json(json)
+      all_json = JSON.cd json, %w(all)
+      all_json['node_counts'] = @all_nodes.to_json
+      all_json['connection_counts'] = @all_edges.to_json
+
+      nodes_json = JSON.cd(json, %w(nodes))
+      @typed_nodes.each_pair do |type, count|
+        skip = skip_from_json(:node, type, count)
+        JSON.cd(nodes_json, [type])['counts'] = count.to_json unless skip
+      end
+
+      nodes_json = JSON.cd(json, %w(nodes))
+      @typed_node_props.each_pair do |type, props|
+        props.each_pair do |name, count|
+          skip = skip_from_json(:node, type, count)
+          JSON.cd(nodes_json, [type, 'properties', name])['counts'] = count.to_json unless skip
+        end
+      end
+
+      edges_json = JSON.cd(json, %w(connections))
+      @typed_edges.each_pair do |type, count|
+        skip = skip_from_json(:edge, type, count)
+        JSON.cd(edges_json, [type])['counts'] = count.to_json unless skip
+      end
+
+      edges_json = JSON.cd(json, %w(connections))
+      @typed_edge_props.each_pair do |type, props|
+        props.each_pair do |name, count|
+          skip = skip_from_json(:edge, type, count)
+          JSON.cd(edges_json, [type, 'properties', name])['counts'] = count.to_json unless skip
+        end
+      end
+
+      add_link_stats_to_json(json)
+    end
+
+    def skip_from_json(kind, type, count)
+      false unless count.empty?
+      case kind
+        when :node
+          @typer.unknown_node_type?(type)
+        when :edge
+          @typer.unknown_edge_type?(type)
+        else
+          raise ArgumentError
+      end
+    end
+
+    protected
+
+    def add_link_stats_to_json(json)
+      nodes_json = JSON.cd(json, %w(nodes))
+
+      @node_link_src_stats.each_pair do |type, hash|
+        JSON.cd(nodes_json, [type])['src_stats'] = hash.to_json
+      end
+
+      @node_link_dst_stats.each_pair do |type, hash|
+        JSON.cd(nodes_json, [type])['dst_stats'] = hash.to_json
+      end
+
+      edges_json = JSON.cd(json, %w(connections))
+      @edge_link_src_stats.each_pair do |type, hash|
+        JSON.cd(edges_json, [type])['src_stats'] = hash_to_json_array('to_dst', hash)
+      end
+
+      @edge_link_dst_stats.each_pair do |type, hash|
+        JSON.cd(edges_json, [type])['dst_stats'] = hash_to_json_array('from_src', hash)
+      end
+    end
+
+    def hash_to_json_array(target, hash)
+      result = []
+      hash.each_pair do |key, value|
+        inner = []
+        value.each_pair do |i_key, i_value|
+          inner << { 'name' => i_key, 'counts' => i_value.to_json }
+        end
+        result << { 'name' => key, target => inner }
+      end
+      result
+    end
+  end
+end

data/lib/neoscout/main.rb

@@ -0,0 +1,205 @@
+require 'java'
+require 'yaml'
+require 'optparse'
+require 'json'
+require 'httparty'
+require 'date'
+require 'active_model'
+require 'active_support/inflector'
+
+module NeoScout
+
+  class Main
+    attr_reader :opt_db
+    attr_reader :opt_schema
+    attr_reader :opt_port
+    attr_reader :opt_webservice
+    attr_reader :opt_bind
+    attr_reader :opt_report
+    attr_reader :opt_no_nodes
+    attr_reader :opt_db_config
+    attr_reader :opt_no_edges
+    attr_reader :opt_type_mapper
+    attr_reader :opt_pre_mapper
+    attr_reader :opt_output_file
+
+
+    def initialize
+      @opt_report      = 0
+      @opt_webservice  = false
+      @opt_output_file = nil
+      @opt_no_nodes    = false
+      @opt_db_config   = nil
+      @opt_no_edges    = false
+      @opt_pre_mapper  = lambda { |t| t }
+      @opt_type_mapper = lambda { |t| t }
+      parse_opts
+    end
+
+    def parse_opts
+      optparse = OptionParser.new do |opts|
+        opts.banner = "Usage: --db <neo4j:path> --schema <url> [--port <port>]"
+        opts.on('-d', '--db DB', 'Path to database in the form neo4j:<path>)') do |db|
+          @opt_db     = db
+        end
+        opts.on('-u', '--schema-url URL', 'URL to database schema') do |url|
+          @opt_schema = lambda { || ::JSON.parse(HTTParty.get(url)) }
+        end
+        opts.on('-s', '--schema-file FILE', 'schema file') do |file|
+          @opt_schema = lambda { || ::JSON.parse(IO::read(file)) }
+        end
+        opts.on('-o', '--output-file FILE', 'output file in standalone mode') do |f|
+          @opt_output_file = f
+        end
+        opts.on('-w', '--webservice', 'Run inside sinatra') do
+          @opt_webservice = true
+        end
+        opts.on('-p', '--port PORT', 'Port to be used') do |port|
+          @opt_port   = port.to_i
+        end
+        opts.on('-b', '--bind ITF', 'Interface to be used') do |itf|
+          @port       = itf
+        end
+        opts.on('-r', '--report NUM', 'Report progress every NUM graph elements') do |num|
+          @opt_report = num.to_i
+        end
+        opts.on('--no-nodes', 'Do not iterate over nodes') do
+          @opt_no_nodes = true
+        end
+        opts.on('--no-edges', 'Do not iterate over edges') do
+          @opt_no_edges = true
+        end
+        opts.on('-P', '--pluralize-types', 'Pluralize type names') do
+          @opt_pre_mapper = lambda { |t| t.pluralize }
+        end
+        opts.on('-S', '--singularize-types', 'Singularize type names') do
+          @opt_pre_mapper = lambda { |t| t.singularize }
+        end
+        opts.on('-M', '--type-mapper MAPPER',
+                'Set the type mapper (underscore, downcase, upcase)') do |mapper|
+          @opt_type_mapper = case mapper
+                               when 'underscore'
+                                 lambda { |t| t.underscore }
+                               when 'downcase'
+                                 lambda { |t| t.downcase }
+                               when 'upcase'
+                                 lambda { |t| t.upcase }
+                               else
+                                 raise ArgumentException('Unsupported mapper')
+                             end
+        end
+        opts.on('-C', '--db-config FILE', 'Set config file for db driver') do |file|
+          @opt_db_config = file
+        end
+        opts.on('-h', '--help', 'Display this screen') do
+          puts opts
+          exit 1
+        end
+      end
+      optparse.parse!
+      @opt_output_file = nil if @opt_webservice
+    end
+
+    def start_db
+      @opt_db.match(/(neo4j:)(.*)/) do |m|
+        Neo4j.config[:storage_path] = m[2] unless (m[2].length == 0)
+        YAML.load_file(@opt_db_config).each_pair { |k,v| Neo4j.config[k] = v } if @opt_db_config
+        Neo4j.start
+        return lambda do
+          scout = ::NeoScout::GDB_Neo4j::Scout.new
+          pre_mapper = self.opt_pre_mapper
+          scout.typer.node_mapper = lambda { |t| self.opt_type_mapper.call(pre_mapper.call(t)) }
+          scout.typer.edge_mapper = lambda { |t| self.opt_type_mapper.call(pre_mapper.call(t)) }
+          scout
+        end
+      end
+
+      raise ArgumentError("Unsupported database type")
+    end
+
+    def shutdown_db
+      @opt_db.match(/(neo4j:)(.*)/) do |m|
+        Neo4j.shutdown
+        return
+      end
+
+      raise ArgumentError("Unsupported database type")
+    end
+
+    class SimpleConsoleLogger
+      def method_missing(key, *args)
+        print key
+        print ': '
+        puts *args
+      end
+    end
+
+    def run
+      ### Load schema at least once to know that we're safe
+      self.opt_schema.call()
+      ### Run as service if requested
+      return run_webservice(self.opt_schema, self.start_db) if self.opt_webservice
+
+      json = run_standalone(self.opt_schema, self.start_db, SimpleConsoleLogger.new)
+      if self.opt_output_file
+        then File.open(self.opt_output_file, 'w') { |f| f.write(json) }
+        else puts(json) end
+      shutdown_db
+    end
+
+    def run_standalone(schema_maker, scout_maker, logger)
+      schema   = schema_maker.call()
+      scout    = scout_maker.call()
+      scout.verifier.init_from_json schema
+      counts   = scout.new_counts
+      logger   = SimpleConsoleLogger.new unless logger
+      progress = lambda do |mode, what, num|
+        if ((num % self.opt_report) == 0) || (mode == :finish)
+          logger.info("#{DateTime.now}: #{what} ITERATOR PROGRESS (#{mode} / #{num})")
+        end
+      end
+      scout.count_edges counts: counts, report_progress: progress unless self.opt_no_edges
+      scout.count_nodes counts: counts, report_progress: progress unless self.opt_no_nodes
+      counts.add_to_json schema
+      schema.to_json
+    end
+
+    def run_webservice(schema_maker, scout_maker)
+      ### Run sinatra
+      require 'sinatra'
+
+      set :port, @opt_port if @opt_port
+      set :bind, @opt_bind if @opt_bind
+      set :show_exceptions, true
+      set :sessions, false
+      set :logging, true
+      set :dump_errors, true
+      set :lock, true # -- really?
+      set :root, File.expand_path("../../root", __FILE__)
+      set :run, true
+      # set :public_folder
+
+      ### Keep self around for calling helpers in sinatra handlers
+      main = self
+
+      ### Return schema
+      get '/schema' do
+        content_type :json
+        schema_maker.call().to_json
+      end
+
+      ### Run verify over database and report results
+      get '/verify' do
+        content_type :json
+        main.run_standalone(schema_maker, scout_maker, self.logger)
+      end
+
+      ### Shutdown server, the hard way
+      get '/shutdown' do
+        main.shutdown_db
+        java.lang.System.exit(0)
+      end
+    end
+  end
+
+end