neoscout 0.1

data/lib/neoscout/model.rb

@@ -0,0 +1,148 @@
+require 'set'
+
+module NeoScout
+
+  class ElementIterator
+    def iter_nodes(args) ; raise NotImplentedError end
+    def iter_edges(args) ; raise NotImplentedError end
+  end
+
+  class Typer
+    def node_type(node) ; raise NotImplementedError end
+    def edge_type(edge) ; raise NotImplementedError end
+
+    def checked_node_type?(node_type) ; raise NotImplementedError end
+    def checked_edge_type?(edge_type) ; raise NotImplementedError end
+
+    def valid_value?(value_type, value) ; true end
+
+    def unknown_node_type?(type) ; false end
+    def unknown_edge_type?(type) ; false end
+  end
+
+  module TyperValueTableMixin
+    def valid_value?(value_type, value)
+      if (entry = self.value_type_table[value_type])
+        entry.call(value_type_name, value)
+      else
+        true
+      end
+    end
+  end
+
+  class Counts
+    attr_reader :all_nodes
+    attr_reader :all_edges
+
+    attr_reader :typed_nodes
+    attr_reader :typed_edges
+
+    attr_reader :typed_node_props
+    attr_reader :typed_edge_props
+
+    attr_reader :node_link_src_stats
+    attr_reader :node_link_dst_stats
+    attr_reader :edge_link_src_stats
+    attr_reader :edge_link_dst_stats
+
+    def initialize(typer)
+      @typer = typer
+      reset
+    end
+
+    def reset
+      @all_nodes        = Counter.new
+      @all_edges        = Counter.new
+
+      @typed_nodes      = Counter.new_multi_keyed :node_type
+      @typed_edges      = Counter.new_multi_keyed :edge_type
+
+      @typed_node_props = Counter.new_multi_keyed :node_type, :prop_constr
+      @typed_edge_props = Counter.new_multi_keyed :node_type, :prop_constr
+
+      @node_link_src_stats = Counter.new_multi_keyed :src_type, :edge_type
+      @node_link_dst_stats = Counter.new_multi_keyed :dst_type, :edge_type
+
+      @edge_link_src_stats = Counter.new_multi_keyed :edge_type, :src_type, :dst_type
+      @edge_link_dst_stats = Counter.new_multi_keyed :edge_type, :dst_type, :src_type
+    end
+
+    def count_node(type, ok)
+      @all_nodes.incr(ok)
+      @typed_nodes[type].incr(ok)
+    end
+
+    def count_node_prop(type, prop, ok)
+      @typed_node_props[type][prop].incr(ok)
+    end
+
+    def count_edge(type, ok)
+      @all_edges.incr(ok)
+      @typed_edges[type].incr(ok)
+    end
+
+    def count_edge_prop(type, prop, ok)
+      @typed_edge_props[type][prop].incr(ok)
+    end
+
+    def count_link_stats(edge_type, src_type, dst_type, ok)
+      # puts "#{src_type} -- #{edge_type} -- #{dst_type} #{if ok then "CHECK" else "FAIL" end}"
+      @node_link_src_stats[src_type][edge_type].incr(ok)
+      @node_link_dst_stats[dst_type][edge_type].incr(ok)
+      @edge_link_src_stats[edge_type][src_type][dst_type].incr(ok)
+      @edge_link_dst_stats[edge_type][dst_type][src_type].incr(ok)
+    end
+
+  end
+
+  class Verifier
+    attr_reader :node_props
+    attr_reader :edge_props
+    attr_reader :allowed_edges
+
+    def initialize
+      @node_props = HashWithDefault.new { |type| ConstrainedSet.new { |o| o.kind_of? Constraints::PropConstraint } }
+      @edge_props = HashWithDefault.new { |type| ConstrainedSet.new { |o| o.kind_of? Constraints::PropConstraint } }
+      @allowed_edges = HashWithDefault.new_multi_keyed(:edge_type, :src_type) { |v| Set.new }
+    end
+
+    def new_node_prop_constr(args={})
+      Constraints::PropConstraint.new args
+    end
+
+    def new_edge_prop_constr(args={})
+      Constraints::PropConstraint.new args
+    end
+
+    def new_card_constr(args={})
+      Constraints::CardConstraint.new args
+    end
+
+    def add_valid_edge(edge_type, src_type, dst_type)
+      @allowed_edges[edge_type][src_type] << dst_type
+    end
+
+    def add_valid_edge_sets(edge_type, src_types, dst_types)
+      src_types.each do |src_type|
+        dst_types.each do |dst_type|
+          add_valid_edge edge_type, src_type, dst_type
+        end
+      end
+    end
+
+    def checked_node_type?(node_type)
+      ! @node_props[node_type].empty?
+    end
+
+    def checked_edge_type?(node_type)
+      ! @node_props[node_type].empty?
+    end
+
+    def allowed_edge?(edge_type, src_type, dst_type)
+      allowed_edges[edge_type].empty? || allowed_edges[edge_type][src_type].member?(dst_type)
+    end
+  end
+
+end
+
+

data/lib/neoscout/scout.rb

@@ -0,0 +1,119 @@
+module NeoScout
+
+  class Scout
+    attr_reader :typer, :verifier, :iterator
+
+    def initialize(args={})
+      @typer    = args[:typer]
+      @typer    = Typer.new unless @typer
+      @verifier = args[:verifier]
+      @verifier = Verifier.new unless @verifier
+      @iterator = args[:iterator]
+      @iterator = ElementIterator.new unless @iterator
+    end
+
+    def checked_node_type?(node_type)
+      @typer.checked_node_type?(node_type) && @verifier.checked_node_type?(node_type)
+    end
+
+    def checked_edge_type?(edge_type)
+      @typer.checked_edge_type?(edge_type) && @verifier.checked_edge_type?(edge_type)
+    end
+
+
+    def new_counts
+      NeoScout::Counts.new(typer)
+    end
+
+    def count_nodes(args)
+      counts = prep_counts(args[:counts])
+      @iterator.iter_nodes(args) do |node|
+        begin
+          node_type = @typer.node_type(node)
+          node_ok   = process_node(counts, node_type, node)
+          counts.count_node(node_type, node_ok)
+        rescue Exception => e
+          puts e
+          counts.count_node(e.class.to_s, false)
+        end
+      end
+      counts
+    end
+
+    def count_edges(args)
+      counts = prep_counts(args[:counts])
+      @iterator.iter_edges(args) do |edge|
+        begin
+          edge_type = @typer.edge_type(edge)
+          edge_ok   = process_edge(counts, edge_type, edge)
+          counts.count_edge(edge_type, edge_ok)
+        rescue Exception => e
+          puts e
+          counts.count_edge(e.class.to_s, false)
+        end
+      end
+      counts
+    end
+
+    def prep_counts(counts) ; counts end
+
+    protected
+
+    def process_node(counts, node_type, node)
+      node_ok    = true
+
+      node_props = Set.new(node.props.keys)
+
+      node_props.delete('_neo_id')
+
+      @verifier.node_props[node_type].each do |constr|
+        prop_ok   = constr.satisfied_by_node?(typer, node)
+        counts.count_node_prop(node_type, constr.name, prop_ok)
+        node_props.delete(constr.name)
+        node_ok &&= prop_ok
+      end
+
+      # Process remaining properties in this node as erroneously missing in the schema
+      # unless the node is untyped
+      node_props.each do |prop_name|
+        prop_ok   = ! checked_node_type?(node_type)
+        counts.count_node_prop(node_type, prop_name, prop_ok)
+        node_ok &&= prop_ok
+      end
+
+      node_ok
+    end
+
+    def process_edge(counts, edge_type, edge)
+      edge_props = Set.new(edge.props.keys)
+      edge_props.delete('_neo_id')
+
+      src_type = @typer.node_type(edge.getStartNode)
+      dst_type = @typer.node_type(edge.getEndNode)
+
+      edge_ok = @verifier.allowed_edge?(edge_type, src_type, dst_type)
+
+      @verifier.edge_props[edge_type].each do |constr|
+        prop_ok   = constr.satisfied_by_edge?(typer, edge)
+        counts.count_edge_prop(edge_type, constr.name, prop_ok)
+        edge_props.delete(constr.name)
+        edge_ok &&= prop_ok
+      end
+
+      # Process remaining properties in this node as erroneously missing in the schema
+      # unless the edge is untyped
+      edge_props.each do |prop_name|
+        prop_ok   = ! checked_edge_type?(edge_type)
+        counts.count_edge_prop(edge_type, prop_name, prop_ok)
+        edge_ok &&= prop_ok
+      end
+
+      # Finally count edge statistics
+      counts.count_link_stats(edge_type, src_type, dst_type, edge_ok)
+
+      edge_ok
+    end
+
+  end
+
+end

data/lib/neoscout/tools.rb

@@ -0,0 +1,156 @@
+module NeoScout
+
+  class Counter
+
+    def initialize
+      reset
+    end
+
+    def reset
+      @ok    = 0
+      @total = 0
+    end
+
+    def incr(ok)
+      if ok then incr_ok else incr_failed end
+    end
+
+    def incr_ok
+      @ok    += 1
+      @total += 1
+    end
+
+    def incr_failed
+      @total +=1
+    end
+
+    def num_ok
+      @ok
+    end
+
+    def num_failed
+      @total - @ok
+    end
+
+    def num_total
+      @total
+    end
+
+    def empty?
+      @total == 0
+    end
+
+    def to_s
+      "(#{num_ok}/#{num_failed}/#{num_total})"
+    end
+
+  end
+
+  class ConstrainedSet < Set
+
+    def initialize(*args, &elem_test)
+      @elem_test = elem_test
+      case
+        when args.length == 0
+          super
+        when args.length == 1
+          args = args[0]
+          raise ArgumentError unless (args.all? &@elem_test)
+          super args
+        else
+          raise ArgumentError
+      end
+    end
+
+    def valid_elem?(elem)
+      @elem_test.call(elem)
+    end
+
+    def <<(elem)
+      raise ArgumentError unless valid_elem?(elem)
+      super elem
+    end
+
+  end
+
+  module HashDefaultsMixin
+
+    def initialize(*args, &blk)
+      super *args
+      @default = blk
+    end
+
+    def default(key)
+      @default.call(key)
+    end
+
+    def [](key)
+      if has_key?(key) then super(key) else self[key]=default(key) end
+    end
+
+    def lookup(key, default_value = nil)
+      if has_key?(key) then self[key] else self[key]=default_value end
+    end
+
+    def key_descr
+      :key
+    end
+
+
+    def self.included(base)
+
+      # defines map_value for mixin target baseclass instances and any subclass instances
+      base.class_exec(base) do |base_class|
+        define_method(:map_value) do |&blk|
+          new_hash = {}
+          each_pair do |k,v|
+            new_hash[k] = if v.kind_of? base_class then v.map_value(&blk) else blk.call(v) end
+          end
+          new_hash
+        end
+      end
+
+      # defines new_multi_keyed on the mixin's target baseclass
+      # (subclasses the baseclass to override key_descr for instances)
+      def base.new_multi_keyed(*list, &blk)
+        new_class = Class.new(self)
+        (class << new_class ; self end).class_exec(list.shift) do |descr|
+          define_method(:key_descr) { || descr }
+        end
+        if list.empty?
+          then new_class.new(&blk)
+          else new_class.new { |key| self.new_multi_keyed(*list, &blk) } end
+      end
+
+    end
+  end
+
+  class HashWithDefault < Hash
+
+    include HashDefaultsMixin
+
+  end
+
+  class Counter
+
+    def self.new_multi_keyed(*list)
+      HashWithDefault.new_multi_keyed(*list) { |key| Counter.new }
+    end
+
+  end
+
+  module JSON
+
+    def self.cd(json, args)
+      current = json
+      args.each do |k|
+        current = if current.has_key? k
+          then current[k]
+          else current[k] = {} end
+      end
+      current
+    end
+
+  end
+
+end

data/lib/neoscout/version.rb

@@ -0,0 +1,3 @@
+module NeoScout
+  VERSION = '0.1'
+end

data/neoscout.gemspec

@@ -0,0 +1,25 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require 'neoscout/version'
+
+Gem::Specification.new do |s|
+  s.name        = 'neoscout'
+  s.version     = NeoScout::VERSION
+  s.summary     = 'Graph database schema extraction and validation tool'
+  s.description = 'Tool for validating the schema of a free form graph databases and for reporting errors, including a REST access layer for runtime checking'
+  s.author      = 'Stefan Plantikow'
+  s.email       = 'stefanp@moviepilot.com'
+  s.homepage    = 'http://moviepilot.github.com/neoscout'
+  s.rubyforge_project = 'neoscout'
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.bindir      = 'script'
+  s.executables = `git ls-files -- script/*`.split("\n").map{ |f| File.basename(f) }
+  s.default_executable = 'neoscout'
+  s.executables = ['neoscout']
+  s.licenses = ['PUBLIC DOMAIN WITHOUT ANY WARRANTY']
+end