neoscout 0.1

data/.gitignore

@@ -0,0 +1,9 @@
+.idea
+.bundle
+doc
+db
+pkg
+html
+package
+coverage
+bin

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--colour
+--tty

data/.rvmrc

@@ -0,0 +1,2 @@
+rvm --create use jruby-1.6.7@neoscout
+export JRUBY_OPTS="$JRUBY_OPTS -Xcompat.version=1.9"

data/AUTHORS

@@ -0,0 +1 @@
+Stefan Plantikow <stefanp@moviepilot.com>

data/Gemfile

@@ -0,0 +1,21 @@
+source "http://rubygems.org"
+
+gem 'sinatra'
+gem 'httparty'
+gem 'json', '>=1.6.5'
+gem 'activemodel', '3.1.3'
+gem 'activesupport', '3.1.3'
+
+group :development do
+  gem 'rake'
+  gem 'irbtools'
+end
+
+group :test do
+  gem 'rspec', '2.6.0'
+  gem 'simplecov'
+end
+
+platforms :jruby do
+  gem 'neo4j', '1.3.1'
+end

data/Gemfile.lock

@@ -0,0 +1,138 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    actionpack (3.1.3)
+      activemodel (= 3.1.3)
+      activesupport (= 3.1.3)
+      builder (~> 3.0.0)
+      erubis (~> 2.7.0)
+      i18n (~> 0.6)
+      rack (~> 1.3.5)
+      rack-cache (~> 1.1)
+      rack-mount (~> 0.8.2)
+      rack-test (~> 0.6.1)
+      sprockets (~> 2.0.3)
+    activemodel (3.1.3)
+      activesupport (= 3.1.3)
+      builder (~> 3.0.0)
+      i18n (~> 0.6)
+    activesupport (3.1.3)
+      multi_json (~> 1.0)
+    awesome_print (1.0.2)
+    boson (1.1.0)
+    builder (3.0.0)
+    clipboard (1.0.1)
+    coderay (1.0.5)
+    diff-lcs (1.1.3)
+    erubis (2.7.0)
+    every_day_irb (1.2.0)
+    fancy_irb (0.7.2)
+      paint (>= 0.8.1)
+      unicode-display_width (>= 0.1.1)
+    g (1.6.0)
+      ruby_gntp
+    hike (1.2.1)
+    hirb (0.6.1)
+    httparty (0.8.1)
+      multi_json
+      multi_xml
+    i18n (0.6.0)
+    interactive_editor (0.0.10)
+      spoon (>= 0.0.1)
+    irbtools (1.2.0)
+      awesome_print (~> 1.0.2)
+      boson (>= 0.3.4)
+      clipboard (~> 1.0.0)
+      coderay (~> 1.0.5)
+      every_day_irb (>= 1.2.0)
+      fancy_irb (>= 0.7.2)
+      g (>= 1.5.0)
+      hirb (~> 0.6.0)
+      interactive_editor (>= 0.0.10)
+      method_locator (>= 0.0.4)
+      method_source (>= 0.7.0)
+      methodfinder (>= 1.2.5)
+      ori (~> 0.1.0)
+      paint (>= 0.8.4)
+      sketches (>= 0.1.1)
+      wirb (>= 0.4.1)
+      zucker (>= 11)
+    json (1.6.5-java)
+    method_locator (0.0.4)
+    method_source (0.7.1)
+    methodfinder (1.2.5)
+    multi_json (1.1.0)
+    multi_xml (0.4.1)
+    neo4j (1.3.1-java)
+      activemodel (>= 3.0.0)
+      orm_adapter (>= 0.0.3)
+      railties (>= 3.0.0)
+      will_paginate (= 3.0.pre4)
+    ori (0.1.0)
+    orm_adapter (0.0.6)
+    paint (0.8.4)
+    rack (1.3.6)
+    rack-cache (1.2)
+      rack (>= 0.4)
+    rack-mount (0.8.3)
+      rack (>= 1.0.0)
+    rack-protection (1.2.0)
+      rack
+    rack-ssl (1.3.2)
+      rack
+    rack-test (0.6.1)
+      rack (>= 1.0)
+    railties (3.1.3)
+      actionpack (= 3.1.3)
+      activesupport (= 3.1.3)
+      rack-ssl (~> 1.3.2)
+      rake (>= 0.8.7)
+      rdoc (~> 3.4)
+      thor (~> 0.14.6)
+    rake (0.9.2.2)
+    rdoc (3.12)
+      json (~> 1.4)
+    rspec (2.6.0)
+      rspec-core (~> 2.6.0)
+      rspec-expectations (~> 2.6.0)
+      rspec-mocks (~> 2.6.0)
+    rspec-core (2.6.4)
+    rspec-expectations (2.6.0)
+      diff-lcs (~> 1.1.2)
+    rspec-mocks (2.6.0)
+    ruby_gntp (0.3.4)
+    simplecov (0.6.1)
+      multi_json (~> 1.0)
+      simplecov-html (~> 0.5.3)
+    simplecov-html (0.5.3)
+    sinatra (1.3.2)
+      rack (~> 1.3, >= 1.3.6)
+      rack-protection (~> 1.2)
+      tilt (~> 1.3, >= 1.3.3)
+    sketches (0.1.1)
+    spoon (0.0.1)
+    sprockets (2.0.3)
+      hike (~> 1.2)
+      rack (~> 1.0)
+      tilt (~> 1.1, != 1.3.0)
+    thor (0.14.6)
+    tilt (1.3.3)
+    unicode-display_width (0.1.1)
+    will_paginate (3.0.pre4)
+    wirb (0.4.1)
+    zucker (12.1)
+
+PLATFORMS
+  java
+
+DEPENDENCIES
+  activemodel (= 3.1.3)
+  activesupport (= 3.1.3)
+  httparty
+  irbtools
+  json (>= 1.6.5)
+  neo4j (= 1.3.1)
+  rake
+  rspec (= 2.6.0)
+  simplecov
+  sinatra

data/LICENSE.txt

@@ -0,0 +1,21 @@
+Copyright (c) 2012 Stefan Plantikow
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+

data/README.md

@@ -0,0 +1,194 @@
+# neoscout
+
+Neoscout is a tool for verifying the schema and visualizing the structure of graph databases. It is currently geared exclusively towards neo4j but could easily be extended for other graph stores.
+
+
+## Overview
+
+neoscout walks a graph by iterating over edges and nodes (= graph element). For each visited graph element, schema properties are verified.  Depending on wether the element passes all requirements, it is considered verified or failed and counted.  Required schema properties are either set programmatically or parsed from a JSON schema file as described below.
+
+The general programmatic use of neoscout is:
+
+```ruby
+require 'json'
+require 'neoscout'
+
+# load schema
+schema_json = JOSN.parse('...schema...')
+# make new scout for use with embedded neo4j database
+scout = ::NeoScout::GDB_Neo4j::Scout.new
+# parse schema
+scout.verifier.init_from_json schema_json
+# iterate over edges and nodes, collecting statistics
+counts = scout.new_counts
+scout.count_edges counts: counts
+scout.count_nodes counts: counts
+# add collected statistics back to schema_json
+counts.add_to_json schema_json
+# print result
+puts "<<RESULT\n#{schema_json.to_json}\nRESULT"
+```
+
+
+## Depends on
+
+jruby, neo4j, sinatra, json
+
+
+## Installation
+
+As a gem or via the typical bundle and rake build test install dance.
+
+
+## Model
+
+neoscout assumes that the underlying graph is directed, that nodes and edges can be assigned a unique type in the
+schema, and that arbitrary properties may be assigned to each node and edge.
+
+
+## JSON Schema
+
+
+### Schema Format
+
+The currently supported schema format is
+
+```json
+{
+"nodes":{
+    "node_type_a": {
+        "properties": {
+            "property_a": { "relevant": false },
+            "property_b": { "relevant": true }
+        }
+    },
+    "node_type_b": {
+        "properties": {
+            "property_a": { "relevant": true },
+            "property_b": { "relevant": false }
+         }
+    }
+},
+"connections":{
+    "edge_type_a": {
+        "properties": {
+            "property_a": { "relevant": false },
+            "property_b": { "relevant": true }
+        },
+        "sources": ["node_type_a", "node_type_b"],
+        "targets": ["node_type_a"]
+    },
+    "edge_type_b": {
+        "properties": {
+            "property_a": { "relevant": false },
+            "property_b": { "relevant": true }
+        }
+    }
+}
+}
+```
+
+Some properties may additionally specify a value type under `type`. Verification of value types needs to be
+specified programmatically.
+
+
+### Schema output
+
+When the validation has been completed, collected statistics may be appended the input JSON schema. For every collected
+statistic, a counter of the form `[num_failed, num_total]` is added. The list of currently collected
+statistics is:
+
+* `nodes/type/counts` number of of nodes of type `type`
+    considered ok iff all relevant node properties are verified succesfully and no additional unknown node properties
+    are found or iff no node properties were specified for this type
+* `connections/type/counts` number of of edges of type `type`
+    verified similar to nodes
+* `nodes/type/properties/prop/counts` number of properties `prop` in nodes of type `type` 
+    considered ok iff the property was found and its conncrete value matched its value type (if given in the schema)
+    or the property was not found and was specified as `"relevant": false`
+* `connections/type/properties/prop/counts` number of properties `prop` in edges of type `type`
+    verified similar to nodes
+* `all/node_counts` number of nodes
+    considered ok iff the node was ok according to its type
+* `all/connection_counts` number of edges
+    considered ok iff the node was ok according to its type
+
+Additionally, for each edge of edge type `edge_type` it is verified, wether it's source and destination node types
+`src_type` and `dst_type` are found  in `connections/type/sources` and `connections/type/targets` respectively.
+Any edge type, for which these arrays are missing is considered to be verified by this test. Again, the results are
+aggregated into various statistics:
+
+* `connections/edge_type/src_stats/[ { "name": src_type, "to_dst": [ { "name": dst_type, "counts": /*count */ } ] } ]`
+* `connections/edge_type/dst_stats/[ { "name": dst_type, "from_src": [ { "name": src_type, "counts": /*count */ } ] } ]`
+* `nodes/src_type/src_stats/edge_type`
+* `nodes/dst_type/dst_stats/edge_type`
+
+
+### Optional type testing
+
+Additionally, schema properties may have a string-valued `type` property for testing property *values.
+To register a type test for property values, your implementation of `Typer` needs to mixin
+`TyperValueTableMixin` (true for NeoScout::GDB_Neo4j::Typer). Then, just call:
+
+```ruby
+typer.value_type_table['string'] = lambda { |n,v| v.kind_of? String }
+```
+
+to register your type tests.  Properties that have a `type` attribute, and for which a type test is
+registered but whose value fails the test are considered as failed and reported accordingly.
+
+
+### Example
+
+Please see `spec/lib/neoscout/gdb_neo4_spec.rb`, `spec/lib/neoscout/gdb_neo4_spec_schema.json`,
+`spec/lib/neoscout/gdb_neo4_spec_counts.json` for an extended example.
+
+
+## Standalone runner
+
+There is a rudimentary, sinatra-based standalone runner that attaches to a local neo4j databases, and upon request
+fetches a schema url and verifies the database against it. It is in `scripts/neoscout` and is installed by default.
+Please consult `neoscout --help` for more details.
+
+### Webservice API
+
+The standalone runner can be run as a RESTful webservice using `-w`. If this is done, it suppors the
+follwing API
+
+* `/schema` retrieve schema
+* `/verify` trigger verification
+* `/shutdown` shutdown
+
+
+## Implementation Notes
+
+This is how things work right now but expect a major change in architecture in the next version.
+
+`Scout` is the main class that implements the generic logic for processing nodes and edges using several
+helper classes
+
+* `Typer` assigns types to nodes and edges
+* `Verifier` checks all schema properties
+* `Iterator` provides iteration constructs for iterating over the nodes and edges of the underlying graph
+* furthermore, `Scout` features overridable factory methods for the construction of subclasses implementing
+various Constraints, most importantly the node and edge propery constraints.  Basic implementations of those
+are provided in `constraints.rb`
+
+`Counts` is used for collecting statistics and heavily tied to logic implemented by `Scout`.
+
+JSON schema handling is in `json_schema.rb`
+
+
+### Specializing for a new database
+
+Please consult `gdb_neo4j.rb` to see how to do that, essentially you subclass `Scout` and potentially override default
+values for the various member fields. The standalone runner currently is heavily tied to neo4j.
+
+
+### Notes on GDB_Neo4j
+
+* Node types are currently derived from a configurable property (defaults to '_classname')
+* Edge types directly correspond to the relationship type in neo4j
+* Unkown nodes/edges are assigned to a reserved `__NOTYPE__` type (the actual string may be overriden, see Typer)
+* You can pass configuration options for neo4j using `-C <path-to-yml>`. This is especially important for larger
+databases. See etc/neo4j.yml for an example.

data/Rakefile

@@ -0,0 +1,30 @@
+require 'bundler/gem_tasks'
+
+require 'rdoc/task'
+require 'rspec'
+require 'rspec/core/rake_task'
+
+desc 'Run all rspecs'
+RSpec::Core::RakeTask.new(:spec) do |spec|
+  spec.fail_on_error = true
+  spec.verbose       = false
+#  spec.rspec_opts    = ['--backtrace']
+end
+
+desc 'Run rdoc over project sources'
+RDoc::Task.new(:rdoc) do |rdoc|
+  # rdoc.main = "README.rdoc"
+  rdoc.rdoc_files.include("lib/**/*.rb")
+end
+
+desc 'Run irb in project environment'
+task :console do
+  require 'irb'
+  ARGV.clear
+  IRB.conf[:USE_READLINE] = false if ENV['JRUBY_OPTS'] =~ /--ng/
+  IRB.start
+end
+
+task :doc => :rdoc
+task :test => :spec
+task :irb => :console

data/TODO.org

@@ -0,0 +1,27 @@
+* Version 0.1
+** DONE Minimal documentation
+** Features
+*** DONE type testing mechanism
+*** DONE cardconstraints if ever: nixed, to be replaced with pacer-jogger constraints in 0.2
+*** visualization
+*** read-only mode for neo4j
+** Integration and (alongside) testing
+*** put into sheldon
+*** limit number of queries / share results(?)
+*** more tests
+* Version 0.2
+** Redesign internal structure
+*** Brainstoerming
+** Features
+*** aggregate min-max-values -> recover largest edge ids
+*** more precise schema input
+*** pacer-jogger constraint support
+*** cypher constraint support
+*** query execution tool (?)
+* Version 0.3
+** Cleanup standalone runner
+*** DONE Refactor into class
+*** Push options to sinatra
+*** Better initialization (i.e. register types)
+** Support for blueprints graphs (?)
+** Parallelize iteration over graph