lexster 0.0.1

data/Rakefile

@@ -0,0 +1,7 @@
+require "bundler/gem_tasks"
+require 'rspec/core/rake_task'
+require 'neography/tasks'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/TODO.md

@@ -0,0 +1,4 @@
+# Lexster - To Do
+
+* Execute queries/scripts from model and not Neography (e.g. `Movie.neo_gremlin(gremlin_query)` with query that outputs IDs, returns a list of `Movie`s)
+* Rake task to index all nodes and relatiohsips in Neo4j

data/lexster.gemspec

@@ -0,0 +1,28 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "lexster/version"
+
+Gem::Specification.new do |s|
+  s.name        = "lexster"
+  s.version     = Lexster::VERSION
+  s.authors     = ["Nelson Wittwer, Elad Ossadon"]
+  s.email       = ["nelsonwittwer@gmail.com"]
+  s.homepage    = ""
+  s.summary     = %q{Titan for ActiveRecord}
+  s.description = %q{Extend Ruby on Rails ActiveRecord with Titan nodes. Keep RDBMS and utilize the power of Gremlin queries. Fork of Neoid.}
+
+  s.rubyforge_project = "lexster"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_development_dependency 'rake'
+  s.add_development_dependency 'rspec'
+  s.add_development_dependency 'rest-client'
+  s.add_development_dependency 'activerecord'
+  s.add_development_dependency 'sqlite3'
+
+  s.add_runtime_dependency 'neography'
+end

data/lib/lexster.rb

@@ -0,0 +1,239 @@
+require 'neography'
+require 'lexster/version'
+require 'lexster/config'
+require 'lexster/model_config'
+require 'lexster/model_additions'
+require 'lexster/search_session'
+require 'lexster/node'
+require 'lexster/relationship'
+require 'lexster/batch'
+require 'lexster/database_cleaner'
+require 'lexster/railtie' if defined?(Rails)
+
+module Lexster
+  DEFAULT_FULLTEXT_SEARCH_INDEX_NAME = :lexster_default_search_index
+  NODE_AUTO_INDEX_NAME = 'node_auto_index'
+  RELATIONSHIP_AUTO_INDEX_NAME = 'relationship_auto_index'
+  UNIQUE_ID_KEY = 'lexster_unique_id'
+
+  class << self
+    attr_accessor :db
+    attr_accessor :logger
+    attr_accessor :ref_node
+    attr_accessor :env_loaded
+    attr_reader :config
+    
+    def node_models
+      @node_models ||= []
+    end
+    
+    def relationship_models
+      @relationship_models ||= []
+    end
+
+    def config
+      @config ||= begin
+        c = Lexster::Config.new
+
+        # default
+        c.enable_subrefs = true
+        c.enable_per_model_indexes = false
+
+        c
+      end
+    end
+
+    def configure
+      yield config
+    end
+
+    def initialize_all
+      @env_loaded = true
+      logger.info "Lexster initialize_all"
+      initialize_relationships
+      initialize_server
+    end
+
+    def initialize_server
+      initialize_auto_index
+      initialize_subrefs
+      initialize_per_model_indexes
+    end
+    
+    def db
+      raise "Must set Lexster.db with a Neography::Rest instance" unless @db
+      @db
+    end
+
+    def batch(options={}, &block)
+      Lexster::Batch.new(options, &block).run
+    end
+    
+    def logger
+      @logger ||= Logger.new(ENV['NEOID_LOG'] ? ENV['NEOID_LOG_FILE'] || $stdout : '/dev/null')
+    end
+    
+    def ref_node
+      @ref_node ||= Neography::Node.load(Lexster.db.get_root['self'])
+    end
+    
+    def reset_cached_variables
+      initialize_subrefs
+    end
+    
+    def clean_db(confirm)
+      puts "must call with confirm: Lexster.clean_db(:yes_i_am_sure)" and return unless confirm == :yes_i_am_sure
+      Lexster::NeoDatabaseCleaner.clean_db
+    end
+    
+
+    def enabled=(flag)
+      Thread.current[:lexster_enabled] = flag
+    end
+
+    def enabled
+      flag = Thread.current[:lexster_enabled]
+      # flag should be set by the middleware. in case it wasn't (non-rails app or console), default it to true
+      flag.nil? ? true : flag
+    end
+    alias enabled? enabled
+
+    def use(flag=true)
+      old, self.enabled = enabled?, flag
+      yield if block_given?
+    ensure
+      self.enabled = old
+    end
+
+    def execute_script_or_add_to_batch(gremlin_query, script_vars)
+      if Lexster::Batch.current_batch
+        # returns a SingleResultPromiseProxy!
+        Lexster::Batch.current_batch << [:execute_script, gremlin_query, script_vars]
+      else
+        value = Lexster.db.execute_script(gremlin_query, script_vars)
+
+        value = yield(value) if block_given?
+
+        Lexster::BatchPromiseProxy.new(value)
+      end
+    end
+
+    # create a fulltext index if not exists
+    def ensure_default_fulltext_search_index
+      Lexster.db.create_node_index(DEFAULT_FULLTEXT_SEARCH_INDEX_NAME, 'fulltext', 'lucene') unless (indexes = Lexster.db.list_node_indexes) && indexes[DEFAULT_FULLTEXT_SEARCH_INDEX_NAME]
+    end
+
+    def search(types, term, options = {})
+      options = options.reverse_merge(limit: 15,match_type: "AND")
+
+      types = [*types]
+
+      query = []
+
+      types.each do |type|
+        query_for_type = []
+
+        query_for_type << "ar_type:#{type.name}"
+
+        case term
+        when String
+          search_in_fields = type.lexster_config.search_options.fulltext_fields.keys
+          next if search_in_fields.empty?
+          query_for_type << search_in_fields.map{ |field| generate_field_query(field, term, true, options[:match_type]) }.join(" OR ")
+        when Hash
+          term.each do |field, value|
+            query_for_type << generate_field_query(field, value, false)
+          end
+        end
+
+        query << "(#{query_for_type.join(") AND (")})"
+      end
+
+      query = "(#{query.join(") OR (")})"
+
+      logger.info "Lexster query #{query}"
+
+      gremlin_query = <<-GREMLIN
+        #{options[:before_query]}
+
+        idx = g.getRawGraph().index().forNodes('#{DEFAULT_FULLTEXT_SEARCH_INDEX_NAME}')
+        hits = idx.query('#{sanitize_query_for_gremlin(query)}')
+
+        hits = #{options[:limit] ? "hits.take(#{options[:limit]})" : "hits"}
+
+        #{options[:after_query]}
+      GREMLIN
+
+      logger.info "[NEOID] search:\n#{gremlin_query}"
+
+      results = Lexster.db.execute_script(gremlin_query)
+
+      SearchSession.new(results, *types)
+    end
+
+    private
+      def sanitize_term(term)
+        # TODO - case sensitive?
+        term.downcase
+      end
+
+      def sanitize_query_for_gremlin(query)
+        # TODO - case sensitive?
+        query.gsub("'", "\\\\'")
+      end
+
+      def generate_field_query(field, term, fulltext = false, match_type = "AND")
+        term = term.to_s if term
+        return "" if term.nil? || term.empty?
+
+        fulltext = fulltext ? "_fulltext" : nil
+        valid_match_types = %w( AND OR )
+        match_type = valid_match_types.delete(match_type)
+        raise "Invalid match_type option. Valid values are #{valid_match_types.join(',')}" unless match_type
+
+        "(" + term.split(/\s+/).reject(&:empty?).map{ |t| "#{field}#{fulltext}:#{sanitize_term(t)}" }.join(" #{match_type} ") + ")"
+      end
+
+      def initialize_relationships
+        logger.info "Lexster initialize_relationships"
+        relationship_models.each do |rel_model|
+          Relationship.initialize_relationship(rel_model)
+        end
+      end
+
+      def initialize_auto_index
+        logger.info "Lexster initialize_auto_index"
+        Lexster.db.set_node_auto_index_status(true)
+        Lexster.db.add_node_auto_index_property(UNIQUE_ID_KEY)
+
+        Lexster.db.set_relationship_auto_index_status(true)
+        Lexster.db.add_relationship_auto_index_property(UNIQUE_ID_KEY)
+      end
+
+      def initialize_subrefs
+        return unless config.enable_subrefs
+        
+        node_models.each do |klass|
+          klass.reset_neo_subref_node
+        end
+
+        logger.info "Lexster initialize_subrefs"
+        batch do
+          node_models.each(&:neo_subref_node)
+        end.then do |results|
+          node_models.zip(results).each do |klass, subref|
+            klass.neo_subref_node = subref
+          end
+        end
+      end
+
+      def initialize_per_model_indexes
+        return unless config.enable_per_model_indexes
+
+        logger.info "Lexster initialize_subrefs"
+        batch do
+          node_models.each(&:neo_model_index)
+        end
+      end
+  end
+end

data/lib/lexster/batch.rb

@@ -0,0 +1,168 @@
+module Lexster
+  class Batch
+    def default_options=(value)
+      @default_options = value
+    end
+
+    def self.default_options
+      @default_options ||= { batch_size: 200, individual_promises: true }
+    end
+
+    def self.current_batch
+      Thread.current[:lexster_current_batch]
+    end
+
+    def self.current_batch=(batch)
+      Thread.current[:lexster_current_batch] = batch
+    end
+
+    def self.reset_current_batch
+      Thread.current[:lexster_current_batch] = nil
+    end
+
+    def initialize(options={}, &block)
+      if options.respond_to?(:call) && !block
+        block = options
+        options = {}
+      end
+
+      options.reverse_merge!(self.class.default_options)
+
+      @options = options
+      @block = block
+    end
+
+    def <<(command)
+      commands << command
+
+      if commands.length >= @options[:batch_size]
+        flush_batch
+      end
+
+      if @options[:individual_promises]
+        promise = SingleResultPromiseProxy.new(command)
+        thens << promise
+        promise
+      end
+    end
+
+    def commands
+      @commands ||= []
+    end
+
+    def thens
+      @thens ||= []
+    end
+
+    def count
+      @commands ? @commands.count : 0
+    end
+
+    def results
+      @results ||= []
+    end
+
+    def run
+      self.class.current_batch = self
+
+      begin
+        @block.call(self)
+      ensure      
+        self.class.reset_current_batch
+      end
+
+      Lexster.logger.info "Lexster batch (#{commands.length} commands)"
+
+      flush_batch
+
+      BatchPromiseProxy.new(results)
+    end
+
+    private
+      def flush_batch
+        return [] if commands.empty?
+        current_results = nil
+
+        # results = Lexster.db.batch(*commands).collect { |result| result['body'] }
+
+        benchmark = Benchmark.measure {
+          current_results = Lexster.db.batch(*commands).collect { |result| result['body'] }
+        }
+        Lexster.logger.info "Lexster batch (#{commands.length} commands) - #{benchmark}"
+        commands.clear
+
+        process_results(current_results)
+
+        thens.zip(current_results).each { |t, result| t.perform(result) }
+
+        thens.clear
+
+        results.concat current_results
+      end
+
+      def process_results(results)
+        results.map! do |result|
+          return result unless result.is_a?(Hash) && result['self'] && result['self'][%r[^https?://.*/(node|relationship)/\d+]]
+
+          type = case $1
+          when 'node' then Lexster::Node
+          when 'relationship' then Lexster::Relationship
+          else return result
+          end
+
+          type.from_hash(result)
+        end
+      end
+  end
+
+  # returned from a full batch, after it has been executed,
+  # so a `.then` can be chained after the batch do ... end
+  # it proxies all methods to the result
+  class BatchPromiseProxy
+    def initialize(results)
+      @results = results
+    end
+
+    def method_missing(method, *args)
+      @results.send(method, *args)
+    end
+
+    def then
+      yield(@results)
+    end
+  end
+
+  # returned from adding (<<) an item to a batch in a batch block:
+  # Lexster.batch { |batch| (batch << [:neography_command, param]).is_a?(SingleResultPromiseProxy) }
+  # so a `.then` can be chained:
+  # Lexster.batch { |batch| (batch << [:neography_command, param]).then { |result| puts result } }
+  # the `then` is called once the batch is flushed with the result of the single job in the batch
+  # it proxies all methods to the result, so in case it is returned (like in Lexster.execute_script_or_add_to_batch)
+  # the result of the method will be proxied to the result from the batch. See Node#neo_save
+  class SingleResultPromiseProxy
+    def initialize(*args)
+    end
+
+    attr_accessor :result
+
+    def result
+      raise "Accessed result too soon" unless @result
+      @result
+    end
+
+    def method_missing(method, *args)
+      result.send(method, *args)
+    end
+
+    def then(&block)
+      @then = block
+      self
+    end
+
+    def perform(result)
+      @result = result
+      return unless @then
+      @then.call(result)
+    end
+  end
+end