syphon 0.0.1 → 0.0.2

data/CHANGELOG

@@ -1,3 +1,13 @@
+== 0.0.2 2013-12-12
+
+ * Add index_settings attribute to control sharding, replication, etc.
+ * Allow dynamic SQL (blocks) for attributes.
+ * Make index_base_name overridable instead of index_name. The former is
+   prefixed with the configured index_namespace.
+ * Restructure configuration under Rails. Single file (syphon.yml) with log
+   settings at top-level, and database and elasticsearch settings under their
+   respective keys. Log file may now be relative to rails root.
+
 == 0.0.1 2013-11-18
 
  * Hi.

data/Gemfile

@@ -3,5 +3,6 @@ gemspec
 
 gem 'debugger', '~> 1.6.0', platform: :ruby_19
 gem 'looksee', '~> 1.1.0'
+gem 'rails'
 gem 'ritual', '~> 0.4.0'
 gem 'temporaries', '~> 0.3.0'

data/README.markdown

@@ -2,4 +2,65 @@
 
 Syphon data from an Arel source into ElasticSearch.
 
-### Work in progress
+## Installation
+
+    gem install syphon
+
+## Usage
+
+    class UsersIndex
+      include Syphon::Index
+
+      define_source do
+        # Define some attributes. Options like "index: :not_analyzed" are passed
+        # to elasticsearch.
+
+        string :login, index: :not_analyzed
+        string :name
+        integer :age
+        geo_point :location
+        string :bio
+        nested_documents do
+          
+        end
+      end
+    end
+
+## Why?
+
+In 2 words: fast indexing.
+
+There is no shortage of ElasticSearch adapters for ruby. The issue is that they
+typically work by defining fields as attributes (methods) on your model
+instances, which means to index a lot of data, you need to roundtrip your data
+through heavy ruby models, which can be quite slow, particularly if your models
+are made of ActiveRecord.
+
+Syphon makes no such assumption. You define your fields to index using SQL
+expressions. These are assembled into a single SQL query, and the resulting rows
+are used to build ElasticSearch documents in an intuitive way using minimal ruby
+data structures. Those coming from ThinkingSphinx might find this notion -- and
+syntax -- familiar.
+
+Syphon is similar in concept to an ElasticSearch river, except it's completely
+handled off the server. In fact it was originally implemented using the
+[JDBC river][jdbc-river], but the current incarnation was found to be a little
+lacking in functionality for a smooth integration (such as documents with
+multiple nested fields, and synchronous updates for testing).
+
+Syphon focuses on the indexing aspect only - you are free to choose another gem
+for the query DSL.
+
+[jdbc-river]: https://github.com/jprante/elasticsearch-river-jdbc
+
+## Contributing
+
+ * [Bug reports](https://github.com/howaboutwe/syphon/issues)
+ * [Source](https://github.com/howaboutwe/syphon)
+ * Patches: Fork on Github, send pull request.
+   * Include tests where practical.
+   * Leave the version alone, or bump it in a separate commit.
+
+## Copyright
+
+Copyright (c) George Ogata. See LICENSE for details.

data/lib/syphon/index.rb

@@ -3,14 +3,16 @@ module Syphon
     def self.included(base)
       base.extend ClassMethods
       base.pre_sql ||= []
+      base.index_settings ||= {}
       super
     end
 
     module ClassMethods
-      attr_accessor :pre_sql
+      attr_accessor :pre_sql, :index_settings
 
       def inherited(subclass)
         subclass.pre_sql = pre_sql.dup
+        subclass.index_settings = index_settings.dup
         super
       end
 
@@ -25,11 +27,19 @@ module Syphon
       def index_name
         @index_name ||=
           begin
-            prefix = Syphon.index_namespace.to_s.empty? ? '' : "#{Syphon.index_namespace}_"
-            prefix + name.sub(/Index\z/, '').underscore.pluralize
+            namespace = Syphon.index_namespace
+            if namespace.to_s.empty?
+              index_base_name
+            else
+              "#{namespace}_#{index_base_name}"
+            end
           end
       end
 
+      def index_base_name
+        @index_base_name ||= name.sub(/Index\z/, '').underscore.pluralize
+      end
+
       def sources
         @sources ||= {}
       end
@@ -38,7 +48,7 @@ module Syphon
         old_internal_name = internal_index_name
         new_internal_name = new_internal_index_name(index_name)
 
-        client.indices.create(index: new_internal_name)
+        client.indices.create(index: new_internal_name, body: {settings: index_settings})
         sources.each do |name, source|
           body = source.mapping
           client.indices.put_mapping(index: new_internal_name, type: source.type, body: body)
@@ -82,9 +92,7 @@ module Syphon
         @warmups ||= []
       end
 
-      protected
-
-      attr_writer :index_name
+      attr_writer :index_base_name
 
       private
 

data/lib/syphon/railtie.rb

@@ -1,27 +1,51 @@
+require 'erb'
+require 'yaml'
+
 module Syphon
   class Railtie < Rails::Railtie
+    this = self
+
     rake_tasks do
       require 'syphon/tasks'
     end
 
     initializer "syphon.initialize" do
-      if Syphon.database_configuration.empty? && defined?(ActiveRecord::Base)
-        db_configs = ActiveRecord::Base.configurations
-        db_config = db_configs["#{Rails.env}_syphon"] || db_configs[Rails.env] and
-          Syphon.database_configuration = db_config.symbolize_keys
-      end
+      this.set_configuration(
+        env: Rails.env,
+        root: Rails.root,
+        app_name: Rails.application.class.parent_name,
+        dbconfig: ActiveRecord::Base.configurations,
+      )
+    end
 
-      path = "#{Rails.root}/config/syphon.yml"
-      if File.exist?(path)
-        erb = File.read(path)
-        yaml = ERB.new(erb).result
-        config = YAML.load(yaml)[Rails.env] and
-          Syphon.configuration = config.symbolize_keys
+    class << self
+      def set_configuration(params = {})
+        env, root, app_name, dbconfig =
+          params.values_at(:env, :root, :app_name, :dbconfig)
+
+        path = "#{root}/config/syphon.yml"
+        if File.exist?(path)
+          erb = File.read(path)
+          yaml = ERB.new(erb).result
+          config = YAML.load(yaml)[env]
+        end
+
+        config ||= {}
+        config.symbolize_keys!
+        config[:log] = normalize_log(env, root, config[:log])
+        config[:database] ||= dbconfig[env].dup
+        config[:index_namespace] ||= "#{app_name.underscore}_#{env}"
+        config[:database].try(:symbolize_keys!)
+        config[:elasticsearch].try(:symbolize_keys!)
+        Syphon.configuration = config
       end
 
-      if Syphon.index_namespace.nil?
-        app_name = Rails.application.class.parent_name.underscore
-        Syphon.index_namespace = "#{app_name}_#{Rails.env}"
+      private
+
+      def normalize_log(env, root, log)
+        return nil if log == false
+        log ||= "#{root}/log/syphon.#{env}.log"
+        log.start_with?('/') ? log : "#{root}/#{log}"
       end
     end
   end

data/lib/syphon/schema.rb

@@ -84,24 +84,25 @@ module Syphon
     end
 
     class Field
-      def initialize(name, type, expression, options = {})
+      def initialize(schema, name, type, expression, options = {})
+        @schema = schema
         @name = name.to_sym
         @type = type
         @expression = expression
         @properties = options.merge(type: type)
       end
 
-      attr_reader :name, :type, :expression, :properties
+      attr_reader :schema, :name, :type, :expression, :properties
 
       def select(outer = nil)
         name = outer ? "#{outer}[#{self.name}]" : self.name
-        "#{expression} AS `#{name}`"
+        "#{schema.send(:query_fragment, expression)} AS `#{name}`"
       end
     end
 
     class NestedField < Field
-      def initialize(name, options = {}, &block)
-        super(name, :nested, nil, options)
+      def initialize(schema, name, options = {}, &block)
+        super(schema, name, :nested, nil, options)
         @nested_schema = Schema.new(&block)
       end
 
@@ -127,7 +128,7 @@ module Syphon
       end
 
       def field(name, type, expression, options = {})
-        schema.fields[name.to_sym] = Field.new(name, type, expression, options)
+        schema.fields[name.to_sym] = Field.new(schema, name, type, expression, options)
       end
 
       %w[string short byte integer long float double date boolean binary geo_point].each do |type|
@@ -139,7 +140,7 @@ module Syphon
       end
 
       def nested(name, options = {}, &block)
-        schema.fields[name.to_sym] = NestedField.new(name, options, &block)
+        schema.fields[name.to_sym] = NestedField.new(schema, name, options, &block)
       end
 
       {

data/lib/syphon/version.rb

@@ -1,5 +1,5 @@
 module Syphon
-  VERSION = [0, 0, 1]
+  VERSION = [0, 0, 2]
 
   class << VERSION
     include Comparable

data/lib/syphon.rb

@@ -10,31 +10,58 @@ module Syphon
   autoload :VERSION, 'syphon/version'
 
   class << self
-    attr_writer :configuration, :database_configuration, :index_namespace
+    attr_writer :configuration
 
     def configuration
       @configuration ||= {}
     end
 
     def database_configuration
-      @database_configuration ||= {}
+      configuration[:database] || {}
+    end
+
+    def elasticsearch_configuration
+      configuration = Syphon.configuration[:elasticsearch].try(:dup) || {}
+      configuration[:logger] = logger
+      configuration
     end
 
     def index_namespace
-      @index_namespace ||= configuration[:index_namespace]
+      configuration[:index_namespace]
     end
 
     def database_connection
-      @database_connection ||= Mysql2::Client.new(database_configuration)
+      Thread.current[:syphon_database_connection] ||= Mysql2::Client.new(database_configuration)
     end
 
     def client
-      Thread.current[:syphon_client] ||= Elasticsearch::Client.new(Syphon.configuration)
+      Thread.current[:syphon_client] ||= Elasticsearch::Client.new(elasticsearch_configuration)
     end
 
     def index_classes
       Syphon.configuration['index_classes'].map(&:constantize)
     end
+
+    def logger
+      Thread.current[:syphon_logger] ||= make_logger
+    end
+
+    def logger=(logger)
+      Thread.current[:syphon_logger] = logger
+    end
+
+    private
+
+    def make_logger
+      log = Syphon.configuration[:log] || STDOUT
+      Logger.new(log).tap do |logger|
+        logger.formatter = lambda do |level, time, progname, message|
+          "#{time.strftime('%Y-%m-%d: %H:%M:%S')}: #{level}: #{message}\n"
+        end
+        level = configuration[:log_level] and
+          logger.level = Logger.const_get(level.upcase)
+      end
+    end
   end
 end
 

data/test/syphon/test_index.rb

@@ -12,15 +12,15 @@ describe Syphon::Index do
 
   describe ".index_name" do
     describe "when no index namespace is set" do
-      use_attribute_value Syphon, :index_namespace, nil
+      use_attribute_value Syphon, :configuration, {index_namespace: nil}
 
-      it "it derived from the class name" do
+      it "it is the index base name" do
         TestIndex.index_name.must_equal 'tests'
       end
     end
 
     describe "when the index namespace is empty" do
-      use_attribute_value Syphon, :index_namespace, ''
+      use_attribute_value Syphon, :configuration, {index_namespace: ''}
 
       it "it is treated the same as nil" do
         TestIndex.index_name.must_equal 'tests'
@@ -32,10 +32,21 @@ describe Syphon::Index do
         TestIndex.index_name.must_equal 'syphon_tests'
       end
     end
+  end
+
+  describe ".index_base_name" do
+    it "is based on the class name" do
+      TestIndex.index_base_name.must_equal 'tests'
+    end
+
+    it "can be overridden" do
+      TestIndex.index_base_name = 'wibble'
+      TestIndex.index_base_name.must_equal 'wibble'
+    end
 
-    it "can be overridden by the class" do
-      TestIndex.class_eval { self.index_name = 'wibble' }
-      TestIndex.index_name.must_equal 'wibble'
+    it "contributes to the index name" do
+      TestIndex.index_base_name = 'wibble'
+      TestIndex.index_name.must_equal 'syphon_wibble'
     end
   end
 
@@ -95,6 +106,14 @@ describe Syphon::Index do
       hits.map { |doc| doc['_source']['login'] }.must_equal ['bob']
     end
 
+    it "passes configured index settings" do
+      TestIndex.index_settings = {number_of_shards: 23}
+      TestIndex.build
+      index = TestIndex.client.indices.get_alias(name: TestIndex.index_name).keys.first
+      num_shards = client.indices.get_settings[index]['settings']['index.number_of_shards']
+      num_shards.must_equal '23'
+    end
+
     it "runs all warmups between building the new index and rotating it in" do
       this = self
       runs = []

data/test/syphon/test_railtie.rb

@@ -0,0 +1,98 @@
+require_relative '../test_helper'
+
+describe Syphon::Railtie do
+  use_attribute_value Syphon, :configuration, {}
+
+  describe ".set_configuration" do
+    use_temporary_directory "#{ROOT}/test/tmp"
+
+    let(:params) do
+      {
+        env: 'test',
+        root: tmp,
+        app_name: 'MyApp',
+        dbconfig: {'test' => {'database' => 'ardb'}},
+      }
+    end
+
+    def write_config(data)
+      FileUtils.mkdir_p "#{tmp}/config"
+      open("#{tmp}/config/syphon.yml", 'w') { |f| f.print data.to_yaml }
+    end
+
+    describe "when no configuration file is present" do
+      it "sets all the defaults" do
+        Syphon::Railtie.set_configuration(params)
+        Syphon.configuration[:log].must_equal "#{tmp}/log/syphon.test.log"
+        Syphon.configuration[:database].must_equal({database: 'ardb'})
+        Syphon.configuration[:index_namespace].must_equal('my_app_test')
+      end
+    end
+
+    describe "when a configuration file is present, but no key for the environment" do
+      it "sets all the defaults" do
+        write_config('other_env' => 'blah')
+        Syphon::Railtie.set_configuration(params)
+        Syphon.configuration[:log].must_equal "#{tmp}/log/syphon.test.log"
+        Syphon.configuration[:database].must_equal({database: 'ardb'})
+        Syphon.configuration[:index_namespace].must_equal('my_app_test')
+      end
+    end
+
+    describe "when a valid configuration is present" do
+      it "sets the configuration" do
+        write_config('test' => {'reload_on_failure' => true})
+        Syphon::Railtie.set_configuration(params)
+        Syphon.configuration[:reload_on_failure].must_equal true
+      end
+
+      it "defaults the index namespace to one based on the app name and Rails env" do
+        Syphon::Railtie.set_configuration(params)
+        Syphon.index_namespace.must_equal 'my_app_test'
+      end
+
+      it "sets a custom index namespace if configured" do
+        write_config('test' => {'index_namespace' => 'my_namespace'})
+        Syphon::Railtie.set_configuration(params)
+        Syphon.index_namespace.must_equal 'my_namespace'
+      end
+
+      it "sets a good default log path" do
+        write_config('test' => {})
+        Syphon::Railtie.set_configuration(params)
+        Syphon.configuration[:log].must_equal "#{tmp}/log/syphon.test.log"
+      end
+
+      it "expands a given log path relative to the rails root" do
+        write_config('test' => {'log' => 'path/to/my.log'})
+        Syphon::Railtie.set_configuration(params)
+        Syphon.configuration[:log].must_equal "#{tmp}/path/to/my.log"
+      end
+
+      it "sets no log if the log option is false" do
+        write_config('test' => {'log' => false})
+        Syphon::Railtie.set_configuration(params)
+        Syphon.configuration[:log].must_be_nil
+      end
+
+      it "sets the given database configuration" do
+        write_config('test' => {'database' => {'database' => 'mydb'}})
+        Syphon::Railtie.set_configuration(params)
+        Syphon.database_configuration.must_equal({database: 'mydb'})
+      end
+
+      it "defaults to the primary ActiveRecord configuration" do
+        write_config('test' => {})
+        Syphon::Railtie.set_configuration(params)
+        Syphon.database_configuration.must_equal({database: 'ardb'})
+      end
+
+      it "sets the given elasticsearch configuration" do
+        FileUtils.mkdir_p "#{tmp}/log"
+        write_config('test' => {'elasticsearch' => {'reload_on_failure' => true}})
+        Syphon::Railtie.set_configuration(params)
+        Syphon.elasticsearch_configuration[:reload_on_failure].must_equal true
+      end
+    end
+  end
+end

data/test/syphon/test_schema.rb

@@ -180,14 +180,38 @@ describe Syphon::Schema do
         having 'count(*) = 1'
       end
       schema.query.must_equal <<-EOS.strip.gsub(/\s+/, ' ')
-          SELECT S AS `s`, T AS `inner[t]`
-          FROM things
-          INNER JOIN a ON 1
-          INNER JOIN b ON 2
-          WHERE a = 1
-          GROUP BY x
-          HAVING count(*) = 1
-        EOS
+        SELECT S AS `s`, T AS `inner[t]`
+        FROM things
+        INNER JOIN a ON 1
+        INNER JOIN b ON 2
+        WHERE a = 1
+        GROUP BY x
+        HAVING count(*) = 1
+      EOS
+    end
+
+    it "calls blocks for dynamic queries" do
+      schema = Syphon::Schema.new do
+        string :s, -> { 'S' }
+        nested :inner do
+          string :t, -> { 'T' }
+        end
+        from -> { 'things' }
+        join -> { 'INNER JOIN a ON 1' }
+        join -> { 'INNER JOIN b ON 2' }
+        where -> { 'a = 1' }
+        group_by -> { 'x' }
+        having -> { 'count(*) = 1' }
+      end
+      schema.query.must_equal <<-EOS.strip.gsub(/\s+/, ' ')
+        SELECT S AS `s`, T AS `inner[t]`
+        FROM things
+        INNER JOIN a ON 1
+        INNER JOIN b ON 2
+        WHERE a = 1
+        GROUP BY x
+        HAVING count(*) = 1
+      EOS
     end
 
     it "omits optional clauses when in the minimal case" do
@@ -268,4 +292,3 @@ describe Syphon::Schema do
     end
   end
 end
-

data/test/test_helper.rb

@@ -3,16 +3,18 @@ ROOT = File.expand_path('..', File.dirname(__FILE__))
 $:.unshift "#{ROOT}/lib"
 require 'minitest/spec'
 require 'yaml'
+require 'fileutils'
 require 'temporaries'
 require 'debugger' if RUBY_VERSION < '2.0'
 require 'looksee'
+require 'rails'
 
 require 'syphon'
 
-config = YAML.load_file("#{ROOT}/test/config.yml")
-Syphon.database_configuration = config['database']
-Syphon.configuration = config['elasticsearch']
-Syphon.index_namespace = 'syphon'
+config = YAML.load_file("#{ROOT}/test/config.yml").symbolize_keys
+config[:database].symbolize_keys!
+Syphon.configuration = config.merge(index_namespace: 'syphon')
+Syphon.logger = Logger.new('/dev/null')
 
 MiniTest::Spec.class_eval do
   def self.uses_users_table

data/test/test_syphon.rb

@@ -10,7 +10,7 @@ describe Syphon do
   end
 
   describe ".database_configuration" do
-    use_instance_variable_value Syphon, :database_configuration, nil
+    use_instance_variable_value Syphon, :configuration, nil
 
     it "defaults to an empty hash" do
       Syphon.database_configuration.must_equal({})
@@ -18,11 +18,61 @@ describe Syphon do
   end
 
   describe ".index_namespace" do
-    use_instance_variable_value Syphon, :index_namespace, nil
-    use_instance_variable_value Syphon, :configuration, {index_namespace: 'NAMESPACE'}
+    describe "when a namespace is configured" do
+      use_instance_variable_value Syphon, :configuration, {index_namespace: 'NAMESPACE'}
 
-    it "defaults to the configured index namespace" do
-      Syphon.index_namespace.must_equal('NAMESPACE')
+      it "is the configured index namespace" do
+        Syphon.index_namespace.must_equal('NAMESPACE')
+      end
+    end
+
+    describe "when no namespace is configured" do
+      use_instance_variable_value Syphon, :configuration, {}
+
+      it "is nil" do
+        Syphon.index_namespace.must_be_nil
+      end
+    end
+  end
+
+  describe ".database_configuration" do
+    use_attribute_value Syphon, :configuration, nil
+
+    it "uses the configured database configuration" do
+      Syphon.configuration = {database: {database: 'mydb'}}
+      Syphon.database_configuration.must_equal({database: 'mydb'})
+    end
+  end
+
+  describe ".elasticsearch_configuration" do
+    use_attribute_value Syphon, :configuration, nil
+    use_attribute_value Syphon, :logger, nil
+    use_temporary_directory "#{ROOT}/test/tmp"
+
+    it "includes all configured elasticserach settings" do
+      Syphon.configuration = {elasticsearch: {reload_on_failure: true}}
+      Syphon.elasticsearch_configuration[:reload_on_failure].must_equal true
+    end
+
+    it "adds the Syphon logger" do
+      logger = Logger.new(logger)
+      Syphon.logger = logger
+      Syphon.elasticsearch_configuration[:logger].must_equal logger
+    end
+  end
+
+  describe ".logger" do
+    use_attribute_value Syphon, :configuration, nil
+    use_attribute_value Syphon, :logger, nil
+    use_temporary_directory "#{ROOT}/test/tmp"
+
+    it "uses the configured log path and log level" do
+      Syphon.configuration[:log] = "#{tmp}/syphon.log"
+      Syphon.configuration[:log_level] = 'info'
+      Syphon.logger.debug 'DEBUG LEVEL'
+      Syphon.logger.info 'INFO LEVEL'
+      File.read("#{ROOT}/test/tmp/syphon.log").wont_include 'DEBUG LEVEL'
+      File.read("#{ROOT}/test/tmp/syphon.log").must_include 'INFO LEVEL'
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: syphon
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-11-18 00:00:00.000000000 Z
+date: 2013-12-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: elasticsearch
@@ -100,6 +100,7 @@ files:
 - test/config.yml.sample
 - test/syphon/test_builder.rb
 - test/syphon/test_index.rb
+- test/syphon/test_railtie.rb
 - test/syphon/test_schema.rb
 - test/syphon/test_source.rb
 - test/test_helper.rb
@@ -118,7 +119,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -585634600777255261
+      hash: -2157484304572429612
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -127,7 +128,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -585634600777255261
+      hash: -2157484304572429612
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.25
@@ -138,6 +139,7 @@ test_files:
 - test/config.yml.sample
 - test/syphon/test_builder.rb
 - test/syphon/test_index.rb
+- test/syphon/test_railtie.rb
 - test/syphon/test_schema.rb
 - test/syphon/test_source.rb
 - test/test_helper.rb