syphon 0.0.2 → 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: f6b4117676c9b04cf839c499c0e1923b39defb30
+  data.tar.gz: c95e03409f0b86a8aa6c404f8ba1583a1e033758
+SHA512:
+  metadata.gz: 7b68fea96fc36d64e29698a4dd217bbadb20d8b6369de324a4f602aee900cb76f568afb312999ef1808c863379c0e541fadfa8eaa7650ece627dc433b43281dc
+  data.tar.gz: 472a520467282278d21b6c8b79610cc6eb807fe5a204d4744de479b34290427de5b8c746c353fa5322bc3c3b4a41bffb50e7f5092249459a3ada0e16a735d468

data/.gitignore

@@ -1,2 +1,3 @@
-Gemfile.lock
-test/config.yml
+/.bundle
+/Gemfile.lock
+/test/config.yml

data/CHANGELOG

@@ -1,3 +1,14 @@
+== 0.1.0 2014-03-18
+
+ * Clean up partially built index if building fails.
+ * Use an ENV_syphon database configuration in database.yml before falling back
+   to the standard one with Rails.
+ * Support ActiveSupport 4.
+ * Add :multi option for fields. This causes values to be indexed as arrays.
+   Arrays are no longer autodetected - fields which may be multivalued must be
+   declared as such, and will always be returned from ElasticSearch as an array
+   in ElasticSearch >= 1.0.
+
 == 0.0.2 2013-12-12
 
  * Add index_settings attribute to control sharding, replication, etc.

data/Gemfile

@@ -1,8 +1,10 @@
 source 'https://rubygems.org'
 gemspec
 
-gem 'debugger', '~> 1.6.0', platform: :ruby_19
-gem 'looksee', '~> 1.1.0'
+gem 'debugger', platform: :ruby_19
+gem 'byebug', platform: [:ruby_20, :ruby_21]
+gem 'minitest', '< 5'
+gem 'looksee'
 gem 'rails'
 gem 'ritual', '~> 0.4.0'
-gem 'temporaries', '~> 0.3.0'
+gem 'temporaries'

data/README.markdown

@@ -2,65 +2,4 @@
 
 Syphon data from an Arel source into ElasticSearch.
 
-## Installation
-
-    gem install syphon
-
-## Usage
-
-    class UsersIndex
-      include Syphon::Index
-
-      define_source do
-        # Define some attributes. Options like "index: :not_analyzed" are passed
-        # to elasticsearch.
-
-        string :login, index: :not_analyzed
-        string :name
-        integer :age
-        geo_point :location
-        string :bio
-        nested_documents do
-          
-        end
-      end
-    end
-
-## Why?
-
-In 2 words: fast indexing.
-
-There is no shortage of ElasticSearch adapters for ruby. The issue is that they
-typically work by defining fields as attributes (methods) on your model
-instances, which means to index a lot of data, you need to roundtrip your data
-through heavy ruby models, which can be quite slow, particularly if your models
-are made of ActiveRecord.
-
-Syphon makes no such assumption. You define your fields to index using SQL
-expressions. These are assembled into a single SQL query, and the resulting rows
-are used to build ElasticSearch documents in an intuitive way using minimal ruby
-data structures. Those coming from ThinkingSphinx might find this notion -- and
-syntax -- familiar.
-
-Syphon is similar in concept to an ElasticSearch river, except it's completely
-handled off the server. In fact it was originally implemented using the
-[JDBC river][jdbc-river], but the current incarnation was found to be a little
-lacking in functionality for a smooth integration (such as documents with
-multiple nested fields, and synchronous updates for testing).
-
-Syphon focuses on the indexing aspect only - you are free to choose another gem
-for the query DSL.
-
-[jdbc-river]: https://github.com/jprante/elasticsearch-river-jdbc
-
-## Contributing
-
- * [Bug reports](https://github.com/howaboutwe/syphon/issues)
- * [Source](https://github.com/howaboutwe/syphon)
- * Patches: Fork on Github, send pull request.
-   * Include tests where practical.
-   * Leave the version alone, or bump it in a separate commit.
-
-## Copyright
-
-Copyright (c) George Ogata. See LICENSE for details.
+### Work in progress

data/lib/syphon/builder.rb

@@ -26,14 +26,17 @@ module Syphon
     def add_to_document(document, row, schema = self.schema, index = 0)
       schema.fields.each do |name, field|
         if field.is_a?(Schema::NestedField)
-          nested_doc = {}
-          index = add_to_document(nested_doc, row, field.nested_schema, index)
-          document[field.name] = combine(document[field.name], nested_doc)
-          index
+          value = {}
+          index = add_to_document(value, row, field.nested_schema, index)
         else
-          document[field.name] = combine(document[field.name], row[index])
+          value = row[index]
           index += 1
         end
+        if field.multi?
+          (document[field.name] ||= []) << value
+        else
+          document[field.name] = value
+        end
       end
       index
     end

data/lib/syphon/index.rb

@@ -48,19 +48,28 @@ module Syphon
         old_internal_name = internal_index_name
         new_internal_name = new_internal_index_name(index_name)
 
-        client.indices.create(index: new_internal_name, body: {settings: index_settings})
-        sources.each do |name, source|
-          body = source.mapping
-          client.indices.put_mapping(index: new_internal_name, type: source.type, body: body)
-          source.import(index: new_internal_name) unless options[:schema_only]
-        end
-
-        warmups.each { |w| w.call(new_internal_name) }
+        made_it = false
+        begin
+          client.indices.create(index: new_internal_name, body: {settings: index_settings})
+          sources.each do |name, source|
+            body = source.mapping
+            client.indices.put_mapping(index: new_internal_name, type: source.type, body: body)
+            source.import(index: new_internal_name) unless options[:schema_only]
+          end
 
-        remove = {remove: {index: old_internal_name, alias: index_name}} if old_internal_name
-        add = {add: {index: new_internal_name, alias: index_name}}
-        client.indices.update_aliases body: {actions: [remove, add].compact}
-        client.indices.delete(index: old_internal_name) if old_internal_name
+          warmups.each { |w| w.call(new_internal_name) }
+
+          remove = {remove: {index: old_internal_name, alias: index_name}} if old_internal_name
+          add = {add: {index: new_internal_name, alias: index_name}}
+          client.indices.update_aliases body: {actions: [remove, add].compact}
+          made_it = true
+        ensure
+          if made_it
+            client.indices.delete(index: old_internal_name) if old_internal_name
+          else
+            client.indices.delete(index: new_internal_name) if new_internal_name
+          end
+        end
       end
 
       def destroy

data/lib/syphon/railtie.rb

@@ -33,7 +33,7 @@ module Syphon
         config ||= {}
         config.symbolize_keys!
         config[:log] = normalize_log(env, root, config[:log])
-        config[:database] ||= dbconfig[env].dup
+        config[:database] ||= (dbconfig["#{env}_syphon"] || dbconfig[env]).dup
         config[:index_namespace] ||= "#{app_name.underscore}_#{env}"
         config[:database].try(:symbolize_keys!)
         config[:elasticsearch].try(:symbolize_keys!)

data/lib/syphon/schema.rb

@@ -90,10 +90,15 @@ module Syphon
         @type = type
         @expression = expression
         @properties = options.merge(type: type)
+        @multi = options[:multi]
       end
 
       attr_reader :schema, :name, :type, :expression, :properties
 
+      def multi?
+        @multi
+      end
+
       def select(outer = nil)
         name = outer ? "#{outer}[#{self.name}]" : self.name
         "#{schema.send(:query_fragment, expression)} AS `#{name}`"

data/lib/syphon/version.rb

@@ -1,5 +1,5 @@
 module Syphon
-  VERSION = [0, 0, 2]
+  VERSION = [0, 1, 0]
 
   class << VERSION
     include Comparable

data/syphon.gemspec

@@ -15,7 +15,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
 
   gem.add_dependency 'elasticsearch', '~> 0.4.0'
-  gem.add_dependency 'activesupport', '~> 3.2.0'
+  gem.add_dependency 'activesupport', '< 5'
   gem.add_dependency 'mysql2', '~> 0.3.12'
 
   gem.add_development_dependency 'bundler'

data/test/syphon/test_builder.rb

@@ -30,37 +30,43 @@ describe Syphon::Builder do
       ]
     end
 
-    it "merges content from rows with the same root id" do
+    it "replaces content from subsequent rows with the same root id for singular fields" do
       schema = Syphon::Schema.new do
         integer :id, 0
         string :name, 'x'
       end
       results = [[1, 'one'], [1, 'two']]
+      Syphon::Builder.new(results, schema).to_a.
+        must_equal [{id: 1, name: 'two'}]
+    end
+
+    it "combines content from subsequent rows with the same root id for multi fields" do
+      schema = Syphon::Schema.new do
+        integer :id, 0
+        string :name, 'x', multi: true
+      end
+      results = [[1, 'one'], [1, 'two']]
       Syphon::Builder.new(results, schema).to_a.
         must_equal [{id: 1, name: ['one', 'two']}]
     end
 
-    it "merges content with the same root id correctly when there are nested fields" do
+    it "replaces content from subsequent rows for singular nested fields" do
       schema = Syphon::Schema.new do
         integer :id, 0
         nested :nested1 do
-          integer :a, 'x'
-        end
-        nested :nested2 do
-          integer :a, 'x'
+          integer :id, 0
+          integer :name, 'x'
         end
       end
-      results = [[1, 10, 11], [2, 20, 21]]
-      Syphon::Builder.new(results, schema).to_a.must_equal [
-        {id: 1, nested1: {a: 10}, nested2: {a: 11}},
-        {id: 2, nested1: {a: 20}, nested2: {a: 21}},
-      ]
+      results = [[1, 2, 'a'], [1, 3, 'b']]
+      Syphon::Builder.new(results, schema).to_a.
+        must_equal [{id: 1, nested1: {id: 3, name: 'b'}}]
     end
 
-    it "supports arrays as nested fields" do
+    it "replaces content from subsequent rows for multi nested fields" do
       schema = Syphon::Schema.new do
         integer :id, 0
-        nested :nested1 do
+        nested :nested1, multi: true do
           integer :id, 0
           integer :name, 'x'
         end

data/test/syphon/test_index.rb

@@ -106,6 +106,13 @@ describe Syphon::Index do
       hits.map { |doc| doc['_source']['login'] }.must_equal ['bob']
     end
 
+    it "deletes the index if building fails" do
+      indices = TestIndex.client.indices.status['indices'].keys.to_set
+      TestIndex.source.instance_eval { def import(*); raise 'fubar'; end }
+      -> { TestIndex.build }.must_raise(RuntimeError)
+      TestIndex.client.indices.status['indices'].keys.to_set.must_equal indices
+    end
+
     it "passes configured index settings" do
       TestIndex.index_settings = {number_of_shards: 23}
       TestIndex.build

data/test/syphon/test_railtie.rb

@@ -81,7 +81,14 @@ describe Syphon::Railtie do
         Syphon.database_configuration.must_equal({database: 'mydb'})
       end
 
-      it "defaults to the primary ActiveRecord configuration" do
+      it "defaults to a configuration for syphon in the current environment" do
+        write_config('test' => {})
+        params[:dbconfig] = {'test_syphon' => {database: 'syphondb'}, 'test' => {database: 'ardb'}}
+        Syphon::Railtie.set_configuration(params)
+        Syphon.database_configuration.must_equal({database: 'syphondb'})
+      end
+
+      it "defaults to the primary ActiveRecord configuration otherwise" do
         write_config('test' => {})
         Syphon::Railtie.set_configuration(params)
         Syphon.database_configuration.must_equal({database: 'ardb'})

data/test/test_helper.rb

@@ -5,7 +5,7 @@ require 'minitest/spec'
 require 'yaml'
 require 'fileutils'
 require 'temporaries'
-require 'debugger' if RUBY_VERSION < '2.0'
+require 'byebug'
 require 'looksee'
 require 'rails'
 
@@ -20,21 +20,29 @@ MiniTest::Spec.class_eval do
   def self.uses_users_table
     let(:db) { Syphon.database_connection }
 
-    before do
-      columns = "id int auto_increment PRIMARY KEY, login VARCHAR(20)"
-      db.query "CREATE TABLE IF NOT EXISTS users(#{columns})"
-    end
+    include Module.new {
+      extend MiniTest::Spec::DSL
 
-    after do
-      db.query "DROP TABLE IF EXISTS users"
-    end
+      before do
+        columns = "id int auto_increment PRIMARY KEY, login VARCHAR(20)"
+        db.query "CREATE TABLE IF NOT EXISTS users(#{columns})"
+      end
+
+      after do
+        db.query "DROP TABLE IF EXISTS users"
+      end
+    }
   end
 
   def self.uses_elasticsearch
     let(:client) { Syphon.client }
 
-    before { clear_indices }
-    after { clear_indices }
+    include Module.new {
+      extend MiniTest::Spec::DSL
+
+      before { clear_indices }
+      after { clear_indices }
+    }
   end
 
   def clear_indices

metadata

@@ -1,78 +1,69 @@
 --- !ruby/object:Gem::Specification
 name: syphon
 version: !ruby/object:Gem::Version
-  version: 0.0.2
-  prerelease: 
+  version: 0.1.0
 platform: ruby
 authors:
 - George Ogata
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-12 00:00:00.000000000 Z
+date: 2014-03-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: elasticsearch
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.4.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.4.0
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 3.2.0
+        version: '5'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 3.2.0
+        version: '5'
 - !ruby/object:Gem::Dependency
   name: mysql2
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.3.12
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.3.12
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: Syphon data from an Arel source into ElasticSearch
@@ -82,7 +73,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
+- ".gitignore"
 - CHANGELOG
 - Gemfile
 - LICENSE
@@ -107,33 +98,26 @@ files:
 - test/test_syphon.rb
 homepage: https://github.com/howaboutwe/syphon
 licenses: []
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: -2157484304572429612
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: -2157484304572429612
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.25
+rubygems_version: 2.2.2
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Syphon data from an Arel source into ElasticSearch
 test_files:
 - test/config.yml.sample