syphon 0.0.1

data/.gitignore

@@ -0,0 +1,2 @@
+Gemfile.lock
+test/config.yml

data/CHANGELOG

@@ -0,0 +1,3 @@
+== 0.0.1 2013-11-18
+
+ * Hi.

data/Gemfile

@@ -0,0 +1,7 @@
+source 'https://rubygems.org'
+gemspec
+
+gem 'debugger', '~> 1.6.0', platform: :ruby_19
+gem 'looksee', '~> 1.1.0'
+gem 'ritual', '~> 0.4.0'
+gem 'temporaries', '~> 0.3.0'

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) George Ogata
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.markdown

@@ -0,0 +1,5 @@
+## Syphon
+
+Syphon data from an Arel source into ElasticSearch.
+
+### Work in progress

data/Rakefile

@@ -0,0 +1 @@
+require 'ritual'

data/lib/syphon/builder.rb

@@ -0,0 +1,57 @@
+module Syphon
+  class Builder
+    def initialize(results, schema)
+      @results = results
+      @schema = schema
+      @nested_docs = {}
+    end
+
+    attr_reader :results, :schema, :nested_docs
+
+    def each
+      current_id = nil
+      current_document = nil
+      results.each_with_index do |row, index|
+        id = row[0]
+        if current_id.nil? || id != current_id
+          yield current_document unless current_document.nil?
+          current_document = {}
+          current_id = id
+        end
+        add_to_document(current_document, row)
+      end
+      yield current_document unless current_document.nil?
+    end
+
+    def add_to_document(document, row, schema = self.schema, index = 0)
+      schema.fields.each do |name, field|
+        if field.is_a?(Schema::NestedField)
+          nested_doc = {}
+          index = add_to_document(nested_doc, row, field.nested_schema, index)
+          document[field.name] = combine(document[field.name], nested_doc)
+          index
+        else
+          document[field.name] = combine(document[field.name], row[index])
+          index += 1
+        end
+      end
+      index
+    end
+
+    include Enumerable
+
+    private
+
+    def combine(existing, incoming)
+      if existing
+        if existing.is_a?(Array)
+          existing << incoming unless existing.include?(incoming)
+        else
+          existing == incoming ? existing : [existing, incoming]
+        end
+      else
+        incoming
+      end
+    end
+  end
+end

data/lib/syphon/index.rb

@@ -0,0 +1,108 @@
+module Syphon
+  module Index
+    def self.included(base)
+      base.extend ClassMethods
+      base.pre_sql ||= []
+      super
+    end
+
+    module ClassMethods
+      attr_accessor :pre_sql
+
+      def inherited(subclass)
+        subclass.pre_sql = pre_sql.dup
+        super
+      end
+
+      def database_connection
+        Syphon.database_connection
+      end
+
+      def client
+        Syphon.client
+      end
+
+      def index_name
+        @index_name ||=
+          begin
+            prefix = Syphon.index_namespace.to_s.empty? ? '' : "#{Syphon.index_namespace}_"
+            prefix + name.sub(/Index\z/, '').underscore.pluralize
+          end
+      end
+
+      def sources
+        @sources ||= {}
+      end
+
+      def build(options = {})
+        old_internal_name = internal_index_name
+        new_internal_name = new_internal_index_name(index_name)
+
+        client.indices.create(index: new_internal_name)
+        sources.each do |name, source|
+          body = source.mapping
+          client.indices.put_mapping(index: new_internal_name, type: source.type, body: body)
+          source.import(index: new_internal_name) unless options[:schema_only]
+        end
+
+        warmups.each { |w| w.call(new_internal_name) }
+
+        remove = {remove: {index: old_internal_name, alias: index_name}} if old_internal_name
+        add = {add: {index: new_internal_name, alias: index_name}}
+        client.indices.update_aliases body: {actions: [remove, add].compact}
+        client.indices.delete(index: old_internal_name) if old_internal_name
+      end
+
+      def destroy
+        internal_name = internal_index_name and
+          client.indices.delete index: internal_name
+      end
+
+      def search(options = {})
+        options[:index] ||= index_name
+        options[:type] ||= source.type
+        client.search(options)
+      end
+
+      def define_source(name = nil, options = {}, &block)
+        source = sources[name] ||= Source.new(self, name, options)
+        source.schema.configure(&block) if block
+        source
+      end
+
+      def define_warmup(&block)
+        warmups << block
+      end
+
+      def source(name = nil)
+        sources[name]
+      end
+
+      def warmups
+        @warmups ||= []
+      end
+
+      protected
+
+      attr_writer :index_name
+
+      private
+
+      def internal_index_name
+        index_name, alias_info = client.indices.get_alias(name: self.index_name).first
+        index_name
+      rescue Elasticsearch::Transport::Transport::Errors::NotFound
+        nil
+      end
+
+      def new_internal_index_name(index_name)
+        i = 0
+        loop do
+          name = "#{index_name}_#{i}"
+          return name if !client.indices.exists(index: name)
+          i += 1
+        end
+      end
+    end
+  end
+end

data/lib/syphon/railtie.rb

@@ -0,0 +1,28 @@
+module Syphon
+  class Railtie < Rails::Railtie
+    rake_tasks do
+      require 'syphon/tasks'
+    end
+
+    initializer "syphon.initialize" do
+      if Syphon.database_configuration.empty? && defined?(ActiveRecord::Base)
+        db_configs = ActiveRecord::Base.configurations
+        db_config = db_configs["#{Rails.env}_syphon"] || db_configs[Rails.env] and
+          Syphon.database_configuration = db_config.symbolize_keys
+      end
+
+      path = "#{Rails.root}/config/syphon.yml"
+      if File.exist?(path)
+        erb = File.read(path)
+        yaml = ERB.new(erb).result
+        config = YAML.load(yaml)[Rails.env] and
+          Syphon.configuration = config.symbolize_keys
+      end
+
+      if Syphon.index_namespace.nil?
+        app_name = Rails.application.class.parent_name.underscore
+        Syphon.index_namespace = "#{app_name}_#{Rails.env}"
+      end
+    end
+  end
+end

data/lib/syphon/schema.rb

@@ -0,0 +1,167 @@
+module Syphon
+  class Schema
+    def initialize(&block)
+      @fields = {}
+      @relation = nil
+      @joins = []
+      @conditions = nil
+      @group_clause = nil
+      @having_clause = nil
+      configure(&block) if block
+    end
+
+    attr_reader :fields, :joins
+    attr_accessor :relation, :conditions, :group_clause, :having_clause
+
+    def configure(&block)
+      DSL.new(self)._eval(&block)
+    end
+
+    def query(options = {})
+      order = options[:order] and
+        order_by_fragment = "ORDER BY #{query_fragment(order)}"
+      limit = options[:limit] and
+        limit_fragment = "LIMIT #{query_fragment(limit)}"
+
+      select_fragment = options[:select] || select_fragments
+      where_fragment = where_fragment(options.slice(:scope, :invert))
+
+      <<-EOS.strip.gsub(/\s+/, ' ')
+        SELECT #{select_fragment}
+        FROM #{query_fragment(relation)}
+        #{joins_fragment}
+        #{where_fragment}
+        #{group_by_fragment}
+        #{having_fragment}
+        #{order_by_fragment}
+        #{limit_fragment}
+      EOS
+    end
+
+    def properties
+      mapping = {}
+      fields.each do |name, field|
+        mapping[name] = field.properties
+      end
+      mapping
+    end
+
+    private
+
+    def select_fragments
+      fields.map { |name, field| field.select }.join(', ')
+    end
+
+    def joins_fragment
+      return nil if joins.empty?
+      joins.map { |j| query_fragment(j) }.join(' ')
+    end
+
+    def where_fragment(options)
+      fragment = query_fragment(conditions) || '1'
+      fragment = "NOT (#{fragment})" if options[:invert]
+      scope = options[:scope] and
+        fragment = "(#{fragment}) AND (#{scope})"
+      fragment == '1' ? nil : "WHERE #{fragment}"
+    end
+
+    def group_by_fragment
+      clause = query_fragment(group_clause) and
+        "GROUP BY #{clause}"
+    end
+
+    def having_fragment
+      clause = query_fragment(having_clause) and
+        "HAVING #{clause}"
+    end
+
+    def query_fragment(string_or_callable)
+      if string_or_callable.respond_to?(:call)
+        string_or_callable.call
+      elsif string_or_callable
+        string_or_callable
+      end
+    end
+
+    class Field
+      def initialize(name, type, expression, options = {})
+        @name = name.to_sym
+        @type = type
+        @expression = expression
+        @properties = options.merge(type: type)
+      end
+
+      attr_reader :name, :type, :expression, :properties
+
+      def select(outer = nil)
+        name = outer ? "#{outer}[#{self.name}]" : self.name
+        "#{expression} AS `#{name}`"
+      end
+    end
+
+    class NestedField < Field
+      def initialize(name, options = {}, &block)
+        super(name, :nested, nil, options)
+        @nested_schema = Schema.new(&block)
+      end
+
+      attr_reader :nested_schema
+
+      def properties
+        super.merge(properties: nested_schema.properties)
+      end
+
+      def select
+        nested_schema.fields.map { |n, f| f.select(name) }.join(', ')
+      end
+    end
+
+    DSL = Struct.new(:schema) do
+      def _eval(&block)
+        if block.arity == 1
+          block.call(self)
+        else
+          instance_eval(&block)
+        end
+        schema
+      end
+
+      def field(name, type, expression, options = {})
+        schema.fields[name.to_sym] = Field.new(name, type, expression, options)
+      end
+
+      %w[string short byte integer long float double date boolean binary geo_point].each do |type|
+        class_eval <<-EOS, __FILE__, __LINE__ + 1
+          def #{type}(name, expression, options = {})
+            field(name, :#{type}, expression, options)
+          end
+        EOS
+      end
+
+      def nested(name, options = {}, &block)
+        schema.fields[name.to_sym] = NestedField.new(name, options, &block)
+      end
+
+      {
+        from: :relation,
+        where: :conditions,
+        group_by: :group_clause,
+        having: :having_clause,
+      }.each do |dsl_method, schema_attribute|
+        class_eval <<-EOS, __FILE__, __LINE__ + 1
+          def #{dsl_method}(string = nil, &block)
+            string && block and
+              raise ArgumentError, "both string and block given"
+            schema.#{schema_attribute} = string || block
+          end
+        EOS
+      end
+
+      def join(string = nil, &block)
+        string && block and
+          raise ArgumentError, "both string and block given"
+        schema.joins << (string || block)
+      end
+    end
+  end
+end

data/lib/syphon/source.rb

@@ -0,0 +1,76 @@
+module Syphon
+  class Source
+    def initialize(index, name, options = {}, &block)
+      @index = index
+      @name = name && name.to_sym
+      @type = options[:type] || default_type
+      @schema = Schema.new(&block)
+    end
+
+    attr_reader :index, :name, :type, :schema
+
+    def mapping
+      {type => {properties: schema.properties}}
+    end
+
+    def import(options = {})
+      db = index.database_connection
+      query = options[:query] || schema.query(order: "`#{schema.relation}`.id")
+      index.pre_sql.each { |sql| db.query(sql) }
+      rows = db.query(query, as: :array, stream: true, cache_rows: false)
+      builder = Builder.new(rows, schema)
+
+      builder.each_slice(1000) do |slice|
+        body = []
+        slice.each do |document|
+          body << {index: meta(document[:id], options)} << document
+        end
+        client.bulk body: body
+      end
+      client.indices.refresh index: options[:index] || index.index_name
+    end
+
+    def update_ids(ids)
+      return if ids.empty?
+      query = schema.query(
+        scope: "`#{schema.relation}`.id IN (#{ids.join(', ')})",
+        order: "`#{schema.relation}`.id",
+      )
+      rows = Syphon.database_connection.query(query, as: :array)
+      docs = Builder.new(rows, schema).to_a
+      body = bulk_indexes(docs) + bulk_deletes(ids, docs)
+      client.bulk body: body, refresh: true unless body.empty?
+    end
+
+    protected
+
+    def client
+      index.client
+    end
+
+    private
+
+    def bulk_indexes(documents, options = {})
+      documents.flat_map do |document|
+        [{index: meta(document[:id], options)}, document]
+      end
+    end
+
+    def bulk_deletes(ids, documents, options = {})
+      ids_to_delete = ids - documents.map { |document| document[:id] }
+      ids_to_delete.map do |id|
+        {delete: meta(id, options)}
+      end
+    end
+
+    def meta(id, options = {})
+      {_index: options[:index] || index.index_name, _type: type, _id: id}
+    end
+
+    private
+
+    def default_type
+      @type_name ||= index.name.sub(/Index\z/, '').underscore.to_sym
+    end
+  end
+end

data/lib/syphon/tasks.rb

@@ -0,0 +1,26 @@
+namespace :syphon do
+  task :build, [:indices] => :environment do |t, args|
+    require 'set'
+    classes = Syphon.index_classes
+
+    if (indices = args[:indices]).present?
+      class_names = indices.scan(/\w+/).to_set
+      classes.select! { |c| class_names.include?(c.name) }
+    end
+
+    n = classes.size
+    if n == 0
+      if indices
+        puts "No index classes found matching '#{indices}'. Available: #{Syphon.index_classes.map(&:name).join(', ')}"
+      else
+        puts "No index classes"
+      end
+    else
+      classes.each_with_index do |klass, i|
+        puts "#{i+1}/#{n}: Building #{klass}..."
+        klass.build
+      end
+      puts "Done."
+    end
+  end
+end

data/lib/syphon/version.rb

@@ -0,0 +1,11 @@
+module Syphon
+  VERSION = [0, 0, 1]
+
+  class << VERSION
+    include Comparable
+
+    def to_s
+      join('.')
+    end
+  end
+end

data/lib/syphon.rb

@@ -0,0 +1,41 @@
+require 'active_support/inflector'
+require 'elasticsearch'
+require 'mysql2'
+
+module Syphon
+  autoload :Builder, 'syphon/builder'
+  autoload :Index, 'syphon/index'
+  autoload :Schema, 'syphon/schema'
+  autoload :Source, 'syphon/source'
+  autoload :VERSION, 'syphon/version'
+
+  class << self
+    attr_writer :configuration, :database_configuration, :index_namespace
+
+    def configuration
+      @configuration ||= {}
+    end
+
+    def database_configuration
+      @database_configuration ||= {}
+    end
+
+    def index_namespace
+      @index_namespace ||= configuration[:index_namespace]
+    end
+
+    def database_connection
+      @database_connection ||= Mysql2::Client.new(database_configuration)
+    end
+
+    def client
+      Thread.current[:syphon_client] ||= Elasticsearch::Client.new(Syphon.configuration)
+    end
+
+    def index_classes
+      Syphon.configuration['index_classes'].map(&:constantize)
+    end
+  end
+end
+
+require 'syphon/railtie' if defined?(Rails)

data/syphon.gemspec

@@ -0,0 +1,22 @@
+$:.unshift File.expand_path('lib', File.dirname(__FILE__))
+require 'syphon/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = 'syphon'
+  gem.version       = Syphon::VERSION
+  gem.authors       = ['George Ogata']
+  gem.email         = ['george.ogata@gmail.com']
+  gem.description   = "Syphon data from an Arel source into ElasticSearch"
+  gem.summary       = "Syphon data from an Arel source into ElasticSearch"
+  gem.homepage      = 'https://github.com/howaboutwe/syphon'
+
+  gem.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.files         = `git ls-files`.split("\n")
+  gem.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+
+  gem.add_dependency 'elasticsearch', '~> 0.4.0'
+  gem.add_dependency 'activesupport', '~> 3.2.0'
+  gem.add_dependency 'mysql2', '~> 0.3.12'
+
+  gem.add_development_dependency 'bundler'
+end

data/test/config.yml.sample

@@ -0,0 +1,8 @@
+database:
+  adapter: mysql2
+  host: localhost
+  username: root
+  password:
+  database: syphon
+
+elasticsearch: {}

data/test/syphon/test_builder.rb

@@ -0,0 +1,73 @@
+require_relative '../test_helper'
+
+describe Syphon::Builder do
+  describe "#each" do
+    it "returns a document for each row with a distinct id" do
+      schema = Syphon::Schema.new do
+        integer :id, 0
+        string :name, 'x'
+      end
+      results = [[1, 'one'], [2, 'two']]
+      Syphon::Builder.new(results, schema).to_a.
+        must_equal [{id: 1, name: 'one'}, {id: 2, name: 'two'}]
+    end
+
+    it "builds nested documents for nested fields" do
+      schema = Syphon::Schema.new do
+        integer :id, 0
+        nested :nested1 do
+          integer :a, 'x'
+          integer :b, 'x'
+        end
+        nested :nested2 do
+          integer :a, 'x'
+        end
+      end
+      results = [[1, 10, 11, 12], [2, 20, 21, 22]]
+      Syphon::Builder.new(results, schema).to_a.must_equal [
+        {id: 1, nested1: {a: 10, b: 11}, nested2: {a: 12}},
+        {id: 2, nested1: {a: 20, b: 21}, nested2: {a: 22}},
+      ]
+    end
+
+    it "merges content from rows with the same root id" do
+      schema = Syphon::Schema.new do
+        integer :id, 0
+        string :name, 'x'
+      end
+      results = [[1, 'one'], [1, 'two']]
+      Syphon::Builder.new(results, schema).to_a.
+        must_equal [{id: 1, name: ['one', 'two']}]
+    end
+
+    it "merges content with the same root id correctly when there are nested fields" do
+      schema = Syphon::Schema.new do
+        integer :id, 0
+        nested :nested1 do
+          integer :a, 'x'
+        end
+        nested :nested2 do
+          integer :a, 'x'
+        end
+      end
+      results = [[1, 10, 11], [2, 20, 21]]
+      Syphon::Builder.new(results, schema).to_a.must_equal [
+        {id: 1, nested1: {a: 10}, nested2: {a: 11}},
+        {id: 2, nested1: {a: 20}, nested2: {a: 21}},
+      ]
+    end
+
+    it "supports arrays as nested fields" do
+      schema = Syphon::Schema.new do
+        integer :id, 0
+        nested :nested1 do
+          integer :id, 0
+          integer :name, 'x'
+        end
+      end
+      results = [[1, 2, 'a'], [1, 3, 'b']]
+      Syphon::Builder.new(results, schema).to_a.
+        must_equal [{id: 1, nested1: [{id: 2, name: 'a'}, {id: 3, name: 'b'}]}]
+    end
+  end
+end