syphon 0.0.1

data/test/syphon/test_index.rb

@@ -0,0 +1,144 @@
+require_relative '../test_helper'
+
+describe Syphon::Index do
+  before do
+    Object.const_set(:TestIndex, Class.new)
+    TestIndex.send :include, Syphon::Index
+  end
+
+  after do
+    Object.send(:remove_const, :TestIndex)
+  end
+
+  describe ".index_name" do
+    describe "when no index namespace is set" do
+      use_attribute_value Syphon, :index_namespace, nil
+
+      it "it derived from the class name" do
+        TestIndex.index_name.must_equal 'tests'
+      end
+    end
+
+    describe "when the index namespace is empty" do
+      use_attribute_value Syphon, :index_namespace, ''
+
+      it "it is treated the same as nil" do
+        TestIndex.index_name.must_equal 'tests'
+      end
+    end
+
+    describe "when an index namespace is set" do
+      it "prefixes with the namespace and an underscore" do
+        TestIndex.index_name.must_equal 'syphon_tests'
+      end
+    end
+
+    it "can be overridden by the class" do
+      TestIndex.class_eval { self.index_name = 'wibble' }
+      TestIndex.index_name.must_equal 'wibble'
+    end
+  end
+
+  describe ".define_source" do
+    it "defaults the name and type" do
+      TestIndex.class_eval do
+        define_source
+      end
+
+      source = TestIndex.source
+      source.name.must_be_nil
+      source.type.must_equal :test
+    end
+
+    it "defines a source with the given name and fields" do
+      TestIndex.class_eval do
+        define_source :custom_name, type: :thing do
+          string :value, 'x'
+        end
+      end
+
+      source = TestIndex.source(:custom_name)
+      source.name.must_equal :custom_name
+      source.type.must_equal :thing
+      source.schema.fields.keys.must_equal [:value]
+    end
+  end
+
+  describe ".build" do
+    uses_users_table
+    uses_elasticsearch
+
+    before do
+      clear_indices
+
+      TestIndex.class_eval do
+        define_source do
+          string :login, "users.login"
+          from 'users'
+        end
+      end
+    end
+
+    it "builds the index (as an alias of an underlying index) if it does not yet exist" do
+      db.query "INSERT INTO users(login) VALUES('alice')"
+      TestIndex.build
+
+      hits = TestIndex.search['hits']['hits']
+      hits.map { |doc| doc['_source']['login'] }.must_equal ['alice']
+
+      db.query "DELETE FROM users"
+      db.query "INSERT INTO users(login) VALUES('bob')"
+
+      TestIndex.build
+
+      hits = TestIndex.search['hits']['hits']
+      hits.map { |doc| doc['_source']['login'] }.must_equal ['bob']
+    end
+
+    it "runs all warmups between building the new index and rotating it in" do
+      this = self
+      runs = []
+      TestIndex.class_eval do
+        define_warmup do |new_index|
+          client.indices.exists(index: new_index)
+          -> { client.indices.get_alias(name: TestIndex.index_name) }.
+            must_raise(Elasticsearch::Transport::Transport::Errors::NotFound)
+          runs << 1
+        end
+
+        define_warmup do |new_index|
+          runs << 2
+        end
+      end
+
+      TestIndex.build
+      runs.must_equal [1, 2]
+    end
+  end
+
+  describe ".destroy" do
+    uses_users_table
+    uses_elasticsearch
+
+    before do
+      TestIndex.class_eval do
+        define_source do
+          string :login, "users.login"
+          from 'users'
+        end
+      end
+      TestIndex.build
+    end
+
+    it "deletes the index and any aliases to it" do
+      client.indices.exists(index: TestIndex.index_name).must_equal true
+      client.indices.get_alias(name: TestIndex.index_name).size.must_equal 1
+
+      TestIndex.destroy
+
+      -> { client.indices.get_alias(name: TestIndex.index_name) }.
+        must_raise(Elasticsearch::Transport::Transport::Errors::NotFound)
+      client.indices.exists(index: TestIndex.index_name).must_equal false
+    end
+  end
+end

data/test/syphon/test_schema.rb

@@ -0,0 +1,271 @@
+require_relative '../test_helper'
+
+describe Syphon::Schema do
+  describe "#initialize" do
+    it "configures the schema with the given block" do
+      schema = Syphon::Schema.new do
+        string :s, 'STRING'
+      end
+      schema.fields.values.map(&:name).must_equal [:s]
+    end
+  end
+
+  describe "#configure" do
+    describe "type methods" do
+      it "builds fields with type methods" do
+        schema = Syphon::Schema.new do
+          string :string_field, 'STRING'
+          short :short_field, 'SHORT'
+          byte :byte_field, 'BYTE'
+          integer :integer_field, 'INTEGER'
+          long :long_field, 'LONG'
+          float :float_field, 'FLOAT'
+          double :double_field, 'DOUBLE'
+          date :date_field, 'DATE'
+          boolean :boolean_field, 'BOOLEAN'
+          binary :binary_field, 'BINARY'
+          geo_point :geo_point_field, 'GEO_POINT'
+          nested :nested_field do
+            string :string_field, 'NESTED.STRING'
+          end
+        end
+        schema.fields.values.map(&:name).must_equal [
+          :string_field, :short_field, :byte_field, :integer_field,
+          :long_field, :float_field, :double_field, :date_field,
+          :boolean_field, :binary_field, :geo_point_field, :nested_field,
+        ]
+        schema.fields.values.map(&:type).must_equal [
+          :string, :short, :byte, :integer, :long, :float, :double, :date,
+          :boolean, :binary, :geo_point, :nested,
+        ]
+        schema.fields.values.map(&:expression).must_equal [
+          'STRING', 'SHORT', 'BYTE', 'INTEGER', 'LONG', 'FLOAT', 'DOUBLE',
+          'DATE', 'BOOLEAN', 'BINARY', 'GEO_POINT', nil,
+        ]
+        nested = schema.fields[:nested_field]
+        nested.nested_schema.fields.values.map(&:name).must_equal [:string_field]
+        nested.nested_schema.fields.values.map(&:type).must_equal [:string]
+        nested.nested_schema.fields.values.map(&:expression).must_equal ['NESTED.STRING']
+      end
+
+      it "supports dynamic expressions by passing a proc for a field expression" do
+        i = nil
+        schema = Syphon::Schema.new do
+          string :string_field, -> { i.to_s }
+        end
+        i = 1
+        schema.fields.values.map { |v| v.expression.call }.must_equal ['1']
+        i = 2
+        schema.fields.values.map { |v| v.expression.call }.must_equal ['2']
+      end
+    end
+
+    describe "#from" do
+      it "sets the relation" do
+        schema = Syphon::Schema.new do
+          from 'things'
+        end
+        schema.relation.must_equal 'things'
+      end
+
+      it "supports a dynamic value by passing a block" do
+        i = nil
+        schema = Syphon::Schema.new do
+          from { i.to_s }
+        end
+        i = 1
+        schema.relation.call.must_equal '1'
+        i = 2
+        schema.relation.call.must_equal '2'
+      end
+    end
+
+    describe "#join" do
+      it "accumulates joins" do
+        schema = Syphon::Schema.new do
+          from 'things'
+          join 'INNER JOIN a ON 1'
+          join 'INNER JOIN b ON 2'
+        end
+        schema.joins.must_equal ['INNER JOIN a ON 1', 'INNER JOIN b ON 2']
+      end
+
+      it "supports a dynamic value by passing a block" do
+        i = j = nil
+        schema = Syphon::Schema.new do
+          from 'things'
+          join { i.to_s }
+          join { j.to_s }
+        end
+        i, j = 1, 2
+        schema.joins.map(&:call).must_equal ['1', '2']
+        i, j = 3, 4
+        schema.joins.map(&:call).must_equal ['3', '4']
+      end
+    end
+
+    describe "#where" do
+      it "sets the conditions" do
+        schema = Syphon::Schema.new do
+          where 'x = 1'
+        end
+        schema.conditions.must_equal 'x = 1'
+      end
+
+      it "supports a dynamic value by passing a block" do
+        i = nil
+        schema = Syphon::Schema.new do
+          where { i.to_s }
+        end
+        i = 1
+        schema.conditions.call.must_equal '1'
+        i = 2
+        schema.conditions.call.must_equal '2'
+      end
+    end
+
+    describe "#group_by" do
+      it "sets the group clause" do
+        schema = Syphon::Schema.new do
+          group_by 'x, y'
+        end
+        schema.group_clause.must_equal 'x, y'
+      end
+
+      it "supports a dynamic value by passing a block" do
+        i = nil
+        schema = Syphon::Schema.new do
+          group_by { i.to_s }
+        end
+        i = 1
+        schema.group_clause.call.must_equal '1'
+        i = 2
+        schema.group_clause.call.must_equal '2'
+      end
+    end
+
+    describe "#having" do
+      it "sets the having clause" do
+        schema = Syphon::Schema.new do
+          having 'x = 1'
+        end
+        schema.having_clause.must_equal 'x = 1'
+      end
+
+      it "supports a dynamic value by passing a block" do
+        i = nil
+        schema = Syphon::Schema.new do
+          having { i.to_s }
+        end
+        i = 1
+        schema.having_clause.call.must_equal '1'
+        i = 2
+        schema.having_clause.call.must_equal '2'
+      end
+    end
+  end
+
+  describe "#query" do
+    it "returns the complete query" do
+      schema = Syphon::Schema.new do
+        string :s, 'S'
+        nested :inner do
+          string :t, 'T'
+        end
+        from 'things'
+        join 'INNER JOIN a ON 1'
+        join 'INNER JOIN b ON 2'
+        where 'a = 1'
+        group_by 'x'
+        having 'count(*) = 1'
+      end
+      schema.query.must_equal <<-EOS.strip.gsub(/\s+/, ' ')
+          SELECT S AS `s`, T AS `inner[t]`
+          FROM things
+          INNER JOIN a ON 1
+          INNER JOIN b ON 2
+          WHERE a = 1
+          GROUP BY x
+          HAVING count(*) = 1
+        EOS
+    end
+
+    it "omits optional clauses when in the minimal case" do
+      schema = Syphon::Schema.new do
+        string :s, 'S'
+        from 'things'
+      end
+      schema.query.must_equal "SELECT S AS `s` FROM things"
+    end
+
+    it "supports overriding the select expression" do
+      schema = Syphon::Schema.new do
+        string :s, 'S'
+        from 'things'
+      end
+      schema.query(select: '1').must_equal "SELECT 1 FROM things"
+    end
+
+    it "applies the given extra scope, order, and limit" do
+      schema = Syphon::Schema.new do
+        string :s, 'S'
+        from 'things'
+        where 'a = 1'
+        group_by 'x'
+        having 'count(*) = 1'
+      end
+      options = {scope: 'b = 2', order: 'c DESC', limit: 2}
+      schema.query(options).must_equal <<-EOS.strip.gsub(/\s+/, ' ')
+          SELECT S AS `s`
+          FROM things
+          WHERE (a = 1) AND (b = 2)
+          GROUP BY x
+          HAVING count(*) = 1
+          ORDER BY c DESC
+          LIMIT 2
+        EOS
+    end
+
+    it "inverts the condition if :invert is true" do
+      schema = Syphon::Schema.new do
+        string :s, 'S'
+        from 'things'
+        where 'a = 1'
+      end
+      schema.query(invert: true).must_equal "SELECT S AS `s` FROM things WHERE NOT (a = 1)"
+    end
+
+    it "does not invert the scope" do
+      schema = Syphon::Schema.new do
+        string :s, 'S'
+        from 'things'
+        where 'a = 1'
+      end
+      schema.query(invert: true, scope: 'id = 1').must_equal "SELECT S AS `s` FROM things WHERE (NOT (a = 1)) AND (id = 1)"
+    end
+  end
+
+  describe "#properties" do
+    it "returns the properties hash for the fields" do
+      schema = Syphon::Schema.new do
+        string :name, 'x'
+        integer :value, 'x'
+        nested :inner do
+          string :s, 'x'
+        end
+      end
+
+      schema.properties.must_equal({
+        name: {type: :string},
+        value: {type: :integer},
+        inner: {
+          type: :nested,
+          properties: {
+            s: {type: :string},
+          },
+        },
+      })
+    end
+  end
+end
+

data/test/syphon/test_source.rb

@@ -0,0 +1,141 @@
+require_relative '../test_helper'
+
+describe Syphon::Source do
+  before do
+    Object.const_set(:TestIndex, Class.new)
+    TestIndex.send :include, Syphon::Index
+  end
+
+  after do
+    Object.send(:remove_const, :TestIndex)
+  end
+
+  let(:source) { TestIndex.source }
+
+  describe "#initialize" do
+    it "sets a default type" do
+      source = Syphon::Source.new(TestIndex, :things_source)
+      source.name.must_equal :things_source
+      source.type.must_equal :test
+    end
+
+    it "sets the name and type" do
+      source = Syphon::Source.new(TestIndex, :things_source, type: :custom_type)
+      source.name.must_equal :things_source
+      source.type.must_equal :custom_type
+    end
+
+    it "initializes the schema from the given block" do
+      source = Syphon::Source.new(TestIndex, :things_source) do
+        string :name, 'x'
+      end
+      source.schema.fields.keys.must_equal [:name]
+    end
+  end
+
+  describe "#mapping" do
+    it "returns the ElasticSearch mapping" do
+      source = Syphon::Source.new(TestIndex, :things_source, type: :thing) do
+        string :name, 'x'
+      end
+
+      source.mapping.must_equal({
+        thing: {
+          properties: {
+            name: {
+              type: :string
+            },
+          },
+        },
+      })
+    end
+  end
+
+  describe "#import" do
+    uses_users_table
+    uses_elasticsearch
+
+    before do
+      TestIndex.class_eval do
+        define_source do
+          integer :id, 'id'
+          string :login, 'login', stored: true
+          from 'users'
+        end
+      end
+      TestIndex.build(schema_only: true)
+    end
+
+    it "imports the data as configured by the SQL query" do
+      db.query "INSERT INTO users(login) VALUES('alice')"
+      source.import
+
+      hits = client.search(index: TestIndex.index_name)['hits']['hits']
+      hits.map { |doc| doc['_source']['login'] }.must_equal ['alice']
+    end
+
+    it "imports data correctly the second time" do
+      db.query "INSERT INTO users(login) VALUES('alice')"
+      source.import
+
+      db.query "INSERT INTO users(login) VALUES('bob')"
+      source.import
+
+      hits = client.search(index: TestIndex.index_name)['hits']['hits']
+      hits.map { |doc| doc['_source']['login'] }.sort.must_equal ['alice', 'bob']
+    end
+
+    it "uses the given index name" do
+      client.indices.create(index: 'syphon_custom')
+      db.query "INSERT INTO users(login) VALUES('alice')"
+      source.import(index: 'syphon_custom')
+
+      hits = client.search(index: 'syphon_custom')['hits']['hits']
+      hits.map { |doc| doc['_source']['login'] }.must_equal ['alice']
+    end
+  end
+
+  describe "#update" do
+    uses_users_table
+    uses_elasticsearch
+
+    before do
+      TestIndex.class_eval do
+        define_source do
+          integer :id, 'id'
+          string :login, 'login', stored: true
+          from 'users'
+        end
+      end
+      TestIndex.build(schema_only: true)
+    end
+
+    it "updates the index from the database, scoping with the given conditions" do
+      db.query "INSERT INTO users(login) VALUES('alice'), ('bob')"
+      TestIndex.build
+
+      hits = client.search(index: TestIndex.index_name)['hits']['hits']
+      hits.map { |doc| doc['_source']['login'] }.sort.must_equal ['alice', 'bob']
+
+      db.query "UPDATE users SET login = 'superalice' WHERE login = 'alice'"
+      db.query "UPDATE users SET login = 'wonderbob' WHERE login = 'bob'"
+      bob_id = db.query("SELECT id FROM users WHERE login = 'wonderbob'", as: :array).to_a[0][0]
+      source.update_ids([bob_id])
+
+      hits = client.search(index: TestIndex.index_name)['hits']['hits']
+      hits.map { |doc| doc['_source']['login'] }.sort.must_equal ['alice', 'wonderbob']
+    end
+
+    it "deletes records that should no longer exist" do
+      db.query "INSERT INTO users(login) VALUES('alice'), ('bob')"
+      TestIndex.build
+
+      bob_id = db.query("SELECT id FROM users WHERE login = 'bob'", as: :array).to_a[0][0]
+      db.query "DELETE FROM users WHERE id = #{bob_id}"
+      source.update_ids([bob_id])
+
+      hits = client.search(index: TestIndex.index_name)['hits']['hits']
+      hits.map { |doc| doc['_source']['login'] }.must_equal ['alice']
+    end
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,42 @@
+ROOT = File.expand_path('..', File.dirname(__FILE__))
+
+$:.unshift "#{ROOT}/lib"
+require 'minitest/spec'
+require 'yaml'
+require 'temporaries'
+require 'debugger' if RUBY_VERSION < '2.0'
+require 'looksee'
+
+require 'syphon'
+
+config = YAML.load_file("#{ROOT}/test/config.yml")
+Syphon.database_configuration = config['database']
+Syphon.configuration = config['elasticsearch']
+Syphon.index_namespace = 'syphon'
+
+MiniTest::Spec.class_eval do
+  def self.uses_users_table
+    let(:db) { Syphon.database_connection }
+
+    before do
+      columns = "id int auto_increment PRIMARY KEY, login VARCHAR(20)"
+      db.query "CREATE TABLE IF NOT EXISTS users(#{columns})"
+    end
+
+    after do
+      db.query "DROP TABLE IF EXISTS users"
+    end
+  end
+
+  def self.uses_elasticsearch
+    let(:client) { Syphon.client }
+
+    before { clear_indices }
+    after { clear_indices }
+  end
+
+  def clear_indices
+    client.indices.status['indices'].keys.grep(/\Asyphon_/).
+      each { |name| client.indices.delete(index: name) }
+  end
+end

data/test/test_syphon.rb

@@ -0,0 +1,28 @@
+require_relative 'test_helper'
+
+describe Syphon do
+  describe ".configuration" do
+    use_attribute_value Syphon, :configuration, nil
+
+    it "defaults to an empty hash" do
+      Syphon.configuration.must_equal({})
+    end
+  end
+
+  describe ".database_configuration" do
+    use_instance_variable_value Syphon, :database_configuration, nil
+
+    it "defaults to an empty hash" do
+      Syphon.database_configuration.must_equal({})
+    end
+  end
+
+  describe ".index_namespace" do
+    use_instance_variable_value Syphon, :index_namespace, nil
+    use_instance_variable_value Syphon, :configuration, {index_namespace: 'NAMESPACE'}
+
+    it "defaults to the configured index namespace" do
+      Syphon.index_namespace.must_equal('NAMESPACE')
+    end
+  end
+end