scalastic 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 8d4a081eb653e2378d6a39f4f77cd51f44b553b0
+  data.tar.gz: c38641e0fe62a596f2d631a7edbfc6c4276785b7
+SHA512:
+  metadata.gz: a46a54dd509265a69b621b5596bd21c23b3d766f1d4c479faa28436e69b5650bb133dba236316860eb8ae48cca864fdf9b1f06b1d20f28bfb898162287cefb03
+  data.tar.gz: 252794ac6ea2ccb301636a90953e8a31ed2a3186ea079ba1c4894ffbc4ea86ddf41075e89f3e37d49a527ef7f5c05992c40bd46e86f4cc9e694d06a5920b330d

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.travis.yml

@@ -0,0 +1,5 @@
+language: ruby
+rvm:
+  - 2.2.0
+  - 1.9.3
+before_install: gem install bundler -v 1.10.6

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in scalastic.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Alex Baturytski
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,156 @@
+# Scalastic
+
+Scalastic is a library that organizes your Elasticsearch documents into partitions, as recommended [here](https://www.elastic.co/guide/en/elasticsearch/guide/current/faking-it.html). A partition provides methods for indexing documents (adding or updating), searching for them, and deleting them. Partitions are isolated: documents stored in one partition are not visible in other partitions.
+
+Initially your partition lives in a single index; however, as it grows, you may extend it to another index, redirecting all new documents there and keeping all documents from the original index available for searches. Partitions use Elasticsearch's filtered aliases; a partition with id 1 will create alias "scalastic_1_index" for indexing, and "scalastic_1_search" for searching. The index alias always points to a single index; the search one may span across multiple indices, with the most recent ones at the top. 
+
+Scalastic relies on the field "scalastic_partition_id" of type "long" to determine partition the document belongs to. The search alias for a partition has a term filter that returns only document belonging to that partition. The index alias, however, does not have a filter; if you're inserting documents into that alias directly (and not using Scalastic API), it becomes your responsibility to set this field to the correct value (which is partition id) for your document to show up in the partition. Note that delete API uses search alias to locate and delete documents to allow deletion of documents created in older indices.
+
+## Configuring the environment for Scalastic
+Every new index must be prepared before it can be used with scalastic:
+
+```ruby
+es_client = Elasticsearch::Client.new
+es_client.indices.create index: 'my_index'
+es_client.partitions.prepare index: 'my_index'
+```
+
+Preparing an index creates mappings for fields required by Scalastic. Note that preparing uses the _default_ mapping, which means that already existing document types will not be affected. Because of that **you should always prepare new indices before using them**.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'scalastic'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install scalastic
+
+## Usage
+
+Scalastic extends functionality of the Elasticsearch client by adding a property called "partitions" to the Elasticsearch client. That property serves as a gateway to all partitions' functionality.
+
+### Creating a partition
+```ruby
+# Connect to Elasticsearch and get the partitions client
+es_client = Elasticsearch::Client.new
+partitions = es_client.partitions
+
+# Create an index for the test.
+es_client.indices.create index: 'create_partition_test'
+partitions.prepare_index index: 'create_partition_test'   # Must be called once per each index
+
+# Create a partition
+partition = partitions.create index: 'create_partition_test', id: 1
+raise 'Partition was not created' unless partition.exists?
+```
+
+### Listing all existing partitions
+```ruby
+# Set everything up
+client = Elasticsearch::Client.new
+client.indices.create index: 'list_partitions_test'
+partitions = client.partitions
+partitions.prepare_index index: 'list_partitions_test'    # Must be called once per each index
+
+# Create a couple of partitions
+partitions.create index: 'list_partitions_test', id: 1
+partitions.create index: 'list_partitions_test', id: 2
+partitions.create index: 'list_partitions_test', id: 3
+
+# List all partitions
+partitions.to_a.each{|p| puts "Found partition #{p.id}"}
+```
+
+### Deleting a partition
+```ruby
+# Connect to Elasticsearch and create an index
+client = Elasticsearch::Client.new
+partitions = client.partitions
+client.indices.create index: 'delete_partition_test'
+partitions.prepare_index index: 'delete_partition_test'
+
+# Create partitions
+partitions.create index: 'delete_partition_test', id: 1
+partitions.create index: 'delete_partition_test', id: 2
+partitions.create index: 'delete_partition_test', id: 3
+
+# Delete one of the partitions
+partitions.delete id: 2
+raise "Partition still exists" if partitions[2].exists?
+```
+
+### Extending partition to another index
+```ruby
+# Connect to Elasticsearch and set up indices
+client = Elasticsearch::Client.new
+partitions = client.partitions
+client.indices.create index: 'extend_partition_1'
+partitions.prepare_index index: 'extend_partition_1'
+client.indices.create index: 'extend_partition_2'
+partitions.prepare_index index: 'extend_partition_2'
+
+# Create a partition residing in extend_partition_1
+partition = partitions.create(index: 'extend_partition_1', id: 1)
+
+# Extend partition to index extend_partition_2. Now search will be performed in both indices, but 
+# all new documents will be indexex into extend_partition_2.
+partition.extend_to(index: 'extend_partition_2')
+```
+
+### Operating documents inside partitions
+```ruby
+client = Elasticsearch::Client.new
+partitions = client.partitions
+
+client.indices.create index: 'partition_index'
+partitions.prepare_index index: 'partition_index'
+partition1 = partitions.create(index: 'partition_index', id: 1)
+partition2 = partitions.create(index: 'partition_index', id: 2)
+
+partition1.index id: 1, type: 'document', body: {subject: 'Subject 1'}
+partition1.index id: 2, type: 'document', body: {subject: 'Subject 2'}
+
+# Partition 2 should have no documents
+count = partition2.search(search_type: 'count', body: {query: {match_all: {}}})['hits']['total']
+raise 'Partition 2 is not empty!' unless count == 0
+
+# Partiton 1 should contain everything we just indexed
+hits = partition1.search(type: 'document', body: {query:{match_all: {}}})['hits']['hits']
+raise "Expected 2 documents, got #{hits.size}" unless hits.size == 2
+h1 = hits.find{|h| h['_id'].to_i == 1}
+raise 'Document 1 cannot be found' unless h1
+raise 'Invalid scalastic_partition_id' unless h1['_source']['scalastic_partition_id'] == 1
+
+# Now delete something from partition 1
+partition1.delete type: 'document', id: 1
+count = partition1.search(search_type: 'count', body: {query: {match_all: {}}})['hits']['total']
+raise "Expected 1 document, got #{count}" unless count == 1
+```
+
+### Notes
+* Indices must be *prepared* before they can be used by Scalastic by calling "prepare" on the partitions client; doing so will create critical field mappings. Each index must be prepared only once. 
+* All hash keys in arguments must be symbols; using anything else may result in unexpected behavior.
+
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/aliakb/scalastic.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,5 @@
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "scalastic"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/elasticsearch/transport_client.rb

@@ -0,0 +1,9 @@
+module Elasticsearch
+  module Transport
+    class Client
+      def partitions
+        @partitions_client ||= Scalastic::PartitionsClient.new(self, Scalastic::Config.default)
+      end
+    end
+  end
+end

data/lib/scalastic.rb

@@ -0,0 +1,9 @@
+require 'scalastic/version'
+require 'scalastic/config'
+require 'scalastic/partitions_client'
+
+require 'elasticsearch/transport_client'
+
+module Scalastic
+  # Your code goes here...
+end

data/lib/scalastic/config.rb

@@ -0,0 +1,37 @@
+module Scalastic
+  class Config
+    attr_reader(:partition_prefix)
+    attr_reader(:partition_selector)
+
+    def self.default
+      @default ||= new
+    end
+
+    def initialize
+      @partition_prefix = "scalastic"
+      @partition_selector = "scalastic_partition_id"
+    end
+
+    def index_endpoint(partition_id)
+      "#{partition_prefix}_#{partition_id}_index"
+    end
+
+    def search_endpoint(partition_id)
+      "#{partition_prefix}_#{partition_id}_search"
+    end
+
+    def get_partition_id(alias_name)
+      m = partition_regex.match(alias_name)
+      m && m[1].to_i
+    end
+
+    private
+
+    def partition_regex
+      @partition_regex ||= begin
+        escaped_prefix = Regexp.escape(partition_prefix)
+        Regexp.new("^#{escaped_prefix}_(\\w+)_(index|search)$")
+      end
+    end
+  end
+end

data/lib/scalastic/es_actions_generator.rb

@@ -0,0 +1,23 @@
+module Scalastic
+  module EsActionsGenerator
+    extend self
+
+    def new_search_alias(config, args)
+      index = args[:index] || raise(ArgumentError, "Missing required argument :index")
+      id = args[:id] || raise(ArgumentError, "Missing required argument :id")
+      routing = args[:routing]
+      {index: index, alias: config.search_endpoint(id), filter: {term: {config.partition_selector => id}}}.tap do |op|
+        op.merge!(routing: routing) if routing
+      end
+    end
+
+    def new_index_alias(config, args)
+      index = args[:index] || raise(ArgumentError, "Missing required argument :index")
+      id = args[:id] || raise(ArgumentError, "Missing required argument :id")
+      routing = args[:routing]
+      {index: index, alias: config.index_endpoint(id)}.tap do |op|
+        op.merge!(routing: routing) if routing
+      end
+    end
+  end
+end

data/lib/scalastic/partition.rb

@@ -0,0 +1,60 @@
+require 'scalastic/es_actions_generator'
+
+module Scalastic
+  class Partition
+    attr_reader(:es_client)
+    attr_reader(:config)
+    attr_reader(:id)
+
+    def initialize(es_client, config, id)
+      raise(ArgumentError, 'ES client is nil!') if es_client.nil?
+      raise(ArgumentError, 'config is nil!') if config.nil?
+      raise(ArgumentError, 'id is empty!') if id.nil? || id.to_s.empty?
+
+      @es_client = es_client
+      @config = config
+      @id = id
+    end
+
+    def extend_to(args)
+      index = args[:index]
+      raise(ArgumentError, 'Missing required argument :index') if index.nil? || index.to_s.empty?
+
+      index_alias = config.index_endpoint(id)
+      indices = es_client.indices.get_aliases(name: index_alias).select{|i, d| d['aliases'].any?}.keys
+      actions = indices.map{|i| {remove: {index: i, alias: index_alias}}}
+      actions << {add: EsActionsGenerator.new_index_alias(config, args.merge(id: id))}
+      actions << {add: EsActionsGenerator.new_search_alias(config, args.merge(id: id))}
+      es_client.indices.update_aliases(body: {actions: actions})
+    end
+
+    def search(args = {})
+      args = args.merge(index: config.search_endpoint(id))
+      es_client.search(args)
+    end
+
+    def index(args)
+      args = {body: {}}.merge(args)
+      args[:body][config.partition_selector.to_sym] = id
+      args = args.merge(index: config.index_endpoint(id))
+      es_client.index(args)
+    end
+
+    def delete(args = {})
+      args = args.merge(index: config.search_endpoint(id))
+      es_client.delete(args)
+    end
+
+    def exists?
+      names = [config.search_endpoint(id), config.index_endpoint(id)]
+      all_aliases = es_client.indices.get_aliases name: names.join(',')
+      all_aliases.any?{|_index, data| data['aliases'].any?}
+    end
+
+    #TODO: add bulk
+
+    def inspect
+      "ES partition #{id}"
+    end
+  end
+end

data/lib/scalastic/partitions_client.rb

@@ -0,0 +1,53 @@
+require "scalastic/partition"
+require "scalastic/es_actions_generator"
+
+module Scalastic
+  class PartitionsClient
+    attr_reader(:es_client)
+    attr_reader(:config)
+
+    def initialize(es_client, config = Config.default)
+      raise(ArgumentError, "ES client is nil") if es_client.nil?
+      raise(ArgumentError, "Config is nil") if config.nil?
+      @es_client = es_client
+      @config = config
+    end
+
+    def create(args = {})
+      actions = [
+        {add: EsActionsGenerator.new_search_alias(config, args)},
+        {add: EsActionsGenerator.new_index_alias(config, args)},
+      ]
+      es_client.indices.update_aliases(body: {actions: actions})
+      self[args[:id]]
+    end
+
+    def delete(args = {})
+      id = args[:id] || raise(ArgumentError, "Missing required argument :id")
+      pairs = es_client.indices.get_aliases.map{|i, d| d["aliases"].keys.select{|a| config.get_partition_id(a) == id}.map{|a| [i, a]}}.flatten(1)
+      unless pairs.any?
+        #TODO: log a warning
+        return
+      end
+      actions = pairs.map{|i, a| {remove: {index: i, alias: a}}}
+      es_client.indices.update_aliases(body: {actions: actions})
+    end
+
+    def [](id)
+      Partition.new(es_client, config, id)
+    end
+
+    def to_a
+      aliases = es_client.indices.get_aliases
+      partition_ids = aliases.map{|_, data| data["aliases"].keys}.flatten.map{|a| config.get_partition_id(a)}.compact.uniq
+      partition_ids.map{|id| Partition.new(es_client, config, id)}
+    end
+
+    def prepare_index(args)
+      index = args[:index] || raise(ArgumentError, "Missing required argument :index")
+      mapping = {properties: {config.partition_selector => {type: "long"}}}
+      es_client.indices.put_mapping(index: index, type: "_default_", body: {"_default_" => mapping})
+      es_client.indices.put_mapping(index: index, type: "scalastic", body: {"scalastic" => mapping})
+    end
+  end
+end

data/lib/scalastic/version.rb

@@ -0,0 +1,3 @@
+module Scalastic
+  VERSION = "0.1.0"
+end

data/scalastic.gemspec

@@ -0,0 +1,29 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'scalastic/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "scalastic"
+  spec.version       = Scalastic::VERSION
+  spec.authors       = ["Aliaksei Baturytski"]
+  spec.email         = ["abaturytski@gmail.com"]
+
+  spec.summary       = "Elasticsearch document partitions"
+  spec.description   = "Elasticsearch alias-based partitions for scalable indexing and searching"
+  spec.homepage      = "https://github.com/aliakb/scalastic"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.10"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.4"
+  spec.add_development_dependency "simplecov", "~> 0.11"
+
+  spec.add_dependency "elasticsearch", "~> 1.0"
+  spec.add_dependency "multi_json", "~> 1.0"
+end

metadata

@@ -0,0 +1,144 @@
+--- !ruby/object:Gem::Specification
+name: scalastic
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Aliaksei Baturytski
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2016-01-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.4'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.11'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.11'
+- !ruby/object:Gem::Dependency
+  name: elasticsearch
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: multi_json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+description: Elasticsearch alias-based partitions for scalable indexing and searching
+email:
+- abaturytski@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/elasticsearch/transport_client.rb
+- lib/scalastic.rb
+- lib/scalastic/config.rb
+- lib/scalastic/es_actions_generator.rb
+- lib/scalastic/partition.rb
+- lib/scalastic/partitions_client.rb
+- lib/scalastic/version.rb
+- scalastic.gemspec
+homepage: https://github.com/aliakb/scalastic
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Elasticsearch document partitions
+test_files: []