tantiny 0.2.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: e082705a8a556a2bf8ebfd08445ea2917e308a1fdd80e2e2dcf75edaa71bc135
+  data.tar.gz: 2098370ed2edcf0aa533abf3327ba4513f76fdf1119ba6f007de68d162510897
+SHA512:
+  metadata.gz: 76171526e6677a13874d8cb2b915496eb844ac120404f220440552d00217b059f3ac55f329318ec020249ec4106a2977ee7251a9c526c0fa237077be8401c937
+  data.tar.gz: 4425ba8f9a882b9e6b2b9eb7c4d0d8aa4a87342249fd0588fd2d82a6f0c8f709fcc2ee5e22e63b66b84256c398decb2a7a207414636cf6d588c64a63c85a496f

data/CHANGELOG.md

@@ -0,0 +1,20 @@
+# Changelog
+
+### [0.2.2](https://github.com/baygeldin/tantiny/compare/v0.2.1...v0.2.2) (2022-03-07)
+
+
+### Bug Fixes
+
+* Fix native extension initialization ([78c7495](https://github.com/baygeldin/tantiny/commit/78c74951a4ade684395f756467aa583aad1f90a8))
+* Include transpiled files in the gem build ([5dba8f6](https://github.com/baygeldin/tantiny/commit/5dba8f6a75f36eb27756c9e8d8f7f3872d73bf97))
+
+### [0.2.1](https://github.com/baygeldin/tantiny/compare/v0.2.0...v0.2.1) (2022-03-07)
+
+
+### Features
+
+* Initial release ([f10ed87](https://github.com/baygeldin/tantiny/commit/f10ed878e0b781580d5a04d854c44e6b868621b1))
+
+### [0.2.0] (2022-03-07)
+
+- Dummy release

data/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "tantiny"
+version = "0.2.2" # {x-release-please-version}
+edition = "2021"
+authors = ["Alexander Baygeldin"]
+repository = "https://github.com/baygeldin/tantiny"
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+rutie = "0.8"
+tantivy = "0.16"
+lazy_static = "1.4"
+paste = "1.0"
+
+[package.metadata.thermite]
+github_releases = true
+github_release_type = "latest"
+git_tag_regex = "^v(\\d+\\.\\d+\\.\\d+)$"

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 Alexander Baygeldin
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,309 @@
+# Tantiny
+
+Need a fast full-text search for your Ruby script, but Solr and Elasticsearch are an overkill? 😏
+
+You're in the right place. **Tantiny** is a minimalistic full-text search library for Ruby based on [Tantivy](https://github.com/quickwit-oss/tantivy) (an awesome alternative to Apache Lucene written in Rust). The greatest advantage of using it is that you don't need *anything* to make it work (no separate server or process), it's purely embeddable. So, if your task at hand requires a full-text search, but a full-blown distributed search engine would be an overkill, Tantiny would be perfect for you.
+
+The main philosophy is to provide low-level access Tantivy's inverted index, but with a nice Ruby-esque API, sensible defaults, and additional functionality sprinkled on top (so, Tantiny not exactly bindings to Tantivy, but it tries to be close).
+
+Take a look at the most basic example:
+
+```ruby
+index = Tantiny::Index.new("/path/to/index") { text :description }
+
+index << { id: 1, description: "Hello World!" }
+index << { id: 2, description: "What's up?" }
+index << { id: 3, description: "Goodbye World!" }
+
+index.commit
+index.reload
+
+index.search("world") # 1, 3
+```
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'tantiny'
+```
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install tantiny
+
+You don't **have to** have Rust installed on your system since Tantiny will try to download the pre-compiled binaries hosted on GitHub releases during the installation. However, if no pre-compiled binaries were found for your system (which is a combination of platform, architecture, and Ruby version) you will need to [install Rust](https://www.rust-lang.org/tools/install) first.
+
+## Defining the index
+
+You have to specify a path to where the index would be stored and a block that defines the schema:
+
+```ruby
+Tantiny::Index.new "/tmp/index" do
+  id :imdb_id
+  facet :category
+  string :title
+  text :description
+  integer :duration
+  double :rating
+  date :release_date
+end
+```
+
+Here are the descriptions for every field type:
+
+| Type | Description |
+| --- | --- |
+| id | Specifies where documents' ids are stored (defaults to `:id`). |
+| facet | Fields with values like `/animals/birds` (i.e. hierarchial categories). |
+| string | Fields with text that are **not** tokenized. |
+| text | Fields with text that are tokenized by the specified tokenizer. |
+| integer | Fields with integer values. |
+| double  | Fields with float values. |
+| date | Fields with either `DateTime` type or something that converts to it. |
+
+## Managing documents
+
+You can feed the index any kind of object that has methods specified in your schema, but plain hashes also work:
+
+```ruby
+rio_bravo = OpenStruct.new(
+  imdb_id: "tt0053221",
+  type: '/western/US',
+  title: "Rio Bravo",
+  description: "A small-town sheriff enlists a drunk, a kid and an old man to help him fight off a ruthless cattle baron.",
+  duration: 141,
+  rating: 8.0,
+  release_date: Date.parse("March 18, 1959")
+)
+
+hanabi = {
+  imdb_id: "tt0119250",
+  type: "/crime/Japan",
+  title: "Hana-bi",
+  description: "Nishi leaves the police in the face of harrowing personal and professional difficulties. Spiraling into depression, he makes questionable decisions.",
+  duration: 103,
+  rating: 7.7,
+  release_date: Date.parse("December 1, 1998")
+}
+
+brother = {
+  imdb_id: "tt0118767",
+  type: "/crime/Russia",
+  title: "Brother",
+  description: "An ex-soldier with a personal honor code enters the family crime business in St. Petersburg, Russia.",
+  duration: 99,
+  rating: 7.9,
+  release_date: Date.parse("December 12, 1997")
+}
+
+index << rio_bravo
+index << hanabi
+index << brother
+```
+
+In order to update the document just add it again (as long as the id is the same):
+
+```ruby
+rio_bravo.rating = 10.0
+index << rio_bravo
+```
+
+You can also delete it if you want:
+
+```ruby
+index.delete(rio_bravo.imdb_id)
+```
+
+After that you need to commit the index for the changes to take place:
+
+```ruby
+index.commit
+```
+
+## Searching
+
+Make sure that your index is up-to-date by reloading it first:
+
+```ruby
+index.reload
+```
+
+And search it (finally!):
+
+```ruby
+index.search("a drunk, a kid, and an old man")
+```
+
+By default it will return ids of 10 best matching documents, but you can customize it:
+
+```ruby
+index.search("a drunk, a kid, and an old man", limit: 100)
+```
+
+You may wonder, how exactly does it conduct the search? Well, the default behavior is to use `smart_query` search (see below for details) over all `text` fields defined in your schema. So, you can pass the parameters that the `smart_query` accepts right here:
+
+```ruby
+index.search("a dlunk, a kib, and an olt mab", fuzzy_distance: 1)
+```
+
+However, you can customize it by composing your own query out of basic building blocks: 
+
+```ruby
+popular_movies = index.range_query(:rating, 8.0..10.0)
+about_sheriffs = index.term_query(:description, "sheriff")
+crime_movies = index.facet_query(:cetegory, "/crime")
+long_ass_movies = index.range_query(:duration, 180..9999)
+something_flashy = index.smart_query(:description, "bourgeoisie")
+
+index.search((popular_movies & about_sheriffs) | (crime_movies & !long_ass_movies) | something_flashy)
+```
+
+I know, weird taste! But pretty cool, huh? Take a look at all the available queries below.
+
+### Supported queries
+
+| Query | Behavior |
+| --- | --- |
+| all_query | Returns all indexed documents. |
+| empty_query | Returns exactly nothing (used internally). |
+| term_query | Documents that contain the specified term. |
+| fuzzy_term_query | Documents that contain the specified term within a Levenshtein distance. |
+| phrase_query | Documents that contain the specified sequence of terms. |
+| regex_query | Documents that contain a term that matches the specified regex. |
+| prefix_query | Documents that contain a term with the specified prefix. |
+| range_query | Documents that with an `integer`, `double` or `date` field within the specified range. |
+| facet_query | Documents that belong to the specified category. |
+| smart_query | A combination of `term_query`, `fuzzy_term_query` and `prefix_query`. |
+
+Take a look at the [signatures file](https://github.com/baygeldin/tantiny/blob/main/sig/tantiny/query.rbs) to see what parameters do queries accept.
+
+### Searching on multiple fields
+
+All queries can search on multuple fields (except for `facet_query` because it doesn't make sense there).
+
+So, the following query:
+
+```ruby
+index.term_query(%i[title, description], "hello")
+```
+
+Is equivalent to:
+
+```ruby
+index.term_query(:title, "hello") | index.term_query(:description, "hello")
+```
+
+### Boosting queries
+
+All queries support the `boost` parameter that allows to bump documents position in the search:
+
+```ruby
+about_cowboys = index.term_query(:description, "cowboy", boost: 2.0)
+about_samurai = index.term_query(:description, "samurai") # sorry, Musashi...
+
+index.search(about_cowboys | about_samurai)
+```
+
+### `smart_query` behavior
+
+The `smart_query` search will extract terms from your query string using the respective field tokenizers and search the index for documents that contain those terms via the `term_query`. If the `fuzzy_distance` parameter is specified it will use the `fuzzy_term_query`. Also, it allows the last term to be unfinished by using the `prefix_query`.
+
+So, the following query:
+
+```ruby
+index.smart_query(%i[en_text ru_text], "dollars рубли eur", fuzzy_distance: 1)
+```
+
+Is equivalent to:
+
+```ruby
+t1_en = index.fuzzy_term_query(:en_text, "dollar")
+t2_en = index.fuzzy_term_query(:en_text, "рубли")
+t3_en = index.fuzzy_term_query(:en_text, "eur")
+t3_prefix_en = index.prefix_query(:en_text, "eur")
+
+t1_ru = index.fuzzy_term_query(:ru_text, "dollars")
+t2_ru = index.fuzzy_term_query(:ru_text, "рубл")
+t3_ru = index.fuzzy_term_query(:ru_text, "eur")
+t3_prefix_ru = index.prefix_query(:ru_text, "eur")
+
+(t1_en & t2_en & (t3_en | t3_prefix_en)) | (t1_ru & t2_ru & (t3_ru | t3_prefix_ru))
+```
+
+Notice how words "dollars" and "рубли" are stemmed differently depending on the field we are searching. This is assuming we have `en_text` and `ru_text` fields in our schema that use English and Russian stemmer tokenizers respectively.
+
+### About `regex_query`
+
+The `regex_query` accepts the regex pattern, but it has to be a [Rust regex](https://docs.rs/regex/latest/regex/#syntax), not a Ruby `Regexp`. So, instead of `index.regex_query(:description, /hel[lp]/)` you need to use `index.regex_query(:description, "hel[lp]")`. As a side note, the `regex_query` is pretty fast because it uses the [fst crate](https://github.com/BurntSushi/fst) internally.
+
+## Tokenizers
+
+So, we've mentioned tokenizers more than once already. What are they?
+
+Tokenizers is what Tantivy uses to chop your text onto terms to build an inverted index. Then you can search the index by these terms. It's an important concept to understand so that you don't get confused when `index.term_query(:description, "Hello")` returns nothing because `Hello` isn't a term, but `hello` is. You have to extract the terms from the query before searching the index. Currently, only `smart_query` does that for you. Also, the only field type that is tokenized is `text`, so for `string` fields you should use the exact match (i.e. `index.term_query(:title, "Hello")`). 
+
+### Specifying the tokenizer
+
+By default the `simple` tokenizer is used, but you can specify the desired tokenizer globally via index options or locally via field specific options:
+
+```ruby
+en_stemmer = Tantiny::Tokenizer.new(:stemmer)
+ru_stemmer = Tantiny::Tokenizer.new(:stemmer, language: :ru)
+
+Tantiny::Index.new "/tmp/index", tokenizer: en_stemmer do
+  text :description_en
+  text :description_ru, tokenizer: ru_stemmer
+end
+```
+
+### Simple tokenizer
+
+Simple tokenizer chops the text on punctuation and whitespaces, removes long tokens, and lowercases the text.
+
+```ruby
+tokenizer = Tantiny::Tokenizer.new(:simple)
+tokenizer.terms("Hello World!") # ["hello", "world"]
+```
+
+### Stemmer tokenizer
+
+Stemmer tokenizers is exactly like simple tokenizer, but with additional stemming according to the specified language (defaults to English).
+
+```ruby
+tokenizer = Tantiny::Tokenizer.new(:stemmer, language: :ru)
+tokenizer.terms("Привет миру сему!") # ["привет", "мир", "сем"]
+```
+
+Take a look at the [source](https://github.com/baygeldin/tantiny/blob/main/src/helpers.rs) to see what languages are supported.
+
+### Ngram tokenizer
+
+Ngram tokenizer chops your text onto ngrams of specified size.
+
+```ruby
+tokenizer = Tantiny::Tokenizer.new(:ngram, min: 5, max: 10, prefix_only: true)
+tokenizer.terms("Morrowind") # ["Morro", "Morrow", "Morrowi", "Morrowin", "Morrowind"]
+```
+## Retrieving documents
+
+You may have noticed that `search` method returns only documents ids. This is by design. The documents themselves are **not** stored in the index. Tantiny is a minimalistic library, so it tries to keep things simple. If you need to retrieve a full document, use a key-value store like Redis alongside.
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+We use [conventional commits](https://www.conventionalcommits.org) to automatically generate the CHANGELOG, bump the semantic version, and to publish and release the gem. All you need to do is stick to the convention and [CI will take care of everything else](https://github.com/baygeldin/tantiny/blob/main/.github/workflows/release.yml) for you.
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/baygeldin/tantiny.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/bin/console

@@ -0,0 +1,59 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "pry"
+
+require "tantiny"
+
+path = File.join(__dir__, "../tmp")
+en_stem = Tantiny::Tokenizer.new(:stemmer, language: :en)
+
+index = Tantiny::Index.new path, tokenizer: en_stem do
+  id :imdb_id
+  facet :category
+  string :title
+  text :description
+  integer :duration
+  double :rating
+  date :release_date
+end
+
+rio_bravo = OpenStruct.new(
+  imdb_id: "tt0053221",
+  type: '/western/US',
+  title: "Rio Bravo",
+  description: "A small-town sheriff enlists a drunk, a kid and an old man to help him fight off a ruthless cattle baron.",
+  duration: 141,
+  rating: 8.0,
+  release_date: Date.parse("March 18, 1959")
+)
+
+hanabi = {
+  imdb_id: "tt0119250",
+  type: "/crime/Japan",
+  title: "Hana-bi",
+  description: "Nishi leaves the police in the face of harrowing personal and professional difficulties. Spiraling into depression, he makes questionable decisions.",
+  duration: 103,
+  rating: 7.7,
+  release_date: Date.parse("December 1, 1998")
+}
+
+brother = {
+  imdb_id: "tt0118767",
+  type: "/crime/Russia",
+  title: "Brother",
+  description: "An ex-soldier with a personal honor code enters the family crime business in St. Petersburg, Russia.",
+  duration: 99,
+  rating: 7.9,
+  release_date: Date.parse("December 12, 1997")
+}
+
+index << rio_bravo
+index << hanabi
+index << brother
+
+index.commit 
+index.reload
+
+binding.pry

data/bin/setup

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install

data/ext/Rakefile

@@ -0,0 +1,5 @@
+require "thermite/tasks"
+
+project_dir = File.dirname(File.dirname(__FILE__))
+Thermite::Tasks.new(cargo_project_path: project_dir, ruby_project_path: project_dir)
+task default: %w[thermite:build]

data/lib/.rbnext/3.0/tantiny/schema.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Tantiny
+  class Schema
+    attr_reader :default_tokenizer,
+      :id_field,
+      :text_fields,
+      :string_fields,
+      :integer_fields,
+      :double_fields,
+      :date_fields,
+      :facet_fields,
+      :field_tokenizers
+
+    def initialize(tokenizer, &block)
+      @default_tokenizer = tokenizer
+      @id_field = :id
+      @text_fields = []
+      @string_fields = []
+      @integer_fields = []
+      @double_fields = []
+      @date_fields = []
+      @facet_fields = []
+      @field_tokenizers = {}
+
+      instance_exec(&block)
+    end
+
+    def tokenizer_for(field)
+      field_tokenizers[field] || default_tokenizer
+    end
+
+    private
+
+    def id(key) ;  @id_field = key; end
+
+    def string(key) ;  @string_fields << key; end
+
+    def integer(key) ;  @integer_fields << key; end
+
+    def double(key) ;  @double_fields << key; end
+
+    def date(key) ;  @date_fields << key; end
+
+    def facet(key) ;  @facet_fields << key; end
+
+    def text(key, tokenizer: nil)
+      @field_tokenizers[key] = tokenizer if tokenizer
+
+      @text_fields << key
+    end
+  end
+end

data/lib/tantiny/errors.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Tantiny
+  class TantivyError < StandardError; end
+
+  class UnknownField < StandardError
+    def initialize
+      super("Can't find the specified field in the schema.")
+    end
+  end
+
+  class UnknownTokenizer < StandardError
+    def initialize(tokenizer_type)
+      super("Can't find \"#{tokenizer_type}\" tokenizer.")
+    end
+  end
+
+  class UnsupportedRange < StandardError
+    def initialize(range_type)
+      super("#{range_type} range is not supported by range_query.")
+    end
+  end
+
+  class UnsupportedField < StandardError
+    def initialize(field)
+      super("Can't search the \"#{field}\" field with this query.")
+    end
+  end
+end

data/lib/tantiny/helpers.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Tantiny
+  module Helpers
+    def self.timestamp(date)
+      date.to_datetime.iso8601
+    end
+  end
+end

data/lib/tantiny/index.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module Tantiny
+  class Index
+    DEFAULT_INDEX_SIZE = 50_000_000
+    DEFAULT_LIMIT = 10
+
+    def self.new(path, **options, &block)
+      index_size = options[:size] || DEFAULT_INDEX_SIZE
+      default_tokenizer = options[:tokenizer] || Tokenizer.default
+
+      schema = Schema.new(default_tokenizer, &block)
+
+      object = __new(
+        path.to_s,
+        index_size,
+        schema.default_tokenizer,
+        schema.field_tokenizers.transform_keys(&:to_s),
+        schema.text_fields.map(&:to_s),
+        schema.string_fields.map(&:to_s),
+        schema.integer_fields.map(&:to_s),
+        schema.double_fields.map(&:to_s),
+        schema.date_fields.map(&:to_s),
+        schema.facet_fields.map(&:to_s)
+      )
+
+      object.send(:schema=, schema)
+
+      object
+    end
+
+    attr_reader :schema
+
+    def commit
+      __commit
+    end
+
+    def reload
+      __reload
+    end
+
+    def <<(document)
+      __add_document(
+        resolve(document, schema.id_field).to_s,
+        slice_document(document, schema.text_fields) { |v| v.to_s },
+        slice_document(document, schema.string_fields) { |v| v.to_s },
+        slice_document(document, schema.integer_fields) { |v| v.to_i },
+        slice_document(document, schema.double_fields) { |v| v.to_f },
+        slice_document(document, schema.date_fields) { |v| Helpers.timestamp(v) },
+        slice_document(document, schema.facet_fields) { |v| v.to_s }
+      )
+    end
+
+    def delete(id)
+      __delete_document(id.to_s)
+    end
+
+    def search(query, limit: DEFAULT_LIMIT, **smart_query_options)
+      unless query.is_a?(Query)
+        fields = schema.text_fields
+        query = Query.smart_query(self, fields, query.to_s, **smart_query_options)
+      end
+
+      __search(query, limit)
+    end
+
+    # Shortcuts for creating queries:
+    Query::TYPES.each do |query_type|
+      method_name = "#{query_type}_query"
+      define_method(method_name) do |*args, **kwargs|
+        # Ruby 2.6 fix (https://www.ruby-lang.org/en/news/2019/12/12/separation-of-positional-and-keyword-arguments-in-ruby-3-0/)
+        if kwargs.empty?
+          Query.send(method_name, self, *args)
+        else
+          Query.send(method_name, self, *args, **kwargs)
+        end
+      end
+    end
+
+    private
+
+    attr_writer :schema
+
+    def slice_document(document, fields, &block)
+      fields.inject({}) do |hash, field|
+        hash.tap { |h| h[field.to_s] = resolve(document, field) }
+      end.compact.transform_values(&block)
+    end
+
+    def resolve(document, field)
+      document.is_a?(Hash) ? document[field] : document.send(field)
+    end
+  end
+end