vinted-blurrily 1.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b9c8978b87d66ef94646d643c471b745c84b2489
+  data.tar.gz: 73c63bbd8614d452ee27c16ff3fa5371b8a48a21
+SHA512:
+  metadata.gz: bccb29b9b6665b9e8606ee483b434e2873815a804faf2d0bd800714c9af2b02f3156694b29cd5f66e9e9173b28adc6119703ee14503e5843c18849b0c47c78cb
+  data.tar.gz: 59bcd0faf278887485033c26a19757ad5d15c01de468e94c27b2115cc04f4fcb65280b0fba0057cb8a28bc2f902dbc2a9f879a50e9101464e07f133c1eed0981

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 HouseTrip Ltd.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,213 @@
+# Blurrily — Millisecond fuzzy string matching
+
+This is a fork made solely for the purpose of adding Ruby 2.4 support and making it available in RubyGems.
+
+[![Gem Version](https://badge.fury.io/rb/blurrily.svg)](http://badge.fury.io/rb/blurrily)
+[![Build Status](https://travis-ci.org/mezis/blurrily.svg?branch=master)](https://travis-ci.org/mezis/blurrily)
+[![Dependency Status](https://gemnasium.com/mezis/blurrily.svg)](https://gemnasium.com/mezis/blurrily)
+[![Code Climate](https://codeclimate.com/github/mezis/blurrily.svg)](https://codeclimate.com/github/mezis/blurrily)
+[![Coverage Status](https://coveralls.io/repos/mezis/blurrily/badge.png)](https://coveralls.io/r/mezis/blurrily)
+
+> Show me photos of **Marakech** !
+>
+> Here are some photos of **Marrakesh**, Morroco.
+> Did you mean **Martanesh**, Albania, **Marakkanam**, India, or **Marasheshty**, Romania?
+
+Blurrily finds misspelled, prefix, or partial needles in a haystack of
+strings, quickly. It scales well: its response time is typically 1-2ms on
+user-input datasets and 75-100ms on pathological datasets
+([more](#benchmarks)).
+
+Blurrily is compatible and tested with all MRI Rubies from 1.9.3 to 2.2.0.
+It is tested on Linux 2.6 (32bit and 64bit) and MacOS X 10.8.
+
+Blurrily uses a tweaked [trigram](http://en.wikipedia.org/wiki/N-gram)-based
+approach to find good matches. If you're using ActiveRecord and looking for
+a lightweight (albeit much slower), in-process, Rails-friendly version of
+this, check out [fuzzily](http://github.com/mezis/fuzzily), a Ruby gem to
+perform fuzzy text searching in ActiveRecord.
+
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'blurrily'
+
+Or install it yourself as:
+
+    $ gem install blurrily
+
+## Docker
+
+You can optionally run [Burrily as a Docker Container](https://github.com/mrmattwright/docker-blurrily). Maintained by [MrMattWright](https://github.com/mrmattwright).
+
+## Usage
+
+You can use blurrily as a client/server combination (recommended in
+production), or use the internals standalone.
+
+See the [API Documentation](http://rubydoc.info/github/mezis/blurrily/frames)
+for more details.
+
+### Client/server
+
+Fire up a blurrily server:
+
+    $ blurrily
+
+Open up a console and connect:
+  
+  	$ irb -rubygems
+  	> require 'blurrily/client'
+  	> client = Blurrily::Client.new
+
+Store a needle with a reference:
+
+    > client.put('London', 1337)
+
+Recover a reference form the haystack:
+
+    > client.find('lonndon')
+    #=> [1337]
+
+### Standalone
+
+Create the in-memory database:
+
+    > map = Blurrily::Map.new
+
+Store a needle with a reference:
+
+    > map.put('London', 1337)
+
+Recover a reference form the haystack:
+
+    > map.find('lonndon')
+    #=> [1337]
+
+Save the database to disk:
+
+    > map.save('/var/db/data.trigrams')
+
+Load a previously saved database:
+
+    > map = Blurrily::Map.load('/var/db/data.trigrams')
+
+
+## Caveats
+
+### Diacritics, non-latin languages
+
+Blurrily forms trigrams from the 26 latin letters and a stop character (used
+to model start-of-string and separation between words in multi-word
+strings).
+
+This means that case and diacritrics are completely ignored by Blurrily. For
+instance, *Puy-de-Dôme* is strictly equivalent to *puy de dome*.
+
+It also means that any non-latin input will probably result in garbage data
+and garbage results (although it won't crash).
+
+### Multi-word needles and edge stickyness.
+
+Multi-word needles (say, *New York*) are supported.
+
+The engine always favours matches that begin and end similarly to the
+needle, with a bias to the beginning of the strings.
+
+This is because internally, the string *New York* is turned into this
+sequence of trigrams: `**n`, `*ne`, `new`, `ew*`, `w*y`, `*yo`, `yor`,
+`ork`, `rk*`.
+
+## Production notes
+
+### Memory usage
+
+Blurrily does not store your original strings but rather a flat map of
+references and weights for each trigram in your input strings.
+
+In practice any database will use up a base 560KB for the index header, plus
+128 bits per trigram.
+
+As a rule of thumb idea memory usages is 40MB + 8 times the size of your
+input data, and 50% extra on top during bulk imports (lots of writes to the
+database).
+
+For instance, `/usr/share/dict/words` is a list of 235k English words, and
+weighs 2.5MB. Importing the whole list uses up 75MB of memory, 51MB of which
+are the database.
+
+Note that once a database has been written to disk and loaded from disk,
+memory usage is minimal (560KB per database) as the database file is memory
+mapped. For performance you do need as much free memory as the database
+size.
+
+### Disk usage
+
+Disk usage is almost exactly like memory usage, since database files are
+nothing more than a memory dump.
+
+In the `/usr/share/dict/words` example, on-disk size is 51MB.
+For the whole list of Geonames places, on-disk size is 1.1GB.
+
+### Read v write
+
+Writing to blurrily (with `#put`) is fairly expensive—it's a search engine
+after all, optimized for intensive reads.
+
+Supporting writes means the engine needs to keep a hash table of all
+references around, typically weighing 50% of your total input. This is build
+lazily while writing however; so if you load a database from disk and only
+ever read, you will not incur the memory penalty.
+
+### Saving & backing up
+
+Blurrily saves atomically (writing to a separate file, then using rename(2)
+to overwrite the old file), meaning you should never lose data.
+
+The server does this for you every 60 seconds and when quitting. If using
+`Blurrily::Map` directly, remember that a map loaded from disk is more
+memory efficient that a map in memory, so if your workload is read-heavy,
+you should `.load` after each `#save`.
+
+Backing up comes with a caveat: database files are only portable across
+architectures if endianness and pointer size are the same (tested between
+darwin-x86_64 and linux-amd64).
+
+Database files are very compressible; `bzip2` typically shrinks them to 20%
+of their original size.
+
+
+## Benchmarks
+
+Blurrily is wicked fast, often 100x faster than it's ancestor,
+[fuzzily](http://github.com/mezis/fuzzily). This is because it's a close-to-
+the-metal, single-purpose index using almost exclusively libc primitives. On
+the inside the only expensive operations it performs are
+
+- memcpy(2) lots of data around (selection);
+- mergesort(3) to aggregate/count similar entries (reduction);
+- qsort(3) to order by counts (sort).
+
+It tends to be faster with large datasets on BSD than on Linux because the
+former has fast quicksort and mergesort, wheras the latter only has `qsort`,
+a slower, catch-all sorter. In complexity terms this is because FIND tends
+to be *O(n)* on BSD and *O(n ln n)* on Linux.
+
+Enough talk, here are the graphs. The `LOAD` and `PUT` operations are O(1)
+and take respectively ~10ms and ~100µs on any platform, so  they aren't
+graphed here.
+
+- [FIND latency](/doc/bench-find.png)
+- [SAVE latency](/doc/bench-save.png)
+- [DELETE latency](/doc/bench-delete.png)
+
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/bin/blurrily

@@ -0,0 +1,43 @@
+#!/usr/bin/env ruby
+$PROGRAM_NAME = 'blurrily'
+
+require "blurrily/server"
+require 'optparse'
+require 'ostruct'
+
+options = OpenStruct.new
+
+# Defaults
+options.port = 12021
+options.directory  = '.'
+options.host = '0.0.0.0'
+
+parser = OptionParser.new do |opts|
+  opts.banner = "Usage: #{$PROGRAM_NAME} [options]"
+
+  opts.on("-p", "--port <PORT>", "Bind to PORT, defaults to 12021") do |port|
+    puts 'Port has to be numeric value' and exit unless port =~ /\d+/
+    options.port = port.to_i
+  end
+
+  opts.on("-d", "--directory <DIRECTORY>", "Work in DIRECTORY, defaults to .") do |directory|
+    options.directory = directory
+  end
+
+  opts.on("-b", "--bind <ADDRESS>", "Bind to ADDRESS, defaults to 0.0.0.0") do |address|
+    options.host = address || '0.0.0.0'
+  end
+
+  opts.on("-V", "--version", "Output version") do |address|
+    puts Blurrily::VERSION
+    exit
+  end
+
+  opts.on_tail("-h", "--help", "Show this message") do
+    puts opts
+    exit
+  end
+end
+
+parser.parse!(ARGV)
+Blurrily::Server.new(:host => options.host, :port => options.port, :directory => options.directory).start

data/ext/blurrily/blurrily.h

@@ -0,0 +1,21 @@
+/*
+
+  blurrily.h --
+
+  Helper macros
+
+*/
+
+#ifndef __BLURRILY_H__
+#define __BLURRILY_H__ 1
+
+#define BR_PACKED_STRUCT __attribute__ ((__packed__))
+#define UNUSED(_IDENT) _IDENT __attribute__ ((unused))
+
+#ifdef DEBUG
+  #define LOG(...) fprintf(stderr, __VA_ARGS__)
+#else
+  #define LOG(...)
+#endif
+
+#endif

data/ext/blurrily/extconf.rb

@@ -0,0 +1,21 @@
+require 'mkmf'
+
+PLATFORM = `uname`.strip.upcase
+SHARED_FLAGS = "-DPLATFORM_#{PLATFORM} --std=c99 -Wall -Wextra"
+
+case PLATFORM
+when 'LINUX'
+  # make sure ftruncate is available
+  SHARED_FLAGS << ' -D_XOPEN_SOURCE=700'
+  SHARED_FLAGS << ' -D_GNU_SOURCE=1'
+  # make sure off_t is 64 bit long
+  SHARED_FLAGS << ' -D_FILE_OFFSET_BITS=64'
+end
+
+# production
+$CFLAGS += " #{SHARED_FLAGS} -Os"
+
+# development
+# $CFLAGS += " #{SHARED_FLAGS} -O0 -g"
+
+create_makefile('blurrily/map_ext')

data/ext/blurrily/map_ext.c

@@ -0,0 +1,230 @@
+#include <ruby.h>
+#include <assert.h>
+#include "storage.h"
+#include "blurrily.h"
+
+static VALUE eClosedError = Qnil;
+static VALUE eBlurrilyModule = Qnil;
+
+/******************************************************************************/
+
+static int raise_if_closed(VALUE self)
+{
+  if (rb_ivar_get(self, rb_intern("@closed")) != Qtrue) return 0;
+  rb_raise(eClosedError, "Map was freed");
+  return 1;
+}
+
+static void mark_as_closed(VALUE self)
+{
+  rb_ivar_set(self, rb_intern("@closed"), Qtrue);
+}
+
+/******************************************************************************/
+
+static void blurrily_free(void* haystack)
+{
+  int res = -1;
+
+  if (haystack == NULL) return;
+  res = blurrily_storage_close((trigram_map*) &haystack);
+  assert(res >= 0);
+}
+
+/******************************************************************************/
+
+static void blurrily_mark(void* haystack)
+{
+  if (haystack == NULL) return;
+  blurrily_storage_mark((trigram_map) haystack);
+}
+
+/******************************************************************************/
+
+static VALUE blurrily_new(VALUE class) {
+  VALUE       wrapper  = Qnil;
+  trigram_map haystack = (trigram_map)NULL;
+  int         res      = -1;
+
+  res = blurrily_storage_new(&haystack);
+  if (res < 0) { rb_sys_fail(NULL); return Qnil; }
+
+  wrapper = Data_Wrap_Struct(class, blurrily_mark, blurrily_free, (void*)haystack);
+  rb_obj_call_init(wrapper, 0, NULL);
+  return wrapper;
+}
+
+/******************************************************************************/
+
+static VALUE blurrily_load(VALUE class, VALUE rb_path) {
+  char*       path     = StringValuePtr(rb_path);
+  VALUE       wrapper  = Qnil;
+  trigram_map haystack = (trigram_map)NULL;
+  int         res      = -1;
+
+  res = blurrily_storage_load(&haystack, path);
+  if (res < 0) { rb_sys_fail(NULL); return Qnil; }
+
+  wrapper = Data_Wrap_Struct(class, blurrily_mark, blurrily_free, (void*)haystack);
+  rb_obj_call_init(wrapper, 0, NULL);
+  return wrapper;
+}
+
+/******************************************************************************/
+
+static VALUE blurrily_initialize(VALUE UNUSED(self)) {
+  return Qtrue;
+}
+
+/******************************************************************************/
+
+static VALUE blurrily_put(VALUE self, VALUE rb_needle, VALUE rb_reference, VALUE rb_weight) {
+  trigram_map  haystack  = (trigram_map)NULL;
+  int          res       = -1;
+  char*        needle    = StringValuePtr(rb_needle);
+  uint32_t     reference = NUM2UINT(rb_reference);
+  uint32_t     weight    = NUM2UINT(rb_weight);
+
+  if (raise_if_closed(self)) return Qnil;
+  Data_Get_Struct(self, struct trigram_map_t, haystack);
+
+  res = blurrily_storage_put(haystack, needle, reference, weight);
+  assert(res >= 0);
+
+  return INT2NUM(res);
+}
+
+/******************************************************************************/
+
+static VALUE blurrily_delete(VALUE self, VALUE rb_reference) {
+  trigram_map  haystack  = (trigram_map)NULL;
+  uint32_t     reference = NUM2UINT(rb_reference);
+  int          res       = -1;
+
+  if (raise_if_closed(self)) return Qnil;
+  Data_Get_Struct(self, struct trigram_map_t, haystack);
+
+  res = blurrily_storage_delete(haystack, reference);
+  assert(res >= 0);
+
+  return INT2NUM(res);
+}
+
+/******************************************************************************/
+
+static VALUE blurrily_save(VALUE self, VALUE rb_path) {
+  trigram_map  haystack  = (trigram_map)NULL;
+  int          res       = -1;
+  const char*  path      = StringValuePtr(rb_path);
+
+  if (raise_if_closed(self)) return Qnil;
+  Data_Get_Struct(self, struct trigram_map_t, haystack);
+
+  res = blurrily_storage_save(haystack, path);
+  if (res < 0) rb_sys_fail(NULL);
+
+  return Qnil;
+}
+
+/******************************************************************************/
+
+static VALUE blurrily_find(VALUE self, VALUE rb_needle, VALUE rb_limit) {
+  trigram_map   haystack   = (trigram_map)NULL;
+  int           res        = -1;
+  const char*   needle     = StringValuePtr(rb_needle);
+  int           limit      = NUM2UINT(rb_limit);
+  trigram_match matches    = NULL;
+  VALUE         rb_matches = Qnil;
+
+  if (raise_if_closed(self)) return Qnil;
+  Data_Get_Struct(self, struct trigram_map_t, haystack);
+
+  if (limit <= 0) {
+    // rb_limit = rb_const_get(eBlurrilyModule, rb_intern('LIMIT_DEFAULT'));
+    rb_limit = rb_const_get(eBlurrilyModule, rb_intern("LIMIT_DEFAULT"));
+    limit = NUM2UINT(rb_limit);
+  }
+  matches = (trigram_match) malloc(limit * sizeof(trigram_match_t));
+
+  res = blurrily_storage_find(haystack, needle, limit, matches);
+  assert(res >= 0);
+
+  /* wrap the matches into a Ruby array */
+  rb_matches = rb_ary_new();
+  for (int k = 0; k < res; ++k) {
+    VALUE rb_match = rb_ary_new();
+    rb_ary_push(rb_match, rb_uint_new(matches[k].reference));
+    rb_ary_push(rb_match, rb_uint_new(matches[k].matches));
+    rb_ary_push(rb_match, rb_uint_new(matches[k].weight));
+    rb_ary_push(rb_matches, rb_match);
+  }
+  return rb_matches;
+}
+
+
+/******************************************************************************/
+
+static VALUE blurrily_stats(VALUE self)
+{
+  trigram_map     haystack = (trigram_map)NULL;
+  trigram_stat_t  stats;
+  VALUE           result   = rb_hash_new();
+  int             res      = -1;
+
+  if (raise_if_closed(self)) return Qnil;
+  Data_Get_Struct(self, struct trigram_map_t, haystack);
+
+  res = blurrily_storage_stats(haystack, &stats);
+  assert(res >= 0);
+
+  (void) rb_hash_aset(result, ID2SYM(rb_intern("references")), UINT2NUM(stats.references));
+  (void) rb_hash_aset(result, ID2SYM(rb_intern("trigrams")),   UINT2NUM(stats.trigrams));
+
+  return result;
+}
+
+/******************************************************************************/
+
+static VALUE blurrily_close(VALUE self)
+{
+  trigram_map     haystack = (trigram_map)NULL;
+  int             res      = -1;
+
+  if (raise_if_closed(self)) return Qnil;
+  Data_Get_Struct(self, struct trigram_map_t, haystack);
+
+  res = blurrily_storage_close(&haystack);
+  if (res < 0) rb_sys_fail(NULL);
+
+  DATA_PTR(self) = NULL;
+  mark_as_closed(self);
+  return Qnil;
+}
+
+/******************************************************************************/
+
+void Init_map_ext(void) {
+  VALUE klass  = Qnil;
+
+  /* assume we haven't yet defined blurrily */
+  eBlurrilyModule = rb_define_module("Blurrily");
+  assert(eBlurrilyModule != Qnil);
+
+  klass = rb_define_class_under(eBlurrilyModule, "RawMap", rb_cObject);
+  assert(klass != Qnil);
+
+  eClosedError = rb_define_class_under(klass, "ClosedError", rb_eRuntimeError);
+  assert(klass != Qnil);
+
+  rb_define_singleton_method(klass, "new",  blurrily_new,  0);
+  rb_define_singleton_method(klass, "load", blurrily_load, 1);
+
+  rb_define_method(klass, "initialize", blurrily_initialize, 0);
+  rb_define_method(klass, "put",        blurrily_put,        3);
+  rb_define_method(klass, "delete",     blurrily_delete,     1);
+  rb_define_method(klass, "save",       blurrily_save,       1);
+  rb_define_method(klass, "find",       blurrily_find,       2);
+  rb_define_method(klass, "stats",      blurrily_stats,      0);
+  rb_define_method(klass, "close",      blurrily_close,      0);
+  return;
+}