vinted-blurrily 1.0.2

data/ext/blurrily/storage.h

@@ -0,0 +1,119 @@
+/*
+  
+  storage.h --
+
+  Trigram map creation, persistence, and qurying.
+
+*/
+#ifndef __STORAGE_H__
+#define __STORAGE_H__
+
+#include <inttypes.h>
+#include "tokeniser.h"
+#include "blurrily.h"
+
+struct trigram_map_t;
+typedef struct trigram_map_t* trigram_map;
+
+struct BR_PACKED_STRUCT trigram_match_t {
+  uint32_t reference;
+  uint32_t matches;
+  uint32_t weight;
+};
+typedef struct trigram_match_t trigram_match_t;
+typedef struct trigram_match_t* trigram_match;
+
+typedef struct trigram_stat_t {
+  uint32_t references;
+  uint32_t trigrams;
+
+} trigram_stat_t;
+
+
+/* 
+  Create a new trigram map, resident in memory.
+*/
+int blurrily_storage_new(trigram_map* haystack);
+
+/* 
+  Load an existing trigram map from disk.
+*/
+int blurrily_storage_load(trigram_map* haystack, const char* path);
+
+/* 
+  Release resources claimed by <new> or <open>.
+*/
+int blurrily_storage_close(trigram_map* haystack);
+
+/* 
+  Mark resources managed by Ruby GC.
+*/
+void blurrily_storage_mark(trigram_map haystack);
+
+
+/* 
+  Persist to disk what <blurrily_storage_new> or <blurrily_storage_open>
+  gave you.
+*/
+int blurrily_storage_save(trigram_map haystack, const char* path);
+
+/*
+  Add a new string to the map. <reference> is your identifier for that
+  string, <weight> will be using to discriminate entries that match "as
+  well" when searching.
+
+  If <weight> is zero, it will be replaced by the number of characters in
+  the <needle>.
+
+  Returns positive on success, negative on failure.
+*/
+int blurrily_storage_put(trigram_map haystack, const char* needle, uint32_t reference, uint32_t weight);
+
+/*
+  Check the map for an existing <reference>.
+
+  Returns < 0 on error, 0 if the reference is not found, the number of trigrams
+  for that reference otherwise.
+
+  If <weight> is not NULL, will be set to the weight value passed to the put
+  method on return (is the reference is found).
+
+  If <trigrams> is not NULL, it should point an array <nb_trigrams> long,
+  and up to <nb_trigrams> will be copied into it matching the <needle>
+  originally passed to the put method.
+
+  Not that this is a O(n) method: the whole map will be read.
+*/
+// int blurrily_storage_get(trigram_map haystack, uint32_t reference, uint32_t* weight, int nb_trigrams, trigram_t* trigrams);
+
+/*
+  Remove a <reference> from the map.
+
+  Note that this is very innefective.
+
+  Returns positive on success, negative on failure.
+*/
+int blurrily_storage_delete(trigram_map haystack, uint32_t reference);
+
+/*
+  Return at most <limit> entries matching <needle> from the <haystack>.
+
+  Results are written to <results>. The first results are the ones entries
+  sharing the most trigrams with the <needle>. Amongst entries with the same
+  number of matches, the lightest ones (lowest <weight>) will be returned
+  first.
+
+  <results> should be allocated by the caller.
+
+  Returns number of matches on success, negative on failure.
+*/
+int blurrily_storage_find(trigram_map haystack, const char* needle, uint16_t limit, trigram_match results);
+
+/*
+  Copies metadata into <stats>
+
+  Returns positive on success, negative on failure.
+*/
+int blurrily_storage_stats(trigram_map haystack, trigram_stat_t* stats);
+
+#endif

data/ext/blurrily/tokeniser.c

@@ -0,0 +1,126 @@
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#include <math.h>
+#include "tokeniser.h"
+#include "blurrily.h"
+
+
+/******************************************************************************/
+
+static int ipow(int a, int b)
+{
+  int result = 1;
+  
+  while (b-- > 0) result = result * a;
+  return result;
+}
+
+/******************************************************************************/
+
+static void string_to_code(const char* input, trigram_t *output)
+{
+  trigram_t result = 0;
+
+  for (int k = 0 ; k < 3; ++k) {
+    if (input[k] == '*' || input[k] < 'a' || input[k] > 'z') continue;
+    result += ipow(TRIGRAM_BASE, k) * (input[k] - 'a' + 1);
+  }
+
+  *output = result;
+}
+
+/******************************************************************************/
+
+static void code_to_string(trigram_t input, char* output)
+{
+  for (int k = 0 ; k < 3; ++k) {
+    uint16_t elem = input / ipow(TRIGRAM_BASE, k) % TRIGRAM_BASE;
+    if (elem == 0) {
+      output[k] = '*';
+    } else {
+      output[k] = ('a' + elem - 1);
+    }
+  }
+  output[3] = 0;
+}
+
+/******************************************************************************/
+
+static int blurrily_compare_trigrams(const void* left_p, const void* right_p)
+{
+  trigram_t* left  = (trigram_t*)left_p;
+  trigram_t* right = (trigram_t*)right_p;
+  return (int)*left - (int)*right;
+}
+
+/******************************************************************************/
+
+int blurrily_tokeniser_parse_string(const char* input, trigram_t* output)
+{
+  size_t length     = strlen(input);
+  char*  normalized = (char*) malloc(length+5);
+  size_t duplicates = 0;
+
+  snprintf(normalized, length+4, "**%s*", input);
+
+  /* replace spaces with '*' */
+  for (size_t k = 0; k < length+3; ++k) {
+    if (normalized[k] == ' ') normalized[k] = '*';
+  }
+
+  /* compute trigrams */
+  for (size_t k = 0; k <= length; ++k) {
+    string_to_code(normalized+k, output+k);
+  }
+
+  /* print results */
+  LOG("-- normalization\n");
+  LOG("%s -> %s\n", input, normalized);
+  LOG("-- tokenisation\n");
+  for (size_t k = 0; k <= length; ++k) {
+    char res[4];
+
+    code_to_string(output[k], res);
+
+    LOG("%c%c%c -> %d -> %s\n",
+      normalized[k], normalized[k+1], normalized[k+2],
+      output[k], res
+    );
+  }
+
+  /* sort */
+  qsort((void*)output, length+1, sizeof(trigram_t), &blurrily_compare_trigrams);
+
+  /* remove duplicates */
+  for (size_t k = 1; k <= length; ++k) {
+    trigram_t* previous = output + k - 1;
+    trigram_t* current  = output + k;
+
+    if (*previous == *current) {
+      *previous = 32768;
+      ++duplicates;
+    }
+  }
+
+  /* compact */
+  qsort((void*)output, length+1, sizeof(trigram_t), &blurrily_compare_trigrams);
+
+  /* print again */
+  LOG("-- after sort/compact\n");
+  for (size_t k = 0; k <= length-duplicates; ++k) {
+    char res[4];
+    code_to_string(output[k], res);
+    LOG("%d -> %s\n", output[k], res);
+  }
+
+  free((void*)normalized);
+  return (int) (length + 1 - duplicates);
+}
+
+/******************************************************************************/
+
+int blurrily_tokeniser_trigram(trigram_t UNUSED(input), char* UNUSED(output))
+{
+  return 0;
+}

data/ext/blurrily/tokeniser.h

@@ -0,0 +1,46 @@
+/*
+  
+  tokeniser.h --
+
+  Split a string into an array of trigrams.
+
+  The input string should be only lowercase latin letters and spaces
+  (convert using iconv).
+
+  Each trigram is a three-symbol tuple consisting of latters and the
+  "epsilon" character used to represent spaces and beginning-of-word/end-of-
+  word anchors.
+
+  Each trigram is represented by a 16-bit integer.
+
+*/
+#ifndef __TOKENISER_H__
+#define __TOKENISER_H__
+
+#include <inttypes.h>
+
+#define TRIGRAM_BASE 28
+
+typedef uint16_t trigram_t;
+
+/* 
+  Parse the <input> string and store the result in <ouput>.
+  <output> must be allocated by the caller and provide at least as many slots
+  as characters in <input>, plus one.
+  (not all will be necessarily be filled)
+
+  Returns the number of trigrams on success, a negative number on failure.
+*/
+int blurrily_tokeniser_parse_string(const char* input, trigram_t* output);
+
+
+/*
+  Given an <input> returns a string representation of the trigram in <output>.
+  <output> must be allocated by caller and will always be exactly 3
+  <characters plus NULL.
+
+  Returns positive on success, negative on failure.
+*/
+int blurrily_tokeniser_trigram(trigram_t input, char* output);
+
+#endif

data/lib/blurrily.rb

@@ -0,0 +1 @@
+require 'blurrily/version'

data/lib/blurrily/client.rb

@@ -0,0 +1,136 @@
+# encoding: utf-8
+
+require 'socket'
+require 'ipaddr'
+require 'blurrily/defaults'
+
+module Blurrily
+  class Client
+    Error = Class.new(RuntimeError)
+
+    # Initialize a new {Blurrily::Client} connection to {Blurrily::Server}.
+    #
+    # @param host IP Address or FQDN of the Blurrily::Server.
+    #           Defaults to Blurrily::DEFAULT_HOST.
+    # @param port Port Blurrily::Server is listening on.
+    #           Defaults to Blurrily::DEFAULT_PORT.
+    # @param db_name Name of the data store being targeted.
+    #           Defaults to Blurrily::DEFAULT_DATABASE.
+    #
+    # Examples
+    #
+    # ```
+    # Blurrily::Client.new('127.0.0.1', 12021, 'location_en')
+    # # => #<Blurrily::Client:0x007fcd0d33e708 @host="127.0.0.1", @port=12021, @db_name="location_en">
+    # ```
+    #
+    # @returns the instance of {Blurrily::Client}
+    def initialize(options = {})
+      @host    = options.fetch(:host,     DEFAULT_HOST)
+      @port    = options.fetch(:port,     DEFAULT_PORT)
+      @db_name = options.fetch(:db_name,  DEFAULT_DATABASE)
+    end
+
+    # Find record references based on a given string (needle)
+    #
+    # @param needle The string you're searching for matches on.
+    #          Must not contain tabs.
+    #          Required
+    # @param limit  Limit the number of results retruned (default: 10).
+    #          Must be numeric.
+    #          Optional
+    #
+    # Examples
+    #
+    # ```
+    # @client.find('London')
+    # # => [[123,6,3],[124,5,3]...]
+    # ```
+    #
+    # @returns an Array of matching [`ref`,`score`,`weight`] ordered by score. `ref` is the identifying value of the original record.
+    # Note that unless modified, `weight` is simply the string length.
+    def find(needle, limit = nil)
+      limit ||= LIMIT_DEFAULT
+      check_valid_needle(needle)
+      raise(ArgumentError, "LIMIT value must be in #{LIMIT_RANGE}") unless LIMIT_RANGE.include?(limit)
+
+      cmd = ["FIND", @db_name, needle, limit]
+      send_cmd_and_get_results(cmd).map(&:to_i).each_slice(3).to_a
+    end
+
+    # Index a given record.
+    #
+    # @param db_name The name of the data store being targeted. Required
+    # @param needle The string you wish to index. Must not contain tabs. Required
+    # @param ref The indentifying value of the record being indexed. Must be numeric. Required
+    # @param weight Weight of this particular reference. Default 0. Don't change unless you know what you're doing. Optional.
+    #
+    # Examples
+    #
+    # ```
+    # @client.put('location_en', 'London', 123, 0)
+    # # => OK
+    # ```
+    #
+    # @returns something to let you know that all is well.
+    def put(needle, ref, weight = 0)
+      check_valid_needle(needle)
+      check_valid_ref(ref)
+      raise(ArgumentError, "WEIGHT value must be in #{WEIGHT_RANGE}") unless WEIGHT_RANGE.include?(weight)
+
+      cmd = ["PUT", @db_name, needle, ref, weight]
+      send_cmd_and_get_results(cmd)
+      return
+    end
+
+    def delete(ref)
+      check_valid_ref(ref)
+      cmd = ['DELETE', @db_name, ref]
+      send_cmd_and_get_results(cmd)
+      return
+    end
+
+    def clear()
+      send_cmd_and_get_results(['CLEAR', @db_name])
+      return
+    end
+
+
+    private
+
+
+    PORT_RANGE = 1025..32768
+
+    def check_valid_needle(needle)
+      raise(ArgumentError, "bad needle") if !needle.kind_of?(String) || needle.empty? || needle.include?("\t")
+    end
+
+    def check_valid_ref(ref)
+      raise(ArgumentError, "REF value must be in #{REF_RANGE}") unless REF_RANGE.include?(ref)
+    end
+
+
+    def connection
+      @connection ||= TCPSocket.new(@host, @port)
+    end
+
+    def send_cmd_and_get_results(argv)
+      output = argv.join("\t")
+      connection.puts output
+      input = connection.gets
+      case input
+      when "OK\n"
+        return []
+      when /^OK\t(.*)\n/
+        return $1.split("\t")
+      when /^ERROR\t(.*)\n/
+        raise Error, $1
+      when nil
+        raise Error, 'Server disconnected'
+      else
+        raise Error, 'Server did not respect protocol'
+      end
+    end
+
+  end
+end

data/lib/blurrily/command_processor.rb

@@ -0,0 +1,53 @@
+# encoding: utf-8
+require 'blurrily/defaults'
+
+module Blurrily
+  class CommandProcessor
+    ProtocolError = Class.new(StandardError)
+
+    def initialize(map_group)
+      @map_group = map_group
+    end
+
+    def process_command(line)
+      command, map_name, *args = line.split(/\t/)
+      raise ProtocolError, 'Unknown command' unless COMMANDS.include? command
+      raise ProtocolError, 'Invalid database name' unless map_name =~ /^[a-z_]+$/
+      result = send("on_#{command}", map_name, *args)
+      ['OK', *result].compact.join("\t")
+    rescue ArgumentError, ProtocolError => e
+      ['ERROR', e.message].join("\t")
+    end
+
+    private
+
+    COMMANDS = %w(FIND PUT DELETE CLEAR)
+
+    def on_PUT(map_name, needle, ref, weight = nil)
+      raise ProtocolError, 'Invalid reference' unless ref =~ /^\d+$/ && REF_RANGE.include?(ref.to_i)
+      raise ProtocolError, 'Invalid weight'    unless weight.nil? || (weight =~ /^\d+$/ && WEIGHT_RANGE.include?(weight.to_i))
+
+      @map_group.map(map_name).put(*[needle, ref.to_i, weight.to_i].compact)
+      return
+    end
+
+    def on_DELETE(map_name, ref)
+      raise ProtocolError, 'Invalid reference' unless ref =~ /^\d+$/ && REF_RANGE.include?(ref.to_i)
+
+      @map_group.map(map_name).delete(ref.to_i)
+      return
+    end
+
+    def on_FIND(map_name, needle, limit = nil)
+      raise ProtocolError, 'Limit must be a number' if limit && !LIMIT_RANGE.include?(limit.to_i)
+
+      results = @map_group.map(map_name).find(*[needle, limit && limit.to_i].compact)
+      return results.flatten
+    end
+
+    def on_CLEAR(map_name)
+      @map_group.clear(map_name)
+      return
+    end
+  end
+end