blurrily 0.0.1 → 0.1.0

data/ext/blurrily/storage.h

@@ -42,6 +42,12 @@ int blurrily_storage_load(trigram_map* haystack, const char* path);
 */
 int blurrily_storage_close(trigram_map* haystack);
 
+/* 
+  Mark resources managed by Ruby GC.
+*/
+void blurrily_storage_mark(trigram_map haystack);
+
+
 /* 
   Persist to disk what <blurrily_storage_new> or <blurrily_storage_open>
   gave you.

data/ext/blurrily/tokeniser.c

@@ -3,7 +3,6 @@
 #include <stdio.h>
 #include <math.h>
 #include "tokeniser.h"
-#include "log.h"
 #include "blurrily.h"
 
 
@@ -59,19 +58,19 @@ static int blurrily_compare_trigrams(const void* left_p, const void* right_p)
 
 int blurrily_tokeniser_parse_string(const char* input, trigram_t* output)
 {
-  int   length     = strlen(input);
-  char* normalized = (char*) malloc(length+5);
-  int   duplicates = 0;
+  size_t length     = strlen(input);
+  char*  normalized = (char*) malloc(length+5);
+  size_t duplicates = 0;
 
   snprintf(normalized, length+4, "**%s*", input);
 
   /* replace spaces with '*' */
-  for (int k = 0; k < length+3; ++k) {
+  for (size_t k = 0; k < length+3; ++k) {
     if (normalized[k] == ' ') normalized[k] = '*';
   }
 
   /* compute trigrams */
-  for (int k = 0; k <= length; ++k) {
+  for (size_t k = 0; k <= length; ++k) {
     string_to_code(normalized+k, output+k);
   }
 
@@ -79,7 +78,7 @@ int blurrily_tokeniser_parse_string(const char* input, trigram_t* output)
   LOG("-- normalization\n");
   LOG("%s -> %s\n", input, normalized);
   LOG("-- tokenisation\n");
-  for (int k = 0; k <= length; ++k) {
+  for (size_t k = 0; k <= length; ++k) {
     char res[4];
 
     code_to_string(output[k], res);
@@ -94,7 +93,7 @@ int blurrily_tokeniser_parse_string(const char* input, trigram_t* output)
   qsort((void*)output, length+1, sizeof(trigram_t), &blurrily_compare_trigrams);
 
   /* remove duplicates */
-  for (int k = 1; k <= length; ++k) {
+  for (size_t k = 1; k <= length; ++k) {
     trigram_t* previous = output + k - 1;
     trigram_t* current  = output + k;
 
@@ -109,14 +108,14 @@ int blurrily_tokeniser_parse_string(const char* input, trigram_t* output)
 
   /* print again */
   LOG("-- after sort/compact\n");
-  for (int k = 0; k <= length-duplicates; ++k) {
+  for (size_t k = 0; k <= length-duplicates; ++k) {
     char res[4];
     code_to_string(output[k], res);
     LOG("%d -> %s\n", output[k], res);
   }
 
   free((void*)normalized);
-  return length+1 - duplicates;
+  return (int) (length + 1 - duplicates);
 }
 
 /******************************************************************************/

data/lib/blurrily.rb

@@ -1,3 +1 @@
-require "blurrily/map_ext"
-require "blurrily/map"
-require "blurrily/version"
+require 'blurrily/version'

data/lib/blurrily/client.rb

@@ -0,0 +1,129 @@
+# encoding: utf-8
+
+require 'socket'
+require 'ipaddr'
+require 'blurrily/defaults'
+
+module Blurrily
+  class Client
+    Error = Class.new(RuntimeError)
+
+    # Public: Initialize a new Blurrily::Client connection to Blurrily::Server.
+    #
+    # host    - IP Address or FQDN of the Blurrily::Server.
+    #           Defaults to Blurrily::DEFAULT_HOST.
+    # port    - Port Blurrily::Server is listening on.
+    #           Defaults to Blurrily::DEFAULT_PORT.
+    # db_name - Name of the data store being targeted.
+    #           Defaults to Blurrily::DEFAULT_DATABASE.
+    #
+    # Examples
+    #
+    #  Blurrily::Client.new('127.0.0.1', 12021, 'location_en')
+    #  # => #<Blurrily::Client:0x007fcd0d33e708 @host="127.0.0.1", @port=12021, @db_name="location_en">
+    #
+    # Returns the instance of Blurrily::Client
+    def initialize(options = {})
+      @host    = options.fetch(:host,     DEFAULT_HOST)
+      @port    = options.fetch(:port,     DEFAULT_PORT)
+      @db_name = options.fetch(:db_name,  DEFAULT_DATABASE)
+    end
+
+    # Public: Find record references based on a given string (needle)
+    #
+    # needle - The string you're searching for matches on.
+    #          Must not contain tabs.
+    #          Required
+    # limit  - Limit the number of results retruned (default: 10).
+    #          MUST be numeric.
+    #          Optional
+    #
+    # Examples
+    #
+    #  @client.find('London')
+    #  # => [[123,6,3],[124,5,3]...]
+    #
+    # Returns an Array of matching [REF,SCORE,WEIGHT] ordered by score. REF is the identifying value of the original record.
+    def find(needle, limit = nil)
+      limit ||= LIMIT_DEFAULT
+      check_valid_needle(needle)
+      raise(ArgumentError, "LIMIT value must be in #{LIMIT_RANGE}") unless LIMIT_RANGE.include?(limit)
+
+      cmd = ["FIND", @db_name, needle, limit]
+      send_cmd_and_get_results(cmd).map(&:to_i)
+    end
+
+    # Public: Index a given record.
+    #
+    # db_name - The name of the data store being targeted. Required
+    # needle  - The string you wish to index. Must not contain tabs. Required
+    # ref     - The indentifying value of the record being indexed. Must be numeric. Required
+    # weight  - Weight of this particular reference. Default 0. Don't change unless you know what you're doing. Optional.
+    #
+    # Examples
+    #
+    #  @client.put('location_en', 'London', 123, 0)
+    #  # => OK
+    #
+    # Returns something to let you know that all is well.
+    def put(needle, ref, weight = 0)
+      check_valid_needle(needle)
+      check_valid_ref(ref)
+      raise(ArgumentError, "WEIGHT value must be in #{WEIGHT_RANGE}") unless WEIGHT_RANGE.include?(weight)
+
+      cmd = ["PUT", @db_name, needle, ref, weight]
+      send_cmd_and_get_results(cmd)
+      return
+    end
+
+    def delete(ref)
+      check_valid_ref(ref)
+      cmd = ['DELETE', @db_name, ref]
+      send_cmd_and_get_results(cmd)
+      return
+    end
+
+    def clear()
+      send_cmd_and_get_results(['CLEAR', @db_name])
+      return
+    end
+
+
+    private
+
+
+    PORT_RANGE = 1025..32768
+
+    def check_valid_needle(needle)
+      raise(ArgumentError, "bad needle") if !needle.kind_of?(String) || needle.empty? || needle.include?("\t")
+    end
+
+    def check_valid_ref(ref)
+      raise(ArgumentError, "REF value must be in #{REF_RANGE}") unless REF_RANGE.include?(ref)
+    end
+
+
+    def connection
+      @connection ||= TCPSocket.new(@host, @port)
+    end
+
+    def send_cmd_and_get_results(argv)
+      output = argv.join("\t")
+      connection.puts output
+      input = connection.gets
+      case input
+      when "OK\n"
+        return []
+      when /^OK\t(.*)\n/
+        return $1.split("\t")
+      when /^ERROR\t(.*)\n/
+        raise Error, $1
+      when nil
+        raise Error, 'Server disconnected'
+      else
+        raise Error, 'Server did not respect protocol'
+      end
+    end
+
+  end
+end

data/lib/blurrily/command_processor.rb

@@ -0,0 +1,54 @@
+# encoding: utf-8
+require 'blurrily/defaults'
+
+module Blurrily
+  class CommandProcessor
+    ProtocolError = Class.new(StandardError)
+
+    def initialize(map_group)
+      @map_group = map_group
+    end
+
+    def process_command(line)
+      command, map_name, *args = line.split(/\t/)
+      raise ProtocolError, 'Unknown command' unless COMMANDS.include? command
+      raise ProtocolError, 'Invalid database name' unless map_name =~ /^[a-z_]+$/
+      result = send("on_#{command}", map_name, *args)
+      ['OK', *result].compact.join("\t")
+    rescue ArgumentError, ProtocolError => e
+      ['ERROR', e.message].join("\t")
+    end
+
+    private
+
+    COMMANDS = %w(FIND PUT DELETE CLEAR)
+
+    def on_PUT(map_name, needle, ref, weight = nil)
+      raise ProtocolError, 'Invalid reference' unless ref =~ /^\d+$/ && REF_RANGE.include?(ref.to_i)
+      raise ProtocolError, 'Invalid weight'    unless weight.nil? || (weight =~ /^\d+$/ && WEIGHT_RANGE.include?(weight.to_i))
+
+      @map_group.map(map_name).put(*[needle, ref.to_i, weight.to_i].compact)
+      return
+    end
+
+    def on_DELETE(map_name, ref)
+      raise ProtocolError, 'Invalid reference' unless ref =~ /^\d+$/ && REF_RANGE.include?(ref.to_i)
+
+      @map_group.map(map_name).delete(ref.to_i)
+      return
+    end
+
+    def on_FIND(map_name, needle, limit = nil)
+      raise ProtocolError, 'Limit must be a number' if limit && !LIMIT_RANGE.include?(limit.to_i)
+
+      results = @map_group.map(map_name).find(*[needle, limit && limit.to_i].compact)
+      refs = results.map{ |result| result.first }
+      return refs
+    end
+
+    def on_CLEAR(map_name)
+      @map_group.clear(map_name)
+      return
+    end
+  end
+end

data/lib/blurrily/defaults.rb

@@ -0,0 +1,10 @@
+module Blurrily
+  DEFAULT_HOST     = 'localhost'
+  DEFAULT_PORT     = 12021
+  DEFAULT_DATABASE = 'words'
+
+  LIMIT_DEFAULT = 10
+  LIMIT_RANGE   = 1..1024
+  REF_RANGE     = 1..(1<<31)
+  WEIGHT_RANGE  = 0..(1<<31)
+end

data/lib/blurrily/map.rb

@@ -1,10 +1,12 @@
 require 'blurrily/map_ext'
-require 'active_support/all' # fixme: we only need enough to get mb_chars and alias_method_chain in
+require 'active_support/core_ext/module/aliasing' # alias_method_chain
+require 'active_support/core_ext/string/multibyte' # mb_chars
 
 module Blurrily
   Map.class_eval do
 
-    def put_with_string_normalize(needle, reference, weight=0)
+    def put_with_string_normalize(needle, reference, weight=nil)
+      weight ||= 0
       needle = normalize_string needle
       put_without_string_normalize(needle, reference, weight)
     end
@@ -25,7 +27,8 @@ module Blurrily
     def normalize_string(needle)
       result = needle.downcase
       unless result =~ /^([a-z ])+$/
-        result = result.mb_chars.normalize(:kd).gsub(/[^\x00-\x7F]/,'').to_s.gsub(/[^a-z]/,' ')
+        result = ActiveSupport::Multibyte::Chars.new(result).mb_chars.normalize(:kd).gsub(/[^\x00-\x7F]/,'').to_s.gsub(/[^a-z]/,' ')
+        # result = result.mb_chars.normalize(:kd).gsub(/[^\x00-\x7F]/,'').to_s.gsub(/[^a-z]/,' ')
       end
       result.gsub(/\s+/,' ').strip
     end

data/lib/blurrily/map_group.rb

@@ -0,0 +1,39 @@
+require 'pathname'
+require 'blurrily/map'
+
+module Blurrily
+  class MapGroup
+
+    def initialize(directory = nil)
+      @directory = Pathname.new(directory || Dir.pwd)
+      @maps = {}
+    end
+
+    def map(name)
+      @maps[name] ||= load_map(name) || Map.new
+    end
+
+    def save
+      @directory.mkpath
+      @maps.each do |name, map|
+        map.save(path_for(name).to_s)
+      end
+    end
+
+    def clear(name)
+      @maps[name] = Map.new
+    end
+
+    private
+
+    def load_map(name)
+      Map.load(path_for(name).to_s)
+    rescue Errno::ENOENT
+      nil
+    end
+
+    def path_for(name)
+      @directory.join("#{name}.trigrams")
+    end
+  end
+end

data/lib/blurrily/server.rb

@@ -0,0 +1,49 @@
+require 'eventmachine'
+require 'blurrily/defaults'
+require 'blurrily/command_processor'
+require 'blurrily/map_group'
+
+module Blurrily
+  class Server
+
+    def initialize(options)
+      @host      = options.fetch(:host,      '0.0.0.0')
+      @port      = options.fetch(:port,      Blurrily::DEFAULT_PORT)
+      directory  = options.fetch(:directory, Dir.pwd)
+
+      @map_group = MapGroup.new(directory)
+      @command_processor = CommandProcessor.new(@map_group)
+    end
+
+    def start
+      EventMachine.run do
+        # hit Control + C to stop
+        Signal.trap("INT")  { EventMachine.stop }
+        Signal.trap("TERM") { EventMachine.stop }
+
+        saver = proc { @map_group.save }
+        EventMachine.add_periodic_timer(60, &saver)
+        EventMachine.add_shutdown_hook(&saver)
+        Signal.trap("USR1", &saver)
+
+        EventMachine.start_server(@host, @port, Handler, @command_processor)
+      end
+    end
+
+    private
+
+    module Handler
+      def initialize(processor)
+        @processor = processor
+      end
+
+      def receive_data(data)
+        data.split("\n").each do |line|
+          output = @processor.process_command(line.strip)
+          output << "\n"
+          send_data(output)
+        end
+      end
+    end
+  end
+end

data/lib/blurrily/version.rb

@@ -1,3 +1,3 @@
 module Blurrily
-  VERSION = "0.0.1"
+  VERSION = "0.1.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: blurrily
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.1.0
 platform: ruby
 authors:
 - Julien Letessier
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-27 00:00:00.000000000 Z
+date: 2013-04-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -150,28 +150,105 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: benchmark-ips
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rb-fsevent
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: terminal-notifier-guard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Native fuzzy string search
 email:
 - julien.letessier@gmail.com
-executables: []
+executables:
+- blurrily
 extensions:
 - ext/blurrily/extconf.rb
 extra_rdoc_files: []
 files:
+- lib/blurrily/client.rb
+- lib/blurrily/command_processor.rb
+- lib/blurrily/defaults.rb
 - lib/blurrily/map.rb
+- lib/blurrily/map_group.rb
 - lib/blurrily/server.rb
 - lib/blurrily/version.rb
 - lib/blurrily.rb
 - ext/blurrily/map_ext.c
+- ext/blurrily/search_tree.c
 - ext/blurrily/storage.c
 - ext/blurrily/tokeniser.c
 - ext/blurrily/blurrily.h
-- ext/blurrily/log.h
+- ext/blurrily/search_tree.h
 - ext/blurrily/storage.h
 - ext/blurrily/tokeniser.h
 - ext/blurrily/extconf.rb
 - README.md
 - LICENSE.txt
+- bin/blurrily
 homepage: http://github.com/mezis/blurrily
 licenses: []
 metadata: {}