melisa 0.1.0

data/lib/melisa/bytes_trie.rb

@@ -0,0 +1,55 @@
+module Melisa
+  VALUE_SEPARATOR = "\xff"
+
+  class BytesTrie < Trie
+    def initialize(hash={}, separator=VALUE_SEPARATOR, opts={})
+      super([], [], opts)
+
+      @sep = separator
+      @sep_c = separator.force_encoding('binary').ord
+
+      add_many(hash, [])
+    end
+
+    def add_many(hash, weights)
+      for key, value in hash
+        push(raw_key(key, value))
+      end
+    end
+
+    def include?(key)
+      super(key + @sep)
+    end
+
+    def get(key)
+      build unless @built
+      agent = Marisa::Agent.new
+      agent.set_query(key + @sep)
+      if @trie.predictive_search(agent)
+        agent_key_value(agent)
+      end
+    end
+    alias :[] :get
+
+    # Search for many results with a given prefix
+    def get_all(key)
+      build unless @built
+      agent = Marisa::Agent.new
+      agent.set_query(key)
+      [].tap do |results|
+        while @trie.predictive_search(agent)
+          results << agent_key_value(agent)
+        end
+      end
+    end
+
+  protected
+    def raw_key(key, value)
+      key + @sep + value
+    end
+
+    def agent_key_value(agent)
+      agent.key_str.split(@sep)[1]
+    end
+  end
+end

data/lib/melisa/int_trie.rb

@@ -0,0 +1,14 @@
+module Melisa
+  class IntTrie < BytesTrie
+  protected
+    def raw_key(key, value)
+      key + @sep + [value.to_i].pack('i*')
+    end
+
+    def agent_key_value(agent)
+      if value = agent.key_str.split(@sep)[1]
+        value.unpack('i*').first
+      end
+    end
+  end
+end

data/lib/melisa/search.rb

@@ -0,0 +1,55 @@
+module Melisa
+  class Search
+    include Enumerable
+    
+    def initialize(trie, prefix)
+      @trie = trie
+      @prefix = prefix
+    end
+
+    def search(prefix)
+      Search.new(@trie, @prefix + prefix)
+    end
+
+    def reset_agent
+      # Reset the agent state so predictive_search iterates through all keys
+      @agent = Marisa::Agent.new
+      @agent.set_query(@prefix)
+    end
+
+    def each(&block)
+      reset_agent
+      # Yield each key
+      yield @agent.key_str while @trie.trie.predictive_search(@agent)
+    end
+
+    def size
+      keys.size
+    end
+
+    def keys
+      @keys ||= [].tap do |arr|
+        reset_agent
+        arr << @agent.key_str while @trie.trie.predictive_search(@agent)
+      end
+    end
+
+    def has_keys?
+      reset_agent
+      return @trie.trie.predictive_search(@agent)
+    end
+
+    def include?(key)
+      a = Marisa::Agent.new
+      a.set_query(key)
+      @trie.trie.lookup(a)
+    end
+
+    def with_prefixes(&block)
+      reset_agent
+      while @trie.trie.common_prefix_search(@agent)
+        block.call(@agent.key_str)
+      end
+    end
+  end
+end

data/lib/melisa/trie.rb

@@ -0,0 +1,96 @@
+require "melisa/base_config_flags"
+require "melisa/search"
+
+module Melisa
+  ImmutableError = Class.new(StandardError)
+
+  class Trie
+    include Enumerable
+
+    attr_reader :trie
+
+    # Initialize a BaseTrie.
+    # @keys    An array of UTF-8 strings
+    # @weights An array of corresponding weights
+    # @opts
+    #   :binary      Boolean, true for a binary Trie, false for text
+    #   :num_tries   An integer from 1 to 127 representing the depth of recursive Tries
+    #   :cache_size  One of [:tiny, :small, :normal, :large, :huge]
+    #   :order       One of [:label, :weight]
+    def initialize(keys=[], weights=[], opts={})
+      @trie = Marisa::Trie.new
+      @keyset = Marisa::Keyset.new
+      @options = opts
+      @built = false
+
+      add_many(keys, weights)
+    end
+
+    def build
+      @trie.build(@keyset, config_flags(@options)) unless @built
+      @built = true
+    end
+
+    def add(key, weight=nil)
+      raise ImmutableError, "Can't add #{key}, Trie already built" if @built
+      self.tap { push(key, weight) }
+    end
+    alias :<< :add
+
+    def add_many(keys, weights)
+      for key, weight in keys.zip(weights)
+        push(key, weight)
+      end
+    end
+
+    def search(prefix)
+      build unless @built
+      Search.new(self, prefix)
+    end
+
+    def each(&block)
+      build unless @built
+      search('').each(&block)
+    end
+
+    def size
+      build unless @built
+      @trie.num_keys()
+    end
+
+    def keys
+      build unless @built
+      search('').keys
+    end
+
+    def has_keys?
+      build unless @built
+      search('').has_keys?
+    end
+
+    def include?(key)
+      build unless @built
+      search('').include?(key)
+    end
+
+    def load(path)
+      self.tap { @trie.load(path); @built = true }
+    end
+
+    def save(path)
+      build unless @built
+      self.tap { @trie.save(path) }
+    end
+
+  protected
+    include BaseConfigFlags
+
+    def push(key, weight=nil)
+      if weight
+        @keyset.push_back(key, weight)
+      else
+        @keyset.push_back(key)
+      end
+    end
+  end
+end

data/lib/melisa/version.rb

@@ -0,0 +1,3 @@
+module Melisa
+  VERSION = '0.1.0'
+end

data/melisa.gemspec

@@ -0,0 +1,36 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'melisa/version'
+
+Gem::Specification.new do |gem|
+  gem.name = "melisa"
+  gem.summary = "Melisa is a Rubyesque wrapper for the Marisa Trie C library"
+  gem.description = "While marisa-trie provides a ruby binding, it is not particularly rubyesque. Melisa fixes that."
+  gem.homepage = "http://github.com/wordtreefoundation/melisa"
+  gem.authors = ['Duane Johnson']
+  gem.email = ['duane.johnson@gmail.com']
+  gem.licenses = ["MIT"]
+
+  gem.files = %w[melisa.gemspec README.md]
+  gem.files += Dir.glob("lib/**/*.rb")
+  gem.files += Dir.glob("spec/**/*")
+
+  # Configure marisa external C library
+  gem.files += Dir.glob("ext/**/*.h")
+  gem.files += Dir.glob("ext/**/*.cc")
+  gem.files += Dir.glob("ext/**/*.cxx")
+  gem.files += Dir.glob("ext/**/extconf.rb")
+  gem.platform = Gem::Platform::RUBY
+  gem.extensions = Dir['ext/**/extconf.rb']
+
+  gem.test_files = Dir.glob("spec/**/*")
+  gem.require_paths = ["lib", "ext"]
+  gem.version = Melisa::VERSION
+  gem.required_ruby_version = '>= 1.9.0'
+
+  gem.add_development_dependency "rake"
+  gem.add_development_dependency "bundler", ">= 1.0.0"
+  gem.add_development_dependency "rspec", "~> 2.6"
+  gem.add_development_dependency "debugger"
+end

data/spec/base_config_flags_spec.rb

@@ -0,0 +1,73 @@
+require_relative "spec_helper"
+
+class ConfigHarness
+  include Melisa::BaseConfigFlags
+end
+
+describe Melisa::BaseConfigFlags do
+  let(:config) { ConfigHarness.new }
+
+  context "config_flags" do
+    it "builds an integer" do
+      config.config_flags.should be_a(Integer)
+    end
+
+    it "accepts a hash of options" do
+      config.config_flags(:binary => false).should == 135683
+      config.config_flags(:binary => true).should == 139779
+    end
+  end
+
+  context "binary_flag" do
+    it "accepts boolean" do
+      config.binary_flag(true).should == Marisa::BINARY_TAIL
+      config.binary_flag(false).should == Marisa::TEXT_TAIL
+      lambda { config.binary_flag(nil) }.should raise_error(ArgumentError)
+    end
+  end
+
+  context "valid_num_tries" do
+    it "must be >= MIN" do
+      min = Marisa::MIN_NUM_TRIES
+      lambda { config.valid_num_tries(min-1) }.should raise_error
+      lambda { config.valid_num_tries(min) }.should_not raise_error
+    end
+
+    it "must be <= MAX" do
+      max = Marisa::MAX_NUM_TRIES
+      lambda { config.valid_num_tries(max+1) }.should raise_error
+      lambda { config.valid_num_tries(max) }.should_not raise_error
+    end
+
+    it "accepts values between MIN and MAX" do
+      value = Marisa::MIN_NUM_TRIES+1
+      config.valid_num_tries(value).should == value
+    end
+
+    it "accepts special :default symbol" do
+      config.valid_num_tries(:default).should == Marisa::DEFAULT_NUM_TRIES
+    end
+  end
+
+  context "lookup_cache_size" do
+    it "must be a valid size" do
+      lambda { config.lookup_cache_size(:tiny) }.should_not raise_error
+      lambda { config.lookup_cache_size(:eentsy_weentsy) }.should raise_error
+    end
+
+    it "returns an integer" do
+      config.lookup_cache_size(:tiny).should be_a(Integer)
+    end
+  end
+
+  context "valid_node_order" do
+    it "must be a valid node order" do
+      lambda { config.valid_node_order(:weight) }.should_not raise_error
+      lambda { config.valid_node_order(:upside_down) }.should raise_error
+    end
+
+    it "returns an integer" do
+      config.valid_node_order(:weight).should be_a(Integer)
+    end
+  end
+end

data/spec/bytes_trie_spec.rb

@@ -0,0 +1,16 @@
+require_relative 'spec_helper'
+
+describe Melisa::BytesTrie do
+  let(:hash) { {'one' => '1', 'two' => '2', 'onetwo' => '3'} }
+  let(:trie) { Melisa::BytesTrie.new(hash) }
+
+  it "stores values" do
+    trie['one'].should == '1'
+    trie['two'].should == '2'
+    trie['onetwo'].should == '3'
+  end
+
+  it "retreives many values by prefix" do
+    trie.get_all('one').should =~ ['1', '3']
+  end
+end

data/spec/int_trie_spec.rb

@@ -0,0 +1,16 @@
+require_relative 'spec_helper'
+
+describe Melisa::IntTrie do
+  let(:hash) { {'one' => 1, 'two' => 2, 'onetwo' => 3} }
+  let(:trie) { Melisa::IntTrie.new(hash) }
+
+  it "stores values" do
+    trie['one'].should == 1
+    trie['two'].should == 2
+    trie['onetwo'].should == 3
+  end
+
+  it "retreives many values by prefix" do
+    trie.get_all('one').should =~ [1, 3]
+  end
+end

data/spec/search_spec.rb

@@ -0,0 +1,29 @@
+require_relative 'spec_helper'
+
+describe Melisa::Search do 
+  let(:keys) { ['one', 'two', 'onetwo'] }
+  let(:trie) { Melisa::Trie.new(keys) }
+  
+  it "iterates with each" do
+    arr = []
+    trie.each do |key|
+      arr << key
+    end
+    arr.should =~ keys
+  end
+
+  it "implements enumerable methods" do
+    trie.all? { |k| k.include? 'o' }.should be_true
+    trie.any? { |k| k.include? 'z' }.should_not be_true
+    trie.map { |k| k.upcase }.should =~ ['ONE', 'TWO', 'ONETWO']
+  end
+
+  it "narrows the search" do
+    subset = trie.search('one')
+    subset.should be_a(Melisa::Search)
+    subset.size.should == 2
+    subset.keys.should =~ ['one', 'onetwo']
+    # subset.map { |k| k.upcase }.should =~ ['ONE', 'ONETWO']
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1 @@
+require 'melisa'

data/spec/trie_spec.rb

@@ -0,0 +1,30 @@
+require_relative "spec_helper"
+require 'tempfile'
+
+describe Melisa::Trie do
+  let(:terms) { ['one', 'two', 'onetwo'] }
+  let(:trie) { Melisa::Trie.new(terms) }
+
+  it "initializes" do
+    trie
+  end
+
+  it "tests for inclusion" do
+    trie.include?('one').should be_true
+    trie.include?('three').should_not be_true
+  end
+
+  it "lists keys" do
+    trie.keys.should =~ ['one', 'two', 'onetwo']
+  end
+
+  it "saves" do
+    tmp = Tempfile.new('melisa')
+    trie.save(tmp.path)
+
+    trie2 = Melisa::Trie.new
+    trie2.load(tmp.path)
+
+    trie2.keys.should =~ ['one', 'two', 'onetwo']
+  end
+end

metadata

@@ -0,0 +1,207 @@
+--- !ruby/object:Gem::Specification
+name: melisa
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Duane Johnson
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-03-30 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.6'
+- !ruby/object:Gem::Dependency
+  name: debugger
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: While marisa-trie provides a ruby binding, it is not particularly rubyesque.
+  Melisa fixes that.
+email:
+- duane.johnson@gmail.com
+executables: []
+extensions:
+- ext/marisa/bindings/ruby/extconf.rb
+extra_rdoc_files: []
+files:
+- melisa.gemspec
+- README.md
+- lib/melisa/base_config_flags.rb
+- lib/melisa/bytes_trie.rb
+- lib/melisa/int_trie.rb
+- lib/melisa/search.rb
+- lib/melisa/trie.rb
+- lib/melisa/version.rb
+- lib/melisa.rb
+- spec/base_config_flags_spec.rb
+- spec/bytes_trie_spec.rb
+- spec/int_trie_spec.rb
+- spec/search_spec.rb
+- spec/spec_helper.rb
+- spec/trie_spec.rb
+- ext/marisa/bindings/marisa-swig.h
+- ext/marisa/bindings/perl/marisa-swig.h
+- ext/marisa/bindings/python/marisa-swig.h
+- ext/marisa/bindings/ruby/marisa-swig.h
+- ext/marisa/lib/marisa/agent.h
+- ext/marisa/lib/marisa/base.h
+- ext/marisa/lib/marisa/exception.h
+- ext/marisa/lib/marisa/grimoire/algorithm/sort.h
+- ext/marisa/lib/marisa/grimoire/algorithm.h
+- ext/marisa/lib/marisa/grimoire/intrin.h
+- ext/marisa/lib/marisa/grimoire/io/mapper.h
+- ext/marisa/lib/marisa/grimoire/io/reader.h
+- ext/marisa/lib/marisa/grimoire/io/writer.h
+- ext/marisa/lib/marisa/grimoire/io.h
+- ext/marisa/lib/marisa/grimoire/trie/cache.h
+- ext/marisa/lib/marisa/grimoire/trie/config.h
+- ext/marisa/lib/marisa/grimoire/trie/entry.h
+- ext/marisa/lib/marisa/grimoire/trie/header.h
+- ext/marisa/lib/marisa/grimoire/trie/history.h
+- ext/marisa/lib/marisa/grimoire/trie/key.h
+- ext/marisa/lib/marisa/grimoire/trie/louds-trie.h
+- ext/marisa/lib/marisa/grimoire/trie/range.h
+- ext/marisa/lib/marisa/grimoire/trie/state.h
+- ext/marisa/lib/marisa/grimoire/trie/tail.h
+- ext/marisa/lib/marisa/grimoire/trie.h
+- ext/marisa/lib/marisa/grimoire/vector/bit-vector.h
+- ext/marisa/lib/marisa/grimoire/vector/flat-vector.h
+- ext/marisa/lib/marisa/grimoire/vector/pop-count.h
+- ext/marisa/lib/marisa/grimoire/vector/rank-index.h
+- ext/marisa/lib/marisa/grimoire/vector/vector.h
+- ext/marisa/lib/marisa/grimoire/vector.h
+- ext/marisa/lib/marisa/iostream.h
+- ext/marisa/lib/marisa/key.h
+- ext/marisa/lib/marisa/keyset.h
+- ext/marisa/lib/marisa/query.h
+- ext/marisa/lib/marisa/scoped-array.h
+- ext/marisa/lib/marisa/scoped-ptr.h
+- ext/marisa/lib/marisa/stdio.h
+- ext/marisa/lib/marisa/trie.h
+- ext/marisa/lib/marisa.h
+- ext/marisa/tests/marisa-assert.h
+- ext/marisa/tools/cmdopt.h
+- ext/marisa/lib/marisa/agent.cc
+- ext/marisa/lib/marisa/grimoire/io/mapper.cc
+- ext/marisa/lib/marisa/grimoire/io/reader.cc
+- ext/marisa/lib/marisa/grimoire/io/writer.cc
+- ext/marisa/lib/marisa/grimoire/trie/louds-trie.cc
+- ext/marisa/lib/marisa/grimoire/trie/tail.cc
+- ext/marisa/lib/marisa/grimoire/vector/bit-vector.cc
+- ext/marisa/lib/marisa/keyset.cc
+- ext/marisa/lib/marisa/trie.cc
+- ext/marisa/tests/base-test.cc
+- ext/marisa/tests/io-test.cc
+- ext/marisa/tests/marisa-test.cc
+- ext/marisa/tests/trie-test.cc
+- ext/marisa/tests/vector-test.cc
+- ext/marisa/tools/cmdopt.cc
+- ext/marisa/tools/marisa-benchmark.cc
+- ext/marisa/tools/marisa-build.cc
+- ext/marisa/tools/marisa-common-prefix-search.cc
+- ext/marisa/tools/marisa-dump.cc
+- ext/marisa/tools/marisa-lookup.cc
+- ext/marisa/tools/marisa-predictive-search.cc
+- ext/marisa/tools/marisa-reverse-lookup.cc
+- ext/marisa/bindings/marisa-swig.cxx
+- ext/marisa/bindings/perl/marisa-swig.cxx
+- ext/marisa/bindings/perl/marisa-swig_wrap.cxx
+- ext/marisa/bindings/python/marisa-swig.cxx
+- ext/marisa/bindings/python/marisa-swig_wrap.cxx
+- ext/marisa/bindings/ruby/marisa-swig.cxx
+- ext/marisa/bindings/ruby/marisa-swig_wrap.cxx
+- ext/marisa/bindings/ruby/extconf.rb
+homepage: http://github.com/wordtreefoundation/melisa
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+- ext
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: 1.9.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Melisa is a Rubyesque wrapper for the Marisa Trie C library
+test_files:
+- spec/base_config_flags_spec.rb
+- spec/bytes_trie_spec.rb
+- spec/int_trie_spec.rb
+- spec/search_spec.rb
+- spec/spec_helper.rb
+- spec/trie_spec.rb
+has_rdoc: