text_nlp 0.0.0 → 0.0.1

data/.gitignore

@@ -7,4 +7,5 @@ tmp/**/*
 bin/*
 vendor/gems/*
 !vendor/gems/cache/
-.sass-cache/*
+.sass-cache/*
+*.gem

data/lib/text_nlp/expressions.rb

@@ -0,0 +1,75 @@
+# encoding: UTF-8
+
+class TextNlp
+  class Expressions
+    
+    attr_accessor :values
+    
+    def initialize(expressions = [])
+      @root, @values = {}, []
+      expressions.each { |expr| self << expr }
+    end
+    
+    def <<(expression)
+      node = @root
+      expression = expression.normalize
+      @values << expression
+      tokens = expression.tokenize
+      tokens_count = tokens.size
+      tokens.each_with_index do |token,i|
+        unless node.key?(token)
+          node[token] = {}
+          node[token][:parent] = node
+        end
+        node = node[token]
+        if (i == (tokens_count-1)) # leaf
+          node[:leaf] = 1
+        end
+      end
+    end
+    
+    def any?(text)
+      (find(text).size > 0)
+    end
+    
+    def expressionize(text)
+      expressions = find(text).sort { |e1,e2| e2.tokenize.size <=> e1.tokenize.size }
+      text = text.tokenize.join(',')
+      expressions.each { |expr| text.gsub!(expr.tokenize.join(','), expr) }
+      text.split(',')
+    end
+    
+    def find(text)
+      find_expressions(0,text.normalize.tokenize.map { |t| t })
+    end
+  
+    private
+    def find_expressions(start_index, tokens, expressions = [])
+      node, leaf, expr = @root, false, []
+      tokens[start_index..-1].each_with_index do |token,i|
+        if (node.key?(token))
+          node = node[token]
+          expr << token
+        else
+          while (expr.size > 0 && node)
+            if node.key?(:leaf)
+              expressions << expr.join(' ')
+              break
+            end
+            expr.pop
+            node = node[:parent]
+          end
+          break
+        end
+      end
+      start_index = expr.size > 0 ? (start_index + expr.size) : (start_index + 1)
+      if (start_index <= (tokens.size - 1))
+        find_expressions(start_index, tokens, expressions)
+      else
+        expressions << expr.join(' ') if (expr.size > 0 && node.key?(:leaf) )
+      end
+      expressions
+    end
+    
+  end
+end

data/lib/text_nlp/normalizer.rb

@@ -1,9 +1,7 @@
 # encoding: UTF-8
 
 class TextNlp
-  class Normalizer
-    def initialize
-    end
+  class Normalizer    
     def normalize(text)
       text.downcase!
       text.tr!("éèàçîêô","eeacieo")

data/lib/text_nlp/pattern.rb

@@ -0,0 +1,123 @@
+# encoding: UTF-8
+
+class TextNlp
+  class Pattern
+    
+    attr_reader :root
+    
+    def initialize(root_or_string = nil)
+      if (root_or_string.is_a?(String))
+        @root = parse(root_or_string)  
+      else
+        @root = root_or_string
+      end
+    end
+    
+    def <<(node)
+      @root << node
+    end
+    
+    def match?(text)
+      @root.evaluate(text)
+    end
+    
+    private
+    def parse(expr)      
+      operators = ['||','&&']
+      current_expression, node, opened, closed = '', nil, 0, 0
+      expr.chars.each_with_index do |char,i|
+        if (char == '(')
+          opened += 1
+          current_expression << char if ((opened - closed) > 1)
+        elsif (char == ')')
+          closed += 1
+          current_expression << char if ((opened - closed) > 0)
+        elsif ((opened == closed) && (operators.include?(expr[i-1..i])))          
+          node = operator_node(expr[i-1..i])
+          node << parse(current_expression[0..-2])
+          node << parse(expr[i+1..-1])
+          break;
+        else          
+          current_expression << char          
+        end
+      end
+      unless node
+        if (current_expression.match(/\|{2}|&{2}/))          
+          node = parse(current_expression)
+        else
+          node = current_expression[0..0] == '!' ? Not.new(current_expression[1..-1]) : Unary.new(current_expression)
+        end
+      end
+      node
+    end
+    
+    def operator_node(operator)
+      node = case operator
+        when '||' then Or.new
+        when '&&' then And.new
+      end
+      node
+    end
+    
+    class Composite
+      attr_reader :nodes
+    
+      def initialize(*nodes)
+        @nodes = nodes || []
+      end
+    
+      def <<(node)
+        @nodes << node
+      end
+    
+      def values
+        @nodes.map { |node| node.values }.flatten
+      end
+    end
+    
+    class And < Composite
+      def evaluate(expr)
+        @nodes.each do |node|
+          return false unless node.evaluate(expr)
+        end
+        return true
+      end
+    end
+    
+    class Or < Composite
+      def evaluate(expr)
+        @nodes.each do |node|
+          return true if node.evaluate(expr)
+        end
+        return false
+      end
+    end
+    
+    class Unary
+      attr_reader :value
+    
+      def initialize(value)
+        @value = value
+        @expressions = Expressions.new([@value])
+      end
+    
+      def evaluate(expr)
+        @expressions.any?(expr)
+      end
+    
+      def values
+        [value]
+      end
+    end
+    
+    class Not < Unary
+      def evaluate(expr)
+        !super(expr)
+      end
+      def values
+        []
+      end
+    end
+    
+  end
+end

data/lib/text_nlp/string.rb

@@ -0,0 +1,36 @@
+# encoding: UTF-8
+
+class String
+  
+  attr_accessor :normalized
+  
+  class << self
+    attr_accessor :normalizer
+    attr_accessor :tokenizer
+  end
+  
+  def normalize
+    unless normalized()
+      new_string = (String.normalizer || TextNlp::Normalizer.new).normalize(self)
+      new_string.normalized = true
+      return new_string
+    end
+    self
+  end
+  
+  def tokenize
+    (String.tokenizer || TextNlp::Tokenizer.new).tokenize(self)
+  end
+  
+  def similarity(text)
+    score = 0.0
+    tokens1 = self.normalize.tokenize
+    tokens2 = text.normalize.tokenize
+    if (tokens1.size > 0 && tokens2.size > 0)
+      intersection = tokens1 & tokens2
+      score = (((intersection.size.to_f / tokens1.size.to_f) + (intersection.size.to_f / tokens2.size.to_f)) / 2)
+    end
+    score
+  end
+  
+end

data/lib/text_nlp/tokenizer.rb

@@ -0,0 +1,9 @@
+# encoding: UTF-8
+
+class TextNlp
+  class Tokenizer    
+    def tokenize(text)
+      text.split(/\s+/)
+    end
+  end
+end

data/lib/text_nlp.rb

@@ -3,6 +3,10 @@
 $:.unshift(File.dirname(__FILE__) + '/../lib')
 
 require "text_nlp/normalizer.rb"
+require "text_nlp/tokenizer.rb"
+require "text_nlp/string.rb"
+require "text_nlp/expressions.rb"
+require "text_nlp/pattern.rb"
 
 class TextNlp
 end

data/spec/expressions_spec.rb

@@ -0,0 +1,33 @@
+# encoding: utf-8
+require "spec_helper"
+
+describe TextNlp::Expressions do
+  
+  it "should find the expressions" do
+    expression_values = ['nicolas sarkozy','nicolas sarkozy 1er de france','carla bruni','carla bruni sarkozy a']
+    etree = TextNlp::Expressions.new(expression_values)
+    expressions = etree.find('nicolas sarkozy 1er de italie est marie a carla bruni qui de fait est devenue carla bruni sarkozy a')
+    expressions.size.should eq 3
+    ['nicolas sarkozy','carla bruni','carla bruni sarkozy a'].each { |e| expressions.include?(e).should be_true }
+    expression_values.size.should eq etree.values.size
+    expression_values.each do |v|
+      etree.values.include?(v).should be_true
+    end
+  end
+  
+  it "should expressionize the text" do
+    expression_values = ['nicolas sarkozy','nicolas sarkozy 1er de france','carla bruni','carla bruni sarkozy a']
+    etree = TextNlp::Expressions.new(expression_values)
+    expressions = etree.expressionize('nicolas sarkozy 1er de italie est marie a carla bruni qui de fait est devenue carla bruni sarkozy a')
+    expressions.should eq ['nicolas sarkozy','1er','de','italie','est','marie','a','carla bruni','qui','de','fait','est','devenue','carla bruni sarkozy a']
+    ['nicolas sarkozy','1er','de','italie','est','marie','a','carla bruni','qui','de','fait','est','devenue','carla bruni sarkozy a'].each { |e| expressions.include?(e).should be_true }
+  end
+  
+  it "should returns true or false if any expression present in text" do
+    etree = TextNlp::Expressions.new(['olympique de marseille','lyon'])
+    etree.any?("l olympique de marseille").should be_true
+    etree.any?("lyon c est plus ce que c etait").should be_true
+    etree.any?("marseille pres du vieux port").should be_false
+  end
+  
+end

data/spec/pattern_spec.rb

@@ -0,0 +1,13 @@
+# encoding: utf-8
+require "spec_helper"
+
+describe TextNlp::Pattern do
+  
+  it "should match or not the pattern" do
+    TextNlp::Pattern.new("((bd)||(bande dessinée))&&!samsung").match?("cette bd est super").should be_true
+    TextNlp::Pattern.new("((bd)||(bande dessinée))&&!samsung").match?("cette bd est illisible sur samsung NTC").should be_false
+    TextNlp::Pattern.new("((bd)||(bande dessinée))&&!samsung").match?("cette bande dessinée est illisible sur samsung NTC").should be_false
+    TextNlp::Pattern.new("((bd)||(bande dessinée))&&!samsung").match?("cette bande dessinée est illisible").should be_true
+  end
+  
+end

data/spec/string_spec.rb

@@ -0,0 +1,36 @@
+# encoding: utf-8
+require "spec_helper"
+
+describe String do  
+  
+  before(:each) { String.normalizer = nil; String.tokenizer = nil; }
+  
+  it "should call normalizer" do
+    text = "TOTO"
+    normalizer = double()
+    String.normalizer = normalizer
+    normalizer.stub(:normalize) { |txt| txt.downcase }
+    normalizer.should_receive(:normalize).with(text)
+    text = text.normalize
+    text.should eq "TOTO".downcase
+    normalizer.should_not_receive(:normalize).with(text)
+    text.normalize.should eq "TOTO".downcase
+  end
+  
+  it "should call tokenizer" do
+    text = "TOTO"
+    tokenizer = double()
+    String.tokenizer = tokenizer
+    tokenizer.should_receive(:tokenize).with(text)
+    text.tokenize
+  end
+  
+  it "should compute similarity" do
+    "il fait chaud".similarity("il fait chaud").should eq 1.0
+    "il fait chaud".similarity("putin c nul ici").should eq 0.0
+    "il fait chaud".similarity("youhou ca le fait").should be_within(0.01).of(0.29)
+    "".similarity("il fait chaud").should eq 0.0
+    "il fait chaud".similarity("").should eq 0.0
+  end
+  
+end

data/spec/tokenizer_spec.rb

@@ -0,0 +1,11 @@
+# encoding: utf-8
+require "spec_helper"
+
+describe TextNlp::Tokenizer do
+  
+  it "should tokenize text" do
+    n = TextNlp::Tokenizer.new
+    n.tokenize("Comment q'ça    se fait ?   blabla?").should eq ["Comment","q'ça","se","fait","?","blabla?"]
+  end
+  
+end

data/text_nlp.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'text_nlp'
-  s.version       = '0.0.0'
+  s.version       = '0.0.1'
   s.date          = '2011-07-05'
   s.summary       = "A minimalist NLP library"
   s.description   = s.summary

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: text_nlp
 version: !ruby/object:Gem::Version
-  version: 0.0.0
+  version: 0.0.1
   prerelease: 
 platform: ruby
 authors:
@@ -26,10 +26,17 @@ files:
 - README
 - Rakefile
 - lib/text_nlp.rb
+- lib/text_nlp/expressions.rb
 - lib/text_nlp/normalizer.rb
+- lib/text_nlp/pattern.rb
+- lib/text_nlp/string.rb
+- lib/text_nlp/tokenizer.rb
+- spec/expressions_spec.rb
 - spec/normalizer_spec.rb
+- spec/pattern_spec.rb
 - spec/spec_helper.rb
-- text_nlp-0.0.0.gem
+- spec/string_spec.rb
+- spec/tokenizer_spec.rb
 - text_nlp.gemspec
 homepage: http://github.com/fonzo14/text_nlp
 licenses: []