freeling-client 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: db5bc7707caee2de4e0609c5fe595ff9213c3f43
-  data.tar.gz: 090a06c682c062753199ec37b21eaeebc868315a
+  metadata.gz: 22175d4562553eaba3d4d095df0cf9277813cfb7
+  data.tar.gz: 3e62780a11bab00955217a164a8aa1dfa342b952
 SHA512:
-  metadata.gz: 5fb75e56e9efaf8842271b502bfb6df06829cb3248959f3a712bf28d0b2f4724ff796218a24334b288b5851b2849e70f91011c3c3751052b5d3b589c4bcd6352
-  data.tar.gz: 841275c8fd541b0600229195f2fc4b82474b17c7d52a08e94a1003d6dfb860ea08f635c851c1b86612a3d6e6fd1cda288449e612301e55a73da04e11bca20486
+  metadata.gz: b4042283e88acaa32d476ce2575bef9f7611a798acb0b00d8c061bfec78c2aed66b8fa0b9295053d484b5a5fd8e25323f70a137a5a5a3205b3e54e349a1c2a69
+  data.tar.gz: 249cf116647b29dba7a8b5d6286cf6a53edf4d1d6cd58ae016ebc5a5afe696e34627c13c6d7831e1c6e18bec73ad5e3f1379a91e6688a281e71626cb3a5c2c01

data/.gitignore

@@ -0,0 +1,34 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalisation:
+/.bundle/
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc

data/Gemfile.lock

@@ -2,7 +2,7 @@ PATH
   remote: .
   specs:
     freeling-client (0.0.1)
-      hashie
+      hashie (~> 3)
 
 GEM
   remote: https://rubygems.org/
@@ -17,5 +17,5 @@ PLATFORMS
 DEPENDENCIES
   bundler (~> 1.7)
   freeling-client!
-  minitest
+  minitest (~> 5)
   rake (~> 10.3)

data/LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 Marcos Vanetta
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

data/README.md

@@ -1,6 +1,6 @@
 # Freeling::Client
 
-Simple client wrapper for Freeling analyzer tool. If you need to install freeling on Ubuntu 14.04 just follow [this](https://gist.github.com/malev/d6a8b51c2ae0a762ab1d) guide.
+Simple client wrapper for Freeling analyzer tool. For only works with [this](http://nlp.lsi.upc.edu/freeling/doc/userman/html/node91.html) options. If you need to install freeling on Ubuntu 14.04 just follow [this](https://gist.github.com/malev/d6a8b51c2ae0a762ab1d) guide.
 
 ## Example of usage:
 
@@ -43,16 +43,20 @@ freeling_client.call(text)
 
 Performing morphological analysis:
 
-    FREELINGSHARE=/usr/local/share/freeling/ analyzer -f config/freeling/analyzer.cfg --server --port 50005 --inpf plain --outf morfo
+    FREELINGSHARE=/usr/local/share/freeling/ analyzer \
+    -f config/freeling/analyzer.cfg --server --port 50005 --inpf plain --outf morfo
 
 Performing morphological with PoS tagging:
 
-    FREELINGSHARE=/usr/local/share/freeling/ analyzer -f config/freeling/analyzer.cfg --server --port 50005 --inpf plain --outf tagged
+    FREELINGSHARE=/usr/local/share/freeling/ analyzer \
+    -f config/freeling/analyzer.cfg --server --port 50005 --inpf plain --outf tagged
 
 Asking for the senses of the tagged words:
 
-    FREELINGSHARE=/usr/local/share/freeling/ analyzer -f config/freeling/analyzer.cfg --server --port 50005 --inpf plain --outf sense --sense all
+    FREELINGSHARE=/usr/local/share/freeling/ analyzer \
+    -f config/freeling/analyzer.cfg --server --port 50005 --inpf plain --outf sense --sense all
 
 With `nec` analysis:
 
-    FREELINGSHARE=/usr/local/share/freeling/ analyzer -f config/freeling/analyzer.cfg --server --port 50005 --inpf plain --outf tagged --nec --noflush
+    FREELINGSHARE=/usr/local/share/freeling/ analyzer \
+    -f config/freeling/analyzer.cfg --server --port 50005 --inpf plain --outf tagged --nec --noflush

data/lib/freeling_client/analyzer.rb

@@ -1,6 +1,10 @@
+# encoding: utf-8
+
+require "enumerator"
 require "open3"
 require "tempfile"
 require "timeout"
+
 require "hashie/mash"
 require "freeling_client/base"
 
@@ -8,6 +12,8 @@ require "freeling_client/base"
 module FreelingClient
   class Analyzer < Base
 
+    Token = Class.new(Hashie::Mash)
+
     def initialize(opt={})
       @config = opt.fetch(:config, 'config/freeling/analyzer.cfg')
       @port = opt[:port]
@@ -43,6 +49,58 @@ module FreelingClient
       output
     end
 
+    def tokens(cmd, text)
+      valide_command!(cmd)
+      Enumerator.new do |yielder|
+        call(cmd, text).each do |freeling_line|
+          yielder << parse_token_line(freeling_line) unless freeling_line.empty?
+        end
+      end
+    end
+
+    def ptokens(cmd, text)
+      Enumerator.new do |yielder|
+        pos = 0
+        tokens(cmd, text).each do |token|
+          ne_text = token['form'].dup
+
+          ne_regexp = build_regexp(ne_text)
+          token_pos = text.index(ne_regexp, pos)
+
+          if token_pos && token_pos < (pos + 5)
+            token.pos = token_pos
+            yielder << token
+
+            pos = token_pos + ne_text.length
+          else
+            pos = pos + ne_text.length
+          end
+        end
+      end
+    end
+
+    def parse_token_line(str)
+      form, lemma, tag, prob = str.split(' ')[0..3]
+      Token.new({
+        :form => form,
+        :lemma => lemma,
+        :tag => tag,
+        :prob => prob.nil? ? nil : prob.to_f,
+      }.reject { |k, v| v.nil? })
+    end
+
+    def build_regexp(ne_text)
+      begin
+        if ne_text =~ /\_/
+           /#{ne_text.split('_').join('\W+')}/i
+        else
+          /#{ne_text}/i
+        end
+      rescue RegexpError => e
+        /./
+      end
+    end
+
     def command(cmd, file_path)
       self.send("command_#{cmd}", file_path)
     end

data/lib/freeling_client/version.rb

@@ -1,3 +1,3 @@
 module FreelingClient
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

data/test/lib/freeling_client_test.rb

@@ -1,23 +1,34 @@
+# encoding: utf-8
+
 require 'test_helper'
 
+text = "El gato come pescado. Pero a Don Jaime no le gustan los gatos."
 
 describe FreelingClient do
   it 'detects language' do
-    text = "El gato come pescado. Pero a Don Jaime no le gustan los gatos."
     lang_detector = FreelingClient::LanguageDetector.new
     lang_detector.detect(text).must_equal :es
   end
 
   it 'Uses freeling as a client (mode server)' do
-    text = "El gato come pescado. Pero a Don Jaime no le gustan los gatos."
     freeling_client = FreelingClient::Client.new
     freeling_client.call(text)[0].must_equal "El el DA0MS0 1"
   end
 
   it 'Uses freeling to get a morphological analysis' do
-    text = "El gato come pescado. Pero a Don Jaime no le gustan los gatos."
     analyzer = FreelingClient::Analyzer.new
-
     analyzer.call(:morfo, text)[0].must_equal "El el DA0MS0 1"
   end
+
+  it "returns parsed tokens" do
+    analyzer = FreelingClient::Analyzer.new
+    analyzer.tokens(:morfo, text).first.lemma.must_equal "el"
+  end
+
+  it "returns positionated tokens" do
+    analyzer = FreelingClient::Analyzer.new
+    tokens = analyzer.ptokens(:morfo, text).to_a
+    tokens[0].pos.must_equal 0
+    tokens[1].pos.must_equal 3
+  end
 end

data/test/test_helper.rb

@@ -1,5 +1,4 @@
 require 'minitest/autorun'
 require 'minitest/unit'
-require 'mocha/mini_test'
 
 require 'freeling_client'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: freeling-client
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Marcos Vanetta
@@ -72,8 +72,10 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".gitignore"
 - Gemfile
 - Gemfile.lock
+- LICENSE
 - README.md
 - Rakefile
 - config/freeling/analyzer.cfg