freeling-client 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: db5bc7707caee2de4e0609c5fe595ff9213c3f43
+  data.tar.gz: 090a06c682c062753199ec37b21eaeebc868315a
+SHA512:
+  metadata.gz: 5fb75e56e9efaf8842271b502bfb6df06829cb3248959f3a712bf28d0b2f4724ff796218a24334b288b5851b2849e70f91011c3c3751052b5d3b589c4bcd6352
+  data.tar.gz: 841275c8fd541b0600229195f2fc4b82474b17c7d52a08e94a1003d6dfb860ea08f635c851c1b86612a3d6e6fd1cda288449e612301e55a73da04e11bca20486

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,21 @@
+PATH
+  remote: .
+  specs:
+    freeling-client (0.0.1)
+      hashie
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    hashie (3.3.2)
+    minitest (5.4.3)
+    rake (10.3.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.7)
+  freeling-client!
+  minitest
+  rake (~> 10.3)

data/README.md

@@ -0,0 +1,58 @@
+# Freeling::Client
+
+Simple client wrapper for Freeling analyzer tool. If you need to install freeling on Ubuntu 14.04 just follow [this](https://gist.github.com/malev/d6a8b51c2ae0a762ab1d) guide.
+
+## Example of usage:
+
+```ruby
+require 'freeling_client'
+
+text <<-EOF
+Malcolm X was effectively orphaned early in life. His father was killed when he was six and his mother was placed in a mental hospital when he was thirteen, after which he lived in a series of foster homes.
+EOF
+
+options = {
+  timeout: 200,
+  config: 'config.cfg',
+  fidn: 'ident.dat'
+}
+
+# Using the language detection tool
+lang_detector = FreelingClient::LanguageDetector.new options
+lang_detector.detect(text) # => :en
+
+# Morphological, morpho with PoS tagging, tagged words and nec analysis
+analyzer = FreelingClient::Analyzer.new options
+analyzer.call(:morfo, text)
+analyzer.call(:tagged, text)
+analyzer.call(:tagged_sense, text)
+analyzer.call(:tagged_ned, text)
+
+# Using as a client
+# You will need to setup the server first. Check bellow
+options = {
+  server: 'localhost',
+  port: 50005
+}
+
+freeling_client = FreelingClient::Client.new options
+freeling_client.call(text)
+```
+
+## Running Freeling as a server
+
+Performing morphological analysis:
+
+    FREELINGSHARE=/usr/local/share/freeling/ analyzer -f config/freeling/analyzer.cfg --server --port 50005 --inpf plain --outf morfo
+
+Performing morphological with PoS tagging:
+
+    FREELINGSHARE=/usr/local/share/freeling/ analyzer -f config/freeling/analyzer.cfg --server --port 50005 --inpf plain --outf tagged
+
+Asking for the senses of the tagged words:
+
+    FREELINGSHARE=/usr/local/share/freeling/ analyzer -f config/freeling/analyzer.cfg --server --port 50005 --inpf plain --outf sense --sense all
+
+With `nec` analysis:
+
+    FREELINGSHARE=/usr/local/share/freeling/ analyzer -f config/freeling/analyzer.cfg --server --port 50005 --inpf plain --outf tagged --nec --noflush

data/Rakefile

@@ -0,0 +1,10 @@
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+  t.test_files = FileList["test/**/*_test.rb"]
+  t.verbose = true
+end
+
+desc "Run tests"
+task :default => :test

data/config/freeling/analyzer.cfg

@@ -0,0 +1,89 @@
+##
+#### default configuration file for Spanish analyzer
+##
+
+TraceLevel=3
+TraceModule=0x0000
+
+## Options to control the applied modules. The input may be partially
+## processed, or not a full analysis may me wanted. The specific
+## formats are a choice of the main program using the library, as well
+## as the responsability of calling only the required modules.  
+## Valid input formats are: plain, token, splitted, morfo, tagged, sense.
+## Valid output formats are: : plain, token, splitted, morfo, tagged,
+## shallow, parsed, dep.
+InputFormat=plain
+OutputFormat=tagged
+
+# consider each newline as a sentence end
+AlwaysFlush=no
+
+#### Tokenizer options
+TokenizerFile=$FREELINGSHARE/es/tokenizer.dat
+
+#### Splitter options
+SplitterFile=$FREELINGSHARE/es/splitter.dat
+
+#### Morfo options
+AffixAnalysis=yes
+MultiwordsDetection=yes
+NumbersDetection=yes
+PunctuationDetection=yes
+DatesDetection=yes
+QuantitiesDetection=yes
+DictionarySearch=yes
+ProbabilityAssignment=yes
+OrthographicCorrection=no
+DecimalPoint=,
+ThousandPoint=.
+LocutionsFile=$FREELINGSHARE/es/locucions.dat 
+QuantitiesFile=$FREELINGSHARE/es/quantities.dat
+AffixFile=$FREELINGSHARE/es/afixos.dat
+ProbabilityFile=$FREELINGSHARE/es/probabilitats.dat
+DictionaryFile=$FREELINGSHARE/es/dicc.src
+PunctuationFile=$FREELINGSHARE/common/punct.dat
+ProbabilityThreshold=0.001
+
+# NER options 
+NERecognition=yes
+NPDataFile=$FREELINGSHARE/es/np.dat
+## comment line above and uncomment that below, if you want 
+## a better NE recognizer (higer accuracy, lower speed)
+#NPDataFile=$FREELINGSHARE/es/ner/ner-ab.dat
+
+#Spelling Corrector config file
+CorrectorFile=$FREELINGSHARE/es/corrector/corrector.dat
+
+## Phonetic encoding of words.
+Phonetics=no
+PhoneticsFile=$FREELINGSHARE/es/phonetics.dat
+
+## NEC options
+NEClassification=no
+NECFile=$FREELINGSHARE/es/nec/nec-svm.dat
+
+## Sense annotation options (none,all,mfs,ukb)
+SenseAnnotation=none
+SenseConfigFile=$FREELINGSHARE/es/senses.dat
+UKBConfigFile=$FREELINGSHARE/es/ukb.dat
+
+#### Tagger options
+Tagger=hmm
+TaggerHMMFile=$FREELINGSHARE/es/tagger.dat
+TaggerRelaxFile=$FREELINGSHARE/es/constr_gram.dat
+TaggerRelaxMaxIter=500
+TaggerRelaxScaleFactor=670.0
+TaggerRelaxEpsilon=0.001
+TaggerRetokenize=yes
+TaggerForceSelect=tagger
+
+#### Parser options
+GrammarFile=$FREELINGSHARE/es/grammar-dep.dat
+
+#### Dependence Parser options
+DepTxalaFile=$FREELINGSHARE/es/dep/dependences.dat
+
+#### Coreference Solver options
+CoreferenceResolution=no
+CorefFile=$FREELINGSHARE/es/coref/coref.dat
+

data/freeling-client.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "freeling_client/version"
+
+
+Gem::Specification.new do |s|
+  s.name        = "freeling-client"
+  s.version     = FreelingClient::VERSION
+  s.date        = "2014-12-10"
+  s.summary     = "Freeling client wrapper"
+  s.description = "Freeling client wrapper with API"
+  s.authors     = ["Marcos Vanetta"]
+  s.email       = "marcosvanetta@gmail.com"
+  s.homepage    = "http://codingnews.info"
+  s.license       = "MIT"
+
+  s.files       = `git ls-files -z`.split("\x0")
+  s.test_files  = s.files.grep(%r{^(test|spec|features)/})
+  s.require_paths = ["lib"]
+
+  s.add_dependency "hashie", "~> 3"
+  s.add_development_dependency "bundler", "~> 1.7"
+  s.add_development_dependency "rake", "~> 10.3"
+  s.add_development_dependency "minitest", "~> 5"
+end

data/lib/freeling_client/analyzer.rb

@@ -0,0 +1,80 @@
+require "open3"
+require "tempfile"
+require "timeout"
+require "hashie/mash"
+require "freeling_client/base"
+
+
+module FreelingClient
+  class Analyzer < Base
+
+    def initialize(opt={})
+      @config = opt.fetch(:config, 'config/freeling/analyzer.cfg')
+      @port = opt[:port]
+      @server = opt[:server]
+      @timeout = opt.fetch(:timeout, 60) # Three hours
+    end
+
+    def call(cmd, text)
+      valide_command!(cmd)
+
+      output = []
+      file = Tempfile.new('foo', encoding: 'utf-8')
+      begin
+        file.write(text)
+        file.close
+        stdin, stdout, stderr = Open3.popen3(command(cmd, file.path))
+        Timeout::timeout(@timeout) {
+          until (line = stdout.gets).nil?
+            output << line.chomp
+          end
+
+          message = stderr.readlines
+          unless message.empty?
+            raise ExtractionError, message.join("\n")
+          end
+        }
+      rescue Timeout::Error
+        raise ExtractionError, "Timeout"
+      ensure
+        file.close
+        file.unlink
+      end
+      output
+    end
+
+    def command(cmd, file_path)
+      self.send("command_#{cmd}", file_path)
+    end
+
+    def command_morfo(file_path)
+      "#{freeling_share} #{freeling_bin} -f #{config} --inpf plain --outf morfo < #{file_path}"
+    end
+
+    def command_tagged(file_path)
+      "#{freeling_share} #{freeling_bin} -f #{config} --inpf plain --outf tagged < #{file_path}"
+    end
+
+    def command_tagged_nec(file_path)
+      "#{freeling_share} #{freeling_bin} -f #{config} --inpf plain --outf tagged --nec --noflush < #{file_path}"
+    end
+
+    def command_tagged_sense(file_path)
+      "#{freeling_share} #{freeling_bin} -f #{config} --inpf plain --outf sense --sense all < #{file_path}"
+    end
+
+    def freeling_share
+      "FREELINGSHARE=/usr/local/share/freeling/"
+    end
+
+    def freeling_bin
+      "/usr/local/bin/analyzer"
+    end
+
+    def valide_command!(cmd)
+      unless [:morfo, :tagged, :tagged_nec, :tagged_sense].include?(cmd)
+        raise CommandError, "#{cmd} does not exist"
+      end
+    end
+  end
+end

data/lib/freeling_client/base.rb

@@ -0,0 +1,9 @@
+module FreelingClient
+  class Base
+    attr_reader :config, :ident, :server, :port
+
+    class ExtractionError < StandardError; end
+    class CommandError < StandardError; end
+
+  end
+end

data/lib/freeling_client/client.rb

@@ -0,0 +1,45 @@
+require "open3"
+require "freeling_client/base"
+
+
+module FreelingClient
+  class Client < Base
+    def initialize(opt = {})
+      @server = opt.fetch(:server, 'localhost')
+      @port = opt.fetch(:port, 50005)
+      @timeout = opt.fetch(:timeout, 120)
+    end
+
+    def call(text)
+      output = []
+      file = Tempfile.new('foo', encoding: 'utf-8')
+
+      begin
+        file.write(text)
+        file.close
+        stdin, stdout, stderr = Open3.popen3(command(file.path))
+
+        Timeout::timeout(@timeout) {
+          until (line = stdout.gets).nil?
+            output << line.chomp
+          end
+
+          message = stderr.readlines
+          unless message.empty?
+            raise ExtractionError, message.join("\n")
+          end
+        }
+      rescue Timeout::Error
+        raise ExtractionError, "Timeout"
+      ensure
+        file.close
+        file.unlink
+      end
+      output
+    end
+
+    def command(file_path)
+      "/usr/local/bin/analyzer_client #{server}:#{port} < #{file_path}"
+    end
+  end
+end

data/lib/freeling_client/language_detector.rb

@@ -0,0 +1,45 @@
+require "open3"
+require "freeling_client/base"
+
+
+module FreelingClient
+  class LanguageDetector < Base
+    def initialize(opt = {})
+      @config = opt.fetch(:config, 'config/freeling/analyzer.cfg')
+      @ident = opt.fetch(:ident, '/usr/local/share/freeling/common/lang_ident/ident.dat')
+      @timeout = opt.fetch(:timeout, 120)
+    end
+
+    def detect(text)
+      output = []
+      file = Tempfile.new('foo', encoding: 'utf-8')
+
+      begin
+        file.write(text)
+        file.close
+        stdin, stdout, stderr = Open3.popen3(command(file.path))
+
+        Timeout::timeout(@timeout) {
+          until (line = stdout.gets).nil?
+            output << line.chomp
+          end
+
+          message = stderr.readlines
+          unless message.empty?
+            raise ExtractionError, message.join("\n")
+          end
+        }
+      rescue Timeout::Error
+        raise ExtractionError, "Timeout"
+      ensure
+        file.close
+        file.unlink
+      end
+      output[0].to_sym
+    end
+
+    def command(file_path)
+      "/usr/local/bin/analyzer --outf ident --fidn #{ident} -f #{config} < #{file_path}"
+    end
+  end
+end

data/lib/freeling_client/version.rb

@@ -0,0 +1,3 @@
+module FreelingClient
+  VERSION = "0.0.1"
+end

data/lib/freeling_client.rb

@@ -0,0 +1,8 @@
+require "tempfile"
+require "freeling_client/version"
+require "freeling_client/analyzer"
+require "freeling_client/client"
+require "freeling_client/language_detector"
+
+
+module FreelingClient; end

data/start-freeling.sh

@@ -0,0 +1,5 @@
+FREELINGSHARE=/usr/local/share/freeling/ analyzer \
+-f config/freeling/analyzer.cfg \
+--server --port 50005 \
+--inpf plain \
+--outf morfo

data/test/lib/freeling_client_test.rb

@@ -0,0 +1,23 @@
+require 'test_helper'
+
+
+describe FreelingClient do
+  it 'detects language' do
+    text = "El gato come pescado. Pero a Don Jaime no le gustan los gatos."
+    lang_detector = FreelingClient::LanguageDetector.new
+    lang_detector.detect(text).must_equal :es
+  end
+
+  it 'Uses freeling as a client (mode server)' do
+    text = "El gato come pescado. Pero a Don Jaime no le gustan los gatos."
+    freeling_client = FreelingClient::Client.new
+    freeling_client.call(text)[0].must_equal "El el DA0MS0 1"
+  end
+
+  it 'Uses freeling to get a morphological analysis' do
+    text = "El gato come pescado. Pero a Don Jaime no le gustan los gatos."
+    analyzer = FreelingClient::Analyzer.new
+
+    analyzer.call(:morfo, text)[0].must_equal "El el DA0MS0 1"
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,5 @@
+require 'minitest/autorun'
+require 'minitest/unit'
+require 'mocha/mini_test'
+
+require 'freeling_client'

metadata

@@ -0,0 +1,116 @@
+--- !ruby/object:Gem::Specification
+name: freeling-client
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Marcos Vanetta
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-12-10 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: hashie
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.3'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5'
+description: Freeling client wrapper with API
+email: marcosvanetta@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- Gemfile.lock
+- README.md
+- Rakefile
+- config/freeling/analyzer.cfg
+- freeling-client.gemspec
+- lib/freeling_client.rb
+- lib/freeling_client/analyzer.rb
+- lib/freeling_client/base.rb
+- lib/freeling_client/client.rb
+- lib/freeling_client/language_detector.rb
+- lib/freeling_client/version.rb
+- start-freeling.sh
+- test/lib/freeling_client_test.rb
+- test/test_helper.rb
+homepage: http://codingnews.info
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Freeling client wrapper
+test_files:
+- test/lib/freeling_client_test.rb
+- test/test_helper.rb