nlp_toolz 1.1.0 → 1.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ce5f4cad49039b0d8cb6d626facc67a4efa32ae4
-  data.tar.gz: 0565742385f0a34aabe4e456cde014ba2673a589
+  metadata.gz: 2ce637d8ddb8a8ad0b62c3998d016d3552b0cf58
+  data.tar.gz: be30bf6df57f309a050ebd0dffa6b6e5bcf85d48
 SHA512:
-  metadata.gz: 1ec11ec4b9b07437fb16f9ab0c181c9cee40a0cc900f90d02d2a6e4fc3bac7efaae890e8eda16bf7dcf8e3595bcb4010cf9d3893bee2a7a937b0fd527c40356f
-  data.tar.gz: 06d53b1bfe11004d0abeba1db130a13f664a054e8ed56f5edb260ec3f8bf189b0f9cb64687a471d2241ffd0d612ae632b50853737282a8d1901ea0645be4426a
+  metadata.gz: 453750f9759a44a70939feca45f9bf85d1ec50cf44c9794da66614dcbf2e10f6f97f7e0b3d2709489547c409a92f67dc2bbaf88b4f494b0f459d1d69269531bb
+  data.tar.gz: ad7179da66ff954010aca8245180cedf688d49531d6963cfaae92bfa944bc4e240fef3cb41cee471757ab893d2dd1847d2ca6fa0a66c33e76541e10513c2845a

data/.gitignore

@@ -26,3 +26,5 @@ teste.rb
 test-data/
 jars/*
 models/*
+jars.zip
+models.zip

data/README.md

@@ -18,7 +18,9 @@ Or install it yourself as:
 
     $ gem install nlp_toolz
 
-Download jars and model files from [Dropbox](https://www.dropbox.com/sh/1layyjgf5h0wwi3/s2SHAnfVhs) and unzip it in gem folder.
+~~Download jars and model files from [Dropbox](https://www.dropbox.com/sh/1layyjgf5h0wwi3/s2SHAnfVhs) and unzip it in gem folder.~~
+
+Simplyfied installing `models` and `jars`, now run `nlp_toolz init` from command line.
 
 ## Usage
 

data/Rakefile

@@ -1,9 +1,11 @@
 #!/usr/bin/env rake
 require "bundler/gem_tasks"
-require "awesome_print"
 require 'rspec/core'
 require 'rspec/core/rake_task'
+require "nlp_toolz"
+
 RSpec::Core::RakeTask.new(:spec) do |spec|
+  NlpToolz.check_dependencies
   spec.pattern = FileList['spec/**/*_spec.rb']
 end
 

data/bin/nlp_toolz

@@ -14,8 +14,7 @@ include GLI::App
 
 # helper methods
 def get_out(this)
-  ap this if $stdout.tty?
-  $stdout.puts this unless $stdout.tty?
+  $stdout.puts this
 end
 
 def get_in(input_arg)
@@ -32,6 +31,23 @@ def get_file(name)
   file.force_encoding("utf-8") unless file.nil?
 end
 
+def unzip_file (file, destination)
+  Zip::ZipFile.open(file) { |zip_file|
+   zip_file.each { |f|
+     f_path=File.join(destination, f.name)
+     FileUtils.mkdir_p(File.dirname(f_path))
+     zip_file.extract(f, f_path) unless File.exist?(f_path)
+   }
+  }
+end
+
+def add_path_in_lanikernel
+  file = IO.read(File.join(NlpToolz::HOME,'models','language','lanikernel.ini'))
+  file.sub!('WordlistDir=models',"WordlistDir=#{NlpToolz::HOME}/models")
+  file.sub!('BlacklistFile=models',"BlacklistFile=#{NlpToolz::HOME}/models")
+  file.sub!('MappingFile=models',"MappingFile=#{NlpToolz::HOME}/models")
+  File.open(File.join(NlpToolz::HOME,'models','language','lanikernel.ini'), 'w') {|f| f.write(file) }
+end
 
 program_desc 'running basic NLP tasks'
 
@@ -40,7 +56,34 @@ version NlpToolz::VERSION
 desc 'initial setup'
 command :init do |c|
   c.action do |global_options,options,args|
-    puts 'setting up app'
+    unless Dir.exist?(File.join(NlpToolz::HOME,'models')) && Dir.exist?(File.join(NlpToolz::HOME,'jars'))
+      gem_home = NlpToolz::HOME
+      $stdout.print "download and unzip to: ".green
+      $stdout.puts "#{gem_home}".blue
+      [
+        'https://dl.dropboxusercontent.com/sh/1layyjgf5h0wwi3/AACw8Y04KnFotOpBkzcfLxmwa/jars.zip',
+        'https://dl.dropboxusercontent.com/sh/1layyjgf5h0wwi3/AADUSMRMVg3n54Djdy9BWYVEa/models.zip',
+      ].each do |link|
+        loaded_file = link.split('/').last.sub('?dl=0','')
+        $stdout.print "download: ".green
+        $stdout.puts "#{loaded_file}".blue
+
+        Schiphol.download(
+          link,
+          # Default values
+          :download_folder => "#{gem_home}",
+          :show_progress => true,
+          :max_tries => 3
+        )
+        $stdout.puts "extracting …".green
+        unzip_file(File.join(gem_home,loaded_file),gem_home)
+        FileUtils.rm(File.join(gem_home,loaded_file))
+      end
+      $stdout.puts "add abbsolute path to language config".green
+      add_path_in_lanikernel
+    else
+      $stdout.puts "files exist".green
+    end
   end
 end
 
@@ -51,6 +94,7 @@ command :sent do |c|
   c.arg_name '<path/to/file>'
   c.flag [:f,:file]
   c.action do |global_options,options,args|
+    NlpToolz.check_dependencies
     input = get_in(options[:f] || args.first)
     get_out NlpToolz.get_sentences(input)
   end
@@ -63,6 +107,7 @@ command :parse do |c|
   c.arg_name '<path/to/file>'
   c.flag [:f,:file]
   c.action do |global_options,options,args|
+    NlpToolz.check_dependencies
     input = get_in(options[:f] || args.first)
     get_out NlpToolz.parse_text(input)
   end
@@ -75,6 +120,7 @@ command :tag do |c|
   c.arg_name '<path/to/file>'
   c.flag [:f,:file]
   c.action do |global_options,options,args|
+    NlpToolz.check_dependencies
     input = get_in(options[:f] || args.first)
     get_out NlpToolz.tag_text(input)
   end
@@ -87,6 +133,7 @@ command :token do |c|
   c.arg_name '<path/to/file>'
   c.flag [:f,:file]
   c.action do |global_options,options,args|
+    NlpToolz.check_dependencies
     input = get_in(options[:f] || args.first)
     get_out NlpToolz.tokenize_text(input)
   end

data/lib/nlp_toolz.rb

@@ -9,6 +9,9 @@ require "rjb"
 # external requirements
 require "awesome_print"
 require "multi_json"
+# for downloading models and jars
+require "schiphol"
+require "zip/zip"
 
 # internal requirements
 require "nlp_toolz/version"
@@ -17,6 +20,7 @@ require "nlp_toolz/helpers/string_extended"
 require "nlp_toolz/helpers/tmp_file"
 
 # NLP Tools
+require "nlp_toolz/home"
 require "nlp_toolz/load_jars"
 require "nlp_toolz/language"
 require "nlp_toolz/sentences"
@@ -24,29 +28,41 @@ require "nlp_toolz/pos_tags"
 require "nlp_toolz/tokens"
 require "nlp_toolz/parser"
 
+
 module NlpToolz
+  
+  
   module_function
   
+  def check_dependencies
+    unless Dir.exist?(File.join(NlpToolz::HOME,'models')) && Dir.exist?(File.join(NlpToolz::HOME,'jars'))
+      $stdout.puts "\n--> models and jars not installed,"
+      $stdout.puts "    install it by running:"
+      $stdout.puts "--> $ nlp_toolz init\n".green
+      exit
+    end
+  end
+  
   def get_lang(input)
     NlpToolz::Language.get_language(input)
   end
-  
+
   def get_sentences(input,lang = nil)
     text = NlpToolz::Sentences.new(input,lang)
     text.split_into_sentences if text.has_model?
   end
-  
+
   def tokenize_sentence(input,lang = nil)
     sentence = NlpToolz::Tokens.new(input,lang)
     sentence.tokenize
   end
-  
+
   def tokenize_text(input,lang = nil)
     tokenized_text = []
     get_sentences(input,lang).each do |sentence|
       tokenized_text << tokenize_sentence(sentence,lang)
     end
-    
+  
     tokenized_text
   end
 
@@ -54,29 +70,30 @@ module NlpToolz
     sentence = NlpToolz::PosTags.new(input,lang)
     sentence.get_pos_tags if sentence.has_model?
   end
-  
+
   def tag_text(input,lang = nil)
     tagged_text = []
     get_sentences(input,lang).each do |sentence|
       tagged_text << tag_sentence(sentence,lang)
     end
-    
+  
     tagged_text
   end
-  
+
   def parse_sentence(input,lang = nil)
     text = NlpToolz::Parser.new(input,lang)
     text.parse_text
-    
+  
     text.parse_hash
   end
-  
+
   def parse_text(input,lang = nil)
     parsed_text = []
     get_sentences(input,lang).each do |sentence|
       parsed_text << parse_sentence(sentence,lang)
     end
-    
+  
     parsed_text
   end
+  
 end

data/lib/nlp_toolz/home.rb

@@ -0,0 +1,3 @@
+module NlpToolz
+  HOME = Gem::Specification.find_by_name("nlp_toolz").gem_dir
+end

data/lib/nlp_toolz/language.rb

@@ -7,17 +7,16 @@ module NlpToolz
   
   class Language
     
-    # load java classes
-    # Enumeration = Rjb::import("java.util.Enumeration")
-    HashSet     = Rjb::import("java.util.HashSet")
-    # Hashtable   = Rjb::import("java.util.Hashtable")
-    # Set         = Rjb::import("java.util.Set")
+    if Dir.exist?(File.join(NlpToolz::HOME,'models')) && Dir.exist?(File.join(NlpToolz::HOME,'jars'))
+      # load java classes
+      HashSet     = Rjb::import("java.util.HashSet")
     
-    DataSourceException = Rjb::import("de.uni_leipzig.asv.toolbox.jLanI.kernel.DataSourceException")
-    LanIKernel = Rjb::import("de.uni_leipzig.asv.toolbox.jLanI.kernel.LanIKernel")
-    Request = Rjb::import("de.uni_leipzig.asv.toolbox.jLanI.kernel.Request")
-    RequestException = Rjb::import("de.uni_leipzig.asv.toolbox.jLanI.kernel.RequestException")
-    Response = Rjb::import("de.uni_leipzig.asv.toolbox.jLanI.kernel.Response")
+      DataSourceException = Rjb::import("de.uni_leipzig.asv.toolbox.jLanI.kernel.DataSourceException")
+      LanIKernel = Rjb::import("de.uni_leipzig.asv.toolbox.jLanI.kernel.LanIKernel")
+      Request = Rjb::import("de.uni_leipzig.asv.toolbox.jLanI.kernel.Request")
+      RequestException = Rjb::import("de.uni_leipzig.asv.toolbox.jLanI.kernel.RequestException")
+      Response = Rjb::import("de.uni_leipzig.asv.toolbox.jLanI.kernel.Response")
+    end
     
     def self.get_language(text = nil)
       return -1 if text.nil? || text.empty?
@@ -33,7 +32,7 @@ module NlpToolz
       
       req = Request.new(text, languages, modus, reduce)
       
-      LanIKernel.propertyFile = File.join(MODELS, 'language', 'lanikernel')
+      LanIKernel.propertyFile = File.join(MODELS,'language','lanikernel')
       kernel = LanIKernel.getInstance()
       res = kernel.evaluate(req)
       

data/lib/nlp_toolz/load_jars.rb

@@ -1,12 +1,7 @@
 module NlpToolz
-  MODELS = File.join(File.dirname(__FILE__), '..', '..', "models")
-  JARS = File.join(File.dirname(__FILE__), '..', '..', "jars")
-  
-  # CLASS_PATH = [
-  #   File.join(JARS, "jwnl-1.3.3.jar"),
-  #   File.join(JARS, "opennlp-tools-1.5.3.jar"),
-  #   File.join(JARS, "opennlp-maxent-3.0.3.jar")
-  # ].join(":")
+  CONFIG = File.join(File.dirname(__FILE__), '..', '..', 'config')
+  MODELS = File.join(File.dirname(__FILE__), '..', '..', 'models')
+  JARS = File.join(File.dirname(__FILE__), '..', '..', 'jars')
   
   CLASS_PATH = Dir.glob(File.join(JARS,'*.jar')).join(':')
   

data/lib/nlp_toolz/pos_tags.rb

@@ -8,10 +8,12 @@ module NlpToolz
 
   class PosTags
     
-    # load java classes
-    FileInputStream = Rjb::import('java.io.FileInputStream')
-    POSModel        = Rjb::import('opennlp.tools.postag.POSModel')
-    POSTaggerME     = Rjb::import('opennlp.tools.postag.POSTaggerME')
+    if Dir.exist?(File.join(NlpToolz::HOME,'models')) && Dir.exist?(File.join(NlpToolz::HOME,'jars'))
+      # load java classes
+      FileInputStream = Rjb::import('java.io.FileInputStream')
+      POSModel        = Rjb::import('opennlp.tools.postag.POSModel')
+      POSTaggerME     = Rjb::import('opennlp.tools.postag.POSTaggerME')
+    end
     
     attr_accessor :input, :lang, :model, :model_name, :tokenized
     

data/lib/nlp_toolz/sentences.rb

@@ -8,10 +8,12 @@ module NlpToolz
 
   class Sentences
     
+    if Dir.exist?(File.join(NlpToolz::HOME,'models')) && Dir.exist?(File.join(NlpToolz::HOME,'jars'))
     # load java classes
-    FileInputStream    = Rjb::import('java.io.FileInputStream')
-    SentenceDetectorME = Rjb::import('opennlp.tools.sentdetect.SentenceDetectorME')
-    SentenceModel      = Rjb::import('opennlp.tools.sentdetect.SentenceModel')
+      FileInputStream    = Rjb::import('java.io.FileInputStream')
+      SentenceDetectorME = Rjb::import('opennlp.tools.sentdetect.SentenceDetectorME')
+      SentenceModel      = Rjb::import('opennlp.tools.sentdetect.SentenceModel')
+    end
     
     attr_accessor :input, :lang, :model, :model_name, :sentences
     

data/lib/nlp_toolz/tokens.rb

@@ -7,10 +7,12 @@ module NlpToolz
 
   class Tokens
     
-    # load java classes
-    FileInputStream = Rjb::import('java.io.FileInputStream')
-    TokenizerModel  = Rjb::import('opennlp.tools.tokenize.TokenizerModel')
-    TokenizerME     = Rjb::import('opennlp.tools.tokenize.TokenizerME')
+    if Dir.exist?(File.join(NlpToolz::HOME,'models')) && Dir.exist?(File.join(NlpToolz::HOME,'jars'))
+      # load java classes
+      FileInputStream = Rjb::import('java.io.FileInputStream')
+      TokenizerModel  = Rjb::import('opennlp.tools.tokenize.TokenizerModel')
+      TokenizerME     = Rjb::import('opennlp.tools.tokenize.TokenizerME')
+    end
     
     attr_accessor :input, :lang, :model, :model_name, :tokens
     

data/lib/nlp_toolz/version.rb

@@ -4,5 +4,5 @@
 # date:   2012-10-23
 
 module NlpToolz
-  VERSION = "1.1.0"
+  VERSION = "1.1.1"
 end

data/nlp_toolz.gemspec

@@ -39,4 +39,8 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency "multi_json"
   gem.add_runtime_dependency "gli"
   gem.add_runtime_dependency "rake"
+  
+  #for downloading models and jars
+  gem.add_runtime_dependency "schiphol"
+  gem.add_runtime_dependency "rubyzip"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: nlp_toolz
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.1.1
 platform: ruby
 authors:
 - LeFnord
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-10-12 00:00:00.000000000 Z
+date: 2014-10-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -192,6 +192,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: schiphol
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubyzip
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: make NLP tools available, from OpenNLP and BerkeleyParser
 email:
 - pscholz.le@gmail.com
@@ -212,6 +240,7 @@ files:
 - lib/nlp_toolz/helpers/string_extended.rb
 - lib/nlp_toolz/helpers/tmp_file.rb
 - lib/nlp_toolz/helpers/url_handler.rb
+- lib/nlp_toolz/home.rb
 - lib/nlp_toolz/language.rb
 - lib/nlp_toolz/load_jars.rb
 - lib/nlp_toolz/parser.rb