opener-opinion-detector-base 2.0.0

data/core/vendor/src/svm_light/svm_loqo.c

@@ -0,0 +1,211 @@
+/***********************************************************************/
+/*                                                                     */
+/*   svm_loqo.c                                                        */
+/*                                                                     */
+/*   Interface to the PR_LOQO optimization package for SVM.            */
+/*                                                                     */
+/*   Author: Thorsten Joachims                                         */
+/*   Date: 19.07.99                                                    */
+/*                                                                     */
+/*   Copyright (c) 1999  Universitaet Dortmund - All rights reserved   */
+/*                                                                     */
+/*   This software is available for non-commercial use only. It must   */
+/*   not be modified and distributed without prior permission of the   */
+/*   author. The author is not responsible for implications from the   */
+/*   use of this software.                                             */
+/*                                                                     */
+/***********************************************************************/
+
+# include <math.h>
+# include "pr_loqo/pr_loqo.h"
+# include "svm_common.h"
+
+/* Common Block Declarations */
+
+long verbosity;
+
+/* /////////////////////////////////////////////////////////////// */
+
+# define DEF_PRECISION_LINEAR    1E-8
+# define DEF_PRECISION_NONLINEAR 1E-14
+
+double *optimize_qp();
+double *primal=0,*dual=0;
+double init_margin=0.15;
+long   init_iter=500,precision_violations=0;
+double model_b;
+double opt_precision=DEF_PRECISION_LINEAR;
+
+/* /////////////////////////////////////////////////////////////// */
+
+void *my_malloc();
+
+double *optimize_qp(qp,epsilon_crit,nx,threshold,learn_parm)
+QP *qp;
+double *epsilon_crit;
+long nx; /* Maximum number of variables in QP */
+double *threshold;
+LEARN_PARM *learn_parm;
+/* start the optimizer and return the optimal values */
+{
+  register long i,j,result;
+  double margin,obj_before,obj_after;
+  double sigdig,dist,epsilon_loqo;
+  int iter;
+ 
+  if(!primal) { /* allocate memory at first call */
+    primal=(double *)my_malloc(sizeof(double)*nx*3);
+    dual=(double *)my_malloc(sizeof(double)*(nx*2+1));
+  }
+  
+  if(verbosity>=4) { /* really verbose */
+    printf("\n\n");
+    for(i=0;i<qp->opt_n;i++) {
+      printf("%f: ",qp->opt_g0[i]);
+      for(j=0;j<qp->opt_n;j++) {
+	printf("%f ",qp->opt_g[i*qp->opt_n+j]);
+      }
+      printf(": a%ld=%.10f < %f",i,qp->opt_xinit[i],qp->opt_up[i]);
+      printf(": y=%f\n",qp->opt_ce[i]);
+    }
+    for(j=0;j<qp->opt_m;j++) {
+      printf("EQ-%ld: %f*a0",j,qp->opt_ce[j]);
+      for(i=1;i<qp->opt_n;i++) {
+	printf(" + %f*a%ld",qp->opt_ce[i],i);
+      }
+      printf(" = %f\n\n",-qp->opt_ce0[0]);
+    }
+}
+
+  obj_before=0; /* calculate objective before optimization */
+  for(i=0;i<qp->opt_n;i++) {
+    obj_before+=(qp->opt_g0[i]*qp->opt_xinit[i]);
+    obj_before+=(0.5*qp->opt_xinit[i]*qp->opt_xinit[i]*qp->opt_g[i*qp->opt_n+i]);
+    for(j=0;j<i;j++) {
+      obj_before+=(qp->opt_xinit[j]*qp->opt_xinit[i]*qp->opt_g[j*qp->opt_n+i]);
+    }
+  }
+
+  result=STILL_RUNNING;
+  qp->opt_ce0[0]*=(-1.0);
+  /* Run pr_loqo. If a run fails, try again with parameters which lead */
+  /* to a slower, but more robust setting. */
+  for(margin=init_margin,iter=init_iter;
+      (margin<=0.9999999) && (result!=OPTIMAL_SOLUTION);) {
+    sigdig=-log10(opt_precision);
+
+    result=pr_loqo((int)qp->opt_n,(int)qp->opt_m,
+		   (double *)qp->opt_g0,(double *)qp->opt_g,
+		   (double *)qp->opt_ce,(double *)qp->opt_ce0,
+		   (double *)qp->opt_low,(double *)qp->opt_up,
+		   (double *)primal,(double *)dual, 
+		   (int)(verbosity-2),
+		   (double)sigdig,(int)iter, 
+		   (double)margin,(double)(qp->opt_up[0])/4.0,(int)0);
+
+    if(isnan(dual[0])) {     /* check for choldc problem */
+      if(verbosity>=2) {
+	printf("NOTICE: Restarting PR_LOQO with more conservative parameters.\n");
+      }
+      if(init_margin<0.80) { /* become more conservative in general */
+	init_margin=(4.0*margin+1.0)/5.0;
+      }
+      margin=(margin+1.0)/2.0;
+      (opt_precision)*=10.0;   /* reduce precision */
+      if(verbosity>=2) {
+	printf("NOTICE: Reducing precision of PR_LOQO.\n");
+      }
+    }
+    else if(result!=OPTIMAL_SOLUTION) {
+      iter+=2000; 
+      init_iter+=10;
+      (opt_precision)*=10.0;   /* reduce precision */
+      if(verbosity>=2) {
+	printf("NOTICE: Reducing precision of PR_LOQO due to (%ld).\n",result);
+      }      
+    }
+  }
+
+  if(qp->opt_m)         /* Thanks to Alex Smola for this hint */
+    model_b=dual[0];
+  else
+    model_b=0;
+
+  /* Check the precision of the alphas. If results of current optimization */
+  /* violate KT-Conditions, relax the epsilon on the bounds on alphas. */
+  epsilon_loqo=1E-10;
+  for(i=0;i<qp->opt_n;i++) {
+    dist=-model_b*qp->opt_ce[i]; 
+    dist+=(qp->opt_g0[i]+1.0);
+    for(j=0;j<i;j++) {
+      dist+=(primal[j]*qp->opt_g[j*qp->opt_n+i]);
+    }
+    for(j=i;j<qp->opt_n;j++) {
+      dist+=(primal[j]*qp->opt_g[i*qp->opt_n+j]);
+    }
+    /*  printf("LOQO: a[%d]=%f, dist=%f, b=%f\n",i,primal[i],dist,dual[0]); */
+    if((primal[i]<(qp->opt_up[i]-epsilon_loqo)) && (dist < (1.0-(*epsilon_crit)))) {
+      epsilon_loqo=(qp->opt_up[i]-primal[i])*2.0;
+    }
+    else if((primal[i]>(0+epsilon_loqo)) && (dist > (1.0+(*epsilon_crit)))) {
+      epsilon_loqo=primal[i]*2.0;
+    }
+  }
+
+  for(i=0;i<qp->opt_n;i++) {  /* clip alphas to bounds */
+    if(primal[i]<=(0+epsilon_loqo)) {
+      primal[i]=0;
+    }
+    else if(primal[i]>=(qp->opt_up[i]-epsilon_loqo)) {
+      primal[i]=qp->opt_up[i];
+    }
+  }
+
+  obj_after=0;  /* calculate objective after optimization */
+  for(i=0;i<qp->opt_n;i++) {
+    obj_after+=(qp->opt_g0[i]*primal[i]);
+    obj_after+=(0.5*primal[i]*primal[i]*qp->opt_g[i*qp->opt_n+i]);
+    for(j=0;j<i;j++) {
+      obj_after+=(primal[j]*primal[i]*qp->opt_g[j*qp->opt_n+i]);
+    }
+  }
+
+  /* if optimizer returned NAN values, reset and retry with smaller */
+  /* working set. */
+  if(isnan(obj_after) || isnan(model_b)) {
+    for(i=0;i<qp->opt_n;i++) {
+      primal[i]=qp->opt_xinit[i];
+    }     
+    model_b=0;
+    if(learn_parm->svm_maxqpsize>2) {
+      learn_parm->svm_maxqpsize--;  /* decrease size of qp-subproblems */
+    }
+  }
+
+  if(obj_after >= obj_before) { /* check whether there was progress */
+    (opt_precision)/=100.0;
+    precision_violations++;
+    if(verbosity>=2) {
+      printf("NOTICE: Increasing Precision of PR_LOQO.\n");
+    }
+  }
+
+  if(precision_violations > 500) { 
+    (*epsilon_crit)*=10.0;
+    precision_violations=0;
+    if(verbosity>=1) {
+      printf("\nWARNING: Relaxing epsilon on KT-Conditions.\n");
+    }
+  }	  
+
+  (*threshold)=model_b;
+
+  if(result!=OPTIMAL_SOLUTION) {
+    printf("\nERROR: PR_LOQO did not converge. \n");
+    return(qp->opt_xinit);
+  }
+  else {
+    return(primal);
+  }
+}
+

data/ext/hack/Rakefile

@@ -0,0 +1,17 @@
+require 'rake'
+require_relative 'support'
+
+desc 'Verifies the requirements'
+task :requirements do
+  verify_requirements
+  require_executable("make")
+end
+
+task :compile => :requirements do
+  compile_vendored_code
+end
+
+task :default => [:compile] do
+  #path = File.join(PYTHON_SITE_PACKAGES, 'pre_install')
+  #pip_install(PRE_INSTALL_REQUIREMENTS, path)
+end

data/ext/hack/support.rb

@@ -0,0 +1,88 @@
+require 'opener/build-tools'
+
+include Opener::BuildTools::Requirements
+include Opener::BuildTools::Python
+include Opener::BuildTools::Files
+
+# Directory of the core
+CORE_DIR = File.expand_path('../../../core', __FILE__)
+
+# Directory where packages will be installed to.
+PYTHON_SITE_PACKAGES = File.expand_path(
+  '../../../core/site-packages',
+  __FILE__
+)
+
+# Directory containing the temporary files.
+TMP_DIRECTORY = File.expand_path('../../../tmp', __FILE__)
+
+# Path to the pip requirements file used to install requirements before
+# packaging the Gem.
+PRE_BUILD_REQUIREMENTS = File.expand_path(
+  '../../../pre_build_requirements.txt',
+  __FILE__
+)
+
+# Path to the pip requirements file used to install requirements upon Gem
+# installation.
+PRE_INSTALL_REQUIREMENTS = File.expand_path(
+  '../../../pre_install_requirements.txt',
+  __FILE__
+)
+
+# Path to the vendor directory for C code.
+VENDOR_DIRECTORY = File.expand_path('../../../core/vendor', __FILE__)
+
+# Path to the directory to install vendored C code into.
+VENDOR_BUILD_DIRECTORY = File.expand_path(
+  '../../../core/vendor/build',
+  __FILE__
+)
+
+# Path to the directory that contains the source C code to compile.
+VENDOR_SRC_DIRECTORY = File.expand_path('../../../core/vendor/src', __FILE__)
+
+
+##
+# Verifies the requirements to install thi Gem.
+#
+def verify_requirements
+  require_executable('python')
+  require_version('python', python_version, '2.6.0')
+  require_executable('pip')
+  require_version('pip', pip_version, '1.3.1')
+end
+
+##
+# Compiles C code in the current directory using `make`.
+#
+# @param [Array] args The arguments to pass to ./configure
+#
+def compile(*args)
+  sh "./configure #{args.join(' ')}"
+  sh 'make'
+  sh 'make install'
+  sh 'make distclean'
+end
+
+##
+# Compiles the C code found in src/vendor.
+#
+def compile_vendored_code
+  src   = VENDOR_SRC_DIRECTORY
+  build = VENDOR_BUILD_DIRECTORY
+
+  Dir.chdir(File.join(src, "liblbfgs")) do
+    compile("--prefix=#{build}")
+  end
+
+  Dir.chdir(File.join(src, "crfsuite")) do
+    compile("--prefix=#{build}", "--with-liblbfgs=#{build}")
+  end
+
+  Dir.chdir(File.join(src, "svm_light")) do
+    sh 'make'
+    sh 'mv svm_classify svm_learn ../../build/bin'
+    sh 'make clean'
+  end
+end

data/lib/opener/opinion_detectors/base.rb

@@ -0,0 +1,112 @@
+require 'nokogiri'
+require 'open3'
+
+require_relative 'base/version'
+require_relative 'configuration_creator'
+require_relative 'en'
+require_relative 'nl'
+require_relative 'de'
+require_relative 'it'
+
+module Opener
+  module OpinionDetectors
+    class ModelsMissing < StandardError; end
+
+    ##
+    # The base Opinion detector that supports English and Dutch.
+    #
+    # @!attribute [r] args
+    #  @return [Array]
+    #
+    # @!attribute [r] options
+    #  @return [Hash]
+    #
+    class Base
+      attr_reader :args, :options, :conf_file, :models_path
+
+      ##
+      # @param [Hash] options
+      #
+      # @option options [Array] :args Extra commandline arguments to use.
+      #
+      def initialize(options = {})
+        @args    = options.delete(:args) || []
+        @options = options
+      end
+
+      ##
+      # Builds the command used to execute the kernel.
+      #
+      # @param [Array] args Commandline arguments passed to the command.
+      #
+      def command
+        return "#{adjust_python_path} python -E -OO #{kernel} -m #{conf_file.path} #{args.join(' ')}"
+      end
+
+      ##
+      # Runs the command and returns the output of STDOUT, STDERR and the
+      # process information.
+      #
+      # @param [String] input The input to tag.
+      # @return [Array]
+      #
+      def run(input)
+        language     = language(input)
+        conf         = ConfigurationCreator.new(language)
+        @conf_file   = conf.config_file_path
+        @models_path = conf.models_path
+
+        return capture(input)
+      end
+
+      protected
+
+      ##
+      # @return [String]
+      #
+      def adjust_python_path
+        site_packages =  File.join(core_dir, 'site-packages')
+
+        return "env PYTHONPATH=#{site_packages}:$PYTHONPATH"
+      end
+
+      ##
+      # capture3 method doesn't work properly with Jruby, so
+      # this is a workaround
+      #
+      def capture(input)
+        return Open3.popen3(*command.split(" ")) {|i, o, e, t|
+          out_reader = Thread.new { o.read }
+          err_reader = Thread.new { e.read }
+          i.write input
+          i.close
+          [out_reader.value, err_reader.value, t.value]
+        }
+      end
+
+      ##
+      # @return [String]
+      #
+      def core_dir
+        return File.expand_path('../../../../core', __FILE__)
+      end
+
+      ##
+      # @return [String]
+      #
+      def kernel
+        return File.join(core_dir, 'python-scripts/classify_kaf_naf_file.py')
+      end
+
+      ##
+      # @return the language from the KAF
+      #
+      def language(input)
+        document = Nokogiri::XML(input)
+        language = document.at('KAF').attr('xml:lang')
+
+        return language
+      end
+    end # Base
+  end # OpinionDetectors
+end # Opener

data/lib/opener/opinion_detectors/base/version.rb

@@ -0,0 +1,7 @@
+module Opener
+  module OpinionDetectors
+    class Base
+      VERSION = "2.0.0"
+    end
+  end
+end

data/lib/opener/opinion_detectors/configuration_creator.rb

@@ -0,0 +1,86 @@
+require 'erb'
+require 'tempfile'
+
+module Opener
+  module OpinionDetectors
+    class ConfigurationCreator
+      attr_reader :language
+
+      include ERB::Util
+
+      def initialize(language)
+        @language = language
+      end
+
+      def config_file_path
+        file = Tempfile.new('opinion-detector-config')
+        file.write(render)
+        file.close
+
+        return file
+      end
+
+      def render
+        ERB.new(template).result(binding)
+      end
+
+      def models_path
+        env_path = ENV["OPINION_DETECTOR_MODELS_PATH"]
+        if env_path.nil?
+          raise ModelsMissing, "Please provide an environment variable named
+            OPINION_DETECTOR_MODELS_PATH that contains the path to the models"
+        end
+
+        path = File.expand_path(language, env_path)
+        return path
+      end
+
+      def crfsuite_path
+        File.expand_path("../../../../core/vendor/build/bin/crfsuite",__FILE__)
+      end
+
+      def svm_learn_path
+        File.expand_path("../../../../core/vendor/build/bin/svm_learn", __FILE__)
+      end
+
+      def svm_classify_path
+        File.expand_path("../../../../core/vendor/build/bin/svm_classify", __FILE__)
+      end
+
+      def template
+        %{
+[general]
+output_folder = <%=models_path%>/hotel_cfg1
+
+[feature_templates]
+expression = <%=models_path%>/hotel_cfg1/feature_templates/feat_template_expr.txt
+holder = <%=models_path%>/hotel_cfg1/feature_templates/feat_template_holder.txt
+target = <%=models_path%>/hotel_cfg1/feature_templates/feat_template_target.txt
+
+[valid_opinions]
+negative = Negative;StrongNegative
+positive = Positive;StrongPositive
+
+[relation_features]
+use_dependencies = True
+use_training_lexicons = True
+use_this_expression_lexicon = <%=models_path%>/hotel_cfg1/lexicons/polarity_lexicon.txt
+use_this_target_lexicon = <%=models_path%>/hotel_cfg1/lexicons/target_lexicon.txt
+use_tokens_lemmas = True
+exp_tar_threshold = -0.75
+exp_hol_threshold = -0.5
+
+[crfsuite]
+path_to_binary = <%= crfsuite_path %>
+parameters = -a lbfgs
+
+[svmlight]
+path_to_binary_learn = <%= svm_learn_path %>
+path_to_binary_classify = <%= svm_classify_path %>
+parameters = -c 0.1
+        }
+      end
+
+    end
+  end
+end