svm_helper 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: bbfc13983f715f2f0ab66d2a9dbfed543896329e
+  data.tar.gz: 2e6f5c32898cc01a2468ebaa2e82470e01012586
+SHA512:
+  metadata.gz: 8c616e62ff4717808e0b9f29e3d6773c03096934194a1fb706e3eb0ded7527c455ea00039e0a4d39fc08e13f735d4f49cd984d26128e624fbe0b956851893c21
+  data.tar.gz: f28756b0f3f9539f69bcda6ce3f79bdf3ef226ee813f0e5ed7b6f38595f1760e05bbd1abe7ec8ccc5c3a8db8b666871a0d33a755c19c4842a85a600e4749498d

data/.gitignore

@@ -0,0 +1,22 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+
+.rbx
+
+.tags
+.tags_sorted_by_file

data/.rspec

@@ -0,0 +1,3 @@
+--tty
+--color
+--format documentation

data/.ruby-version

@@ -0,0 +1 @@
+2.0.0

data/.travis.yml

@@ -0,0 +1,9 @@
+language: ruby
+rvm:
+  - 2.0.0
+  - 1.9.3
+  - 1.9.2
+  - jruby-19mode
+  - rbx-19mode
+  - ruby-head
+  - jruby-head

data/.versions.conf

@@ -0,0 +1,4 @@
+ruby=ruby-2.0.0
+ruby-gemset=svm_helper
+#ruby-gem-install=bundler rake
+#ruby-bundle-install=true

data/.yardopts

@@ -0,0 +1,3 @@
+--no-private
+--markup-provider=kramdown
+--markup=markdown

data/Gemfile

@@ -0,0 +1,24 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in svm_helper.gemspec
+gemspec
+
+group :development do
+  gem 'yard'
+  gem 'kramdown'
+  gem 'github-markup'
+
+  gem 'pry'
+  gem 'guard-rspec'
+  gem 'guard-yard'
+
+  gem 'rb-inotify', '~> 0.9', :require => false
+  gem 'rb-fsevent', :require => false
+  gem 'rb-fchange', :require => false
+end
+
+group :test do
+  gem 'rake'
+  gem 'mocha', require: 'mocha/api'
+  gem 'factory_girl', '~> 4.0'
+end

data/Guardfile

@@ -0,0 +1,17 @@
+# guard 'rspec', cli: "--color --format p", all_after_pass: false do
+guard 'rspec', cli: "--color --format p", all_after_pass: false, rvm:['2.0.0@svm_helper', 'jruby@svm_helper'] do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/svm_helper/(.+)\.rb$})               { |m| "spec/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')            { 'spec' }
+  watch('spec/factories.rb')              { 'spec' }
+  watch(%r{^spec/factories/(.+)\.rb})     { 'spec' }
+  watch(%r{^spec/support/(.+)_spec\.rb})  { |m| "spec/#{m[1]}s/*" }
+end
+
+notification :tmux,
+  :display_message => true,
+  :timeout => 3 # in seconds
+
+# guard 'yard' do
+#   watch(%r{lib/.+\.rb})
+# end

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Andreas Eger
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,41 @@
+# SvmHelper
+
+Shared helper classes for usage in context of SVM at experteer
+
+[![Build Status](https://travis-ci.org/sch1zo/svm_helper.png?branch=master)](https://travis-ci.org/sch1zo/svm_helper)
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'svm_helper'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install svm_helper
+
+## Usage
+
+Dataflow is normally something like this:
+
+    Job --Preprocessor--> PreprocessedData --Selector--> FeatureVector
+
+The FeatureVector can now be used for training or prediction in a (libsvm) SVM.
+
+Be aware that a FeatureVector has two Attributes:
+
+    data: the feature array itself
+    label: 1 for true, 0 for false
+
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,7 @@
+require "bundler/gem_tasks"
+
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new('spec')
+
+# If you want to make this the default task
+task :default => :spec

data/lib/svm_helper.rb

@@ -0,0 +1,8 @@
+require "svm_helper/version"
+require 'parallel'
+
+require "svm_helper/preprocessed_data"
+require "svm_helper/feature_vector"
+require "svm_helper/preprocessors"
+require "svm_helper/selectors"
+

data/lib/svm_helper/feature_vector.rb

@@ -0,0 +1,17 @@
+require_relative 'interface_helper'
+#
+# FeatureVector interface
+#
+# @author Andreas Eger
+class FeatureVector < InterfaceHelper
+  attribute :word_data
+  attribute :classification_arrays
+  attribute :labels
+
+  def label
+    labels[classification]
+  end
+  def data
+    word_data + classification_arrays[classification]
+  end
+end

data/lib/svm_helper/interface_helper.rb

@@ -0,0 +1,57 @@
+#
+# @abstract Subclass and define attributes
+#
+# @author Andreas Eger
+class InterfaceHelper
+  @@_attributes = Hash.new { |hash, key| hash[key] = [] }
+
+  #
+  # creates setter/getter similar to attr_accesor
+  # @param  name [Symbol]
+  # @macro [attach] attribute
+  #   @method $1
+  #     reads $1
+  #   @method $1=
+  #     saves $1
+  def self.attribute name
+    define_method(name) do
+      @_attributes[name]
+    end
+    define_method(:"#{name}=") do |v|
+      @_attributes[name] = v
+    end
+    attributes << name unless attributes.include? name
+  end
+  def self.attributes
+    @@_attributes[self]
+  end
+  def initialize(params={})
+    @_attributes = {}
+    params.each do |key, value|
+      send("#{key}=", value)
+    end
+    @_attributes[:classification] ||= :function
+  end
+
+  #
+  # custom comperator
+  # @param anOther [InterfaceHelper]
+  #
+  # @return [Boolean] result after comparing each attribute
+  def == anOther
+    @_attributes.keys.map{ |sym| self.send(sym) == anOther.send(sym)}.reduce(true){|a,e| a && e }
+  end
+
+  def industry!
+    @_attributes[:classification] = :industry
+  end
+  def function!
+    @_attributes[:classification] = :function
+  end
+  def career_level!
+    @_attributes[:classification] = :career_level
+  end
+  def classification
+    @_attributes[:classification]
+  end
+end

data/lib/svm_helper/preprocessed_data.rb

@@ -0,0 +1,17 @@
+require_relative 'interface_helper'
+#
+# PreprocessedData interface
+#
+# @author Andreas Eger
+class PreprocessedData < InterfaceHelper
+  attribute :data
+  attribute :ids
+  attribute :labels
+
+  def id
+    ids[classification]
+  end
+  def label
+    labels[classification]
+  end
+end

data/lib/svm_helper/preprocessors.rb

@@ -0,0 +1,2 @@
+require_relative 'preprocessors/simple'
+require_relative 'preprocessors/with_industry_map'

data/lib/svm_helper/preprocessors/simple.rb

@@ -0,0 +1,111 @@
+# encoding: UTF-8
+module Preprocessor
+  #
+  # Preprocessor which just cleans to text
+  #
+  # @author Andreas Eger
+  #
+  class Simple
+    THREAD_COUNT = (ENV['OMP_NUM_THREADS'] || 2).to_i
+    # filters most gender stuff
+    GENDER_FILTER = %r{(\(*(m|w)(\/|\|)(w|m)\)*)|(/-*in)|\(in\)}
+    # filters most wierd symbols
+    SYMBOL_FILTER = %r{/|-|–|:|\+|!|,|\.|\*|\?|/|·|\"|„|•||\||(\S*(&|;)\S*)}
+    # urls and email filter
+    URL_FILTER = /(https?:\/\/)?([\da-z\.-]+)\.([a-z\.]{2,6})([\/\w \.-]*)*\/?/
+    EMAIL_FILTER = /([a-z0-9_\.-]+)@([\da-z\.-]+)\.([a-z\.]{2,6})/
+    # filter for new lines
+    NEW_LINES = /(\r\n)|\r|\n/
+    # extract words from brackets
+    WORDS_IN_BRACKETS = /\(([a-zA-Z]+)\)/
+    # filters multiple whitesspace
+    WHITESPACE = /(\s| )+/
+    # filters all kind of XMl/HTML tags
+    XML_TAG_FILTER = /<(.*?)>/
+    # filter for used job tokens
+    CODE_TOKEN_FILTER = /\[[^\]]*\]|\([^\)]*\)|\{[^\}]*\}|\S*\d+\w+/
+
+    def initialize args={}
+      @parallel = args.fetch(:parallel){false}
+    end
+
+    def label
+      "simple"
+    end
+    #
+    # cleans provided jobs
+    # @overload process(jobs, classification)
+    #   @param  jobs [Job] single Job
+    #   @param  classification [Symbol] in `:industry`, `:function`, `:career_level`
+    # @overload process(jobs, classification)
+    #   @param  jobs [Array<Job>] list of Jobs
+    #   @param  classification [Symbol] in `:industry`, `:function`, `:career_level`
+    #
+    # @return [Array<PreprocessedData>] list of processed job data - or singe job data
+    def process jobs, classification=:function
+      if jobs.respond_to? :map
+        process_jobs jobs, classification
+      else
+        process_job jobs, classification
+      end
+    end
+
+    #
+    # converts string into a cleaner version
+    # @param  title [String] job title
+    #
+    # @return [String] clean and lowercase version of input
+    def clean_title title
+      title.gsub(GENDER_FILTER,'').
+            gsub(SYMBOL_FILTER,'').
+            gsub(WORDS_IN_BRACKETS, '\1').
+            gsub(CODE_TOKEN_FILTER,'').
+            gsub(WHITESPACE,' ').
+            downcase.
+            strip
+    end
+    #
+    # converts string into a cleaner version
+    # @param  desc [String] job description
+    #
+    # @return [String] clean and lowercase version of input
+    def clean_description desc
+      desc.gsub(XML_TAG_FILTER,' ')
+          .gsub(EMAIL_FILTER,'')
+          .gsub(URL_FILTER,'')
+          .gsub(GENDER_FILTER,'')
+          .gsub(NEW_LINES,'')
+          .gsub(SYMBOL_FILTER,' ')
+          .gsub(WHITESPACE,' ')
+          .gsub(WORDS_IN_BRACKETS, '\1')
+          .gsub(CODE_TOKEN_FILTER,'')
+          .downcase
+          .strip
+    end
+
+    private
+    def process_jobs jobs, classification
+      if @parallel && RUBY_PLATFORM == 'java'
+        Parallel.map(jobs, in_threads: THREAD_COUNT ) {|job| process_job job, classification }
+      elsif @parallel
+        Parallel.map(jobs, in_processes: THREAD_COUNT ) {|job| process_job job, classification }
+      else
+        jobs.map {|job| process_job job, classification }
+      end
+    end
+
+    def process_job job, classification
+      PreprocessedData.new(
+        data: [ clean_title(job.title), clean_description(job.description) ],
+        ids: {
+          industry: job.classification_id(:industry),
+          function: job.classification_id(:function),
+          career_level: job.classification_id(:career_level) },
+        labels: {
+          industry: job.label(:industry),
+          function: job.label(:function),
+          career_level: job.label(:career_level) }
+      ).tap{|e| e.send("#{classification}!")}
+    end
+  end
+end

data/lib/svm_helper/preprocessors/with_industry_map.rb

@@ -0,0 +1,40 @@
+require_relative 'simple'
+module Preprocessor
+  #
+  # Preprocessor Base Class
+  #
+  # @author Andreas Eger
+  #
+  class WithIndustryMap < Simple
+    attr_reader :industry_map
+
+    #
+    # @param  args [Hash] options hash
+    # @option args [Hash] :industry_map mapping for the tree like industry ids to continuous ones
+    def initialize args={}
+      @industry_map = args.fetch(:industry_map){ Hash[Pjpp::Industry.select(:id).all.map(&:id).sort.map.with_index{|e,i| [e,i]}] }
+    end
+
+    def map_industry_id(id)
+      @industry_map[id]
+    end
+    def label
+      "with_industry_map"
+    end
+
+    private
+    def process_job job, classification
+      PreprocessedData.new(
+        data: [ clean_title(job.title), clean_description(job.description) ],
+        ids: {
+          industry: map_industry_id(job.classification_id(:industry)),
+          function: job.classification_id(:function),
+          career_level: job.classification_id(:career_level) },
+        labels: {
+          industry: job.label(:industry),
+          function: job.label(:function),
+          career_level: job.label(:career_level) }
+      ).tap{|e| e.send("#{classification}!")}
+    end
+  end
+end