svm_helper 0.1.0 → 0.1.1

data/Guardfile

@@ -1,5 +1,5 @@
-# guard 'rspec', cli: "--color --format p", all_after_pass: false do
-guard 'rspec', cli: "--color --format p", all_after_pass: false, rvm:['2.0.0@svm_helper', 'jruby@svm_helper'] do
+guard 'rspec', cli: "--color --format p", all_after_pass: false do
+# guard 'rspec', cli: "--color --format p", all_after_pass: false, rvm:['2.0.0@svm_helper', 'jruby@svm_helper'] do
   watch(%r{^spec/.+_spec\.rb$})
   watch(%r{^lib/svm_helper/(.+)\.rb$})               { |m| "spec/#{m[1]}_spec.rb" }
   watch('spec/spec_helper.rb')            { 'spec' }

data/lib/svm_helper/feature_vector.rb

@@ -5,13 +5,10 @@ require_relative 'interface_helper'
 # @author Andreas Eger
 class FeatureVector < InterfaceHelper
   attribute :word_data
-  attribute :classification_arrays
-  attribute :labels
+  attribute :classification
+  attribute :label
 
-  def label
-    labels[classification]
-  end
   def data
-    word_data + classification_arrays[classification]
+    word_data + classification
   end
 end

data/lib/svm_helper/interface_helper.rb

@@ -30,7 +30,6 @@ class InterfaceHelper
     params.each do |key, value|
       send("#{key}=", value)
     end
-    @_attributes[:classification] ||= :function
   end
 
   #
@@ -41,17 +40,4 @@ class InterfaceHelper
   def == anOther
     @_attributes.keys.map{ |sym| self.send(sym) == anOther.send(sym)}.reduce(true){|a,e| a && e }
   end
-
-  def industry!
-    @_attributes[:classification] = :industry
-  end
-  def function!
-    @_attributes[:classification] = :function
-  end
-  def career_level!
-    @_attributes[:classification] = :career_level
-  end
-  def classification
-    @_attributes[:classification]
-  end
 end

data/lib/svm_helper/preprocessed_data.rb

@@ -5,13 +5,6 @@ require_relative 'interface_helper'
 # @author Andreas Eger
 class PreprocessedData < InterfaceHelper
   attribute :data
-  attribute :ids
-  attribute :labels
-
-  def id
-    ids[classification]
-  end
-  def label
-    labels[classification]
-  end
+  attribute :id
+  attribute :label
 end

data/lib/svm_helper/preprocessors/id_mapping.rb

@@ -0,0 +1,35 @@
+require_relative 'simple'
+module Preprocessor
+  #
+  # Preprocessor Base Class
+  #
+  # @author Andreas Eger
+  #
+  class IDMapping < Simple
+    attr_reader :id_map
+
+    #
+    # @param  args [Hash] options hash
+    # @option args [Hash] :industry_map mapping for the tree like industry ids to continuous ones
+    def initialize id_map, args={}
+      super(args)
+      @id_map = id_map
+    end
+
+    def map_id(id)
+      @id_map[id]
+    end
+    def label
+      "with_id_mapping"
+    end
+
+    private
+    def process_job job
+      PreprocessedData.new(
+        data: [clean_title(job[:title]), clean_description(job[:description])],
+        id: map_id(job[:id]),
+        label: job[:label]
+      )
+    end
+  end
+end

data/lib/svm_helper/preprocessors/simple.rb

@@ -35,21 +35,26 @@ module Preprocessor
     #
     # cleans provided jobs
     # @overload process(jobs, classification)
-    #   @param  jobs [Job] single Job
+    #   @param  jobs [Hash] single Job
+    #   @option title
+    #   @option description
+    #   @option id
+    #   @option label
     #   @param  classification [Symbol] in `:industry`, `:function`, `:career_level`
     # @overload process(jobs, classification)
-    #   @param  jobs [Array<Job>] list of Jobs
+    #   @param  jobs [Array<Hash>] list of Jobs
     #   @param  classification [Symbol] in `:industry`, `:function`, `:career_level`
     #
     # @return [Array<PreprocessedData>] list of processed job data - or singe job data
-    def process jobs, classification=:function
-      if jobs.respond_to? :map
-        process_jobs jobs, classification
+    def process jobs
+      if jobs.is_a? Array
+        process_jobs jobs
       else
-        process_job jobs, classification
+        process_job jobs
       end
     end
 
+
     #
     # converts string into a cleaner version
     # @param  title [String] job title
@@ -84,28 +89,22 @@ module Preprocessor
     end
 
     private
-    def process_jobs jobs, classification
+    def process_jobs jobs
       if @parallel && RUBY_PLATFORM == 'java'
-        Parallel.map(jobs, in_threads: THREAD_COUNT ) {|job| process_job job, classification }
+        Parallel.map(jobs, in_threads: THREAD_COUNT ) {|job| process_job job }
       elsif @parallel
-        Parallel.map(jobs, in_processes: THREAD_COUNT ) {|job| process_job job, classification }
+        Parallel.map(jobs, in_processes: THREAD_COUNT ) {|job| process_job job }
       else
-        jobs.map {|job| process_job job, classification }
+        jobs.map {|job| process_job job }
       end
     end
 
-    def process_job job, classification
+    def process_job job
       PreprocessedData.new(
-        data: [ clean_title(job.title), clean_description(job.description) ],
-        ids: {
-          industry: job.classification_id(:industry),
-          function: job.classification_id(:function),
-          career_level: job.classification_id(:career_level) },
-        labels: {
-          industry: job.label(:industry),
-          function: job.label(:function),
-          career_level: job.label(:career_level) }
-      ).tap{|e| e.send("#{classification}!")}
+        data: [clean_title(job[:title]), clean_description(job[:description])],
+        id: job[:id],
+        label: job[:label]
+      )
     end
   end
 end

data/lib/svm_helper/preprocessors.rb

@@ -1,2 +1,2 @@
 require_relative 'preprocessors/simple'
-require_relative 'preprocessors/with_industry_map'
+require_relative 'preprocessors/id_mapping'

data/lib/svm_helper/selectors/n_gram.rb

@@ -8,7 +8,7 @@ module Selector
   class NGram < Selector::Simple
     attr_reader :gram_size
 
-    def initialize args={}
+    def initialize classification, args={}
       super
       @gram_size = args.fetch(:gram_size) { 2 }
     end

data/lib/svm_helper/selectors/simple.rb

@@ -26,7 +26,8 @@ module Selector
 
     attr_accessor :global_dictionary
 
-    def initialize args={}
+    def initialize classification, args={}
+      @classification = classification
       @global_dictionary = args.fetch(:global_dictionary) {[]}
       @language = args.fetch(:language){'en'}
       @parallel = args.fetch(:parallel){false}
@@ -43,13 +44,13 @@ module Selector
     # @param  dictionary_size [Integer] Size of a dictionary to create if non exists
     #
     # @return [Array<FeatureVector>] list of feature vectors and labels
-    def generate_vectors data_set, classification=:function, dictionary_size=DEFAULT_DICTIONARY_SIZE
+    def generate_vectors data_set, dictionary_size=DEFAULT_DICTIONARY_SIZE
       words_per_data = extract_words data_set
       generate_global_dictionary words_per_data, dictionary_size
 
       make_vectors(words_per_data) do |words,index|
         word_set = words.uniq
-        make_vector word_set, data_set[index], classification
+        make_vector word_set, data_set[index]
       end
     end
 
@@ -60,9 +61,9 @@ module Selector
     # @param  dictionary [Array] dictionary to use for this selection
     #
     # @return [FeatureVector]
-    def generate_vector data, classification=:function, dictionary=global_dictionary
+    def generate_vector data, dictionary=global_dictionary
       word_set = Set.new extract_words_from_data(data)
-      make_vector word_set, data, classification, dictionary
+      make_vector word_set, data, dictionary
     end
 
     #
@@ -109,8 +110,9 @@ module Selector
       (data.data.flat_map(&:split) - stopwords).delete_if { |e| e.size <= 3 }
     end
 
-    def reset
+    def reset classification
       @global_dictionary = []
+      @classification = classification
     end
 
     private
@@ -120,24 +122,17 @@ module Selector
     # also adds the label
     # @param  words [Array<String>] list of words
     # @param  data [PreprocessedData]
-    # @param  classification [Symbol] in `:industry`, `:function`, `:career_level`
     # @param  dictionary
     #
     # @return [FeatureVector]
-    def make_vector words, data, classification, dictionary=global_dictionary
+    def make_vector words, data, dictionary=global_dictionary
       FeatureVector.new(
         word_data: dictionary.map{|dic_word|
-                words.include?(dic_word) ? 1 : 0
-              },
-        classification_arrays: {
-          function: classification_array(data.ids, :function),
-          industry: classification_array(data.ids, :industry),
-          career_level: classification_array(data.ids, :career_level) },
-        labels: {
-          function: data.labels[:function] ? 1 : 0,
-          industry: data.labels[:industry] ? 1 : 0,
-          career_level: data.labels[:career_level] ? 1 : 0 }
-      ).tap{|e| e.send("#{classification}!")}
+                     words.include?(dic_word) ? 1 : 0
+                   },
+        classification: classification_array(data.id),
+        label: data.label ? 1 : 0
+      )
     end
 
     def make_vectors data, &block
@@ -155,9 +150,8 @@ module Selector
     # @param  ids [Hash] hash with classification ids
     #
     # @return [Array<Integer>] list of size=count(classifcation_ids) with only one not zero item
-    def classification_array(ids, classification)
-      id = ids[classification]
-      Array.new(CLASSIFICATIONS_SIZE[classification]){|n| n==(id-1) ? 1 : 0}
+    def classification_array(id)
+      Array.new(CLASSIFICATIONS_SIZE[@classification]){|n| n==(id-1) ? 1 : 0}
     end
   end
 end

data/lib/svm_helper/selectors/with_binary_encoding.rb

@@ -12,7 +12,7 @@ module Selector
           industry: 16,       # max id 65535, currently 14370
           career_level: 4 }   # max id 15, currently 8
 
-    def initialize args={}
+    def initialize *args
       super
     end
 
@@ -26,9 +26,8 @@ module Selector
     # @param  ids [Hash] hash with classification ids
     #
     # @return [Array<Integer>] binary encoded classification id
-    def classification_array(ids, classification)
-      id = ids[classification]
-      number_to_binary_array(id, CLASSIFICATIONS_SIZE[classification])
+    def classification_array(id)
+      number_to_binary_array(id, CLASSIFICATIONS_SIZE[@classification])
     end
 
     def number_to_binary_array(number, size=8)

data/lib/svm_helper/version.rb

@@ -1,3 +1,3 @@
 module SvmHelper
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

data/spec/factories.rb

@@ -7,24 +7,20 @@ FactoryGirl.define do
     wrong_function_id 4
     wrong_career_level nil
   end
-  factory :job, class: OpenStruct do
+  factory :job, class: Hash do
     title "Meh"
     description "Foo Bar"
-    summary "Really lot of work to do"
-    qc_job_check
-  end
+    id 4
+    label true
 
-  factory :job_without_job_check, class: OpenStruct do
-    title "Meh"
-    description "Foo Bar"
-    summary "Really lot of work to do"
-    original_industry_id 1423
+    initialize_with { attributes }
   end
 
+
   factory :data, class: PreprocessedData do
     data ["haus fooo garten baaz pferd fooo"]
-    ids {{function:  3, industry: 43, career_level: 7}}
-    labels {{function:  true, industry: false, career_level: true}}
+    id 7
+    label true
   end
   factory :data_w_short_words, parent: :data do
     data ["auto foo pferd bz gooo fooo 2"]

data/spec/preprocessors/id_mapping_spec.rb

@@ -0,0 +1,12 @@
+require 'spec_helper'
+
+describe Preprocessor::IDMapping do
+  it_behaves_like 'a preprocessor'
+  let(:preprocessor) { Preprocessor::IDMapping.new(1423=>3, 523=>54) }
+  let(:job) { FactoryGirl.build(:job) }
+  let(:jobs) { [job] }
+  it "should make use of a industry_map" do
+    preprocessor.expects(:map_id)
+    preprocessor.process(jobs)
+  end
+end

data/spec/preprocessors/simple_spec.rb

@@ -11,31 +11,18 @@ describe Preprocessor::Simple do
   context do
     before(:each) do
       @jobs = FactoryGirl.build_list :job, 3
-      @jobs.each{|e| e.stubs(:classification_id)}
-      @jobs.each{|e| e.stubs(:label)}
     end
     it "should work with jobs with quality check" do
       -> {simple.process(@jobs) }.should_not raise_error
     end
-    # it "should set labels to true if quality check exists and no wrong_ label set" do
-    #   simple.process(@jobs).each{|e| e.career_level!; e.label.should be_true}
-    # end
-    it "should set labels to false if quality check exists and wrong_ label is set" do
-      simple.process(@jobs).each{|e| e.function!; e.label.should be_false}
+    it "should set labels to true if quality check exists and label was true" do
+      @jobs.map!{|e| e[:label] = true;e }
+      simple.process(@jobs).each{|e| e.label.should be_true}
+    end
+    it "should set labels to false if quality check exists and label false" do
+      @jobs.map!{|e| e[:label] = false;e }
+      simple.process(@jobs).each{|e| e.label.should be_false}
     end
-  end
-
-  it "should work with jobs without quality check" do
-    jobs = FactoryGirl.build_list :job_without_job_check, 3
-    jobs.each{|e| e.stubs(:classification_id)}
-    jobs.each{|e| e.stubs(:label)}
-    -> {simple.process(jobs) }.should_not raise_error
-  end
-  it "should set labels to false if no quality check" do
-    jobs = FactoryGirl.build_list :job_without_job_check, 3
-    jobs.each{|e| e.stubs(:classification_id)}
-    jobs.each{|e| e.stubs(:label)}
-    simple.process(jobs).each{|e| e.career_level!; e.label.should be_false}
   end
 
   context "processing" do
@@ -43,8 +30,6 @@ describe Preprocessor::Simple do
     before(:each) do
       simple.stubs(:clean_title)
       simple.stubs(:clean_description)
-      jobs.each{|e| e.stubs(:classification_id)}
-      jobs.each{|e| e.stubs(:label)}
     end
     it "should call clean_title on each job" do
       simple.expects(:clean_title).times(3)
@@ -59,7 +44,7 @@ describe Preprocessor::Simple do
   context "#clean_title" do
     it "should be downcased" do
       job = FactoryGirl.build(:job_title_downcasing)
-      simple.clean_title(job.title).should eq(job.clean_title)
+      simple.clean_title(job[:title]).should eq(job[:clean_title])
     end
     [ FactoryGirl.build(:job_title_w_gender),
       FactoryGirl.build(:job_title_w_gender_brackets),
@@ -77,8 +62,8 @@ describe Preprocessor::Simple do
       FactoryGirl.build(:job_title_var_0),
       FactoryGirl.build(:job_title_w_special),
       FactoryGirl.build(:job_title_w_percent)].each do |job|
-      it "should cleanup '#{job.title}'" do
-        simple.clean_title(job.title).should eq(job.clean_title)
+      it "should cleanup '#{job[:title]}'" do
+        simple.clean_title(job[:title]).should eq(job[:clean_title])
       end
     end
   end
@@ -91,27 +76,27 @@ describe Preprocessor::Simple do
         FactoryGirl.build(:job_description_w_gender) ]
     }
     it "should remove html/xml tags" do
-      desc = simple.clean_description(jobs[0].description)
+      desc = simple.clean_description(jobs[0][:description])
       desc.should_not match(/<(.*?)>/)
     end
     it "should remove new lines" do
-      desc = simple.clean_description(jobs[0].description)
+      desc = simple.clean_description(jobs[0][:description])
       desc.should_not match(/\r\n|\n|\r/)
     end
     it "should remove all special characters" do
-      desc = simple.clean_description(jobs[2].description)
+      desc = simple.clean_description(jobs[2][:description])
       desc.should_not match(/[^a-z öäü]/i)
     end
     it "should remove gender tokens" do
-      desc = simple.clean_description(jobs[3].description)
+      desc = simple.clean_description(jobs[3][:description])
       desc.should_not match(%r{(\(*(m|w)(\/|\|)(w|m)\)*)|(/-*in)|\(in\)})
     end
     it "should remove job code token" do
-      desc = simple.clean_description(jobs[4].description)
+      desc = simple.clean_description(jobs[4][:description])
       desc.should_not match(/\[.*\]|\(.*\)|\{.*\}|\d+\w+/)
     end
     it "should be downcased" do
-      desc = simple.clean_description(jobs[2].description)
+      desc = simple.clean_description(jobs[2][:description])
       desc.should_not match(/[^a-z öäü]/)
     end
   end
@@ -125,13 +110,9 @@ describe Preprocessor::Simple do
         FactoryGirl.build(:job_description_w_code_token),
         FactoryGirl.build(:job_description_w_gender) ]
     }
-    before(:each) do
-      jobs.each{|e| e.stubs(:classification_id)}
-      jobs.each{|e| e.stubs(:label)}
-    end
     it "should be the same parallelized" do
-      single = simple.process(jobs, :function)
-      p_data = parallel.process(jobs, :function)
+      single = simple.process(jobs)
+      p_data = parallel.process(jobs)
       single.each.with_index { |e,i| e.data.should == p_data[i].data }
     end
   end

data/spec/selectors/n_gram_spec.rb

@@ -3,7 +3,7 @@ require "spec_helper"
 describe Selector::NGram do
   it_behaves_like 'a selector'
 
-  let(:ngram) { Selector::NGram.new(gram_size: 3) }
+  let(:ngram) { Selector::NGram.new(:function, gram_size: 3) }
   context "#extract_words_from_data" do
     it "should generate a list of words from the data" do
       words = ngram.extract_words_from_data(FactoryGirl.build(:data))

data/spec/selectors/simple_spec.rb

@@ -2,8 +2,8 @@ require "spec_helper"
 
 describe Selector::Simple do
   it_behaves_like 'a selector'
-  
-  let(:simple) { Selector::Simple.new }
+
+  let(:simple) { Selector::Simple.new(:function) }
   it "should have select_feature_vector implemented" do
     expect { simple.generate_vectors([]) }.to_not raise_error
   end
@@ -53,7 +53,8 @@ describe Selector::Simple do
   context "#generate_vector" do
     let(:dictionary) { %w(auto pferd haus hase garten) }
     let(:data) { FactoryGirl.build(:data) }
-    let(:vector) { simple.generate_vector(data).tap{|e| e.career_level! } }
+    let(:simple) { Selector::Simple.new(:career_level) }
+    let(:vector) { simple.generate_vector(data) }
 
     before(:each) do
       simple.stubs(:global_dictionary).returns(dictionary)
@@ -79,7 +80,7 @@ describe Selector::Simple do
     end
     context "custom dictionary" do
       it "should accept a custom dictionary" do
-        vector = simple.generate_vector(data, :career_level, %w(pferd flasche glas))
+        vector = simple.generate_vector(data, %w(pferd flasche glas))
         vector.data.should eq([[1,0,0],[0,0,0,0,0,0,1,0]].flatten)
       end
     end
@@ -106,7 +107,7 @@ describe Selector::Simple do
       simple.generate_vectors(data)
     end
     context "parallel" do
-      let(:parallel) { Selector::Simple.new(parallel: true) }
+      let(:parallel) { Selector::Simple.new(:function, parallel: true) }
       before(:each) do
         simple.stubs(:global_dictionary).returns(dictionary)
         parallel.stubs(:global_dictionary).returns(dictionary)

data/spec/selectors/with_binary_encoding_spec.rb

@@ -2,11 +2,11 @@ require "spec_helper"
 
 describe Selector::WithBinaryEncoding do
   it_behaves_like 'a selector'
-  let(:simple) { Selector::WithBinaryEncoding.new }
+  let(:simple) { Selector::WithBinaryEncoding.new(:career_level) }
 
   let(:dictionary) { %w(auto pferd haus hase garten) }
   let(:data) { FactoryGirl.build(:data) }
-  let(:vector) { simple.generate_vector(data).tap{|e| e.career_level! } }
+  let(:vector) { simple.generate_vector(data) }
 
   before(:each) do
     simple.stubs(:global_dictionary).returns(dictionary)
@@ -32,7 +32,7 @@ describe Selector::WithBinaryEncoding do
   end
   context "custom dictionary" do
     it "should accept a custom dictionary" do
-      vector = simple.generate_vector(data, :career_level, %w(pferd flasche glas))
+      vector = simple.generate_vector(data, %w(pferd flasche glas))
       vector.data.should eq([[1,0,0],[0,1,1,1]].flatten)
     end
   end

data/spec/support/preprocessor_spec.rb

@@ -1,14 +1,10 @@
 require 'spec_helper'
 
 shared_examples_for 'a preprocessor' do
-  let(:preprocessor) { described_class.new(industry_map: {1423=>3, 523=>54}) }
+  let(:preprocessor) { described_class.new(1423=>3, 523=>54) }
   let(:job) { FactoryGirl.build(:job) }
   let(:jobs) { [job] }
 
-  before(:each) do
-    job.stubs(:classification_id)
-    job.stubs(:label)
-  end
   it { preprocessor.should respond_to :process }
   it "should return a PreprocessedData object" do
     preprocessor.process(job).should be_a(PreprocessedData)

data/spec/support/selector_spec.rb

@@ -1,7 +1,7 @@
 require "spec_helper"
 
 shared_examples_for 'a selector' do
-  let(:selector) { described_class.new }
+  let(:selector) { described_class.new(:function) }
   let(:data) { FactoryGirl.build(:data) }
 
   it "should return a FeatureVector object" do

metadata

@@ -1,18 +1,20 @@
 --- !ruby/object:Gem::Specification
 name: svm_helper
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
+  prerelease: 
 platform: ruby
 authors:
 - Andreas Eger
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-13 00:00:00.000000000 Z
+date: 2013-03-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: parallel
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -20,6 +22,7 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -47,8 +50,8 @@ files:
 - lib/svm_helper/interface_helper.rb
 - lib/svm_helper/preprocessed_data.rb
 - lib/svm_helper/preprocessors.rb
+- lib/svm_helper/preprocessors/id_mapping.rb
 - lib/svm_helper/preprocessors/simple.rb
-- lib/svm_helper/preprocessors/with_industry_map.rb
 - lib/svm_helper/selectors.rb
 - lib/svm_helper/selectors/n_gram.rb
 - lib/svm_helper/selectors/simple.rb
@@ -62,8 +65,8 @@ files:
 - spec/factories/jobs/tmp3.html
 - spec/factories/jobs_with_description.rb
 - spec/factories/jobs_with_title.rb
+- spec/preprocessors/id_mapping_spec.rb
 - spec/preprocessors/simple_spec.rb
-- spec/preprocessors/with_industry_map_spec.rb
 - spec/selectors/n_gram_spec.rb
 - spec/selectors/simple_spec.rb
 - spec/selectors/with_binary_encoding_spec.rb
@@ -73,26 +76,33 @@ files:
 - svm_helper.gemspec
 homepage: https://github.com/sch1zo/svm_helper
 licenses: []
-metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
   - - '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: 2037039748537332986
 required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
   - - '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: 2037039748537332986
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.0.rc.2
+rubygems_version: 1.8.25
 signing_key: 
-specification_version: 4
+specification_version: 3
 summary: Preprocessor and Selector classes to generate FeatureVectors from Job data
 test_files:
 - spec/factories.rb
@@ -101,8 +111,8 @@ test_files:
 - spec/factories/jobs/tmp3.html
 - spec/factories/jobs_with_description.rb
 - spec/factories/jobs_with_title.rb
+- spec/preprocessors/id_mapping_spec.rb
 - spec/preprocessors/simple_spec.rb
-- spec/preprocessors/with_industry_map_spec.rb
 - spec/selectors/n_gram_spec.rb
 - spec/selectors/simple_spec.rb
 - spec/selectors/with_binary_encoding_spec.rb

checksums.yaml

@@ -1,7 +0,0 @@
----
-SHA1:
-  metadata.gz: bbfc13983f715f2f0ab66d2a9dbfed543896329e
-  data.tar.gz: 2e6f5c32898cc01a2468ebaa2e82470e01012586
-SHA512:
-  metadata.gz: 8c616e62ff4717808e0b9f29e3d6773c03096934194a1fb706e3eb0ded7527c455ea00039e0a4d39fc08e13f735d4f49cd984d26128e624fbe0b956851893c21
-  data.tar.gz: f28756b0f3f9539f69bcda6ce3f79bdf3ef226ee813f0e5ed7b6f38595f1760e05bbd1abe7ec8ccc5c3a8db8b666871a0d33a755c19c4842a85a600e4749498d

data/lib/svm_helper/preprocessors/with_industry_map.rb

@@ -1,40 +0,0 @@
-require_relative 'simple'
-module Preprocessor
-  #
-  # Preprocessor Base Class
-  #
-  # @author Andreas Eger
-  #
-  class WithIndustryMap < Simple
-    attr_reader :industry_map
-
-    #
-    # @param  args [Hash] options hash
-    # @option args [Hash] :industry_map mapping for the tree like industry ids to continuous ones
-    def initialize args={}
-      @industry_map = args.fetch(:industry_map){ Hash[Pjpp::Industry.select(:id).all.map(&:id).sort.map.with_index{|e,i| [e,i]}] }
-    end
-
-    def map_industry_id(id)
-      @industry_map[id]
-    end
-    def label
-      "with_industry_map"
-    end
-
-    private
-    def process_job job, classification
-      PreprocessedData.new(
-        data: [ clean_title(job.title), clean_description(job.description) ],
-        ids: {
-          industry: map_industry_id(job.classification_id(:industry)),
-          function: job.classification_id(:function),
-          career_level: job.classification_id(:career_level) },
-        labels: {
-          industry: job.label(:industry),
-          function: job.label(:function),
-          career_level: job.label(:career_level) }
-      ).tap{|e| e.send("#{classification}!")}
-    end
-  end
-end

data/spec/preprocessors/with_industry_map_spec.rb

@@ -1,16 +0,0 @@
-require 'spec_helper'
-
-describe Preprocessor::WithIndustryMap do
-  it_behaves_like 'a preprocessor'
-  let(:preprocessor) { Preprocessor::WithIndustryMap.new(industry_map: {1423=>3, 523=>54}) }
-  let(:job) { FactoryGirl.build(:job) }
-  let(:jobs) { [job] }
-  before(:each) do
-    job.stubs(:classification_id)
-    job.stubs(:label)
-  end
-  it "should make use of a industry_map" do
-    preprocessor.expects(:map_industry_id)
-    preprocessor.process(jobs)
-  end
-end