thera 0.0.1 → 0.0.2

data/ext/quarry/quarry_toolkit.cpp

@@ -31,6 +31,20 @@ Object model_rank_text(Object self, Object text) {
   return indexes;
 }
 
+Object model_rank_text_names(Object self, Object text) {
+  Model::Model *model = from_ruby<Model::Model *>(self);
+  string example_text = from_ruby<string>(text);
+  Array names;
+  
+  vector<Classifier::Score> *ranks = model->rank_text(example_text);
+  DataSet::NominalFeature *categories = model->data_set->category_feature();
+  for(unsigned int i = 0; i < ranks->size(); i++)
+    names.push(categories->names[ranks->at(i).category]);
+  
+  delete ranks;
+  return names;
+}
+
 
 extern "C" {
 	
@@ -45,8 +59,8 @@ extern "C" {
     // text pipeline
     rb_mText.define_module_function("standard_pipeline", &Preprocessing::Text::StandardPipeline);
     Data_Type<Preprocessing::Text::TextPipeline> rb_cTextPipeline = define_class_under<Preprocessing::Text::TextPipeline>(rb_mQuarry, "ImplTextPipeline")
-      .define_constructor(Constructor<Preprocessing::Text::TextPipeline>())
-      .define_method("process_text", &Preprocessing::Text::TextPipeline::process_text);
+      .define_constructor(Constructor<Preprocessing::Text::TextPipeline>());
+//      .define_method("process_text", &Preprocessing::Text::TextPipeline::process_text);
     
     // storage
     Data_Type<Storage::Storage> rb_cStorage = define_class_under<Storage::Storage>(rb_mQuarry, "ImplStorage");
@@ -83,7 +97,8 @@ extern "C" {
       .define_method("set_text_pipeline", &Model::Model::set_text_pipeline)
       .define_method("get_text_pipeline", &Model::Model::get_text_pipeline)
       .define_method("rank", &model_rank)
-      .define_method("rank_text", &model_rank_text);
+      .define_method("rank_text", &model_rank_text)
+      .define_method("rank_text_names", &model_rank_text_names);
       
 
     

data/lib/quarry/src/model/model.cpp

@@ -11,7 +11,7 @@ int Model::Model::classify(DataSet::Example *example) {
 }
 
 int Model::Model::classify_text(string text) {
-  DataSet::SparseExample *example = text_pipeline->process_text((DataSet::SparseDataSet *)data_set, (char *)text.c_str());
+  DataSet::SparseExample *example = text_pipeline->process_text((DataSet::SparseDataSet *)data_set, (char *)text.c_str(), false);
   int category = classifier->classify(example);
   delete example;
   return category;
@@ -22,7 +22,7 @@ vector<Classifier::Score> *Model::Model::rank(DataSet::Example *example) {
 }
 
 vector<Classifier::Score> *Model::Model::rank_text(string text) {
-  DataSet::SparseExample *example = text_pipeline->process_text((DataSet::SparseDataSet *)data_set, (char *)text.c_str());
+  DataSet::SparseExample *example = text_pipeline->process_text((DataSet::SparseDataSet *)data_set, (char *)text.c_str(), false);
   vector<Classifier::Score> *ranks = classifier->rank(example);
   delete example;
   return ranks;

data/lib/quarry/src/preprocessing/text/example_generator/example_generator.h

@@ -9,7 +9,7 @@ namespace Preprocessing {
     class ExampleGenerator {
     public:
       ExampleGenerator() {}
-      virtual DataSet::SparseExample *generate(DataSet::SparseDataSet *data_set, vector<char *> *tokens) { return NULL; }
+      virtual DataSet::SparseExample *generate(DataSet::SparseDataSet *data_set, vector<char *> *tokens, bool create_features) { return NULL; }
       virtual uint32_t mark() = 0;
     };
     

data/lib/quarry/src/preprocessing/text/example_generator/token_counter.h

@@ -22,7 +22,7 @@ namespace Preprocessing {
       
       TokenCounter(TokenCounterWeight weight = Count) : ExampleGenerator(), token_counts(), weight(weight) {}
       
-      DataSet::SparseExample *generate(DataSet::SparseDataSet *data_set, vector<char *> *tokens) {
+      DataSet::SparseExample *generate(DataSet::SparseDataSet *data_set, vector<char *> *tokens, bool create_features) {
         int max_count = 0, count = 0;
         double value = 0.0;
         token_counts.clear();
@@ -38,6 +38,8 @@ namespace Preprocessing {
         
         // construct the example
         DataSet::SparseExample *example = data_set->new_example(token_counts.size());
+        DataSet::Feature *feature = NULL;
+        
         for(map<string, int>::iterator token_counts_it = token_counts.begin(); token_counts_it != token_counts.end(); token_counts_it++) {
           value = token_counts_it->second;
           
@@ -46,7 +48,13 @@ namespace Preprocessing {
           else if(weight == Binary)
             value = 1;
           
-          example->set_value(data_set->get_or_create_numeric_feature_by_name(token_counts_it->first)->index, value);
+          if(create_features) {
+            example->set_value(data_set->get_or_create_numeric_feature_by_name(token_counts_it->first)->index, value);
+          } else {
+            feature = data_set->get_feature_by_name(token_counts_it->first);
+            if(feature)
+              example->set_value(feature->index, value);
+          }
         }
         
         return example;

data/lib/quarry/src/preprocessing/text/text_pipeline.cpp

@@ -1,10 +1,10 @@
 #include "text_pipeline.h"
 #include <iostream>
 
-DataSet::SparseExample *Preprocessing::Text::TextPipeline::process_text(DataSet::SparseDataSet *data_set, char *text) {
+DataSet::SparseExample *Preprocessing::Text::TextPipeline::process_text(DataSet::SparseDataSet *data_set, char *text, bool create_features) {
   tokens.clear();
   tokeniser->tokenise(text);
-  return generator->generate(data_set, &tokens);
+  return generator->generate(data_set, &tokens, create_features);
 }
 
 void Preprocessing::Text::TextPipeline::process_token(char *start, char *end) {

data/lib/quarry/src/preprocessing/text/text_pipeline.h

@@ -26,7 +26,7 @@ namespace Preprocessing {
       vector<char *> tokens;
       
       TextPipeline() : tokeniser(NULL), processors(), selectors(), generator(NULL), tokens() {}      
-      DataSet::SparseExample *process_text(DataSet::SparseDataSet *data_set, char *text);
+      DataSet::SparseExample *process_text(DataSet::SparseDataSet *data_set, char *text, bool create_features);
       void process_token(char *start, char *end);
     };
     

data/lib/quarry/src/storage/folders.cpp

@@ -52,7 +52,7 @@ void Storage::Folders::load_directory(string path, DataSet::SparseDataSet *data_
     fclose(file);
     
     // insert a new example into the dataset
-    example = pipeline->process_text(data_set, file_data);
+    example = pipeline->process_text(data_set, file_data, true);
     example->set_category_index(data_set, category_index);
     
     file_count++;

data/lib/quarry_rb/model/model.rb

@@ -52,5 +52,9 @@ module Quarry
     def rank_text(text)
       @model.rank_text(text)
     end
+    
+    def rank_text_names(text)
+      @model.rank_text_names(text)
+    end
   end
 end

data/lib/quarry_rb/text_pipeline.rb

@@ -5,8 +5,8 @@ module Quarry
       @text_pipeline = tp || Quarry::ImplTextPipeline.new
     end
     
-    def process_text(data_set, text)
-      Example.new(@text_pipeline.process_text(data_set, text))
+    def process_text(data_set, text, create_features = false)
+      Example.new(@text_pipeline.process_text(data_set, text, create_features))
     end
     
     def self.standard_pipeline

data/thera.gemspec

@@ -9,7 +9,7 @@ Gem::Specification.new do |s|
   s.description = "C++ Data Mining Library for Ruby"
   s.email = "me@willcannings.com"
   s.authors = ["Will Cannings"]
-  s.version = '0.0.1'
+  s.version = '0.0.2'
   s.extensions = ["ext/extconf.rb"]
 
   s.files         = `git ls-files`.split("\n")

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: thera
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
   prerelease: 
 platform: ruby
 authors:
@@ -13,7 +13,7 @@ date: 2011-12-07 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rice
-  requirement: &70234812248380 !ruby/object:Gem::Requirement
+  requirement: &70171050802120 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,7 +21,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70234812248380
+  version_requirements: *70171050802120
 description: C++ Data Mining Library for Ruby
 email: me@willcannings.com
 executables: []