wapiti 1.0.4 → 1.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: eefd2c624bb02b635f41b9577d303abc352783d2530ef42f3db7f91db2384174
-  data.tar.gz: 73c766d6e05599b5167743dfc53daf20f74db0af0fe22d3ea6a947e9882189ab
+  metadata.gz: d1b615815731e51a90b7043b7ad91c97f0386bf99435c78f18ab23cf9df212cc
+  data.tar.gz: ee11a7e6f76f382847f721f8388b9c7f314e46a90ab391b463aa7e60ef641d41
 SHA512:
-  metadata.gz: 012c48b99ce4d6af1223f97fd03308f5af0c833e4db250842546b7f75de830fa700f71d52043fdd6c2b54f1f24cc83c0a0929211bf23153113d6efadf619fb68
-  data.tar.gz: 1a78e6de9025f6f6e199ff70e0973a7aeb42beda218220f3c63801a8239cd5371bf4546796db82134646d4491fa9cf00072371cfe058a4632ab3946b621a32bd
+  metadata.gz: a7852d9566407e93c9d66a641dd8551e3d4666449b9aad01ef434f822c78955d887911e59adb0418751ec4183d1ffa32504ec525fe3b16dbf05018c650c143c2
+  data.tar.gz: '0762885a4ef9265917756d983932da613ebb2cf32c9f18db76afcdac933ae710fe457ba8b431bdf537b986c9c54f7a9908b605627a72e000dd59f6f9cebab034'

data/HISTORY.md

@@ -1,6 +1,12 @@
+1.0.5 / 2019-05-13
+==================
+* Updated examples in README.md to current API
+* Fix crash when Model#label(input, check: true) is called on input with new labels 
+
 1.0.3 / 2018-07-10
 ==================
 * Fix compile time format-security
+
 1.0.0 / 2017-12-xx
 ==================
 * Added support for Windows platform

data/README.md

@@ -23,9 +23,12 @@ Quickstart
 
 ### Creating a Model
 
+You can run the following examples starting the ruby interpreter (irb or pry) inside spec/fixtures directory.
+ 
 Using a pattern and training data stored in a file:
 
-    model = Wapiti.train('train.txt', pattern: 'pattern.txt')
+    require 'wapiti'
+    model = Wapiti.train('chtrain.txt', pattern: 'chpattern.txt')
     #=> #<Wapiti::Model:0x0000010188f868>
     model.labels
     #=> ["B-ADJP", "B-ADVP", "B-CONJP" ...]
@@ -34,11 +37,21 @@ Using a pattern and training data stored in a file:
 
 Alternatively, you can pass in the training data as a `Wapiti::Dataset`;
 this class supports the default text format used by Wapiti as well as
-additiional formats (such as YAML or XML) and an API to make it easier
+additional formats (such as YAML or XML) and an API, to make it easier
 to manage data sets used for input and training.
 
-    data = Wapiti::Dataset.open('chtrain.xml')
-    model = Wapiti.train(data, options)
+    options = {threads:3, pattern: 'chpattern.txt'} 
+    
+    data_text = Wapiti::Dataset.open('chtrain.txt',tagged:true)
+    model2= Wapiti.train(data_text,options)
+    model2.labels
+    => ["B-ADJP", "B-ADVP", "B-CONJP" ...]
+
+    options = {threads:3, pattern: 'chpattern_only_tag.txt'} 
+    
+    data_xml    = Wapiti::Dataset.open('chtrain.xml')
+    #=> #<Wapiti::Dataset sequences={823}>
+    model3   = Wapiti.train(data_xml, options)
 
 You can consult the `Wapiti::Options.attribute_names` class for a list of
 supported configuration options and `Wapiti::Options.algorithms` for
@@ -64,9 +77,9 @@ Before saving your model you can use `compact` to reduce the model's size:
 
 By calling `#label` on a Model instance you can add labels to a dataset:
 
-    model = Wapiti.load('m2.mod')
-    input = Wapiti::Dataset.load('chtest.txt')
-    output = model.label(input, tagged: true)
+    model = Wapiti.load('ch.mod')
+    input = Wapiti::Dataset.open('chtrain.txt',tagged:true)
+    output = model.label(input)
 
 The result is a new `Wapiti::Dataset` with the predicted labels for each
 token. If your input data was already tagged, you can compare the input
@@ -87,9 +100,11 @@ when calling `#label`), the score for each label will be appended to
 each token/label tuple as a floating point number or passed as a third
 argument to the passed-in block.
 
-    model.label input, score: true
+    output_with_score = model.label input, score: true
     # => Dataset where each token will include a score
-
+    output_with_score.first.map(&:score)
+    # => [5.950832716249245, 8.870883529621942, ...]
+    
 ### Statistics
 
 By setting the *:check* option you can tell Wapiti to keep statistics during
@@ -105,8 +120,8 @@ are also available through the associated attribute readers).
 
     model.label input, check: true
     model.stats
-    => {:token=>{:count=>1896, :errors=>137, :rate=>7.225738396624472},
-    :sequence=>{:count=>77, :errors=>50, :rate=>64.93506493506494}}
+    => {:token=>{:count=>19172, :errors=>36, :rate=>0.18777383684539956},
+        :sequence=>{:count=>823, :errors=>28, :rate=>3.402187120291616}}
 
 For convenience, you can also use the `#check` method, which
 will reset the counters, check your input, and return the stats.

data/ext/wapiti/native.c

@@ -911,7 +911,6 @@ static VALUE model_labels(VALUE self) {
   qrk_t *lp = model->reader->lbl;
 
   VALUE labels = rb_ary_new2(Y);
-
   for (unsigned int i = 0; i < Y; ++i) {
     rb_ary_push(labels, rb_str_new2(qrk_id2str(lp, i)));
   }
@@ -979,14 +978,21 @@ static VALUE decode_sequence(VALUE self, mdl_t *model, raw_t *raw) {
   }
 
   // Statistics
+
   if (model->opt->check) {
     int err = 0;
+    uint32_t lbl = 0;
 
     for (t = 0; t < T; ++t) {
-      stat[0][seq->pos[t].lbl]++;
+      lbl = seq->pos[t].lbl;
+
+      // ((uint32_t)-1) is a magic value for no asigned token
+      if (lbl != ((uint32_t)-1)) {
+        stat[0][lbl]++;
+      }
       stat[1][out[t * N]]++;
 
-      if (seq->pos[t].lbl != out[t * N]) {
+      if (lbl != out[t * N]) {
         terr++;
         err = 1;
       } else {

data/lib/wapiti/version.rb

@@ -1,3 +1,3 @@
 module Wapiti
-  VERSION = '1.0.4'.freeze
+  VERSION = '1.0.5'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: wapiti
 version: !ruby/object:Gem::Version
-  version: 1.0.4
+  version: 1.0.5
 platform: ruby
 authors:
 - Sylvester Keil
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-09-14 00:00:00.000000000 Z
+date: 2019-05-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: builder