wapiti 0.0.5 → 0.1.0

data/ext/wapiti/decoder.c

@@ -1,7 +1,7 @@
 /*
  *      Wapiti - A linear-chain CRF tool
  *
- * Copyright (c) 2009-2011  CNRS
+ * Copyright (c) 2009-2013  CNRS
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -25,10 +25,14 @@
  * POSSIBILITY OF SUCH DAMAGE.
  */
 
+#include <inttypes.h>
 #include <float.h>
+#include <stdint.h>
 #include <stddef.h>
 #include <stdlib.h>
 #include <stdio.h>
+#include <string.h>
+#include <math.h>
 
 #include "wapiti.h"
 #include "gradient.h"
@@ -39,6 +43,7 @@
 #include "thread.h"
 #include "tools.h"
 #include "decoder.h"
+#include "vmath.h"
 
 /******************************************************************************
  * Sequence tagging
@@ -60,9 +65,9 @@
  *   works in log-space.
  */
 static int tag_expsc(mdl_t *mdl, const seq_t *seq, double *vpsi) {
-	const double *x = mdl->theta;
-	const size_t  Y = mdl->nlbl;
-	const int     T = seq->len;
+	const double  *x = mdl->theta;
+	const uint32_t Y = mdl->nlbl;
+	const uint32_t T = seq->len;
 	double (*psi)[T][Y][Y] = (void *)vpsi;
 	// We first have to compute the Ψ_t(y',y,x_t) weights defined as
 	//   Ψ_t(y',y,x_t) = \exp( ∑_k θ_k f_k(y',y,x_t) )
@@ -88,25 +93,25 @@ static int tag_expsc(mdl_t *mdl, const seq_t *seq, double *vpsi) {
 	//   2/ we add the bigrams features weights by looping over actives
 	//        bigrams observations (we don't have to do this for t=0 since
 	//        there is no bigrams here)
-	for (int t = 0; t < T; t++) {
+	for (uint32_t t = 0; t < T; t++) {
 		const pos_t *pos = &(seq->pos[t]);
-		for (size_t y = 0; y < Y; y++) {
+		for (uint32_t y = 0; y < Y; y++) {
 			double sum = 0.0;
-			for (size_t n = 0; n < pos->ucnt; n++) {
-				const size_t o = pos->uobs[n];
+			for (uint32_t n = 0; n < pos->ucnt; n++) {
+				const uint64_t o = pos->uobs[n];
 				sum += x[mdl->uoff[o] + y];
 			}
-			for (size_t yp = 0; yp < Y; yp++)
+			for (uint32_t yp = 0; yp < Y; yp++)
 				(*psi)[t][yp][y] = sum;
 		}
 	}
-	for (int t = 1; t < T; t++) {
+	for (uint32_t t = 1; t < T; t++) {
 		const pos_t *pos = &(seq->pos[t]);
-		for (size_t yp = 0, d = 0; yp < Y; yp++) {
-			for (size_t y = 0; y < Y; y++, d++) {
+		for (uint32_t yp = 0, d = 0; yp < Y; yp++) {
+			for (uint32_t y = 0; y < Y; y++, d++) {
 				double sum = 0.0;
-				for (size_t n = 0; n < pos->bcnt; n++) {
-					const size_t o = pos->bobs[n];
+				for (uint32_t n = 0; n < pos->bcnt; n++) {
+					const uint64_t o = pos->bobs[n];
 					sum += x[mdl->boff[o] + d];
 				}
 				(*psi)[t][yp][y] += sum;
@@ -116,6 +121,30 @@ static int tag_expsc(mdl_t *mdl, const seq_t *seq, double *vpsi) {
 	return 0;
 }
 
+/* tag_memmsc:
+ *   Compute the score for viterbi decoding of MEMM models. This use the
+ *   previous function to compute the classical score and then normalize them
+ *   relative to the previous label. This normalization must be done in linear
+ *   space, not in logarithm one.
+ */
+static int tag_memmsc(mdl_t *mdl, const seq_t *seq, double *vpsi) {
+	const uint32_t Y = mdl->nlbl;
+	const uint32_t T = seq->len;
+	tag_expsc(mdl, seq, vpsi);
+	xvm_expma(vpsi, vpsi, 0.0, T * Y * Y);
+	double (*psi)[T][Y][Y] = (void *)vpsi;
+	for (uint32_t t = 0; t < T; t++) {
+		for (uint32_t yp = 0; yp < Y; yp++) {
+			double sum = 0.0;
+			for (uint32_t y = 0; y < Y; y++)
+				sum += (*psi)[t][yp][y];
+			for (uint32_t y = 0; y < Y; y++)
+				(*psi)[t][yp][y] /= sum;
+		}
+	}
+	return 1;
+}
+
 /* tag_postsc:
  *   This function compute score lattice with posteriors. This generally result
  *   in a slightly best labelling and allow to output normalized score for the
@@ -123,34 +152,71 @@ static int tag_expsc(mdl_t *mdl, const seq_t *seq, double *vpsi) {
  *   a full forward backward instead of just the forward pass.
  */
 static int tag_postsc(mdl_t *mdl, const seq_t *seq, double *vpsi) {
-	const size_t  Y = mdl->nlbl;
-	const int     T = seq->len;
+	const uint32_t Y = mdl->nlbl;
+	const uint32_t T = seq->len;
 	double (*psi)[T][Y][Y] = (void *)vpsi;
-	grd_t *grd = grd_new(mdl, NULL);
-	grd->first = 0;
-	grd->last  = T - 1;
-	grd_check(grd, seq->len);
+	grd_st_t *grd_st = grd_stnew(mdl, NULL);
+	grd_st->first = 0;
+	grd_st->last  = T - 1;
+	grd_stcheck(grd_st, seq->len);
 	if (mdl->opt->sparse) {
-		grd_spdopsi(grd, seq);
-		grd_spfwdbwd(grd, seq);
+		grd_spdopsi(grd_st, seq);
+		grd_spfwdbwd(grd_st, seq);
 	} else {
-		grd_fldopsi(grd, seq);
-		grd_flfwdbwd(grd, seq);
+		grd_fldopsi(grd_st, seq);
+		grd_flfwdbwd(grd_st, seq);
 	}
-	double (*alpha)[T][Y] = (void *)grd->alpha;
-	double (*beta )[T][Y] = (void *)grd->beta;
-	double  *unorm        =         grd->unorm;
-	for (int t = 0; t < T; t++) {
-		for (size_t y = 0; y < Y; y++) {
+	double (*alpha)[T][Y] = (void *)grd_st->alpha;
+	double (*beta )[T][Y] = (void *)grd_st->beta;
+	double  *unorm        =         grd_st->unorm;
+	for (uint32_t t = 0; t < T; t++) {
+		for (uint32_t y = 0; y < Y; y++) {
 			double e = (*alpha)[t][y] * (*beta)[t][y] * unorm[t];
-			for (size_t yp = 0; yp < Y; yp++)
+			for (uint32_t yp = 0; yp < Y; yp++)
 				(*psi)[t][yp][y] = e;
 		}
 	}
-	grd_free(grd);
+	grd_stfree(grd_st);
 	return 1;
 }
 
+/* tag_forced:
+ *   This function apply correction to the psi table to take account of already
+ *   known labels. If a label is known, all arcs leading or comming from other
+ *   labels at this position are NULLified and will not be selected by the
+ *   decoder.
+ */
+static void tag_forced(mdl_t *mdl, const seq_t *seq, double *vpsi, int op) {
+	const uint32_t Y = mdl->nlbl;
+	const uint32_t T = seq->len;
+	const double v = op ? 0.0 : -HUGE_VAL;
+	double (*psi)[T][Y][Y] = (void *)vpsi;
+	for (uint32_t t = 0; t < T; t++) {
+		const uint32_t yr = seq->pos[t].lbl;
+		if (yr == (uint32_t)-1)
+			continue;
+		if (t != 0)
+			for (uint32_t y = 0; y < Y; y++)
+				if (y != yr)
+					for (uint32_t yp = 0; yp < Y; yp++)
+						(*psi)[t][yp][y] = v;
+		if (t != T - 1)
+			for (uint32_t y = 0; y < Y; y++)
+				if (y != yr)
+					for (uint32_t yn = 0; yn < Y; yn++)
+						(*psi)[t + 1][y][yn] = v;
+	}
+	const uint32_t yr = seq->pos[0].lbl;
+	if (yr != (uint32_t)-1) {
+		for (uint32_t y = 0; y < Y; y++) {
+			if (yr == y)
+				continue;
+			for (uint32_t yp = 0; yp < Y; yp++)
+				(*psi)[0][yp][y] = v;
+		}
+	}
+}
+
 /* tag_viterbi:
  *   This function implement the Viterbi algorithm in order to decode the most
  *   probable sequence of labels according to the model. Some part of this code
@@ -160,22 +226,26 @@ static int tag_postsc(mdl_t *mdl, const seq_t *seq, double *vpsi) {
  *   enough stack space.
  */
 void tag_viterbi(mdl_t *mdl, const seq_t *seq,
-	         size_t out[], double *sc, double psc[]) {
-	const size_t  Y = mdl->nlbl;
-	const int     T = seq->len;
-	double  *vpsi  = wapiti_xmalloc(sizeof(double) * T * Y * Y);
-	size_t  *vback = wapiti_xmalloc(sizeof(size_t) * T * Y);
-	double (*psi) [T][Y][Y] = (void *)vpsi;
-	size_t (*back)[T][Y]    = (void *)vback;
-	double  *cur = wapiti_xmalloc(sizeof(double) * Y);
-	double  *old = wapiti_xmalloc(sizeof(double) * Y);
+	         uint32_t out[], double *sc, double psc[]) {
+	const uint32_t Y = mdl->nlbl;
+	const uint32_t T = seq->len;
+	double   *vpsi  = xvm_new(T * Y * Y);
+	uint32_t *vback = wapiti_xmalloc(sizeof(uint32_t) * T * Y);
+	double   (*psi) [T][Y][Y] = (void *)vpsi;
+	uint32_t (*back)[T][Y]    = (void *)vback;
+	double *cur = wapiti_xmalloc(sizeof(double) * Y);
+	double *old = wapiti_xmalloc(sizeof(double) * Y);
 	// We first compute the scores for each transitions in the lattice of
 	// labels.
 	int op;
-	if (mdl->opt->lblpost)
+	if (mdl->type == 1)
+		op = tag_memmsc(mdl, seq, vpsi);
+	else if (mdl->opt->lblpost)
 		op = tag_postsc(mdl, seq, vpsi);
 	else
 		op = tag_expsc(mdl, seq, vpsi);
+	if (mdl->opt->force)
+		tag_forced(mdl, seq, vpsi, op);
 	// Now we can do the Viterbi algorithm. This is very similar to the
 	// forward pass
 	//   | α_1(y) = Ψ_1(y,x_1)
@@ -189,15 +259,15 @@ void tag_viterbi(mdl_t *mdl, const seq_t *seq,
 	// the indice of the y value selected by the max. This also mean that
 	// we only need the current and previous value of the α vectors, not
 	// the full matrix.
-	for (size_t y = 0; y < Y; y++)
+	for (uint32_t y = 0; y < Y; y++)
 		cur[y] = (*psi)[0][0][y];
-	for (int t = 1; t < T; t++) {
-		for (size_t y = 0; y < Y; y++)
+	for (uint32_t t = 1; t < T; t++) {
+		for (uint32_t y = 0; y < Y; y++)
 			old[y] = cur[y];
-		for (size_t y = 0; y < Y; y++) {
-			double bst = -1.0;
-			int    idx = 0;
-			for (size_t yp = 0; yp < Y; yp++) {
+		for (uint32_t y = 0; y < Y; y++) {
+			double   bst = -HUGE_VAL;
+			uint32_t idx = 0;
+			for (uint32_t yp = 0; yp < Y; yp++) {
 				double val = old[yp];
 				if (op)
 					val *= (*psi)[t][yp][y];
@@ -216,15 +286,15 @@ void tag_viterbi(mdl_t *mdl, const seq_t *seq,
 	// this we search in the last α vector the best value. Using this index
 	// as a starting point in the back-pointer array we finally can decode
 	// the best sequence.
-	int bst = 0;
-	for (size_t y = 1; y < Y; y++)
+	uint32_t bst = 0;
+	for (uint32_t y = 1; y < Y; y++)
 		if (cur[y] > cur[bst])
 			bst = y;
 	if (sc != NULL)
 		*sc = cur[bst];
-	for (int t = T; t > 0; t--) {
-		const size_t yp = (t != 1) ? (*back)[t - 1][bst] : 0;
-		const size_t y  = bst;
+	for (uint32_t t = T; t > 0; t--) {
+		const uint32_t yp = (t != 1) ? (*back)[t - 1][bst] : 0;
+		const uint32_t y  = bst;
 		out[t - 1] = y;
 		if (psc != NULL)
 			psc[t - 1] = (*psi)[t - 1][yp][y];
@@ -233,7 +303,7 @@ void tag_viterbi(mdl_t *mdl, const seq_t *seq,
 	free(old);
 	free(cur);
 	free(vback);
-	free(vpsi);
+	xvm_free(vpsi);
 }
 
 /* tag_nbviterbi:
@@ -242,23 +312,27 @@ void tag_viterbi(mdl_t *mdl, const seq_t *seq,
  *   compute only the best one and will return the same sequence than the
  *   previous function but will be slower to do it.
  */
-void tag_nbviterbi(mdl_t *mdl, const seq_t *seq, size_t N,
-	           size_t out[][N], double sc[], double psc[][N]) {
-	const size_t  Y = mdl->nlbl;
-	const int     T = seq->len;
-	double  *vpsi  = wapiti_xmalloc(sizeof(double) * T * Y * Y);
-	size_t  *vback = wapiti_xmalloc(sizeof(size_t) * T * Y * N);
-	double (*psi) [T][Y    ][Y] = (void *)vpsi;
-	size_t (*back)[T][Y * N]    = (void *)vback;
-	double  *cur = wapiti_xmalloc(sizeof(double) * Y * N);
-	double  *old = wapiti_xmalloc(sizeof(double) * Y * N);
+void tag_nbviterbi(mdl_t *mdl, const seq_t *seq, uint32_t N,
+                   uint32_t out[][N], double sc[], double psc[][N]) {
+	const uint32_t Y = mdl->nlbl;
+	const uint32_t T = seq->len;
+	double   *vpsi  = xvm_new(T * Y * Y);
+	uint32_t *vback = wapiti_xmalloc(sizeof(uint32_t) * T * Y * N);
+	double   (*psi) [T][Y    ][Y] = (void *)vpsi;
+	uint32_t (*back)[T][Y * N]    = (void *)vback;
+	double *cur = wapiti_xmalloc(sizeof(double) * Y * N);
+	double *old = wapiti_xmalloc(sizeof(double) * Y * N);
 	// We first compute the scores for each transitions in the lattice of
 	// labels.
 	int op;
-	if (mdl->opt->lblpost)
+	if (mdl->type == 1)
+		op = tag_memmsc(mdl, seq, vpsi);
+	else if (mdl->opt->lblpost)
 		op = tag_postsc(mdl, seq, (double *)psi);
 	else
 		op = tag_expsc(mdl, seq, (double *)psi);
+	if (mdl->opt->force)
+		tag_forced(mdl, seq, vpsi, op);
 	// Here also, it's classical but we have to keep the N best paths
 	// leading to each nodes of the lattice instead of only the best one.
 	// This mean that code is less trivial and the current implementation is
@@ -267,19 +341,19 @@ void tag_nbviterbi(mdl_t *mdl, const seq_t *seq, size_t N,
 	// We first build the list of all incoming arcs from all paths from all
 	// N-best nodes and next select the N-best one. There is a lot of room
 	// here for later optimisations if needed.
-	for (size_t y = 0, d = 0; y < Y; y++) {
+	for (uint32_t y = 0, d = 0; y < Y; y++) {
 		cur[d++] = (*psi)[0][0][y];
-		for (size_t n = 1; n < N; n++)
+		for (uint32_t n = 1; n < N; n++)
 			cur[d++] = -DBL_MAX;
 	}
-	for (int t = 1; t < T; t++) {
-		for (size_t d = 0; d < Y * N; d++)
+	for (uint32_t t = 1; t < T; t++) {
+		for (uint32_t d = 0; d < Y * N; d++)
 			old[d] = cur[d];
-		for (size_t y = 0; y < Y; y++) {
+		for (uint32_t y = 0; y < Y; y++) {
 			// 1st, build the list of all incoming
 			double lst[Y * N];
-			for (size_t yp = 0, d = 0; yp < Y; yp++) {
-				for (size_t n = 0; n < N; n++, d++) {
+			for (uint32_t yp = 0, d = 0; yp < Y; yp++) {
+				for (uint32_t n = 0; n < N; n++, d++) {
 					lst[d] = old[d];
 					if (op)
 						lst[d] *= (*psi)[t][yp][y];
@@ -288,14 +362,14 @@ void tag_nbviterbi(mdl_t *mdl, const seq_t *seq, size_t N,
 				}
 			}
 			// 2nd, init the back with the N first
-			size_t *bk = &(*back)[t][y * N];
-			for (size_t n = 0; n < N; n++)
+			uint32_t *bk = &(*back)[t][y * N];
+			for (uint32_t n = 0; n < N; n++)
 				bk[n] = n;
 			// 3rd, search the N highest values
-			for (size_t i = N; i < N * Y; i++) {
+			for (uint32_t i = N; i < N * Y; i++) {
 				// Search the smallest current value
-				size_t idx = 0;
-				for (size_t n = 1; n < N; n++)
+				uint32_t idx = 0;
+				for (uint32_t n = 1; n < N; n++)
 					if (lst[bk[n]] < lst[bk[idx]])
 						idx = n;
 				// And replace it if needed
@@ -303,24 +377,24 @@ void tag_nbviterbi(mdl_t *mdl, const seq_t *seq, size_t N,
 					bk[idx] = i;
 			}
 			// 4th, get the new scores
-			for (size_t n = 0; n < N; n++)
+			for (uint32_t n = 0; n < N; n++)
 				cur[y * N + n] = lst[bk[n]];
 		}
 	}
 	// Retrieving the best paths is similar to classical Viterbi except that
 	// we have to search for the N bet ones and there is N time more
 	// possibles starts.
-	for (size_t n = 0; n < N; n++) {
-		int bst = 0;
-		for (size_t d = 1; d < Y * N; d++)
+	for (uint32_t n = 0; n < N; n++) {
+		uint32_t bst = 0;
+		for (uint32_t d = 1; d < Y * N; d++)
 			if (cur[d] > cur[bst])
 				bst = d;
 		if (sc != NULL)
 			sc[n] = cur[bst];
 		cur[bst] = -DBL_MAX;
-		for (int t = T; t > 0; t--) {
-			const size_t yp = (t != 1) ? (*back)[t - 1][bst] / N: 0;
-			const size_t y  = bst / N;
+		for (uint32_t t = T; t > 0; t--) {
+			const uint32_t yp = (t != 1) ? (*back)[t - 1][bst] / N: 0;
+			const uint32_t y  = bst / N;
 			out[t - 1][n] = y;
 			if (psc != NULL)
 				psc[t - 1][n] = (*psi)[t - 1][yp][y];
@@ -330,7 +404,7 @@ void tag_nbviterbi(mdl_t *mdl, const seq_t *seq, size_t N,
 	free(old);
 	free(cur);
 	free(vback);
-	free(vpsi);
+	xvm_free(vpsi);
 }
 
 /* tag_label:
@@ -343,17 +417,17 @@ void tag_nbviterbi(mdl_t *mdl, const seq_t *seq, size_t N,
  */
 void tag_label(mdl_t *mdl, FILE *fin, FILE *fout) {
 	qrk_t *lbls = mdl->reader->lbl;
-	const size_t Y = mdl->nlbl;
-	const size_t N = mdl->opt->nbest;
+	const uint32_t Y = mdl->nlbl;
+	const uint32_t N = mdl->opt->nbest;
 	// We start by preparing the statistic collection to be ready if check
 	// option is used. The stat array hold the following for each label
 	//   [0] # of reference with this label
 	//   [1] # of token we have taged with this label
 	//   [2] # of match of the two preceding
-	size_t tcnt = 0, terr = 0;
-	size_t scnt = 0, serr = 0;
-	size_t stat[3][Y];
-	for (size_t y = 0; y < Y; y++)
+	uint64_t tcnt = 0, terr = 0;
+	uint64_t scnt = 0, serr = 0;
+	uint64_t stat[3][Y];
+	for (uint32_t y = 0; y < Y; y++)
 		stat[0][y] = stat[1][y] = stat[2][y] = 0;
 	// Next read the input file sequence by sequence and label them, we have
 	// to take care of not discarding the raw input as we want to send it
@@ -364,24 +438,25 @@ void tag_label(mdl_t *mdl, FILE *fin, FILE *fout) {
 		raw_t *raw = rdr_readraw(mdl->reader, fin);
 		if (raw == NULL)
 			break;
-		seq_t *seq = rdr_raw2seq(mdl->reader, raw, mdl->opt->check);
-		const int T = seq->len;
-		size_t *out = wapiti_xmalloc(sizeof(size_t) * T * N);
-		double *psc = wapiti_xmalloc(sizeof(double) * T * N);
-		double *scs = wapiti_xmalloc(sizeof(double) * N);
+		seq_t *seq = rdr_raw2seq(mdl->reader, raw,
+			mdl->opt->check | mdl->opt->force);
+		const uint32_t T = seq->len;
+		uint32_t *out = wapiti_xmalloc(sizeof(uint32_t) * T * N);
+		double   *psc = wapiti_xmalloc(sizeof(double  ) * T * N);
+		double   *scs = wapiti_xmalloc(sizeof(double  ) * N);
 		if (N == 1)
-			tag_viterbi(mdl, seq, (size_t*)out, scs, (double*)psc);
+			tag_viterbi(mdl, seq, (uint32_t*)out, scs, (double*)psc);
 		else
 			tag_nbviterbi(mdl, seq, N, (void*)out, scs, (void*)psc);
 		// Next we output the raw sequence with an aditional column for
 		// the predicted labels
-		for (size_t n = 0; n < N; n++) {
+		for (uint32_t n = 0; n < N; n++) {
 			if (mdl->opt->outsc)
 				fprintf(fout, "# %d %f\n", (int)n, scs[n]);
-			for (int t = 0; t < T; t++) {
+			for (uint32_t t = 0; t < T; t++) {
 				if (!mdl->opt->label)
 					fprintf(fout, "%s\t", raw->lines[t]);
-				size_t lbl = out[t * N + n];
+				uint32_t lbl = out[t * N + n];
 				const char *lblstr = qrk_id2str(lbls, lbl);
 				fprintf(fout, "%s", lblstr);
 				if (mdl->opt->outsc) {
@@ -394,10 +469,13 @@ void tag_label(mdl_t *mdl, FILE *fin, FILE *fout) {
 		}
 		fflush(fout);
 		// If user provided reference labels, use them to collect
-		// statistics about how well we have performed here.
+		// statistics about how well we have performed here. Labels
+		// unseen at training time are discarded.
 		if (mdl->opt->check) {
 			bool err = false;
-			for (int t = 0; t < T; t++) {
+			for (uint32_t t = 0; t < T; t++) {
+				if (seq->pos[t].lbl == (uint32_t)-1)
+					continue;
 				stat[0][seq->pos[t].lbl]++;
 				stat[1][out[t * N]]++;
 				if (seq->pos[t].lbl != out[t * N])
@@ -405,7 +483,7 @@ void tag_label(mdl_t *mdl, FILE *fin, FILE *fout) {
 				else
 					stat[2][out[t * N]]++;
 			}
-			tcnt += (size_t)T;
+			tcnt += T;
 			serr += err;
 		}
 		// Cleanup memory used for this sequence
@@ -418,7 +496,7 @@ void tag_label(mdl_t *mdl, FILE *fin, FILE *fout) {
 		// much sequence are labelled and if possible the current tokens
 		// and sequence error rates.
 		if (++scnt % 1000 == 0) {
-			info("%10zu sequences labeled", scnt);
+			info("%10"PRIu64" sequences labeled", scnt);
 			if (mdl->opt->check) {
 				const double te = (double)terr  / tcnt * 100.0;
 				const double se = (double)serr  / scnt * 100.0;
@@ -433,11 +511,11 @@ void tag_label(mdl_t *mdl, FILE *fin, FILE *fout) {
 	if (mdl->opt->check) {
 		const double te = (double)terr  / tcnt * 100.0;
 		const double se = (double)serr  / scnt * 100.0;
-		info("    Nb sequences  : %zu\n", scnt);
+		info("    Nb sequences  : %"PRIu64"\n", scnt);
 		info("    Token error   : %5.2f%%\n", te);
 		info("    Sequence error: %5.2f%%\n", se);
 		info("* Per label statistics\n");
-		for (size_t y = 0; y < Y; y++) {
+		for (uint32_t y = 0; y < Y; y++) {
 			const char   *lbl = qrk_id2str(lbls, y);
 			const double  Rc  = (double)stat[2][y] / stat[0][y];
 			const double  Pr  = (double)stat[2][y] / stat[1][y];
@@ -458,12 +536,12 @@ void tag_label(mdl_t *mdl, FILE *fin, FILE *fout) {
  */
 typedef struct eval_s eval_t;
 struct eval_s {
-	mdl_t  *mdl;
-	dat_t  *dat;
-	size_t  tcnt;  // Processed tokens count
-	size_t  terr;  // Tokens error found
-	size_t  scnt;  // Processes sequences count
-	size_t  serr;  // Sequence error found
+	mdl_t    *mdl;
+	dat_t    *dat;
+	uint64_t  tcnt;  // Processed tokens count
+	uint64_t  terr;  // Tokens error found
+	uint64_t  scnt;  // Processes sequences count
+	uint64_t  serr;  // Sequence error found
 };
 
 /* tag_evalsub:
@@ -471,7 +549,7 @@ struct eval_s {
  *   by batch and for each batch do a simple Viterbi and scan the result to find
  *   errors.
  */
-static void tag_evalsub(job_t *job, int id, int cnt, eval_t *eval) {
+static void tag_evalsub(job_t *job, uint32_t id, uint32_t cnt, eval_t *eval) {
 	unused(id && cnt);
 	mdl_t *mdl = eval->mdl;
 	dat_t *dat = eval->dat;
@@ -480,22 +558,23 @@ static void tag_evalsub(job_t *job, int id, int cnt, eval_t *eval) {
 	eval->scnt = 0;
 	eval->serr = 0;
 	// We just get a job a process all the squence in it.
-	size_t count, pos;
+	uint32_t count, pos;
 	while (mth_getjob(job, &count, &pos)) {
-		for (size_t s = pos; s < pos + count; s++) {
+		for (uint32_t s = pos; s < pos + count; s++) {
 			// Tag the sequence with the viterbi
 			const seq_t *seq = dat->seq[s];
-			const int    T   = seq->len;
-			size_t out[T];
+			const uint32_t T = seq->len;
+			uint32_t *out = wapiti_xmalloc(sizeof(uint32_t) * T);
 			tag_viterbi(mdl, seq, out, NULL, NULL);
 			// And check for eventual (probable ?) errors
 			bool err = false;
-			for (int t = 0; t < T; t++)
+			for (uint32_t t = 0; t < T; t++)
 				if (seq->pos[t].lbl != out[t])
 					eval->terr++, err = true;
-			eval->tcnt += (size_t)T;
+			eval->tcnt += T;
 			eval->scnt += 1;
 			eval->serr += err;
+			free(out);
 		}
 	}
 }
@@ -505,13 +584,13 @@ static void tag_evalsub(job_t *job, int id, int cnt, eval_t *eval) {
  *   taining set if not available).
  */
 void tag_eval(mdl_t *mdl, double *te, double *se) {
-	const size_t W = mdl->opt->nthread;
+	const uint32_t W = mdl->opt->nthread;
 	dat_t *dat = (mdl->devel == NULL) ? mdl->train : mdl->devel;
 	// First we prepare the eval state for all the workers threads, we just
 	// have to give them the model and dataset to use. This state will be
 	// used to retrieve partial result they computed.
 	eval_t *eval[W];
-	for (size_t w = 0; w < W; w++) {
+	for (uint32_t w = 0; w < W; w++) {
 		eval[w] = wapiti_xmalloc(sizeof(eval_t));
 		eval[w]->mdl = mdl;
 		eval[w]->dat = dat;
@@ -520,9 +599,9 @@ void tag_eval(mdl_t *mdl, double *te, double *se) {
 	// result by summing them and computing the final error rates.
 	mth_spawn((func_t *)tag_evalsub, W, (void *)eval, dat->nseq,
 		mdl->opt->jobsize);
-	size_t tcnt = 0, terr = 0;
-	size_t scnt = 0, serr = 0;
-	for (size_t w = 0; w < W; w++) {
+	uint64_t tcnt = 0, terr = 0;
+	uint64_t scnt = 0, serr = 0;
+	for (uint32_t w = 0; w < W; w++) {
 		tcnt += eval[w]->tcnt;
 		terr += eval[w]->terr;
 		scnt += eval[w]->scnt;