wapiti 0.0.1

data/ext/wapiti/gradient.h

@@ -0,0 +1,81 @@
+/*
+ *      Wapiti - A linear-chain CRF tool
+ *
+ * Copyright (c) 2009-2011  CNRS
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef gradient_h
+#define gradient_h
+
+#include "wapiti.h"
+#include "model.h"
+#include "sequence.h"
+
+/* grd_t:
+ *   State tracker for the gradient computation. To compute the gradient we need
+ *   to perform several steps and communicate between them a lot of intermediate
+ *   values, all these temporary are store in this object.
+ *   A tracker can be used to compute sequence of length <len> at most, before
+ *   using it you must call grd_check to ensure that the tracker is big enough
+ *   for your sequence.
+ */
+typedef struct grd_s grd_t;
+struct grd_s {
+	mdl_t  *mdl;
+	int     len;     // =T        max length of sequence
+	double *g;       // [F]       vector where to put gradient updates
+	double  lloss;   //           loss value for the sequence
+	double *psi;     // [T][Y][Y] the transitions scores
+	double *psiuni;  // [T][Y]    | Same as psi in sparse format
+	size_t *psiyp;   // [T][Y][Y] |
+	size_t *psiidx;  // [T][Y]    |
+	size_t *psioff;  // [T]
+	double *alpha;   // [T][Y]    forward scores
+	double *beta;    // [T][Y]    backward scores
+	double *scale;   // [T]       scaling factors of forward scores
+	double *unorm;   // [T]       normalization factors for unigrams
+	double *bnorm;   // [T]       normalization factors for bigrams
+	int     first;   //           first position where gradient is needed
+	int     last;    //           last position where gradient is needed
+};
+
+grd_t *grd_new(mdl_t *mdl, double *g);
+void grd_free(grd_t *grd);
+void grd_check(grd_t *grd, int len);
+
+void grd_fldopsi(grd_t *grd, const seq_t *seq);
+void grd_flfwdbwd(grd_t *grd, const seq_t *seq);
+void grd_flupgrad(grd_t *grd, const seq_t *seq);
+
+void grd_spdopsi(grd_t *grd, const seq_t *seq);
+void grd_spfwdbwd(grd_t *grd, const seq_t *seq);
+void grd_spupgrad(grd_t *grd, const seq_t *seq);
+
+void grd_logloss(grd_t *grd, const seq_t *seq);
+
+void grd_dospl(grd_t *grd, const seq_t *seq);
+double grd_gradient(mdl_t *mdl, double *g, grd_t *grds[]);
+
+#endif
+

data/ext/wapiti/lbfgs.c

@@ -0,0 +1,294 @@
+/*
+ *      Wapiti - A linear-chain CRF tool
+ *
+ * Copyright (c) 2009-2011  CNRS
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+#include <math.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "wapiti.h"
+#include "gradient.h"
+#include "model.h"
+#include "options.h"
+#include "progress.h"
+#include "tools.h"
+#include "thread.h"
+#include "vmath.h"
+
+/******************************************************************************
+ * Quasi-Newton optimizer
+ *
+ *   This section implement the quasi-Newton optimizer. We use the L-BFGS
+ *   algorithm described by Liu and Nocedal in [1] and [2]. If an l1-norm must
+ *   be applyed we fallback on the OWL-QN variant described in [3] by Galen and
+ *   Jianfeng which allow to use L-BFGS for function not differentiable in 0.0.
+ *
+ *   [1] Updating quasi-Newton matrices with limited storage, Jorge Nocedal, in
+ *       Mathematics of Computation, vol. 35(151) 773-782, July 1980.
+ *   [2] On the limited memory BFGS method for large scale optimization, Dong C.
+ *       Liu and Jorge Nocedal, in Mathematical Programming, vol. 45(1) 503-528,
+ *       January 1989.
+ *   [3] Scalable Training of L1-Regularized Log-Linear Models, Andrew Galen and
+ *       Gao Jianfeng, in Proceedings of the 24th International Conference on
+ *       Machine Learning (ICML), Corvallis, OR, 2007.
+ ******************************************************************************/
+
+void trn_lbfgs(mdl_t *mdl) {
+	const size_t F  = mdl->nftr;
+	const int    K  = mdl->opt->maxiter;
+	const int    C  = mdl->opt->objwin;
+	const int    M  = mdl->opt->lbfgs.histsz;
+	const size_t W  = mdl->opt->nthread;
+	const bool   l1 = mdl->opt->rho1 != 0.0;
+	double *x, *xp; // Current and previous value of the variables
+	double *g, *gp; // Current and previous value of the gradient
+	double *pg;     // The pseudo-gradient (only for owl-qn)
+	double *d;      // The search direction
+	double *s[M];   // History value s_k = Δ(x,px)
+	double *y[M];   // History value y_k = Δ(g,pg)
+	double  p[M];   // ρ_k
+	double  fh[C];  // f(x) history
+	grd_t  *grds[W];
+	// Initialization: Here, we have to allocate memory on the heap as we
+	// cannot request so much memory on the stack as this will have a too
+	// big impact on performance and will be refused by the system on non-
+	// trivial models.
+	x  = mdl->theta;
+	xp = xvm_new(F); g = xvm_new(F);
+	gp = xvm_new(F); d = xvm_new(F);
+	for (int m = 0; m < M; m++) {
+		s[m] = xvm_new(F);
+		y[m] = xvm_new(F);
+	}
+	pg = l1 ? xvm_new(F) : NULL;
+	grds[0] = grd_new(mdl, g);
+	for (size_t w = 1; w < W; w++)
+		grds[w] = grd_new(mdl, xvm_new(F));
+	// Minimization: This is the heart of the function. (a big heart...) We
+	// will perform iterations until one these conditions is reached
+	//   - the maximum iteration count is reached
+	//   - we have converged (upto numerical precision)
+	//   - the report function return false
+	//   - an error happen somewhere
+	double fx = grd_gradient(mdl, g, grds);
+	for (int k = 0; !uit_stop && k < K; k++) {
+		// We first compute the pseudo-gradient of f for owl-qn. It is
+		// defined in [3, pp 335(4)]
+		//              | ∂_i^- f(x)   if ∂_i^- f(x) > 0
+		//   ◇_i f(x) = | ∂_i^+ f(x)   if ∂_i^+ f(x) < 0
+		//              | 0            otherwise
+		// with
+		//   ∂_i^± f(x) = ∂/∂x_i l(x) + | Cσ(x_i) if x_i ≠ 0
+		//                              | ±C      if x_i = 0
+		if (l1) {
+			const double rho1 = mdl->opt->rho1;
+			for (unsigned f = 0; f < F; f++) {
+				if (x[f] < 0.0)
+					pg[f] = g[f] - rho1;
+				else if (x[f] > 0.0)
+					pg[f] = g[f] + rho1;
+				else if (g[f] < -rho1)
+					pg[f] = g[f] + rho1;
+				else if (g[f] > rho1)
+					pg[f] = g[f] - rho1;
+				else
+					pg[f] = 0.0;
+			}
+		}
+		// 1st step: We compute the search direction. We search in the
+		// direction who minimize the second order approximation given
+		// by the Taylor series which give
+		//   d_k = - H_k^{-1} g_k
+		// But computing the inverse of the hessian is intractable so
+		// the l-bfgs only approximate it's diagonal. The exact
+		// computation is well described in [1, pp 779].
+		// The only special thing for owl-qn here is to use the pseudo
+		// gradient instead of the true one.
+		xvm_neg(d, l1 ? pg : g, F);
+		if (k != 0) {
+			const int km = k % M;
+			const int bnd = (k <= M) ? k : M;
+			double alpha[M], beta;
+			// α_i = ρ_j s_j^T q_{i+1}
+			// q_i = q_{i+1} - α_i y_i
+			for (int i = bnd; i > 0; i--) {
+				const int j = (k - i + M + 1) % M;
+				alpha[i - 1] = p[j] * xvm_dot(s[j], d, F);
+				xvm_axpy(d, -alpha[i - 1], y[j], d, F);
+			}
+			// r_0 = H_0 q_0
+			//     Scaling is described in [2, pp 515]
+			//     for k = 0: H_0 = I
+			//     for k > 0: H_0 = I * y_k^T s_k / ||y_k||²
+			//                    = I * 1 / ρ_k ||y_k||²
+			const double y2 = xvm_dot(y[km], y[km], F);
+			const double v = 1.0 / (p[km] * y2);
+			for (size_t f = 0; f < F; f++)
+				d[f] *= v;
+			// β_j     = ρ_j y_j^T r_i
+			// r_{i+1} = r_i + s_j (α_i - β_i)
+			for (int i = 0; i < bnd; i++) {
+				const int j = (k - i + M) % M;
+				beta = p[j] * xvm_dot(y[j], d, F);
+				xvm_axpy(d, alpha[i] - beta, s[j], d, F);
+			}
+		}
+		// For owl-qn, we must remain in the same orthant than the
+		// pseudo-gradient, so we have to constrain the search
+		// direction as described in [3, pp 35(3)]
+		//   d^k = π(d^k ; v^k)
+		//       = π(d^k ; -◇f(x^k))
+		if (l1)
+			for (size_t f = 0; f < F; f++)
+				if (d[f] * pg[f] >= 0.0)
+					d[f] = 0.0;
+		// 2nd step: we perform a linesearch in the computed direction,
+		// we search a step value that satisfy the constrains using a
+		// backtracking algorithm. Much elaborated algorithm can perform
+		// better in the general case, but for CRF training, bactracking
+		// is very efficient and simple to implement.
+		// For quasi-Newton, the natural step is 1.0 so we start with
+		// this one and reduce it only if it fail with an exception for
+		// the first step where a better guess can be done.
+		// We have to keep track of the current point and gradient as we
+		// will need to compute the delta between those and the found
+		// point, and perhaps need to restore them if linesearch fail.
+		memcpy(xp, x, sizeof(double) * F);
+		memcpy(gp, g, sizeof(double) * F);
+		double sc  = (k == 0) ? 0.1 : 0.5;
+		double stp = (k == 0) ? 1.0 / xvm_norm(d, F) : 1.0;
+		double gd  = l1 ? 0.0 : xvm_dot(g, d, F); // gd = g_k^T d_k
+		double fi  = fx;
+		bool err = false;
+		for (int ls = 1; !uit_stop; ls++, stp *= sc) {
+			// We compute the new point using the current step and
+			// search direction
+			xvm_axpy(x, stp, d, xp, F);
+			// For owl-qn, we have to project back the point in the
+			// current orthant [3, pp 35]
+			//   x^{k+1} = π(x^k + αp^k ; ξ)
+			if (l1) {
+				for (size_t f = 0; f < F; f++) {
+					double or = xp[f];
+					if (or == 0.0)
+						or = -pg[f];
+					if (x[f] * or <= 0.0)
+						x[f] = 0.0;
+				}
+			}
+			// And we ask for the value of the objective function
+			// and its gradient.
+			fx = grd_gradient(mdl, g, grds);
+			// Now we check if the step satisfy the conditions. For
+			// l-bfgs, we check the classical decrease and curvature
+			// known as the Wolfe conditions [2, pp 506]
+			//   f(x_k + α_k d_k) ≤ f(x_k) + β' α_k g_k^T d_k
+			//   g(x_k + α_k d_k)^T d_k ≥ β g_k^T d_k
+			//
+			// And for owl-qn we check a variant of the Armijo rule
+			// described in [3, pp 36]
+			//   f(π(x^k+αp^k;ξ)) ≤ f(x^k) - γv^T[π(x^k+αp^k;ξ)-x^k]
+			if (!l1) {
+				if (fx > fi + stp * gd * 1e-4)
+					sc = 0.5;
+				else if (xvm_dot(g, d, F) < gd * 0.9)
+					sc = 2.1;
+				else
+					break;
+			} else {
+				double vp = 0.0;
+				for (size_t f = 0; f < F; f++)
+					vp += (x[f] - xp[f]) * d[f];
+				if (fx < fi + vp * 1e-4)
+					break;
+			}
+			// If we reach the maximum number of linesearsh steps
+			// without finding a good one, we just fail.
+			if (ls == mdl->opt->lbfgs.maxls) {
+				warning("maximum linesearch reached");
+				err = true;
+				break;
+			}
+		}
+		// If linesearch failed or user interupted training, we return
+		// to the last valid point and stop the training. The model is
+		// probably not fully optimized but we let the user decide what
+		// to do with it.
+		if (err || uit_stop) {
+			memcpy(x, xp, sizeof(double) * F);
+			break;
+		}
+		if (uit_progress(mdl, k + 1, fx) == false)
+			break;
+		// 3rd step: we update the history used for approximating the
+		// inverse of the diagonal of the hessian
+		//   s_k = x_{k+1} - x_k
+		//   y_k = g_{k+1} - g_k
+		//   ρ_k = 1 / y_k^T s_k
+		const int kn = (k + 1) % M;
+		xvm_sub(s[kn], x, xp, F);
+		xvm_sub(y[kn], g, gp, F);
+		p[kn] = 1.0 / xvm_dot(y[kn], s[kn], F);
+		// And last, we check for convergence. The convergence check is
+		// quite simple [2, pp 508]
+		//   ||g|| / max(1, ||x||) ≤ ε
+		// with ε small enough so we stop when numerical precision is
+		// reached. For owl-qn we just have to check against the pseudo-
+		// gradient instead of the true one.
+		const double xn = xvm_norm(x, F);
+		const double gn = xvm_norm(l1 ? pg : g, F);
+		if (gn / max(xn, 1.0) <= 1e-5)
+			break;
+		if (k + 1 == K)
+			break;
+		// Second stoping criterion tested is a check for improvement of
+		// the function value over the past W iteration. When this come
+		// under an epsilon, we also stop the minimization.
+		fh[k % C] = fx;
+		double dlt = 1.0;
+		if (k >= C) {
+			const double of = fh[(k + 1) % C];
+			dlt = fabs(of - fx) / of;
+			if (dlt < mdl->opt->stopeps)
+				break;
+		}
+	}
+	// Cleanup: We free all the vectors we have allocated.
+	xvm_free(xp); xvm_free(g);
+	xvm_free(gp); xvm_free(d);
+	for (int m = 0; m < M; m++) {
+		xvm_free(s[m]);
+		xvm_free(y[m]);
+	}
+	if (l1)
+		xvm_free(pg);
+	for (size_t w = 1; w < W; w++)
+		xvm_free(grds[w]->g);
+	for (size_t w = 0; w < W; w++)
+		grd_free(grds[w]);
+}
+

data/ext/wapiti/model.c

@@ -0,0 +1,296 @@
+/*
+ *      Wapiti - A linear-chain CRF tool
+ *
+ * Copyright (c) 2009-2011  CNRS
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+
+#include "wapiti.h"
+#include "model.h"
+#include "options.h"
+#include "quark.h"
+#include "reader.h"
+#include "tools.h"
+#include "vmath.h"
+
+/*******************************************************************************
+ * Linear chain CRF model
+ *
+ *   There is three concept that must be well understand here, the labels,
+ *   observations, and features. The labels are the values predicted by the
+ *   model at each point of the sequence and denoted by Y. The observations are
+ *   the values, at each point of the sequence, given to the model in order to
+ *   predict the label and denoted by O. A feature is a test on both labels and
+ *   observations, denoted by F. In linear chain CRF there is two kinds of
+ *   features :
+ *     - unigram feature who represent a test on the observations at the current
+ *       point and the label at current point.
+ *     - bigram feature who represent a test on the observation at the current
+ *       point and two labels : the current one and the previous one.
+ *   So for each observation, there Y possible unigram features and Y*Y possible
+ *   bigram features. The kind of features used by the model for a given
+ *   observation depend on the pattern who generated it.
+ ******************************************************************************/
+
+/* mdl_new:
+ *   Allocate a new empty model object linked with the given reader. The model
+ *   have to be synchronized before starting training or labelling. If you not
+ *   provide a reader (as it will loaded from file for example) you must be sure
+ *   to set one in the model before any attempts to synchronize it.
+ */
+mdl_t *mdl_new(rdr_t *rdr) {
+	mdl_t *mdl = wapiti_xmalloc(sizeof(mdl_t));
+	mdl->nlbl   = mdl->nobs  = mdl->nftr = 0;
+	mdl->kind   = NULL;
+	mdl->uoff   = mdl->boff  = NULL;
+	mdl->theta  = NULL;
+	mdl->train  = mdl->devel = NULL;
+	mdl->reader = rdr;
+	mdl->werr   = NULL;
+	mdl->total  = 0.0;
+	return mdl;
+}
+
+/* mdl_free:
+ *   Free all memory used by a model object inculding the reader and datasets
+ *   loaded in the model.
+ */
+void mdl_free(mdl_t *mdl) {
+	free(mdl->kind);
+	free(mdl->uoff);
+	free(mdl->boff);
+	if (mdl->theta != NULL)
+		xvm_free(mdl->theta);
+	if (mdl->train != NULL)
+		rdr_freedat(mdl->train);
+	if (mdl->devel != NULL)
+		rdr_freedat(mdl->devel);
+	if (mdl->reader != NULL)
+		rdr_free(mdl->reader);
+	if (mdl->werr != NULL)
+		free(mdl->werr);
+	free(mdl);
+}
+
+/* mdl_sync:
+ *   Synchronize the model with its reader. As the model is just a placeholder
+ *   for features weights and interned sequences, it know very few about the
+ *   labels and observations, all the informations are kept in the reader. A
+ *   sync will get the labels and observations count as well as the observation
+ *   kind from the reader and build internal structures representing the model.
+ *
+ *   If the model was already synchronized before, there is an existing model
+ *   incompatible with the new one to be created. In this case there is two
+ *   possibility :
+ *     - If only new observations was added, the weights of the old ones remain
+ *       valid and are kept as they form a probably good starting point for
+ *       training the new model, the new observation get a 0 weight ;
+ *     - If new labels was added, the old model are trully meaningless so we
+ *       have to fully discard them and build a new empty model.
+ *   In any case, you must never change existing labels or observations, if this
+ *   happen, you need to create a new model and destroy this one.
+ *
+ *   After synchronization, the labels and observations databases are locked to
+ *   prevent new one to be created. You must unlock them explicitly if needed.
+ *   This reduce the risk of mistakes.
+ */
+void mdl_sync(mdl_t *mdl) {
+	const size_t Y = qrk_count(mdl->reader->lbl);
+	const size_t O = qrk_count(mdl->reader->obs);
+	// If model is already synchronized, do nothing and just return
+	if (mdl->nlbl == Y && mdl->nobs == O)
+		return;
+	if (Y == 0 || O == 0)
+		fatal("cannot synchronize an empty model");
+	// If new labels was added, we have to discard all the model. In this
+	// case we also display a warning as this is probably not expected by
+	// the user. If only new observations was added, we will try to expand
+	// the model.
+	size_t oldF = mdl->nftr;
+	size_t oldO = mdl->nobs;
+	if (mdl->nlbl != Y && mdl->nlbl != 0) {
+		warning("labels count changed, discarding the model");
+		free(mdl->kind);  mdl->kind  = NULL;
+		free(mdl->uoff);  mdl->uoff  = NULL;
+		free(mdl->boff);  mdl->boff  = NULL;
+		if (mdl->theta != NULL) {
+			xvm_free(mdl->theta);
+			mdl->theta = NULL;
+		}
+		oldF = oldO = 0;
+	}
+	mdl->nlbl = Y;
+	mdl->nobs = O;
+	// Allocate the observations datastructure. If the model is empty or
+	// discarded, a new one iscreated, else the old one is expanded.
+	char   *kind = wapiti_xrealloc(mdl->kind, sizeof(char  ) * O);
+	size_t *uoff = wapiti_xrealloc(mdl->uoff, sizeof(size_t) * O);
+	size_t *boff = wapiti_xrealloc(mdl->boff, sizeof(size_t) * O);
+	mdl->kind = kind;
+	mdl->uoff = uoff;
+	mdl->boff = boff;
+	// Now, we can setup the features. For each new observations we fill the
+	// kind and offsets arrays and count total number of features as well.
+	size_t F = oldF;
+	for (size_t o = oldO; o < O; o++) {
+		const char *obs = qrk_id2str(mdl->reader->obs, o);
+		switch (obs[0]) {
+			case 'u': kind[o] = 1; break;
+			case 'b': kind[o] = 2; break;
+			case '*': kind[o] = 3; break;
+		}
+		if (kind[o] & 1)
+			uoff[o] = F, F += Y;
+		if (kind[o] & 2)
+			boff[o] = F, F += Y * Y;
+	}
+	mdl->nftr = F;
+	// We can finally grow the features weights vector itself. We set all
+	// the new features to 0.0 but don't touch the old ones.
+	// This is a bit tricky as aligned malloc cannot be simply grown so we
+	// have to allocate a new vector and copy old values ourself.
+	if (oldF != 0) {
+		double *new = xvm_new(F);
+		for (size_t f = 0; f < oldF; f++)
+			new[f] = mdl->theta[f];
+		xvm_free(mdl->theta);
+		mdl->theta = new;
+	} else {
+		mdl->theta = xvm_new(F);
+	}
+	for (size_t f = oldF; f < F; f++)
+		mdl->theta[f] = 0.0;
+	// And lock the databases
+	qrk_lock(mdl->reader->lbl, true);
+	qrk_lock(mdl->reader->obs, true);
+}
+
+/* mdl_compact:
+ *   Comapct the given model by removing from it all observation who lead to
+ *   zero actives features. On model trained with l1 regularization this can
+ *   lead to a drastic model size reduction and so to faster loading, training
+ *   and labeling.
+ */
+void mdl_compact(mdl_t *mdl) {
+	const size_t Y = mdl->nlbl;
+	// We first build the new observation list with only observations which
+	// lead to at least one active feature. At the same time we build the
+	// translation table which map the new observations index to the old
+	// ones.
+	info("    - Scan the model\n");
+	qrk_t *old_obs = mdl->reader->obs;
+	qrk_t *new_obs = qrk_new();
+	size_t *trans = wapiti_xmalloc(sizeof(size_t) * mdl->nobs);
+	for (size_t oldo = 0; oldo < mdl->nobs; oldo++) {
+		bool active = false;
+		if (mdl->kind[oldo] & 1)
+			for (size_t y = 0; y < Y; y++)
+				if (mdl->theta[mdl->uoff[oldo] + y] != 0.0)
+					active = true;
+		if (mdl->kind[oldo] & 2)
+			for (size_t d = 0; d < Y * Y; d++)
+				if (mdl->theta[mdl->boff[oldo] + d] != 0.0)
+					active = true;
+		if (!active)
+			continue;
+		const char   *str  = qrk_id2str(old_obs, oldo);
+		const size_t  newo = qrk_str2id(new_obs, str);
+		trans[newo] = oldo;
+	}
+	mdl->reader->obs = new_obs;
+	// Now we save the old model features informations and build a new one
+	// corresponding to the compacted model.
+	size_t *old_uoff  = mdl->uoff;  mdl->uoff  = NULL;
+	size_t *old_boff  = mdl->boff;  mdl->boff  = NULL;
+	double *old_theta = mdl->theta; mdl->theta = NULL;
+	free(mdl->kind);
+	mdl->kind = NULL;
+	mdl->nlbl = mdl->nobs = mdl->nftr = 0;
+	mdl_sync(mdl);
+	// The model is now ready, so we copy in it the features weights from
+	// the old model for observations we have kept.
+	info("    - Compact it\n");
+	for (size_t newo = 0; newo < mdl->nobs; newo++) {
+		const size_t oldo = trans[newo];
+		if (mdl->kind[newo] & 1) {
+			double *src = old_theta  + old_uoff[oldo];
+			double *dst = mdl->theta + mdl->uoff[newo];
+			for (size_t y = 0; y < Y; y++)
+				dst[y] = src[y];
+		}
+		if (mdl->kind[newo] & 2) {
+			double *src = old_theta  + old_boff[oldo];
+			double *dst = mdl->theta + mdl->boff[newo];
+			for (size_t d = 0; d < Y * Y; d++)
+				dst[d] = src[d];
+		}
+	}
+	// And cleanup
+	free(trans);
+	qrk_free(old_obs);
+	free(old_uoff);
+	free(old_boff);
+	xvm_free(old_theta);
+}
+
+/* mdl_save:
+ *   Save a model to be restored later in a platform independant way.
+ */
+void mdl_save(mdl_t *mdl, FILE *file) {
+	size_t nact = 0;
+	for (size_t f = 0; f < mdl->nftr; f++)
+		if (mdl->theta[f] != 0.0)
+			nact++;
+	fprintf(file, "#mdl#%zu\n", nact);
+	rdr_save(mdl->reader, file);
+	for (size_t f = 0; f < mdl->nftr; f++)
+		if (mdl->theta[f] != 0.0)
+			fprintf(file, "%zu=%la\n", f, mdl->theta[f]);
+}
+
+/* mdl_load:
+ *   Read back a previously saved model to continue training or start labeling.
+ *   The returned model is synced and the quarks are locked. You must give to
+ *   this function an empty model fresh from mdl_new.
+ */
+void mdl_load(mdl_t *mdl, FILE *file) {
+	const char *err = "invalid model format";
+	size_t nact = 0;
+	if (fscanf(file, "#mdl#%zu\n", &nact) != 1)
+		fatal(err);
+	rdr_load(mdl->reader, file);
+	mdl_sync(mdl);
+	for (size_t i = 0; i < nact; i++) {
+		size_t f;
+		double v;
+		if (fscanf(file, "%zu=%la\n", &f, &v) != 2)
+			fatal(err);
+		mdl->theta[f] = v;
+	}
+}
+