wapiti 0.0.1

data/ext/wapiti/reader.h

@@ -0,0 +1,73 @@
+/*
+ *      Wapiti - A linear-chain CRF tool
+ *
+ * Copyright (c) 2009-2011  CNRS
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef reader_h
+#define reader_h
+
+#include <stdbool.h>
+#include <stdio.h>
+
+#include "wapiti.h"
+#include "pattern.h"
+#include "quark.h"
+#include "sequence.h"
+
+/* rdr_t:
+ *   The reader object who hold all informations needed to parse the input file:
+ *   the patterns and quark for labels and observations. We keep separate count
+ *   for unigrams and bigrams pattern for simpler allocation of sequences. We
+ *   also store the expected number of column in the input data to check that
+ *   pattern are appliables.
+ */
+typedef struct rdr_s rdr_t;
+struct rdr_s {
+	bool    maxent;     //      Is this a maxent reader
+	int     npats;      //  P   Total number of patterns
+	int     nuni, nbi;  //      Number of unigram and bigram patterns
+	int     ntoks;      //      Expected number of tokens in input
+	pat_t **pats;       // [P]  List of precompiled patterns
+	qrk_t  *lbl;        //      Labels database
+	qrk_t  *obs;        //      Observation database
+};
+
+rdr_t *rdr_new(bool maxent);
+void rdr_free(rdr_t *rdr);
+void rdr_freeraw(raw_t *raw);
+void rdr_freeseq(seq_t *seq);
+void rdr_freedat(dat_t *dat);
+
+void rdr_loadpat(rdr_t *rdr, FILE *file);
+raw_t *rdr_readraw(rdr_t *rdr, FILE *file);
+seq_t *rdr_raw2seq(rdr_t *rdr, const raw_t *raw, bool lbl);
+seq_t *rdr_readseq(rdr_t *rdr, FILE *file, bool lbl);
+dat_t *rdr_readdat(rdr_t *rdr, FILE *file, bool lbl);
+
+void rdr_load(rdr_t *rdr, FILE *file);
+void rdr_save(const rdr_t *rdr, FILE *file);
+
+#endif
+

data/ext/wapiti/rprop.c

@@ -0,0 +1,191 @@
+/*
+ *      Wapiti - A linear-chain CRF tool
+ *
+ * Copyright (c) 2009-2011  CNRS
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+#include <math.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "wapiti.h"
+#include "gradient.h"
+#include "model.h"
+#include "options.h"
+#include "progress.h"
+#include "tools.h"
+#include "thread.h"
+#include "vmath.h"
+
+#define sign(v) ((v) < 0.0 ? -1.0 : ((v) > 0.0 ? 1.0 : 0.0))
+#define sqr(v)  ((v) * (v))
+
+/******************************************************************************
+ * Resilient propagation optimizer
+ *
+ *   This is an implementation of the RPROP algorithm (resilient propagation)
+ *   described by Riedmiller and Braun in [1] with an adaptation to be useable
+ *   with l1 regularization.
+ *   The adaptation consist of using a pseudo-gradient similar to the one used
+ *   in OWL-QN to choose an orthant at iterations steps and projecting the step
+ *   in this orthant before the weight update.
+ *
+ *   [1] A direct adaptive method for faster backpropagation learning: The RPROP
+ *       algorithm, Martin Riedmiller and Heinrich Braun, IEEE International
+ *       Conference on Neural Networks, San Francisco, USA, 586-591, March 1993.
+ ******************************************************************************/
+typedef struct rprop_s rprop_t;
+struct rprop_s {
+	mdl_t *mdl;
+	double *xp;
+	double *stp;
+	double *g;
+	double *gp;
+};
+
+/* trn_rpropsub:
+ *   Partial update of the weight vector including partial gradient in case of
+ *   l1 regularisation. The sub vector updated depend on the id and cnt
+ *   parameter given, the job scheduling system is not used here as we can
+ *   easily split processing in equals parts.
+ */
+static void trn_rpropsub(job_t *job, int id, int cnt, rprop_t *st) {
+	unused(job);
+	mdl_t *mdl = st->mdl;
+	const size_t F = mdl->nftr;
+	const double stpmin = mdl->opt->rprop.stpmin;
+	const double stpmax = mdl->opt->rprop.stpmax;
+	const double stpinc = mdl->opt->rprop.stpinc;
+	const double stpdec = mdl->opt->rprop.stpdec;
+	const bool   wbt    = strcmp(mdl->opt->algo, "rprop-");
+	const double rho1   = mdl->opt->rho1;
+	const int    l1     = (rho1 != 0.0) ? mdl->opt->rprop.cutoff + 1: 0;
+	double *x = mdl->theta;
+	double *xp  = st->xp,   *stp = st->stp;
+	double *g   = st->g,    *gp  = st->gp;
+	const size_t from = F * id / cnt;
+	const size_t to   = F * (id + 1) / cnt;
+	for (size_t f = from; f < to; f++) {
+		double pg = g[f];
+		// If there is a l1 component in the regularization component,
+		// we either project the gradient in the current orthant or
+		// check for cutdown depending on the projection scheme wanted.
+		if (l1 == 1) {
+			if (x[f] < 0.0)        pg -= rho1;
+			else if (x[f] > 0.0)   pg += rho1;
+			else if (g[f] < -rho1) pg += rho1;
+			else if (g[f] > rho1)  pg -= rho1;
+			else                   pg  = 0.0;
+		} else if (l1 && sqr(g[f] + rho1 * sign(x[f])) < sqr(rho1)) {
+			if (x[f] == 0.0 || (   gp[f] * g[f] < 0.0
+			                    && xp[f] * x[f] < 0.0)) {
+				if (wbt)
+					xp[f] = x[f];
+				x[f]  = 0.0;
+				gp[f] = g[f];
+				continue;
+			}
+		}
+		// Next we adjust the step depending of the new and
+		// previous gradient values.
+		if (gp[f] * pg > 0.0)
+			stp[f] = min(stp[f] * stpinc, stpmax);
+		else if (gp[f] * pg < 0.0)
+			stp[f] = max(stp[f] * stpdec, stpmin);
+		// Finally update the weight. if there is l1 penalty
+		// and the pseudo gradient projection is used, we have to
+		// project back the update in the choosen orthant.
+		if (!wbt || gp[f] * pg > 0.0) {
+			double dlt = stp[f] * -sign(g[f]);
+			if (l1 == 1 && dlt * pg >= 0.0)
+				dlt = 0.0;
+			if (wbt)
+				xp[f] = x[f];
+			x[f] += dlt;
+		} else if (gp[f] * pg < 0.0) {
+			x[f] = xp[f];
+			g[f] = 0.0;
+		} else {
+			xp[f] = x[f];
+			if (l1 != 1)
+				x[f] += stp[f] * -sign(pg);
+		}
+		gp[f] = g[f];
+	}
+}
+
+void trn_rprop(mdl_t *mdl) {
+	const size_t F   = mdl->nftr;
+	const int    K   = mdl->opt->maxiter;
+	const size_t W   = mdl->opt->nthread;
+	const bool   wbt = strcmp(mdl->opt->algo, "rprop-");
+	const int    cut = mdl->opt->rprop.cutoff;
+	// Allocate state memory and initialize it
+	double *xp  = NULL,       *stp = xvm_new(F);
+	double *g   = xvm_new(F), *gp  = xvm_new(F);
+	if (wbt && !cut)
+		xp = xvm_new(F);
+	for (unsigned f = 0; f < F; f++) {
+		if (wbt && !cut)
+			xp[f]  = 0.0;
+		gp[f]  = 0.0;
+		stp[f] = 0.1;
+	}
+	// Prepare the rprop state used to send information to the rprop worker
+	// about updating weight using the gradient.
+	rprop_t *st = wapiti_xmalloc(sizeof(rprop_t));
+	st->mdl = mdl;
+	st->xp  = xp;  st->stp = stp;
+	st->g   = g;   st->gp  = gp;
+	rprop_t *rprop[W];
+	for (size_t w = 0; w < W; w++)
+		rprop[w] = st;
+	// Prepare the gradient state for the distributed gradient computation.
+	grd_t *grds[W];
+	grds[0] = grd_new(mdl, g);
+	for (size_t w = 1; w < W; w++)
+		grds[w] = grd_new(mdl, xvm_new(F));
+	// And iterate the gradient computation / weight update process until
+	// convergence or stop request
+	for (int k = 0; !uit_stop && k < K; k++) {
+		double fx = grd_gradient(mdl, g, grds);
+		if (uit_stop)
+			break;
+		mth_spawn((func_t *)trn_rpropsub, W, (void **)rprop, 0, 0);
+		if (uit_progress(mdl, k + 1, fx) == false)
+			break;
+	}
+	// Free all allocated memory
+	if (wbt && !cut)
+		xvm_free(xp);
+	xvm_free(g);
+	xvm_free(gp);
+	for (size_t w = 1; w < W; w++)
+		xvm_free(grds[w]->g);
+	for (size_t w = 0; w < W; w++)
+		grd_free(grds[w]);
+	free(st);
+}
+

data/ext/wapiti/sequence.h

@@ -0,0 +1,148 @@
+/*
+ *      Wapiti - A linear-chain CRF tool
+ *
+ * Copyright (c) 2009-2011  CNRS
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef sequence_h
+#define sequence_h
+
+#include <stddef.h>
+
+#include "wapiti.h"
+
+/*******************************************************************************
+ * Sequences and Dataset objects
+ *
+ *   Sequences represent the input data feeded by the user in Wapiti either for
+ *   training or labelling. The internal form used here is very different from
+ *   the data read from files and the convertion process is done in three steps
+ *   illustrated here:
+ *         +------+     +-------+     +-------+     +-------+
+ *         | FILE | --> | raw_t | --> | tok_t | --> | seq_t |
+ *         +------+     +-------+     +-------+     +-------+
+ *   First the sequence is read as a set of lines from the input file, this
+ *   give a raw_t object. Next this set of lines is split in tokens and
+ *   eventually the last one is separated as it will become a label, this result
+ *   in a tok_t object.
+ *   The last step consist in applying all the patterns givens by the user to
+ *   extract from these tokens the observations made on the sequence in order to
+ *   build the seq_t object which can be used by the trainer and tagger.
+ *
+ *   A dataset object is just a container for a list of sequences in internal
+ *   form used to store either training or development set.
+ *
+ *   All the convertion process is driven by the reader object and, as it is
+ *   responsible for creating the objects with a quite special allocation
+ *   scheme, we just have to implement function for freeing these objects here.
+ ******************************************************************************/
+
+/* raw_t:
+ *   Data-structure representing a raw sequence as a set of lines read from the
+ *   input file. This is the result of the first step of the interning process.
+ *   We keep this form separate from the tokenized one as we want to be able to
+ *   output the sequence as it was read in the labelling mode.
+ *
+ *   This represent a sequence of lengths <len> and for each position 't' you
+ *   find the corresponding line at <lines>[t].
+ *
+ *   The <lines> array is allocated with data structure, and the different lines
+ *   are allocated separatly.
+ */
+typedef struct raw_s raw_t;
+struct raw_s {
+	int   len;      //   T     Sequence length
+	char *lines[];  //  [T]    Raw lines directly from file
+};
+
+/* tok_t:
+ *   Data-structure representing a tokenized sequence. This is the result of the
+ *   second step of the interning process after the raw sequence have been split
+ *   in tokens and eventual labels separated from the observations.
+ *
+ *   For each position 't' in the sequence of length <len>, you find at <lbl>[t]
+ *   the eventual label provided in input file, and at <toks>[t] a list of
+ *   string tokens of length <cnts>[t].
+ *
+ *   Memory allocation here is a bit special as the first token at each position
+ *   point to a memory block who hold a copy of the raw line. Each other tokens
+ *   and the label are pointer in this block. This reduce memory fragmentation.
+ */
+typedef struct tok_s tok_t;
+struct tok_s {
+	int    len;     //   T     Sequence length
+	char **lbl;     //  [T]    List of labels strings
+	int   *cnts;    //  [T]    Length of tokens lists
+	char **toks[];  //  [T][]  Tokens lists
+};
+
+/* seq_t:
+ *   Data-structure representing a sequence of length <len> in the internal form
+ *   used by the trainers and the tagger. For each position 't' in the sequence
+ *   (0 <= t < <len>) there is some observations made on the data and an
+ *   eventual label if provided in the input file.
+ *
+ *   There is two kind of features: unigrams and bigrams one, build by combining
+ *   one observation and one or two labels. At position 't', the unigrams
+ *   features are build using the list of observations from <uobs>[t] which
+ *   contains <ucnt>[t] items, and the observation at <lbl>[t]. The bigrams
+ *   features are obtained in the same way using <bobs> and <bcnt>, and have to
+ *   be combined also with <lbl>[t-1].
+ *
+ *   If the sequence is read from a file without label, as it is the case in
+ *   labelling mode, the <lbl> field will be NULL and so, the sequence cannot be
+ *   used for training.
+ *
+ *   The raw field is private and used internaly for efficient memory
+ *   allocation. This allow to allocate <lbl>, <*cnt>, and all the list in
+ *   <*obs> with the datastructure itself.
+ */
+typedef struct pos_s pos_t;
+typedef struct seq_s seq_t;
+struct seq_s {
+	int     len;
+	size_t *raw;
+	struct pos_s {
+		size_t  lbl;
+		size_t  ucnt,  bcnt;
+		size_t *uobs, *bobs;
+	} pos[];
+};
+
+/* dat_t:
+ *   Data-structure representing a full dataset: a collection of sequences ready
+ *   to be used for training or to be labelled. It keep tracks of the maximum
+ *   sequence length as the trainer need this for memory allocation. The dataset
+ *   contains <nseq> sequence stored in <seq>. These sequences are labeled only
+ *   if <lbl> is true.
+ */
+typedef struct dat_s dat_t;
+struct dat_s {
+	bool     lbl;   //         True iff sequences are labelled
+	int      mlen;  //         Length of the longest sequence in the set
+	size_t   nseq;  //   S     Number of sequences in the set
+	seq_t  **seq;   //  [S]    List of sequences
+};
+
+#endif

data/ext/wapiti/sgdl1.c

@@ -0,0 +1,218 @@
+/*
+ *      Wapiti - A linear-chain CRF tool
+ *
+ * Copyright (c) 2009-2011  CNRS
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+#include <math.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "wapiti.h"
+#include "gradient.h"
+#include "model.h"
+#include "options.h"
+#include "progress.h"
+#include "sequence.h"
+#include "tools.h"
+
+/******************************************************************************
+ * The SGD-L1 trainer
+ *
+ *   Implementation of the stochatic gradient descend with L1 penalty described
+ *   in [1] by Tsurukoa et al. This allow to build really sparse models with the
+ *   SGD method.
+ *
+ *   [1] Stochastic gradient descent training for L1-regularized log-linear
+ *       models with cumulative penalty, Yoshimasa Tsuruoka and Jun'ichi Tsuji
+ *       and Sophia Ananiadou, in Proceedings of the ACL and the 4th IJCNLP of
+ *       the AFNLP, pages 477-485, August 2009
+ ******************************************************************************/
+typedef struct sgd_idx_s {
+	size_t *uobs;
+	size_t *bobs;
+} sgd_idx_t;
+
+/* applypenalty:
+ *   This macro is quite ugly as it make a lot of things and use local variables
+ *   of the function below. I'm sorry for this but this is allow to not
+ *   duplicate the code below. Due to the way unigrams and bigrams observation
+ *   are stored we must use this two times. As this macro is dangerous when
+ *   called outsize of sgd-l1 we undef it just after.
+ *   This function match exactly the APPLYPENALTY function defined in [1] pp 481
+ *   and the formula on the middle of the page 480.
+ */
+#define applypenalty(f) do {                               \
+	const double z = w[f];                             \
+	if      (z > 0.0) w[f] = max(0.0, z - (u + q[f])); \
+	else if (z < 0.0) w[f] = min(0.0, z + (u - q[f])); \
+	q[f] += w[f] - z;                                  \
+} while (false)
+
+/* sgd_add:
+ *   Add the <new> value in the array <obs> of size <cnt>. If the value is
+ *   already present, we do nothing, else we add it.
+ */
+static void sgd_add(size_t *obs, size_t *cnt, size_t new) {
+	// First check if value is already in the array, we do a linear probing
+	// as it is simpler and since these array will be very short in
+	// practice, it's efficient enough.
+	for (size_t p = 0; p < *cnt; p++)
+		if (obs[p] == new)
+			return;
+	// Insert the new value at the end since we have not found it.
+	obs[*cnt] = new;
+	*cnt = *cnt + 1;
+}
+
+/* trn_sgdl1:
+ *   Train the model with the SGD-l1 algorithm described by tsurukoa et al.
+ */
+void trn_sgdl1(mdl_t *mdl) {
+	const size_t  Y = mdl->nlbl;
+	const size_t  F = mdl->nftr;
+	const int     U = mdl->reader->nuni;
+	const int     B = mdl->reader->nbi;
+	const int     S = mdl->train->nseq;
+	const int     K = mdl->opt->maxiter;
+	      double *w = mdl->theta;
+	// First we have to build and index who hold, for each sequences, the
+	// list of actives observations.
+	// The index is a simple table indexed by sequences number. Each entry
+	// point to two lists of observations terminated by <none>, one for
+	// unigrams obss and one for bigrams obss.
+	info("    - Build the index\n");
+	sgd_idx_t *idx  = wapiti_xmalloc(sizeof(sgd_idx_t) * S);
+	for (int s = 0; s < S; s++) {
+		const seq_t *seq = mdl->train->seq[s];
+		const int T = seq->len;
+		size_t uobs[U * T + 1], ucnt = 0;
+		size_t bobs[B * T + 1], bcnt = 0;
+		for (int t = 0; t < seq->len; t++) {
+			const pos_t *pos = &seq->pos[t];
+			for (size_t p = 0; p < pos->ucnt; p++)
+				sgd_add(uobs, &ucnt, pos->uobs[p]);
+			for (size_t p = 0; p < pos->bcnt; p++)
+				sgd_add(bobs, &bcnt, pos->bobs[p]);
+		}
+		uobs[ucnt++] = none;
+		bobs[bcnt++] = none;
+		idx[s].uobs = wapiti_xmalloc(sizeof(size_t) * ucnt);
+		idx[s].bobs = wapiti_xmalloc(sizeof(size_t) * bcnt);
+		memcpy(idx[s].uobs, uobs, ucnt * sizeof(size_t));
+		memcpy(idx[s].bobs, bobs, bcnt * sizeof(size_t));
+	}
+	info("      Done\n");
+	// We will process sequences in random order in each iteration, so we
+	// will have to permute them. The current permutation is stored in a
+	// vector called <perm> shuffled at the start of each iteration. We
+	// just initialize it with the identity permutation.
+	// As we use the same gradient function than the other trainers, we need
+	// an array to store it. These functions accumulate the gradient so we
+	// need to clear it at start and before each new computation. As we now
+	// which features are active and so which gradient cell are updated, we
+	// can clear them selectively instead of fully clear the gradient each
+	// time.
+	// We also need an aditional vector named <q> who hold the penalty
+	// already applied to each features.
+	int *perm = wapiti_xmalloc(sizeof(int) * S);
+	for (int s = 0; s < S; s++)
+		perm[s] = s;
+	double *g = wapiti_xmalloc(sizeof(double) * F);
+	double *q = wapiti_xmalloc(sizeof(double) * F);
+	for (size_t f = 0; f < F; f++)
+		g[f] = q[f] = 0.0;
+	// We can now start training the model, we perform the requested number
+	// of iteration, each of these going through all the sequences. For
+	// computing the decay, we will need to keep track of the number of
+	// already processed sequences, this is tracked by the <i> variable.
+	double u = 0.0;
+	grd_t *grd = grd_new(mdl, g);
+	for (int k = 0, i = 0; k < K && !uit_stop; k++) {
+		// First we shuffle the sequence by making a lot of random swap
+		// of entry in the permutation index.
+		for (int s = 0; s < S; s++) {
+			const int a = rand() % S;
+			const int b = rand() % S;
+			const int t = perm[a];
+			perm[a] = perm[b];
+			perm[b] = t;
+		}
+		// And so, we can process sequence in a random order
+		for (int sp = 0; sp < S && !uit_stop; sp++, i++) {
+			const int s = perm[sp];
+			const seq_t *seq = mdl->train->seq[s];
+			grd_dospl(grd, seq);
+			// Before applying the gradient, we have to compute the
+			// learning rate to apply to this sequence. For this we
+			// use an exponential decay [1, pp 481(5)]
+			//   η_i = η_0 * α^{i/S}
+			// And at the same time, we update the total penalty
+			// that must have been applied to each features.
+			//   u <- u + η * rho1 / S
+			const double n0    = mdl->opt->sgdl1.eta0;
+			const double alpha = mdl->opt->sgdl1.alpha;
+			const double nk = n0 * pow(alpha, (double)i / S);
+			u = u + nk * mdl->opt->rho1 / S;
+			// Now we apply the update to all unigrams and bigrams
+			// observations actives in the current sequence. We must
+			// not forget to clear the gradient for the next
+			// sequence.
+			for (size_t n = 0; idx[s].uobs[n] != none; n++) {
+				size_t f = mdl->uoff[idx[s].uobs[n]];
+				for (size_t y = 0; y < Y; y++, f++) {
+					w[f] -= nk * g[f];
+					applypenalty(f);
+					g[f] = 0.0;
+				}
+			}
+			for (size_t n = 0; idx[s].bobs[n] != none; n++) {
+				size_t f = mdl->boff[idx[s].bobs[n]];
+				for (size_t d = 0; d < Y * Y; d++, f++) {
+					w[f] -= nk * g[f];
+					applypenalty(f);
+					g[f] = 0.0;
+				}
+			}
+		}
+		if (uit_stop)
+			break;
+		// Repport progress back to the user
+		if (!uit_progress(mdl, k + 1, -1.0))
+			break;
+	}
+	grd_free(grd);
+	// Cleanup allocated memory before returning
+	for (int s = 0; s < S; s++) {
+		free(idx[s].uobs);
+		free(idx[s].bobs);
+	}
+	free(idx);
+	free(perm);
+	free(g);
+	free(q);
+}
+#undef applypenalty
+