ruby-minigraph 0.0.20.0

data/ext/minigraph/gmap.c

@@ -0,0 +1,211 @@
+#include <stdlib.h>
+#include <assert.h>
+#include "kthread.h"
+#include "kalloc.h"
+#include "bseq.h"
+#include "sys.h"
+#include "mgpriv.h"
+#include "gfa-priv.h"
+
+typedef struct {
+	int64_t mini_batch_size;
+	int n_processed, n_threads, n_fp;
+	const mg_mapopt_t *opt;
+	mg_bseq_file_t **fp;
+	const mg_idx_t *gi;
+	kstring_t str;
+	double *c_seg, *c_link;
+} pipeline_t;
+
+typedef struct {
+	const pipeline_t *p;
+    int n_seq, n_frag;
+	mg_bseq1_t *seq;
+	int *seg_off, *n_seg;
+	mg_gchains_t **gcs;
+	mg_tbuf_t **buf;
+} step_t;
+
+static void worker_for(void *_data, long i, int tid) // kt_for() callback
+{
+    step_t *s = (step_t*)_data;
+	int qlens[MG_MAX_SEG], j, off = s->seg_off[i], pe_ori = s->p->opt->pe_ori;
+	const char *qseqs[MG_MAX_SEG];
+	mg_tbuf_t *b = s->buf[tid];
+	assert(s->n_seg[i] <= MG_MAX_SEG);
+	if (mg_dbg_flag & MG_DBG_QNAME)
+		fprintf(stderr, "QR\t%s\t%d\t%d\n", s->seq[off].name, tid, s->seq[off].l_seq);
+	for (j = 0; j < s->n_seg[i]; ++j) {
+		if (s->n_seg[i] == 2 && ((j == 0 && (pe_ori>>1&1)) || (j == 1 && (pe_ori&1))))
+			mg_revcomp_bseq(&s->seq[off + j]);
+		qlens[j] = s->seq[off + j].l_seq;
+		qseqs[j] = s->seq[off + j].seq;
+	}
+	if (s->p->opt->flag & MG_M_INDEPEND_SEG) {
+		for (j = 0; j < s->n_seg[i]; ++j)
+			mg_map_frag(s->p->gi, 1, &qlens[j], &qseqs[j], &s->gcs[off+j], b, s->p->opt, s->seq[off+j].name);
+	} else {
+		mg_map_frag(s->p->gi, s->n_seg[i], qlens, qseqs, &s->gcs[off], b, s->p->opt, s->seq[off].name);
+	}
+#if 0 // for paired-end reads
+	for (j = 0; j < s->n_seg[i]; ++j) // flip the query strand and coordinate to the original read strand
+		if (s->n_seg[i] == 2 && ((j == 0 && (pe_ori>>1&1)) || (j == 1 && (pe_ori&1)))) {
+			int k, t;
+			mg_revcomp_bseq(&s->seq[off + j]);
+			for (k = 0; k < s->n_reg[off + j]; ++k) {
+				mg_lchain_t *r = &s->reg[off + j][k];
+				t = r->qs;
+				r->qs = qlens[j] - r->qe;
+				r->qe = qlens[j] - t;
+				r->v ^= 1;
+			}
+		}
+#endif
+}
+
+static void *worker_pipeline(void *shared, int step, void *in)
+{
+	int i, j, k;
+    pipeline_t *p = (pipeline_t*)shared;
+    if (step == 0) { // step 0: read sequences
+		int with_qual = !(p->opt->flag & MG_M_NO_QUAL);
+		int with_comment = !!(p->opt->flag & MG_M_COPY_COMMENT);
+		int frag_mode = (p->n_fp > 1 || !!(p->opt->flag & MG_M_FRAG_MODE));
+        step_t *s;
+        s = (step_t*)calloc(1, sizeof(step_t));
+		if (p->n_fp > 1) s->seq = mg_bseq_read_frag(p->n_fp, p->fp, p->mini_batch_size, with_qual, with_comment, &s->n_seq);
+		else s->seq = mg_bseq_read(p->fp[0], p->mini_batch_size, with_qual, with_comment, frag_mode, &s->n_seq);
+		if (s->seq) {
+			s->p = p;
+			for (i = 0; i < s->n_seq; ++i)
+				mg_toupper(s->seq[i].l_seq, s->seq[i].seq);
+			for (i = 0; i < s->n_seq; ++i)
+				s->seq[i].rid = p->n_processed++;
+			s->buf = (mg_tbuf_t**)calloc(p->n_threads, sizeof(mg_tbuf_t*));
+			for (i = 0; i < p->n_threads; ++i)
+				s->buf[i] = mg_tbuf_init();
+			s->seg_off = (int*)calloc(2 * s->n_seq, sizeof(int));
+			s->n_seg = s->seg_off + s->n_seq; // n_seg, rep_len and frag_gap are allocated together with seg_off
+			KCALLOC(0, s->gcs, s->n_seq);
+			for (i = 1, j = 0; i <= s->n_seq; ++i)
+				if (i == s->n_seq || !frag_mode || !mg_qname_same(s->seq[i-1].name, s->seq[i].name)) {
+					s->n_seg[s->n_frag] = i - j;
+					s->seg_off[s->n_frag++] = j;
+					j = i;
+				}
+			return s;
+		} else free(s);
+    } else if (step == 1) { // step 1: map
+		kt_for(p->n_threads, worker_for, in, ((step_t*)in)->n_frag);
+		return in;
+    } else if (step == 2) { // step 2: output
+		void *km = 0;
+        step_t *s = (step_t*)in;
+		for (i = 0; i < p->n_threads; ++i) mg_tbuf_destroy(s->buf[i]);
+		free(s->buf);
+		if (!(mg_dbg_flag & MG_DBG_NO_KALLOC)) km = km_init();
+		for (k = 0; k < s->n_frag; ++k) {
+			int seg_st = s->seg_off[k], seg_en = s->seg_off[k] + s->n_seg[k];
+			if ((p->opt->flag & MG_M_FRAG_MODE) && (p->opt->flag & MG_M_FRAG_MERGE)) {
+				mg_bseq1_t *t = &s->seq[seg_st];
+				int32_t *qlens;
+				KMALLOC(km, qlens, seg_en - seg_st); // TODO: if this is an issue (quite unlikely), preallocate
+				for (i = seg_st; i < seg_en; ++i)
+					qlens[i - seg_st] = s->seq[i].l_seq;
+				if (p->opt->flag & MG_M_CAL_COV)
+					mg_cov_map(p->gi->g, s->gcs[seg_st], p->opt->min_cov_mapq, p->opt->min_cov_blen, p->c_seg, p->c_link, t->name);
+				else mg_write_gaf(&p->str, p->gi->g, s->gcs[seg_st], seg_en - seg_st, qlens, t->name, p->opt->flag, km);
+				kfree(km, qlens);
+				if (p->str.l) mg_err_fputs(p->str.s, stdout);
+			} else {
+				for (i = seg_st; i < seg_en; ++i) {
+					mg_bseq1_t *t = &s->seq[i];
+					if (p->opt->flag & MG_M_CAL_COV)
+						mg_cov_map(p->gi->g, s->gcs[i], p->opt->min_cov_mapq, p->opt->min_cov_blen, p->c_seg, p->c_link, t->name);
+					else mg_write_gaf(&p->str, p->gi->g, s->gcs[i], 1, &t->l_seq, t->name, p->opt->flag, km);
+					if (p->str.l) mg_err_fputs(p->str.s, stdout);
+				}
+			}
+			for (i = seg_st; i < seg_en; ++i) {
+				mg_gchain_free(s->gcs[i]);
+				free(s->seq[i].seq); free(s->seq[i].name);
+				if (s->seq[i].qual) free(s->seq[i].qual);
+				if (s->seq[i].comment) free(s->seq[i].comment);
+			}
+		}
+		free(s->gcs); free(s->seg_off); free(s->seq); // n_seg, rep_len and frag_gap were allocated with seg_off; no memory leak here
+		if (km) km_destroy(km);
+		if (mg_verbose >= 3)
+			fprintf(stderr, "[M::%s::%.3f*%.2f] mapped %d sequences\n", __func__, realtime() - mg_realtime0, cputime() / (realtime() - mg_realtime0), s->n_seq);
+		free(s);
+	}
+    return 0;
+}
+
+static mg_bseq_file_t **open_bseqs(int n, const char **fn)
+{
+	mg_bseq_file_t **fp;
+	int i, j;
+	fp = (mg_bseq_file_t**)calloc(n, sizeof(mg_bseq_file_t*));
+	for (i = 0; i < n; ++i) {
+		if ((fp[i] = mg_bseq_open(fn[i])) == 0) {
+			if (mg_verbose >= 1)
+				fprintf(stderr, "ERROR: failed to open file '%s'\n", fn[i]);
+			for (j = 0; j < i; ++j)
+				mg_bseq_close(fp[j]);
+			free(fp);
+			return 0;
+		}
+	}
+	return fp;
+}
+
+int mg_map_file_frag(const mg_idx_t *idx, int n_segs, const char **fn, const mg_mapopt_t *opt, int n_threads, double *c_seg, double *c_link)
+{
+	int i, pl_threads;
+	pipeline_t pl;
+	if (n_segs < 1) return -1;
+	memset(&pl, 0, sizeof(pipeline_t));
+	pl.n_fp = n_segs;
+	pl.fp = open_bseqs(pl.n_fp, fn);
+	if (pl.fp == 0) return -1;
+	pl.opt = opt, pl.gi = idx;
+	pl.n_threads = n_threads > 1? n_threads : 1;
+	pl.mini_batch_size = opt->mini_batch_size;
+	pl.c_seg = c_seg, pl.c_link = c_link;
+	pl_threads = n_threads == 1? 1 : (opt->flag&MG_M_2_IO_THREADS)? 3 : 2;
+	kt_pipeline(pl_threads, worker_pipeline, &pl, 3);
+
+	free(pl.str.s);
+	for (i = 0; i < pl.n_fp; ++i)
+		mg_bseq_close(pl.fp[i]);
+	free(pl.fp);
+	return 0;
+}
+
+int mg_map_files(gfa_t *g, int n_fn, const char **fn, const mg_idxopt_t *ipt, const mg_mapopt_t *opt0, int n_threads)
+{
+	mg_mapopt_t opt = *opt0;
+	mg_idx_t *gi;
+	int i, ret = 0;
+	double *cov_seg = 0, *cov_link = 0;
+	if ((gi = mg_index(g, ipt, n_threads, &opt)) == 0) return -1;
+	if (opt.flag & MG_M_CAL_COV) {
+		KCALLOC(0, cov_seg,  g->n_seg);
+		KCALLOC(0, cov_link, g->n_arc);
+	}
+	if (opt.flag & MG_M_FRAG_MODE) {
+		ret = mg_map_file_frag(gi, n_fn, fn, &opt, n_threads, cov_seg, cov_link);
+	} else {
+		for (i = 0; i < n_fn; ++i) {
+			ret = mg_map_file_frag(gi, 1, &fn[i], &opt, n_threads, cov_seg, cov_link);
+			if (ret != 0) break;
+		}
+	}
+	if (opt.flag & MG_M_CAL_COV) {
+		gfa_aux_update_cv(g, "dc", cov_seg, cov_link);
+		free(cov_seg); free(cov_link);
+	}
+	mg_idx_destroy(gi);
+	return ret;
+}

data/ext/minigraph/index.c

@@ -0,0 +1,230 @@
+#include <assert.h>
+#include "mgpriv.h"
+#include "khashl.h"
+#include "kthread.h"
+#include "kvec-km.h"
+#include "sys.h"
+
+#define idx_hash(a) ((a)>>1)
+#define idx_eq(a, b) ((a)>>1 == (b)>>1)
+KHASHL_MAP_INIT(KH_LOCAL, idxhash_t, mg_hidx, uint64_t, uint64_t, idx_hash, idx_eq)
+
+typedef struct mg_idx_bucket_s {
+	mg128_v a;   // (minimizer, position) array
+	int32_t n;   // size of the _p_ array
+	uint64_t *p; // position array for minimizers appearing >1 times
+	void *h;     // hash table indexing _p_ and minimizers appearing once
+} mg_idx_bucket_t;
+
+mg_idx_t *mg_idx_init(int k, int w, int b)
+{
+	mg_idx_t *gi;
+	if (k*2 < b) b = k * 2;
+	if (w < 1) w = 1;
+	KCALLOC(0, gi, 1);
+	gi->w = w, gi->k = k, gi->b = b;
+	KCALLOC(0, gi->B, 1<<b);
+	return gi;
+}
+
+void mg_idx_destroy(mg_idx_t *gi)
+{
+	uint32_t i;
+	if (gi == 0) return;
+	if (gi->B) {
+		for (i = 0; i < 1U<<gi->b; ++i) {
+			free(gi->B[i].p);
+			free(gi->B[i].a.a);
+			mg_hidx_destroy((idxhash_t*)gi->B[i].h);
+		}
+		free(gi->B);
+	}
+	gfa_edseq_destroy(gi->n_seg, gi->es);
+	free(gi);
+}
+
+/****************
+ * Index access *
+ ****************/
+
+const uint64_t *mg_idx_hget(const void *h_, const uint64_t *q, int suflen, uint64_t minier, int *n)
+{
+	khint_t k;
+	const idxhash_t *h = (const idxhash_t*)h_;
+	*n = 0;
+	if (h == 0) return 0;
+	k = mg_hidx_get(h, minier>>suflen<<1);
+	if (k == kh_end(h)) return 0;
+	if (kh_key(h, k)&1) { // special casing when there is only one k-mer
+		*n = 1;
+		return &kh_val(h, k);
+	} else {
+		*n = (uint32_t)kh_val(h, k);
+		return &q[kh_val(h, k)>>32];
+	}
+}
+
+const uint64_t *mg_idx_get(const mg_idx_t *gi, uint64_t minier, int *n)
+{
+	int mask = (1<<gi->b) - 1;
+	mg_idx_bucket_t *b = &gi->B[minier&mask];
+	return mg_idx_hget(b->h, b->p, gi->b, minier, n);
+}
+
+void mg_idx_cal_quantile(const mg_idx_t *gi, int32_t m, float f[], int32_t q[])
+{
+	int32_t i;
+	uint64_t n = 0;
+	khint_t *a, k;
+	for (i = 0; i < 1<<gi->b; ++i)
+		if (gi->B[i].h) n += kh_size((idxhash_t*)gi->B[i].h);
+	a = (uint32_t*)malloc(n * 4);
+	for (i = 0, n = 0; i < 1<<gi->b; ++i) {
+		idxhash_t *h = (idxhash_t*)gi->B[i].h;
+		if (h == 0) continue;
+		for (k = 0; k < kh_end(h); ++k) {
+			if (!kh_exist(h, k)) continue;
+			a[n++] = kh_key(h, k)&1? 1 : (uint32_t)kh_val(h, k);
+		}
+	}
+	for (i = 0; i < m; ++i)
+		q[i] = ks_ksmall_uint32_t(n, a, (size_t)((1.0 - (double)f[i]) * n));
+	free(a);
+}
+
+/***************
+ * Index build *
+ ***************/
+
+static void mg_idx_add(mg_idx_t *gi, int n, const mg128_t *a)
+{
+	int i, mask = (1<<gi->b) - 1;
+	for (i = 0; i < n; ++i) {
+		mg128_v *p = &gi->B[a[i].x>>8&mask].a;
+		kv_push(mg128_t, 0, *p, a[i]);
+	}
+}
+
+void mg_idx_hfree(void *h_)
+{
+	idxhash_t *h = (idxhash_t*)h_;
+	if (h == 0) return;
+	mg_hidx_destroy(h);
+}
+
+void *mg_idx_a2h(void *km, int32_t n_a, mg128_t *a, int suflen, uint64_t **q_, int32_t *n_)
+{
+	int32_t N, n, n_keys;
+	int32_t j, start_a, start_q;
+	idxhash_t *h;
+	uint64_t *q;
+
+	*q_ = 0, *n_ = 0;
+	if (n_a == 0) return 0;
+
+	// sort by minimizer
+	radix_sort_128x(a, a + n_a);
+
+	// count and preallocate
+	for (j = 1, n = 1, n_keys = 0, N = 0; j <= n_a; ++j) {
+		if (j == n_a || a[j].x>>8 != a[j-1].x>>8) {
+			++n_keys;
+			if (n > 1) N += n;
+			n = 1;
+		} else ++n;
+	}
+	h = mg_hidx_init2(km);
+	mg_hidx_resize(h, n_keys);
+	KCALLOC(km, q, N);
+	*q_ = q, *n_ = N;
+
+	// create the hash table
+	for (j = 1, n = 1, start_a = start_q = 0; j <= n_a; ++j) {
+		if (j == n_a || a[j].x>>8 != a[j-1].x>>8) {
+			khint_t itr;
+			int absent;
+			mg128_t *p = &a[j-1];
+			itr = mg_hidx_put(h, p->x>>8>>suflen<<1, &absent);
+			assert(absent && j == start_a + n);
+			if (n == 1) {
+				kh_key(h, itr) |= 1;
+				kh_val(h, itr) = p->y;
+			} else {
+				int k;
+				for (k = 0; k < n; ++k)
+					q[start_q + k] = a[start_a + k].y;
+				radix_sort_gfa64(&q[start_q], &q[start_q + n]); // sort by position; needed as in-place radix_sort_128x() is not stable
+				kh_val(h, itr) = (uint64_t)start_q<<32 | n;
+				start_q += n;
+			}
+			start_a = j, n = 1;
+		} else ++n;
+	}
+	assert(N == start_q);
+	return h;
+}
+
+static void worker_post(void *g, long i, int tid)
+{
+	mg_idx_t *gi = (mg_idx_t*)g;
+	mg_idx_bucket_t *b = &gi->B[i];
+	if (b->a.n == 0) return;
+	b->h = (idxhash_t*)mg_idx_a2h(0, b->a.n, b->a.a, gi->b, &b->p, &b->n);
+	kfree(0, b->a.a);
+	b->a.n = b->a.m = 0, b->a.a = 0;
+}
+
+int mg_gfa_overlap(const gfa_t *g)
+{
+	int64_t i;
+	for (i = 0; i < g->n_arc; ++i) // non-zero overlap
+		if (g->arc[i].ov != 0 || g->arc[i].ow != 0)
+			return 1;
+	return 0;
+}
+
+mg_idx_t *mg_index_core(gfa_t *g, int k, int w, int b, int n_threads)
+{
+	mg_idx_t *gi;
+	mg128_v a = {0,0,0};
+	int i;
+
+	if (mg_gfa_overlap(g)) {
+		if (mg_verbose >= 1)
+			fprintf(stderr, "[E::%s] minigraph doesn't work with graphs containing overlapping segments\n", __func__);
+		return 0;
+	}
+	gi = mg_idx_init(k, w, b);
+	gi->g = g;
+
+	for (i = 0; i < g->n_seg; ++i) {
+		gfa_seg_t *s = &g->seg[i];
+		a.n = 0;
+		mg_sketch(0, s->seq, s->len, w, k, i, &a); // TODO: this can be parallelized
+		mg_idx_add(gi, a.n, a.a);
+	}
+	free(a.a);
+	kt_for(n_threads, worker_post, gi, 1<<gi->b);
+	return gi;
+}
+
+mg_idx_t *mg_index(gfa_t *g, const mg_idxopt_t *io, int n_threads, mg_mapopt_t *mo)
+{
+	int32_t i, j;
+	mg_idx_t *gi;
+	for (i = 0; i < g->n_seg; ++i) { // uppercase
+		gfa_seg_t *s = &g->seg[i];
+		for (j = 0; j < s->len; ++j)
+			if (s->seq[j] >= 'a' && s->seq[j] <= 'z')
+				s->seq[j] -= 32;
+	}
+	gi = mg_index_core(g, io->k, io->w, io->bucket_bits, n_threads);
+	if (gi == 0) return 0;
+	gi->es = gfa_edseq_init(gi->g);
+	gi->n_seg = g->n_seg;
+	if (mg_verbose >= 3)
+		fprintf(stderr, "[M::%s::%.3f*%.2f] indexed the graph\n", __func__,
+				realtime() - mg_realtime0, cputime() / (realtime() - mg_realtime0));
+	if (mo) mg_opt_update(gi, mo, 0);
+	return gi;
+}

data/ext/minigraph/kalloc.c

@@ -0,0 +1,224 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "kalloc.h"
+
+/* In kalloc, a *core* is a large chunk of contiguous memory. Each core is
+ * associated with a master header, which keeps the size of the current core
+ * and the pointer to next core. Kalloc allocates small *blocks* of memory from
+ * the cores and organizes free memory blocks in a circular single-linked list.
+ *
+ * In the following diagram, "@" stands for the header of a free block (of type
+ * header_t), "#" for the header of an allocated block (of type size_t), "-"
+ * for free memory, and "+" for allocated memory.
+ *
+ * master        This region is core 1.          master           This region is core 2.
+ *      |                                             |
+ *      *@-------#++++++#++++++++++++@--------        *@----------#++++++++++++#+++++++@------------
+ *       |                           |                 |                               |
+ *       p=p->ptr->ptr->ptr->ptr     p->ptr            p->ptr->ptr                     p->ptr->ptr->ptr
+ */
+typedef struct header_t {
+	size_t size;
+	struct header_t *ptr;
+} header_t;
+
+typedef struct {
+	void *par;
+	size_t min_core_size;
+	header_t base, *loop_head, *core_head; /* base is a zero-sized block always kept in the loop */
+} kmem_t;
+
+static void panic(const char *s)
+{
+	fprintf(stderr, "%s\n", s);
+	abort();
+}
+
+void *km_init2(void *km_par, size_t min_core_size)
+{
+	kmem_t *km;
+	km = (kmem_t*)kcalloc(km_par, 1, sizeof(kmem_t));
+	km->par = km_par;
+	if (km_par) km->min_core_size = min_core_size > 0? min_core_size : ((kmem_t*)km_par)->min_core_size - 2;
+	else km->min_core_size = min_core_size > 0? min_core_size : 0x80000;
+	return (void*)km;
+}
+
+void *km_init(void) { return km_init2(0, 0); }
+
+void km_destroy(void *_km)
+{
+	kmem_t *km = (kmem_t*)_km;
+	void *km_par;
+	header_t *p, *q;
+	if (km == NULL) return;
+	km_par = km->par;
+	for (p = km->core_head; p != NULL;) {
+		q = p->ptr;
+		kfree(km_par, p);
+		p = q;
+	}
+	kfree(km_par, km);
+}
+
+static header_t *morecore(kmem_t *km, size_t nu)
+{
+	header_t *q;
+	size_t bytes, *p;
+	nu = (nu + 1 + (km->min_core_size - 1)) / km->min_core_size * km->min_core_size; /* the first +1 for core header */
+	bytes = nu * sizeof(header_t);
+	q = (header_t*)kmalloc(km->par, bytes);
+	if (!q) panic("[morecore] insufficient memory");
+	q->ptr = km->core_head, q->size = nu, km->core_head = q;
+	p = (size_t*)(q + 1);
+	*p = nu - 1; /* the size of the free block; -1 because the first unit is used for the core header */
+	kfree(km, p + 1); /* initialize the new "core"; NB: the core header is not looped. */
+	return km->loop_head;
+}
+
+void kfree(void *_km, void *ap) /* kfree() also adds a new core to the circular list */
+{
+	header_t *p, *q;
+	kmem_t *km = (kmem_t*)_km;
+	
+	if (!ap) return;
+	if (km == NULL) {
+		free(ap);
+		return;
+	}
+	p = (header_t*)((size_t*)ap - 1);
+	p->size = *((size_t*)ap - 1);
+	/* Find the pointer that points to the block to be freed. The following loop can stop on two conditions:
+	 *
+	 * a) "p>q && p<q->ptr": @------#++++++++#+++++++@-------    @---------------#+++++++@-------
+	 *    (can also be in    |      |                |        -> |                       |
+	 *     two cores)        q      p           q->ptr           q                  q->ptr
+	 *
+	 *                       @--------    #+++++++++@--------    @--------    @------------------
+	 *                       |            |         |         -> |            |
+	 *                       q            p    q->ptr            q       q->ptr
+	 *
+	 * b) "q>=q->ptr && (p>q || p<q->ptr)":  @-------#+++++   @--------#+++++++     @-------#+++++   @----------------
+	 *                                       |                |        |         -> |                |
+	 *                                  q->ptr                q        p       q->ptr                q
+	 *
+	 *                                       #+++++++@-----   #++++++++@-------     @-------------   #++++++++@-------
+	 *                                       |       |                 |         -> |                         |
+	 *                                       p  q->ptr                 q       q->ptr                         q
+	 */
+	for (q = km->loop_head; !(p > q && p < q->ptr); q = q->ptr)
+		if (q >= q->ptr && (p > q || p < q->ptr)) break;
+	if (p + p->size == q->ptr) { /* two adjacent blocks, merge p and q->ptr (the 2nd and 4th cases) */
+		p->size += q->ptr->size;
+		p->ptr = q->ptr->ptr;
+	} else if (p + p->size > q->ptr && q->ptr >= p) {
+		panic("[kfree] The end of the allocated block enters a free block.");
+	} else p->ptr = q->ptr; /* backup q->ptr */
+
+	if (q + q->size == p) { /* two adjacent blocks, merge q and p (the other two cases) */
+		q->size += p->size;
+		q->ptr = p->ptr;
+		km->loop_head = q;
+	} else if (q + q->size > p && p >= q) {
+		panic("[kfree] The end of a free block enters the allocated block.");
+	} else km->loop_head = p, q->ptr = p; /* in two cores, cannot be merged; create a new block in the list */
+}
+
+void *kmalloc(void *_km, size_t n_bytes)
+{
+	kmem_t *km = (kmem_t*)_km;
+	size_t n_units;
+	header_t *p, *q;
+
+	if (n_bytes == 0) return 0;
+	if (km == NULL) return malloc(n_bytes);
+	n_units = (n_bytes + sizeof(size_t) + sizeof(header_t) - 1) / sizeof(header_t); /* header+n_bytes requires at least this number of units */
+
+	if (!(q = km->loop_head)) /* the first time when kmalloc() is called, intialize it */
+		q = km->loop_head = km->base.ptr = &km->base;
+	for (p = q->ptr;; q = p, p = p->ptr) { /* search for a suitable block */
+		if (p->size >= n_units) { /* p->size if the size of current block. This line means the current block is large enough. */
+			if (p->size == n_units) q->ptr = p->ptr; /* no need to split the block */
+			else { /* split the block. NB: memory is allocated at the end of the block! */
+				p->size -= n_units; /* reduce the size of the free block */
+				p += p->size; /* p points to the allocated block */
+				*(size_t*)p = n_units; /* set the size */
+			}
+			km->loop_head = q; /* set the end of chain */
+			return (size_t*)p + 1;
+		}
+		if (p == km->loop_head) { /* then ask for more "cores" */
+			if ((p = morecore(km, n_units)) == 0) return 0;
+		}
+	}
+}
+
+void *kcalloc(void *_km, size_t count, size_t size)
+{
+	kmem_t *km = (kmem_t*)_km;
+	void *p;
+	if (size == 0 || count == 0) return 0;
+	if (km == NULL) return calloc(count, size);
+	p = kmalloc(km, count * size);
+	memset(p, 0, count * size);
+	return p;
+}
+
+void *krealloc(void *_km, void *ap, size_t n_bytes) // TODO: this can be made more efficient in principle
+{
+	kmem_t *km = (kmem_t*)_km;
+	size_t cap, *p, *q;
+
+	if (n_bytes == 0) {
+		kfree(km, ap); return 0;
+	}
+	if (km == NULL) return realloc(ap, n_bytes);
+	if (ap == NULL) return kmalloc(km, n_bytes);
+	p = (size_t*)ap - 1;
+	cap = (*p) * sizeof(header_t) - sizeof(size_t);
+	if (cap >= n_bytes) return ap; /* TODO: this prevents shrinking */
+	q = (size_t*)kmalloc(km, n_bytes);
+	memcpy(q, ap, cap);
+	kfree(km, ap);
+	return q;
+}
+
+void *krelocate(void *km, void *ap, size_t n_bytes)
+{
+	void *p;
+	if (km == 0 || ap == 0) return ap;
+	p = kmalloc(km, n_bytes);
+	memcpy(p, ap, n_bytes);
+	kfree(km, ap);
+	return p;
+}
+
+void km_stat(const void *_km, km_stat_t *s)
+{
+	kmem_t *km = (kmem_t*)_km;
+	header_t *p;
+	memset(s, 0, sizeof(km_stat_t));
+	if (km == NULL || km->loop_head == NULL) return;
+	for (p = km->loop_head;; p = p->ptr) {
+		s->available += p->size * sizeof(header_t);
+		if (p->size != 0) ++s->n_blocks; /* &kmem_t::base is always one of the cores. It is zero-sized. */
+		if (p->ptr > p && p + p->size > p->ptr)
+			panic("[km_stat] The end of a free block enters another free block.");
+		if (p->ptr == km->loop_head) break;
+	}
+	for (p = km->core_head; p != NULL; p = p->ptr) {
+		size_t size = p->size * sizeof(header_t);
+		++s->n_cores;
+		s->capacity += size;
+		s->largest = s->largest > size? s->largest : size;
+	}
+}
+
+void km_stat_print(const void *km)
+{
+	km_stat_t st;
+	km_stat(km, &st);
+	fprintf(stderr, "[km_stat] cap=%ld, avail=%ld, largest=%ld, n_core=%ld, n_block=%ld\n",
+			st.capacity, st.available, st.largest, st.n_blocks, st.n_cores);
+}