rugged 0.19.0 → 0.28.0

data/vendor/libgit2/src/pack-objects.c

@@ -7,13 +7,16 @@
 
 #include "pack-objects.h"
 
-#include "compress.h"
+#include "zstream.h"
 #include "delta.h"
 #include "iterator.h"
 #include "netops.h"
 #include "pack.h"
 #include "thread-utils.h"
 #include "tree.h"
+#include "util.h"
+#include "revwalk.h"
+#include "commit_list.h"
 
 #include "git2/pack.h"
 #include "git2/commit.h"
@@ -25,7 +28,7 @@ struct unpacked {
 	git_pobject *object;
 	void *data;
 	struct git_delta_index *index;
-	unsigned int depth;
+	size_t depth;
 };
 
 struct tree_walk_context {
@@ -34,10 +37,16 @@ struct tree_walk_context {
 };
 
 struct pack_write_context {
-	git_indexer_stream *indexer;
+	git_indexer *indexer;
 	git_transfer_progress *stats;
 };
 
+struct walk_object {
+	git_oid id;
+	unsigned int uninteresting:1,
+		seen:1;
+};
+
 #ifdef GIT_THREADS
 
 #define GIT_PACKBUILDER__MUTEX_OP(pb, mtx, op) do { \
@@ -57,6 +66,12 @@ struct pack_write_context {
 #define git_packbuilder__progress_lock(pb) GIT_PACKBUILDER__MUTEX_OP(pb, progress_mutex, lock)
 #define git_packbuilder__progress_unlock(pb) GIT_PACKBUILDER__MUTEX_OP(pb, progress_mutex, unlock)
 
+/* The minimal interval between progress updates (in seconds). */
+#define MIN_PROGRESS_UPDATE_INTERVAL 0.5
+
+/* Size of the buffer to feed to zlib */
+#define COMPRESS_BUFLEN (1024 * 1024)
+
 static unsigned name_hash(const char *name)
 {
 	unsigned c, hash = 0;
@@ -80,17 +95,26 @@ static unsigned name_hash(const char *name)
 static int packbuilder_config(git_packbuilder *pb)
 {
 	git_config *config;
-	int ret;
+	int ret = 0;
 	int64_t val;
 
-	if (git_repository_config__weakptr(&config, pb->repo) < 0)
-		return -1;
+	if ((ret = git_repository_config_snapshot(&config, pb->repo)) < 0)
+		return ret;
 
 #define config_get(KEY,DST,DFLT) do { \
 	ret = git_config_get_int64(&val, config, KEY); \
-	if (!ret) (DST) = val; \
-	else if (ret == GIT_ENOTFOUND) (DST) = (DFLT); \
-	else if (ret < 0) return -1; } while (0)
+	if (!ret) { \
+		if (!git__is_sizet(val)) { \
+			git_error_set(GIT_ERROR_CONFIG, \
+				"configuration value '%s' is too large", KEY); \
+			ret = -1; \
+			goto out; \
+		} \
+		(DST) = (size_t)val; \
+	} else if (ret == GIT_ENOTFOUND) { \
+	    (DST) = (DFLT); \
+	    ret = 0; \
+	} else if (ret < 0) goto out; } while (0)
 
 	config_get("pack.deltaCacheSize", pb->max_delta_cache_size,
 		   GIT_PACK_DELTA_CACHE_SIZE);
@@ -102,7 +126,10 @@ static int packbuilder_config(git_packbuilder *pb)
 
 #undef config_get
 
-	return 0;
+out:
+	git_config_free(config);
+
+	return ret;
 }
 
 int git_packbuilder_new(git_packbuilder **out, git_repository *repo)
@@ -112,17 +139,23 @@ int git_packbuilder_new(git_packbuilder **out, git_repository *repo)
 	*out = NULL;
 
 	pb = git__calloc(1, sizeof(*pb));
-	GITERR_CHECK_ALLOC(pb);
+	GIT_ERROR_CHECK_ALLOC(pb);
 
 	pb->object_ix = git_oidmap_alloc();
-
 	if (!pb->object_ix)
 		goto on_error;
 
+	pb->walk_objects = git_oidmap_alloc();
+	if (!pb->walk_objects)
+		goto on_error;
+
+	git_pool_init(&pb->object_pool, sizeof(struct walk_object));
+
 	pb->repo = repo;
 	pb->nr_threads = 1; /* do not spawn any thread by default */
 
 	if (git_hash_ctx_init(&pb->ctx) < 0 ||
+		git_zstream_init(&pb->zstream, GIT_ZSTREAM_DEFLATE) < 0 ||
 		git_repository_odb(&pb->odb, repo) < 0 ||
 		packbuilder_config(pb) < 0)
 		goto on_error;
@@ -133,7 +166,7 @@ int git_packbuilder_new(git_packbuilder **out, git_repository *repo)
 		git_mutex_init(&pb->progress_mutex) ||
 		git_cond_init(&pb->progress_cond))
 	{
-		giterr_set(GITERR_OS, "Failed to initialize packbuilder mutex");
+		git_error_set(GIT_ERROR_OS, "failed to initialize packbuilder mutex");
 		goto on_error;
 	}
 
@@ -164,14 +197,13 @@ unsigned int git_packbuilder_set_threads(git_packbuilder *pb, unsigned int n)
 static void rehash(git_packbuilder *pb)
 {
 	git_pobject *po;
-	khiter_t pos;
-	unsigned int i;
+	size_t pos, i;
 	int ret;
 
-	kh_clear(oid, pb->object_ix);
+	git_oidmap_clear(pb->object_ix);
 	for (i = 0, po = pb->object_list; i < pb->nr_objects; i++, po++) {
-		pos = kh_put(oid, pb->object_ix, &po->id, &ret);
-		kh_value(pb->object_ix, pos) = po;
+		pos = git_oidmap_put(pb->object_ix, &po->id, &ret);
+		git_oidmap_set_value_at(pb->object_ix, pos, po);
 	}
 }
 
@@ -179,82 +211,78 @@ int git_packbuilder_insert(git_packbuilder *pb, const git_oid *oid,
 			   const char *name)
 {
 	git_pobject *po;
-	khiter_t pos;
+	size_t newsize, pos;
 	int ret;
 
 	assert(pb && oid);
 
 	/* If the object already exists in the hash table, then we don't
 	 * have any work to do */
-	pos = kh_get(oid, pb->object_ix, oid);
-	if (pos != kh_end(pb->object_ix))
+	if (git_oidmap_exists(pb->object_ix, oid))
 		return 0;
 
 	if (pb->nr_objects >= pb->nr_alloc) {
-		pb->nr_alloc = (pb->nr_alloc + 1024) * 3 / 2;
-		pb->object_list = git__realloc(pb->object_list,
-					       pb->nr_alloc * sizeof(*po));
-		GITERR_CHECK_ALLOC(pb->object_list);
+		GIT_ERROR_CHECK_ALLOC_ADD(&newsize, pb->nr_alloc, 1024);
+		GIT_ERROR_CHECK_ALLOC_MULTIPLY(&newsize, newsize, 3 / 2);
+
+		if (!git__is_uint32(newsize)) {
+			git_error_set(GIT_ERROR_NOMEMORY, "packfile too large to fit in memory.");
+			return -1;
+		}
+
+		pb->nr_alloc = newsize;
+
+		pb->object_list = git__reallocarray(pb->object_list,
+			pb->nr_alloc, sizeof(*po));
+		GIT_ERROR_CHECK_ALLOC(pb->object_list);
 		rehash(pb);
 	}
 
 	po = pb->object_list + pb->nr_objects;
 	memset(po, 0x0, sizeof(*po));
 
-	if (git_odb_read_header(&po->size, &po->type, pb->odb, oid) < 0)
-		return -1;
+	if ((ret = git_odb_read_header(&po->size, &po->type, pb->odb, oid)) < 0)
+		return ret;
 
 	pb->nr_objects++;
 	git_oid_cpy(&po->id, oid);
 	po->hash = name_hash(name);
 
-	pos = kh_put(oid, pb->object_ix, &po->id, &ret);
+	pos = git_oidmap_put(pb->object_ix, &po->id, &ret);
+	if (ret < 0) {
+		git_error_set_oom();
+		return ret;
+	}
 	assert(ret != 0);
-	kh_value(pb->object_ix, pos) = po;
+	git_oidmap_set_value_at(pb->object_ix, pos, po);
 
 	pb->done = false;
-	return 0;
-}
-
-/*
- * The per-object header is a pretty dense thing, which is
- *  - first byte: low four bits are "size",
- *    then three bits of "type",
- *    with the high bit being "size continues".
- *  - each byte afterwards: low seven bits are size continuation,
- *    with the high bit being "size continues"
- */
-static int gen_pack_object_header(
-		unsigned char *hdr,
-		unsigned long size,
-		git_otype type)
-{
-	unsigned char *hdr_base;
-	unsigned char c;
 
-	assert(type >= GIT_OBJ_COMMIT && type <= GIT_OBJ_REF_DELTA);
+	if (pb->progress_cb) {
+		double current_time = git__timer();
+		double elapsed = current_time - pb->last_progress_report_time;
 
-	/* TODO: add support for chunked objects; see git.git 6c0d19b1 */
+		if (elapsed >= MIN_PROGRESS_UPDATE_INTERVAL) {
+			pb->last_progress_report_time = current_time;
 
-	c = (unsigned char)((type << 4) | (size & 15));
-	size >>= 4;
-	hdr_base = hdr;
+			ret = pb->progress_cb(
+				GIT_PACKBUILDER_ADDING_OBJECTS,
+				pb->nr_objects, 0, pb->progress_cb_payload);
 
-	while (size) {
-		*hdr++ = c | 0x80;
-		c = size & 0x7f;
-		size >>= 7;
+			if (ret)
+				return git_error_set_after_callback(ret);
+		}
 	}
-	*hdr++ = c;
 
-	return (int)(hdr - hdr_base);
+	return 0;
 }
 
 static int get_delta(void **out, git_odb *odb, git_pobject *po)
 {
 	git_odb_object *src = NULL, *trg = NULL;
-	unsigned long delta_size;
+	size_t delta_size;
 	void *delta_buf;
+	int error;
 
 	*out = NULL;
 
@@ -262,13 +290,16 @@ static int get_delta(void **out, git_odb *odb, git_pobject *po)
 	    git_odb_read(&trg, odb, &po->id) < 0)
 		goto on_error;
 
-	delta_buf = git_delta(
-		git_odb_object_data(src), (unsigned long)git_odb_object_size(src),
-		git_odb_object_data(trg), (unsigned long)git_odb_object_size(trg),
-		&delta_size, 0);
+	error = git_delta(&delta_buf, &delta_size,
+		git_odb_object_data(src), git_odb_object_size(src),
+		git_odb_object_data(trg), git_odb_object_size(trg),
+		0);
+
+	if (error < 0 && error != GIT_EBUFS)
+		goto on_error;
 
-	if (!delta_buf || delta_size != po->delta_size) {
-		giterr_set(GITERR_INVALID, "Delta size changed");
+	if (error == GIT_EBUFS || delta_size != po->delta_size) {
+		git_error_set(GIT_ERROR_INVALID, "delta size changed");
 		goto on_error;
 	}
 
@@ -284,76 +315,96 @@ on_error:
 	return -1;
 }
 
-static int write_object(git_buf *buf, git_packbuilder *pb, git_pobject *po)
+static int write_object(
+	git_packbuilder *pb,
+	git_pobject *po,
+	int (*write_cb)(void *buf, size_t size, void *cb_data),
+	void *cb_data)
 {
 	git_odb_object *obj = NULL;
-	git_buf zbuf = GIT_BUF_INIT;
-	git_otype type;
-	unsigned char hdr[10];
-	unsigned int hdr_len;
-	unsigned long size;
-	void *data;
+	git_object_t type;
+	unsigned char hdr[10], *zbuf = NULL;
+	void *data = NULL;
+	size_t hdr_len, zbuf_len = COMPRESS_BUFLEN, data_len;
+	int error;
 
+	/*
+	 * If we have a delta base, let's use the delta to save space.
+	 * Otherwise load the whole object. 'data' ends up pointing to
+	 * whatever data we want to put into the packfile.
+	 */
 	if (po->delta) {
 		if (po->delta_data)
 			data = po->delta_data;
-		else if (get_delta(&data, pb->odb, po) < 0)
-				goto on_error;
-		size = po->delta_size;
-		type = GIT_OBJ_REF_DELTA;
+		else if ((error = get_delta(&data, pb->odb, po)) < 0)
+				goto done;
+
+		data_len = po->delta_size;
+		type = GIT_OBJECT_REF_DELTA;
 	} else {
-		if (git_odb_read(&obj, pb->odb, &po->id))
-			goto on_error;
+		if ((error = git_odb_read(&obj, pb->odb, &po->id)) < 0)
+			goto done;
 
 		data = (void *)git_odb_object_data(obj);
-		size = (unsigned long)git_odb_object_size(obj);
+		data_len = git_odb_object_size(obj);
 		type = git_odb_object_type(obj);
 	}
 
 	/* Write header */
-	hdr_len = gen_pack_object_header(hdr, size, type);
+	hdr_len = git_packfile__object_header(hdr, data_len, type);
 
-	if (git_buf_put(buf, (char *)hdr, hdr_len) < 0)
-		goto on_error;
-
-	if (git_hash_update(&pb->ctx, hdr, hdr_len) < 0)
-		goto on_error;
+	if ((error = write_cb(hdr, hdr_len, cb_data)) < 0 ||
+		(error = git_hash_update(&pb->ctx, hdr, hdr_len)) < 0)
+		goto done;
 
-	if (type == GIT_OBJ_REF_DELTA) {
-		if (git_buf_put(buf, (char *)po->delta->id.id, GIT_OID_RAWSZ) < 0 ||
-			git_hash_update(&pb->ctx, po->delta->id.id, GIT_OID_RAWSZ) < 0)
-			goto on_error;
+	if (type == GIT_OBJECT_REF_DELTA) {
+		if ((error = write_cb(po->delta->id.id, GIT_OID_RAWSZ, cb_data)) < 0 ||
+			(error = git_hash_update(&pb->ctx, po->delta->id.id, GIT_OID_RAWSZ)) < 0)
+			goto done;
 	}
 
 	/* Write data */
-	if (po->z_delta_size)
-		size = po->z_delta_size;
-	else if (git__compress(&zbuf, data, size) < 0)
-		goto on_error;
-	else {
-		if (po->delta)
-			git__free(data);
-		data = zbuf.ptr;
-		size = (unsigned long)zbuf.size;
-	}
+	if (po->z_delta_size) {
+		data_len = po->z_delta_size;
 
-	if (git_buf_put(buf, data, size) < 0 ||
-		git_hash_update(&pb->ctx, data, size) < 0)
-		goto on_error;
+		if ((error = write_cb(data, data_len, cb_data)) < 0 ||
+			(error = git_hash_update(&pb->ctx, data, data_len)) < 0)
+			goto done;
+	} else {
+		zbuf = git__malloc(zbuf_len);
+		GIT_ERROR_CHECK_ALLOC(zbuf);
 
-	if (po->delta_data)
-		git__free(po->delta_data);
+		git_zstream_reset(&pb->zstream);
+		git_zstream_set_input(&pb->zstream, data, data_len);
 
-	git_odb_object_free(obj);
-	git_buf_free(&zbuf);
+		while (!git_zstream_done(&pb->zstream)) {
+			if ((error = git_zstream_get_output(zbuf, &zbuf_len, &pb->zstream)) < 0 ||
+				(error = write_cb(zbuf, zbuf_len, cb_data)) < 0 ||
+				(error = git_hash_update(&pb->ctx, zbuf, zbuf_len)) < 0)
+				goto done;
+
+			zbuf_len = COMPRESS_BUFLEN; /* reuse buffer */
+		}
+	}
+
+	/*
+	 * If po->delta is true, data is a delta and it is our
+	 * responsibility to free it (otherwise it's a git_object's
+	 * data). We set po->delta_data to NULL in case we got the
+	 * data from there instead of get_delta(). If we didn't,
+	 * there's no harm.
+	 */
+	if (po->delta) {
+		git__free(data);
+		po->delta_data = NULL;
+	}
 
 	pb->nr_written++;
-	return 0;
 
-on_error:
+done:
+	git__free(zbuf);
 	git_odb_object_free(obj);
-	git_buf_free(&zbuf);
-	return -1;
+	return error;
 }
 
 enum write_one_status {
@@ -363,9 +414,15 @@ enum write_one_status {
 	WRITE_ONE_RECURSIVE = 2 /* already scheduled to be written */
 };
 
-static int write_one(git_buf *buf, git_packbuilder *pb, git_pobject *po,
-		     enum write_one_status *status)
+static int write_one(
+	enum write_one_status *status,
+	git_packbuilder *pb,
+	git_pobject *po,
+	int (*write_cb)(void *buf, size_t size, void *cb_data),
+	void *cb_data)
 {
+	int error;
+
 	if (po->recursing) {
 		*status = WRITE_ONE_RECURSIVE;
 		return 0;
@@ -376,25 +433,24 @@ static int write_one(git_buf *buf, git_packbuilder *pb, git_pobject *po,
 
 	if (po->delta) {
 		po->recursing = 1;
-		if (write_one(buf, pb, po->delta, status) < 0)
-			return -1;
-		switch (*status) {
-		case WRITE_ONE_RECURSIVE:
-			/* we cannot depend on this one */
+
+		if ((error = write_one(status, pb, po->delta, write_cb, cb_data)) < 0)
+			return error;
+
+		/* we cannot depend on this one */
+		if (*status == WRITE_ONE_RECURSIVE)
 			po->delta = NULL;
-			break;
-		default:
-			break;
-		}
 	}
 
+	*status = WRITE_ONE_WRITTEN;
 	po->written = 1;
 	po->recursing = 0;
-	return write_object(buf, pb, po);
+
+	return write_object(pb, po, write_cb, cb_data);
 }
 
-GIT_INLINE(void) add_to_write_order(git_pobject **wo, unsigned int *endp,
-				    git_pobject *po)
+GIT_INLINE(void) add_to_write_order(git_pobject **wo, size_t *endp,
+	git_pobject *po)
 {
 	if (po->filled)
 		return;
@@ -402,8 +458,8 @@ GIT_INLINE(void) add_to_write_order(git_pobject **wo, unsigned int *endp,
 	po->filled = 1;
 }
 
-static void add_descendants_to_write_order(git_pobject **wo, unsigned int *endp,
-					   git_pobject *po)
+static void add_descendants_to_write_order(git_pobject **wo, size_t *endp,
+	git_pobject *po)
 {
 	int add_to_order = 1;
 	while (po) {
@@ -444,8 +500,8 @@ static void add_descendants_to_write_order(git_pobject **wo, unsigned int *endp,
 	};
 }
 
-static void add_family_to_write_order(git_pobject **wo, unsigned int *endp,
-				      git_pobject *po)
+static void add_family_to_write_order(git_pobject **wo, size_t *endp,
+	git_pobject *po)
 {
 	git_pobject *root;
 
@@ -458,15 +514,15 @@ static int cb_tag_foreach(const char *name, git_oid *oid, void *data)
 {
 	git_packbuilder *pb = data;
 	git_pobject *po;
-	khiter_t pos;
+	size_t pos;
 
 	GIT_UNUSED(name);
 
-	pos = kh_get(oid, pb->object_ix, oid);
-	if (pos == kh_end(pb->object_ix))
+	pos = git_oidmap_lookup_index(pb->object_ix, oid);
+	if (!git_oidmap_valid_index(pb->object_ix, pos))
 		return 0;
 
-	po = kh_value(pb->object_ix, pos);
+	po = git_oidmap_value_at(pb->object_ix, pos);
 	po->tagged = 1;
 
 	/* TODO: peel objects */
@@ -476,9 +532,11 @@ static int cb_tag_foreach(const char *name, git_oid *oid, void *data)
 
 static git_pobject **compute_write_order(git_packbuilder *pb)
 {
-	unsigned int i, wo_end, last_untagged;
+	size_t i, wo_end, last_untagged;
+	git_pobject **wo;
 
-	git_pobject **wo = git__malloc(sizeof(*wo) * pb->nr_objects);
+	if ((wo = git__mallocarray(pb->nr_objects, sizeof(*wo))) == NULL)
+		return NULL;
 
 	for (i = 0; i < pb->nr_objects; i++) {
 		git_pobject *po = pb->object_list + i;
@@ -505,8 +563,10 @@ static git_pobject **compute_write_order(git_packbuilder *pb)
 	/*
 	 * Mark objects that are at the tip of tags.
 	 */
-	if (git_tag_foreach(pb->repo, &cb_tag_foreach, pb) < 0)
+	if (git_tag_foreach(pb->repo, &cb_tag_foreach, pb) < 0) {
+		git__free(wo);
 		return NULL;
+	}
 
 	/*
 	 * Give the objects in the original recency order until
@@ -534,8 +594,8 @@ static git_pobject **compute_write_order(git_packbuilder *pb)
 	 */
 	for (i = last_untagged; i < pb->nr_objects; i++) {
 		git_pobject *po = pb->object_list + i;
-		if (po->type != GIT_OBJ_COMMIT &&
-		    po->type != GIT_OBJ_TAG)
+		if (po->type != GIT_OBJECT_COMMIT &&
+		    po->type != GIT_OBJECT_TAG)
 			continue;
 		add_to_write_order(wo, &wo_end, po);
 	}
@@ -545,7 +605,7 @@ static git_pobject **compute_write_order(git_packbuilder *pb)
 	 */
 	for (i = last_untagged; i < pb->nr_objects; i++) {
 		git_pobject *po = pb->object_list + i;
-		if (po->type != GIT_OBJ_TREE)
+		if (po->type != GIT_OBJECT_TREE)
 			continue;
 		add_to_write_order(wo, &wo_end, po);
 	}
@@ -560,7 +620,8 @@ static git_pobject **compute_write_order(git_packbuilder *pb)
 	}
 
 	if (wo_end != pb->nr_objects) {
-		giterr_set(GITERR_INVALID, "invalid write order");
+		git__free(wo);
+		git_error_set(GIT_ERROR_INVALID, "invalid write order");
 		return NULL;
 	}
 
@@ -568,58 +629,65 @@ static git_pobject **compute_write_order(git_packbuilder *pb)
 }
 
 static int write_pack(git_packbuilder *pb,
-		      int (*cb)(void *buf, size_t size, void *data),
-		      void *data)
+	int (*write_cb)(void *buf, size_t size, void *cb_data),
+	void *cb_data)
 {
 	git_pobject **write_order;
 	git_pobject *po;
-	git_buf buf = GIT_BUF_INIT;
 	enum write_one_status status;
 	struct git_pack_header ph;
-	unsigned int i = 0;
+	git_oid entry_oid;
+	size_t i = 0;
+	int error = 0;
 
 	write_order = compute_write_order(pb);
 	if (write_order == NULL)
-		goto on_error;
+		return -1;
+
+	if (!git__is_uint32(pb->nr_objects)) {
+		git_error_set(GIT_ERROR_INVALID, "too many objects");
+		return -1;
+	}
 
 	/* Write pack header */
 	ph.hdr_signature = htonl(PACK_SIGNATURE);
 	ph.hdr_version = htonl(PACK_VERSION);
 	ph.hdr_entries = htonl(pb->nr_objects);
 
-	if (cb(&ph, sizeof(ph), data) < 0)
-		goto on_error;
-
-	if (git_hash_update(&pb->ctx, &ph, sizeof(ph)) < 0)
-		goto on_error;
+	if ((error = write_cb(&ph, sizeof(ph), cb_data)) < 0 ||
+		(error = git_hash_update(&pb->ctx, &ph, sizeof(ph))) < 0)
+		goto done;
 
 	pb->nr_remaining = pb->nr_objects;
 	do {
 		pb->nr_written = 0;
 		for ( ; i < pb->nr_objects; ++i) {
 			po = write_order[i];
-			if (write_one(&buf, pb, po, &status) < 0)
-				goto on_error;
-			if (cb(buf.ptr, buf.size, data) < 0)
-				goto on_error;
-			git_buf_clear(&buf);
+
+			if ((error = write_one(&status, pb, po, write_cb, cb_data)) < 0)
+				goto done;
 		}
 
 		pb->nr_remaining -= pb->nr_written;
 	} while (pb->nr_remaining && i < pb->nr_objects);
 
-	git__free(write_order);
-	git_buf_free(&buf);
+	if ((error = git_hash_final(&entry_oid, &pb->ctx)) < 0)
+		goto done;
 
-	if (git_hash_final(&pb->pack_oid, &pb->ctx) < 0)
-		goto on_error;
+	error = write_cb(entry_oid.id, GIT_OID_RAWSZ, cb_data);
 
-	return cb(pb->pack_oid.id, GIT_OID_RAWSZ, data);
+done:
+	/* if callback cancelled writing, we must still free delta_data */
+	for ( ; i < pb->nr_objects; ++i) {
+		po = write_order[i];
+		if (po->delta_data) {
+			git__free(po->delta_data);
+			po->delta_data = NULL;
+		}
+	}
 
-on_error:
 	git__free(write_order);
-	git_buf_free(&buf);
-	return -1;
+	return error;
 }
 
 static int write_pack_buf(void *buf, size_t size, void *data)
@@ -656,11 +724,18 @@ static int type_size_sort(const void *_a, const void *_b)
 	return a < b ? -1 : (a > b); /* newest first */
 }
 
-static int delta_cacheable(git_packbuilder *pb, unsigned long src_size,
-			   unsigned long trg_size, unsigned long delta_size)
+static int delta_cacheable(
+	git_packbuilder *pb,
+	size_t src_size,
+	size_t trg_size,
+	size_t delta_size)
 {
-	if (pb->max_delta_cache_size &&
-		pb->delta_cache_size + delta_size > pb->max_delta_cache_size)
+	size_t new_size;
+
+	if (git__add_sizet_overflow(&new_size, pb->delta_cache_size, delta_size))
+		return 0;
+
+	if (pb->max_delta_cache_size && new_size > pb->max_delta_cache_size)
 		return 0;
 
 	if (delta_size < pb->cache_max_small_delta_size)
@@ -674,15 +749,14 @@ static int delta_cacheable(git_packbuilder *pb, unsigned long src_size,
 }
 
 static int try_delta(git_packbuilder *pb, struct unpacked *trg,
-		     struct unpacked *src, unsigned int max_depth,
-		     unsigned long *mem_usage, int *ret)
+		     struct unpacked *src, size_t max_depth,
+			 size_t *mem_usage, int *ret)
 {
 	git_pobject *trg_object = trg->object;
 	git_pobject *src_object = src->object;
 	git_odb_object *obj;
-	unsigned long trg_size, src_size, delta_size,
-		      sizediff, max_size, sz;
-	unsigned int ref_depth;
+	size_t trg_size, src_size, delta_size, sizediff, max_size, sz;
+	size_t ref_depth;
 	void *delta_buf;
 
 	/* Don't bother doing diffs between different types */
@@ -700,7 +774,7 @@ static int try_delta(git_packbuilder *pb, struct unpacked *trg,
 		return 0;
 
 	/* Now some size filtering heuristics. */
-	trg_size = (unsigned long)trg_object->size;
+	trg_size = trg_object->size;
 	if (!trg_object->delta) {
 		max_size = trg_size/2 - 20;
 		ref_depth = 1;
@@ -714,7 +788,7 @@ static int try_delta(git_packbuilder *pb, struct unpacked *trg,
 	if (max_size == 0)
 		return 0;
 
-	src_size = (unsigned long)src_object->size;
+	src_size = src_object->size;
 	sizediff = src_size < trg_size ? trg_size - src_size : 0;
 	if (sizediff >= max_size)
 		return 0;
@@ -726,51 +800,52 @@ static int try_delta(git_packbuilder *pb, struct unpacked *trg,
 		if (git_odb_read(&obj, pb->odb, &trg_object->id) < 0)
 			return -1;
 
-		sz = (unsigned long)git_odb_object_size(obj);
+		sz = git_odb_object_size(obj);
 		trg->data = git__malloc(sz);
-		GITERR_CHECK_ALLOC(trg->data);
+		GIT_ERROR_CHECK_ALLOC(trg->data);
 		memcpy(trg->data, git_odb_object_data(obj), sz);
 
 		git_odb_object_free(obj);
 
 		if (sz != trg_size) {
-			giterr_set(GITERR_INVALID,
-				   "Inconsistent target object length");
+			git_error_set(GIT_ERROR_INVALID,
+				   "inconsistent target object length");
 			return -1;
 		}
 
 		*mem_usage += sz;
 	}
 	if (!src->data) {
-		if (git_odb_read(&obj, pb->odb, &src_object->id) < 0)
+		size_t obj_sz;
+
+		if (git_odb_read(&obj, pb->odb, &src_object->id) < 0 ||
+			!git__is_ulong(obj_sz = git_odb_object_size(obj)))
 			return -1;
 
-		sz = (unsigned long)git_odb_object_size(obj);
+		sz = obj_sz;
 		src->data = git__malloc(sz);
-		GITERR_CHECK_ALLOC(src->data);
+		GIT_ERROR_CHECK_ALLOC(src->data);
 		memcpy(src->data, git_odb_object_data(obj), sz);
 
 		git_odb_object_free(obj);
 
 		if (sz != src_size) {
-			giterr_set(GITERR_INVALID,
-				   "Inconsistent source object length");
+			git_error_set(GIT_ERROR_INVALID,
+				   "inconsistent source object length");
 			return -1;
 		}
 
 		*mem_usage += sz;
 	}
 	if (!src->index) {
-		src->index = git_delta_create_index(src->data, src_size);
-		if (!src->index)
+		if (git_delta_index_init(&src->index, src->data, src_size) < 0)
 			return 0; /* suboptimal pack - out of memory */
 
-		*mem_usage += git_delta_sizeof_index(src->index);
+		*mem_usage += git_delta_index_size(src->index);
 	}
 
-	delta_buf = git_delta_create(src->index, trg->data, trg_size,
-				     &delta_size, max_size);
-	if (!delta_buf)
+	if (git_delta_create_from_index(&delta_buf, &delta_size, src->index, trg->data, trg_size,
+		max_size) < 0)
 		return 0;
 
 	if (trg_object->delta) {
@@ -785,15 +860,23 @@ static int try_delta(git_packbuilder *pb, struct unpacked *trg,
 	git_packbuilder__cache_lock(pb);
 	if (trg_object->delta_data) {
 		git__free(trg_object->delta_data);
+		assert(pb->delta_cache_size >= trg_object->delta_size);
 		pb->delta_cache_size -= trg_object->delta_size;
 		trg_object->delta_data = NULL;
 	}
 	if (delta_cacheable(pb, src_size, trg_size, delta_size)) {
-		pb->delta_cache_size += delta_size;
+		bool overflow = git__add_sizet_overflow(
+			&pb->delta_cache_size, pb->delta_cache_size, delta_size);
+
 		git_packbuilder__cache_unlock(pb);
 
+		if (overflow) {
+			git__free(delta_buf);
+			return -1;
+		}
+
 		trg_object->delta_data = git__realloc(delta_buf, delta_size);
-		GITERR_CHECK_ALLOC(trg_object->delta_data);
+		GIT_ERROR_CHECK_ALLOC(trg_object->delta_data);
 	} else {
 		/* create delta when writing the pack */
 		git_packbuilder__cache_unlock(pb);
@@ -808,13 +891,13 @@ static int try_delta(git_packbuilder *pb, struct unpacked *trg,
 	return 0;
 }
 
-static unsigned int check_delta_limit(git_pobject *me, unsigned int n)
+static size_t check_delta_limit(git_pobject *me, size_t n)
 {
 	git_pobject *child = me->delta_child;
-	unsigned int m = n;
+	size_t m = n;
 
 	while (child) {
-		unsigned int c = check_delta_limit(child, n + 1);
+		size_t c = check_delta_limit(child, n + 1);
 		if (m < c)
 			m = c;
 		child = child->delta_sibling;
@@ -822,13 +905,18 @@ static unsigned int check_delta_limit(git_pobject *me, unsigned int n)
 	return m;
 }
 
-static unsigned long free_unpacked(struct unpacked *n)
+static size_t free_unpacked(struct unpacked *n)
 {
-	unsigned long freed_mem = git_delta_sizeof_index(n->index);
-	git_delta_free_index(n->index);
+	size_t freed_mem = 0;
+
+	if (n->index) {
+		freed_mem += git_delta_index_size(n->index);
+		git_delta_index_free(n->index);
+	}
 	n->index = NULL;
+
 	if (n->data) {
-		freed_mem += (unsigned long)n->object->size;
+		freed_mem += n->object->size;
 		git__free(n->data);
 		n->data = NULL;
 	}
@@ -837,25 +925,47 @@ static unsigned long free_unpacked(struct unpacked *n)
 	return freed_mem;
 }
 
+static int report_delta_progress(
+	git_packbuilder *pb, uint32_t count, bool force)
+{
+	int ret;
+
+	if (pb->progress_cb) {
+		double current_time = git__timer();
+		double elapsed = current_time - pb->last_progress_report_time;
+
+		if (force || elapsed >= MIN_PROGRESS_UPDATE_INTERVAL) {
+			pb->last_progress_report_time = current_time;
+
+			ret = pb->progress_cb(
+				GIT_PACKBUILDER_DELTAFICATION,
+				count, pb->nr_objects, pb->progress_cb_payload);
+
+			if (ret)
+				return git_error_set_after_callback(ret);
+		}
+	}
+
+	return 0;
+}
+
 static int find_deltas(git_packbuilder *pb, git_pobject **list,
-		       unsigned int *list_size, unsigned int window,
-		       unsigned int depth)
+	size_t *list_size, size_t window, size_t depth)
 {
 	git_pobject *po;
 	git_buf zbuf = GIT_BUF_INIT;
 	struct unpacked *array;
-	uint32_t idx = 0, count = 0;
-	unsigned long mem_usage = 0;
-	unsigned int i;
+	size_t idx = 0, count = 0;
+	size_t mem_usage = 0;
+	size_t i;
 	int error = -1;
 
 	array = git__calloc(window, sizeof(struct unpacked));
-	GITERR_CHECK_ALLOC(array);
+	GIT_ERROR_CHECK_ALLOC(array);
 
 	for (;;) {
 		struct unpacked *n = array + idx;
-		unsigned int max_depth;
-		int j, best_base = -1;
+		size_t max_depth, j, best_base = SIZE_MAX;
 
 		git_packbuilder__progress_lock(pb);
 		if (!*list_size) {
@@ -863,6 +973,9 @@ static int find_deltas(git_packbuilder *pb, git_pobject **list,
 			break;
 		}
 
+		pb->nr_deltified += 1;
+		report_delta_progress(pb, pb->nr_deltified, false);
+
 		po = *list++;
 		(*list_size)--;
 		git_packbuilder__progress_unlock(pb);
@@ -873,7 +986,7 @@ static int find_deltas(git_packbuilder *pb, git_pobject **list,
 		while (pb->window_memory_limit &&
 		       mem_usage > pb->window_memory_limit &&
 		       count > 1) {
-			uint32_t tail = (idx + window - count) % window;
+			size_t tail = (idx + window - count) % window;
 			mem_usage -= free_unpacked(array + tail);
 			count--;
 		}
@@ -885,15 +998,18 @@ static int find_deltas(git_packbuilder *pb, git_pobject **list,
 		 */
 		max_depth = depth;
 		if (po->delta_child) {
-			max_depth -= check_delta_limit(po, 0);
-			if (max_depth <= 0)
+			size_t delta_limit = check_delta_limit(po, 0);
+
+			if (delta_limit > max_depth)
 				goto next;
+
+			max_depth -= delta_limit;
 		}
 
 		j = window;
 		while (--j > 0) {
 			int ret;
-			uint32_t other_idx = idx + j;
+			size_t other_idx = idx + j;
 			struct unpacked *m;
 
 			if (other_idx >= window)
@@ -926,15 +1042,15 @@ static int find_deltas(git_packbuilder *pb, git_pobject **list,
 		 * between writes at that moment.
 		 */
 		if (po->delta_data) {
-			if (git__compress(&zbuf, po->delta_data, po->delta_size) < 0)
+			if (git_zstream_deflatebuf(&zbuf, po->delta_data, po->delta_size) < 0)
 				goto on_error;
 
 			git__free(po->delta_data);
 			po->delta_data = git__malloc(zbuf.size);
-			GITERR_CHECK_ALLOC(po->delta_data);
+			GIT_ERROR_CHECK_ALLOC(po->delta_data);
 
 			memcpy(po->delta_data, zbuf.ptr, zbuf.size);
-			po->z_delta_size = (unsigned long)zbuf.size;
+			po->z_delta_size = zbuf.size;
 			git_buf_clear(&zbuf);
 
 			git_packbuilder__cache_lock(pb);
@@ -958,10 +1074,10 @@ static int find_deltas(git_packbuilder *pb, git_pobject **list,
 		 */
 		if (po->delta) {
 			struct unpacked swap = array[best_base];
-			int dist = (window + idx - best_base) % window;
-			int dst = best_base;
+			size_t dist = (window + idx - best_base) % window;
+			size_t dst = best_base;
 			while (dist--) {
-				int src = (dst + 1) % window;
+				size_t src = (dst + 1) % window;
 				array[dst] = array[src];
 				dst = src;
 			}
@@ -983,7 +1099,7 @@ on_error:
 		git__free(array[i].data);
 	}
 	git__free(array);
-	git_buf_free(&zbuf);
+	git_buf_dispose(&zbuf);
 
 	return error;
 }
@@ -999,13 +1115,13 @@ struct thread_params {
 	git_cond cond;
 	git_mutex mutex;
 
-	unsigned int list_size;
-	unsigned int remaining;
+	size_t list_size;
+	size_t remaining;
 
-	int window;
-	int depth;
-	int working;
-	int data_ready;
+	size_t window;
+	size_t depth;
+	size_t working;
+	size_t data_ready;
 };
 
 static void *threaded_find_deltas(void *arg)
@@ -1024,7 +1140,7 @@ static void *threaded_find_deltas(void *arg)
 		git_packbuilder__progress_unlock(me->pb);
 
 		if (git_mutex_lock(&me->mutex)) {
-			giterr_set(GITERR_THREAD, "unable to lock packfile condition mutex");
+			git_error_set(GIT_ERROR_THREAD, "unable to lock packfile condition mutex");
 			return NULL;
 		}
 
@@ -1047,11 +1163,11 @@ static void *threaded_find_deltas(void *arg)
 }
 
 static int ll_find_deltas(git_packbuilder *pb, git_pobject **list,
-			  unsigned int list_size, unsigned int window,
-			  unsigned int depth)
+			  size_t list_size, size_t window, size_t depth)
 {
 	struct thread_params *p;
-	int i, ret, active_threads = 0;
+	size_t i;
+	int ret, active_threads = 0;
 
 	if (!pb->nr_threads)
 		pb->nr_threads = git_online_cpus();
@@ -1061,12 +1177,12 @@ static int ll_find_deltas(git_packbuilder *pb, git_pobject **list,
 		return 0;
 	}
 
-	p = git__malloc(pb->nr_threads * sizeof(*p));
-	GITERR_CHECK_ALLOC(p);
+	p = git__mallocarray(pb->nr_threads, sizeof(*p));
+	GIT_ERROR_CHECK_ALLOC(p);
 
 	/* Partition the work among the threads */
 	for (i = 0; i < pb->nr_threads; ++i) {
-		unsigned sub_size = list_size / (pb->nr_threads - i);
+		size_t sub_size = list_size / (pb->nr_threads - i);
 
 		/* don't use too small segments or no deltas will be found */
 		if (sub_size < 2*window && i+1 < pb->nr_threads)
@@ -1100,10 +1216,10 @@ static int ll_find_deltas(git_packbuilder *pb, git_pobject **list,
 		git_mutex_init(&p[i].mutex);
 		git_cond_init(&p[i].cond);
 
-		ret = git_thread_create(&p[i].thread, NULL,
+		ret = git_thread_create(&p[i].thread,
 					threaded_find_deltas, &p[i]);
 		if (ret) {
-			giterr_set(GITERR_THREAD, "unable to create thread");
+			git_error_set(GIT_ERROR_THREAD, "unable to create thread");
 			return -1;
 		}
 		active_threads++;
@@ -1120,7 +1236,7 @@ static int ll_find_deltas(git_packbuilder *pb, git_pobject **list,
 	while (active_threads) {
 		struct thread_params *target = NULL;
 		struct thread_params *victim = NULL;
-		unsigned sub_size = 0;
+		size_t sub_size = 0;
 
 		/* Start by locating a thread that has transitioned its
 		 * 'working' flag from 1 -> 0. This indicates that it is
@@ -1172,7 +1288,7 @@ static int ll_find_deltas(git_packbuilder *pb, git_pobject **list,
 		git_packbuilder__progress_unlock(pb);
 
 		if (git_mutex_lock(&target->mutex)) {
-			giterr_set(GITERR_THREAD, "unable to lock packfile condition mutex");
+			git_error_set(GIT_ERROR_THREAD, "unable to lock packfile condition mutex");
 			git__free(p);
 			return -1;
 		}
@@ -1182,7 +1298,7 @@ static int ll_find_deltas(git_packbuilder *pb, git_pobject **list,
 		git_mutex_unlock(&target->mutex);
 
 		if (!sub_size) {
-			git_thread_join(target->thread, NULL);
+			git_thread_join(&target->thread, NULL);
 			git_cond_free(&target->cond);
 			git_mutex_free(&target->mutex);
 			active_threads--;
@@ -1200,13 +1316,20 @@ static int ll_find_deltas(git_packbuilder *pb, git_pobject **list,
 static int prepare_pack(git_packbuilder *pb)
 {
 	git_pobject **delta_list;
-	unsigned int i, n = 0;
+	size_t i, n = 0;
 
 	if (pb->nr_objects == 0 || pb->done)
 		return 0; /* nothing to do */
 
-	delta_list = git__malloc(pb->nr_objects * sizeof(*delta_list));
-	GITERR_CHECK_ALLOC(delta_list);
+	/*
+	 * Although we do not report progress during deltafication, we
+	 * at least report that we are in the deltafication stage
+	 */
+	if (pb->progress_cb)
+			pb->progress_cb(GIT_PACKBUILDER_DELTAFICATION, 0, pb->nr_objects, pb->progress_cb_payload);
+
+	delta_list = git__mallocarray(pb->nr_objects, sizeof(*delta_list));
+	GIT_ERROR_CHECK_ALLOC(delta_list);
 
 	for (i = 0; i < pb->nr_objects; ++i) {
 		git_pobject *po = pb->object_list + i;
@@ -1228,6 +1351,8 @@ static int prepare_pack(git_packbuilder *pb)
 		}
 	}
 
+	report_delta_progress(pb, pb->nr_objects, true);
+
 	pb->done = true;
 	git__free(delta_list);
 	return 0;
@@ -1244,61 +1369,80 @@ int git_packbuilder_foreach(git_packbuilder *pb, int (*cb)(void *buf, size_t siz
 int git_packbuilder_write_buf(git_buf *buf, git_packbuilder *pb)
 {
 	PREPARE_PACK;
+	git_buf_sanitize(buf);
 	return write_pack(pb, &write_pack_buf, buf);
 }
 
 static int write_cb(void *buf, size_t len, void *payload)
 {
 	struct pack_write_context *ctx = payload;
-	return git_indexer_stream_add(ctx->indexer, buf, len, ctx->stats);
+	return git_indexer_append(ctx->indexer, buf, len, ctx->stats);
 }
 
 int git_packbuilder_write(
 	git_packbuilder *pb,
 	const char *path,
-	git_transfer_progress_callback progress_cb,
+	unsigned int mode,
+	git_transfer_progress_cb progress_cb,
 	void *progress_cb_payload)
 {
-	git_indexer_stream *indexer;
+	git_indexer_options opts = GIT_INDEXER_OPTIONS_INIT;
+	git_indexer *indexer;
 	git_transfer_progress stats;
 	struct pack_write_context ctx;
+	int t;
 
 	PREPARE_PACK;
 
-	if (git_indexer_stream_new(
-		&indexer, path, progress_cb, progress_cb_payload) < 0)
+	opts.progress_cb = progress_cb;
+	opts.progress_cb_payload = progress_cb_payload;
+
+	if (git_indexer_new(
+		&indexer, path, mode, pb->odb, &opts) < 0)
 		return -1;
 
+	if (!git_repository__cvar(&t, pb->repo, GIT_CVAR_FSYNCOBJECTFILES) && t)
+		git_indexer__set_fsync(indexer, 1);
+
 	ctx.indexer = indexer;
 	ctx.stats = &stats;
 
 	if (git_packbuilder_foreach(pb, write_cb, &ctx) < 0 ||
-		git_indexer_stream_finalize(indexer, &stats) < 0) {
-		git_indexer_stream_free(indexer);
+		git_indexer_commit(indexer, &stats) < 0) {
+		git_indexer_free(indexer);
 		return -1;
 	}
 
-	git_indexer_stream_free(indexer);
+	git_oid_cpy(&pb->pack_oid, git_indexer_hash(indexer));
+
+	git_indexer_free(indexer);
 	return 0;
 }
 
 #undef PREPARE_PACK
 
-static int cb_tree_walk(const char *root, const git_tree_entry *entry, void *payload)
+const git_oid *git_packbuilder_hash(git_packbuilder *pb)
+{
+	return &pb->pack_oid;
+}
+
+
+static int cb_tree_walk(
+	const char *root, const git_tree_entry *entry, void *payload)
 {
+	int error;
 	struct tree_walk_context *ctx = payload;
 
 	/* A commit inside a tree represents a submodule commit and should be skipped. */
-	if (git_tree_entry_type(entry) == GIT_OBJ_COMMIT)
+	if (git_tree_entry_type(entry) == GIT_OBJECT_COMMIT)
 		return 0;
 
-	if (git_buf_sets(&ctx->buf, root) < 0 ||
-		git_buf_puts(&ctx->buf, git_tree_entry_name(entry)) < 0)
-		return -1;
+	if (!(error = git_buf_sets(&ctx->buf, root)) &&
+		!(error = git_buf_puts(&ctx->buf, git_tree_entry_name(entry))))
+		error = git_packbuilder_insert(
+			ctx->pb, git_tree_entry_id(entry), git_buf_cstr(&ctx->buf));
 
-	return git_packbuilder_insert(ctx->pb,
-		git_tree_entry_id(entry),
-		git_buf_cstr(&ctx->buf));
+	return error;
 }
 
 int git_packbuilder_insert_commit(git_packbuilder *pb, const git_oid *oid)
@@ -1318,32 +1462,307 @@ int git_packbuilder_insert_commit(git_packbuilder *pb, const git_oid *oid)
 
 int git_packbuilder_insert_tree(git_packbuilder *pb, const git_oid *oid)
 {
-	git_tree *tree;
+	int error;
+	git_tree *tree = NULL;
 	struct tree_walk_context context = { pb, GIT_BUF_INIT };
 
-	if (git_tree_lookup(&tree, pb->repo, oid) < 0 ||
-	    git_packbuilder_insert(pb, oid, NULL) < 0)
-		return -1;
+	if (!(error = git_tree_lookup(&tree, pb->repo, oid)) &&
+	    !(error = git_packbuilder_insert(pb, oid, NULL)))
+		error = git_tree_walk(tree, GIT_TREEWALK_PRE, cb_tree_walk, &context);
+
+	git_tree_free(tree);
+	git_buf_dispose(&context.buf);
+	return error;
+}
+
+int git_packbuilder_insert_recur(git_packbuilder *pb, const git_oid *id, const char *name)
+{
+	git_object *obj;
+	int error;
+
+	assert(pb && id);
+
+	if ((error = git_object_lookup(&obj, pb->repo, id, GIT_OBJECT_ANY)) < 0)
+		return error;
+
+	switch (git_object_type(obj)) {
+	case GIT_OBJECT_BLOB:
+		error = git_packbuilder_insert(pb, id, name);
+		break;
+	case GIT_OBJECT_TREE:
+		error = git_packbuilder_insert_tree(pb, id);
+		break;
+	case GIT_OBJECT_COMMIT:
+		error = git_packbuilder_insert_commit(pb, id);
+		break;
+	case GIT_OBJECT_TAG:
+		if ((error = git_packbuilder_insert(pb, id, name)) < 0)
+			goto cleanup;
+		error = git_packbuilder_insert_recur(pb, git_tag_target_id((git_tag *) obj), NULL);
+		break;
+
+	default:
+		git_error_set(GIT_ERROR_INVALID, "unknown object type");
+		error = -1;
+	}
+
+cleanup:
+	git_object_free(obj);
+	return error;
+}
+
+size_t git_packbuilder_object_count(git_packbuilder *pb)
+{
+	return pb->nr_objects;
+}
+
+size_t git_packbuilder_written(git_packbuilder *pb)
+{
+	return pb->nr_written;
+}
+
+static int lookup_walk_object(struct walk_object **out, git_packbuilder *pb, const git_oid *id)
+{
+	struct walk_object *obj;
 
-	if (git_tree_walk(tree, GIT_TREEWALK_PRE, cb_tree_walk, &context) < 0) {
-		git_tree_free(tree);
-		git_buf_free(&context.buf);
+	obj = git_pool_mallocz(&pb->object_pool, 1);
+	if (!obj) {
+		git_error_set_oom();
 		return -1;
 	}
 
+	git_oid_cpy(&obj->id, id);
+
+	*out = obj;
+	return 0;
+}
+
+static int retrieve_object(struct walk_object **out, git_packbuilder *pb, const git_oid *id)
+{
+	int error;
+	size_t pos;
+	struct walk_object *obj;
+
+	pos = git_oidmap_lookup_index(pb->walk_objects, id);
+	if (git_oidmap_valid_index(pb->walk_objects, pos)) {
+		obj = git_oidmap_value_at(pb->walk_objects, pos);
+	} else {
+		if ((error = lookup_walk_object(&obj, pb, id)) < 0)
+			return error;
+
+		git_oidmap_insert(pb->walk_objects, &obj->id, obj, &error);
+	}
+
+	*out = obj;
+	return 0;
+}
+
+static int mark_blob_uninteresting(git_packbuilder *pb, const git_oid *id)
+{
+	int error;
+	struct walk_object *obj;
+
+	if ((error = retrieve_object(&obj, pb, id)) < 0)
+		return error;
+
+	obj->uninteresting = 1;
+
+	return 0;
+}
+
+static int mark_tree_uninteresting(git_packbuilder *pb, const git_oid *id)
+{
+	struct walk_object *obj;
+	git_tree *tree;
+	int error;
+	size_t i;
+
+	if ((error = retrieve_object(&obj, pb, id)) < 0)
+		return error;
+
+	if (obj->uninteresting)
+		return 0;
+
+	obj->uninteresting = 1;
+
+	if ((error = git_tree_lookup(&tree, pb->repo, id)) < 0)
+		return error;
+
+	for (i = 0; i < git_tree_entrycount(tree); i++) {
+		const git_tree_entry *entry = git_tree_entry_byindex(tree, i);
+		const git_oid *entry_id = git_tree_entry_id(entry);
+		switch (git_tree_entry_type(entry)) {
+		case GIT_OBJECT_TREE:
+			if ((error = mark_tree_uninteresting(pb, entry_id)) < 0)
+				goto cleanup;
+			break;
+		case GIT_OBJECT_BLOB:
+			if ((error = mark_blob_uninteresting(pb, entry_id)) < 0)
+				goto cleanup;
+			break;
+		default:
+			/* it's a submodule or something unknown, we don't want it */
+			;
+		}
+	}
+
+cleanup:
 	git_tree_free(tree);
-	git_buf_free(&context.buf);
+	return error;
+}
+
+/*
+ * Mark the edges of the graph uninteresting. Since we start from a
+ * git_revwalk, the commits are already uninteresting, but we need to
+ * mark the trees and blobs.
+ */
+static int mark_edges_uninteresting(git_packbuilder *pb, git_commit_list *commits)
+{
+	int error;
+	git_commit_list *list;
+	git_commit *commit;
+
+	for (list = commits; list; list = list->next) {
+		if (!list->item->uninteresting)
+			continue;
+
+		if ((error = git_commit_lookup(&commit, pb->repo, &list->item->oid)) < 0)
+			return error;
+
+		error = mark_tree_uninteresting(pb, git_commit_tree_id(commit));
+		git_commit_free(commit);
+
+		if (error < 0)
+			return error;
+	}
+
 	return 0;
 }
 
-uint32_t git_packbuilder_object_count(git_packbuilder *pb)
+int insert_tree(git_packbuilder *pb, git_tree *tree)
 {
-	return pb->nr_objects;
+	size_t i;
+	int error;
+	git_tree *subtree;
+	struct walk_object *obj;
+	const char *name;
+
+	if ((error = retrieve_object(&obj, pb, git_tree_id(tree))) < 0)
+		return error;
+
+	if (obj->seen || obj->uninteresting)
+		return 0;
+
+	obj->seen = 1;
+
+	if ((error = git_packbuilder_insert(pb, &obj->id, NULL)))
+		return error;
+
+	for (i = 0; i < git_tree_entrycount(tree); i++) {
+		const git_tree_entry *entry = git_tree_entry_byindex(tree, i);
+		const git_oid *entry_id = git_tree_entry_id(entry);
+		switch (git_tree_entry_type(entry)) {
+		case GIT_OBJECT_TREE:
+			if ((error = git_tree_lookup(&subtree, pb->repo, entry_id)) < 0)
+				return error;
+
+			error = insert_tree(pb, subtree);
+			git_tree_free(subtree);
+
+			if (error < 0)
+				return error;
+
+			break;
+		case GIT_OBJECT_BLOB:
+			if ((error = retrieve_object(&obj, pb, entry_id)) < 0)
+				return error;
+			if (obj->uninteresting)
+				continue;
+			name = git_tree_entry_name(entry);
+			if ((error = git_packbuilder_insert(pb, entry_id, name)) < 0)
+				return error;
+			break;
+		default:
+			/* it's a submodule or something unknown, we don't want it */
+			;
+		}
+	}
+
+
+	return error;
 }
 
-uint32_t git_packbuilder_written(git_packbuilder *pb)
+int insert_commit(git_packbuilder *pb, struct walk_object *obj)
 {
-	return pb->nr_written;
+	int error;
+	git_commit *commit = NULL;
+	git_tree *tree = NULL;
+
+	obj->seen = 1;
+
+	if ((error = git_packbuilder_insert(pb, &obj->id, NULL)) < 0)
+		return error;
+
+	if ((error = git_commit_lookup(&commit, pb->repo, &obj->id)) < 0)
+		return error;
+
+	if ((error = git_tree_lookup(&tree, pb->repo, git_commit_tree_id(commit))) < 0)
+		goto cleanup;
+
+	if ((error = insert_tree(pb, tree)) < 0)
+		goto cleanup;
+
+cleanup:
+	git_commit_free(commit);
+	git_tree_free(tree);
+	return error;
+}
+
+int git_packbuilder_insert_walk(git_packbuilder *pb, git_revwalk *walk)
+{
+	int error;
+	git_oid id;
+	struct walk_object *obj;
+
+	assert(pb && walk);
+
+	if ((error = mark_edges_uninteresting(pb, walk->user_input)) < 0)
+		return error;
+
+	/*
+	 * TODO: git marks the parents of the edges
+	 * uninteresting. This may provide a speed advantage, but does
+	 * seem to assume the remote does not have a single-commit
+	 * history on the other end.
+	 */
+
+	/* walk down each tree up to the blobs and insert them, stopping when uninteresting */
+	while ((error = git_revwalk_next(&id, walk)) == 0) {
+		if ((error = retrieve_object(&obj, pb, &id)) < 0)
+			return error;
+
+		if (obj->seen || obj->uninteresting)
+			continue;
+
+		if ((error = insert_commit(pb, obj)) < 0)
+			return error;
+	}
+
+	if (error == GIT_ITEROVER)
+		error = 0;
+
+	return error;
+}
+
+int git_packbuilder_set_callbacks(git_packbuilder *pb, git_packbuilder_progress progress_cb, void *progress_cb_payload)
+{
+	if (!pb)
+		return -1;
+
+	pb->progress_cb = progress_cb;
+	pb->progress_cb_payload = progress_cb_payload;
+
+	return 0;
 }
 
 void git_packbuilder_free(git_packbuilder *pb)
@@ -1368,7 +1787,11 @@ void git_packbuilder_free(git_packbuilder *pb)
 	if (pb->object_list)
 		git__free(pb->object_list);
 
+	git_oidmap_free(pb->walk_objects);
+	git_pool_clear(&pb->object_pool);
+
 	git_hash_ctx_cleanup(&pb->ctx);
+	git_zstream_free(&pb->zstream);
 
 	git__free(pb);
 }