rugged 1.1.1 → 1.2.0

data/vendor/libgit2/src/odb.h

@@ -12,16 +12,21 @@
 #include "git2/odb.h"
 #include "git2/oid.h"
 #include "git2/types.h"
+#include "git2/sys/commit_graph.h"
 
-#include "vector.h"
 #include "cache.h"
-#include "posix.h"
+#include "commit_graph.h"
 #include "filter.h"
+#include "posix.h"
+#include "vector.h"
 
 #define GIT_OBJECTS_DIR "objects/"
 #define GIT_OBJECT_DIR_MODE 0777
 #define GIT_OBJECT_FILE_MODE 0444
 
+#define GIT_ODB_DEFAULT_LOOSE_PRIORITY 1
+#define GIT_ODB_DEFAULT_PACKED_PRIORITY 2
+
 extern bool git_odb__strict_hash_verification;
 
 /* DO NOT EXPORT */
@@ -40,8 +45,10 @@ struct git_odb_object {
 /* EXPORT */
 struct git_odb {
 	git_refcount rc;
+	git_mutex lock;  /* protects backends */
 	git_vector backends;
 	git_cache own_cache;
+	git_commit_graph *cgraph;
 	unsigned int do_fsync :1;
 };
 
@@ -126,6 +133,13 @@ int git_odb__read_header_or_object(
 	git_odb_object **out, size_t *len_p, git_object_t *type_p,
 	git_odb *db, const git_oid *id);
 
+/*
+ * Attempt to get the ODB's commit-graph file. This object is still owned by
+ * the ODB. If the repository does not contain a commit-graph, it will return
+ * GIT_ENOTFOUND.
+ */
+int git_odb__get_commit_graph_file(git_commit_graph_file **out, git_odb *odb);
+
 /* freshen an entry in the object database */
 int git_odb__freshen(git_odb *db, const git_oid *id);
 

data/vendor/libgit2/src/odb_loose.c

@@ -309,7 +309,7 @@ static int read_loose_standard(git_rawobj *out, git_buf *obj)
 		goto done;
 	}
 
-	assert(decompressed >= head_len);
+	GIT_ASSERT(decompressed >= head_len);
 	body_len = decompressed - head_len;
 
 	if (body_len)
@@ -344,7 +344,8 @@ static int read_loose(git_rawobj *out, git_buf *loc)
 	int error;
 	git_buf obj = GIT_BUF_INIT;
 
-	assert(out && loc);
+	GIT_ASSERT_ARG(out);
+	GIT_ASSERT_ARG(loc);
 
 	if (git_buf_oom(loc))
 		return -1;
@@ -411,7 +412,8 @@ static int read_header_loose(git_rawobj *out, git_buf *loc)
 	ssize_t obj_len;
 	int fd, error;
 
-	assert(out && loc);
+	GIT_ASSERT_ARG(out);
+	GIT_ASSERT_ARG(loc);
 
 	if (git_buf_oom(loc))
 		return -1;
@@ -585,7 +587,8 @@ static int loose_backend__read_header(size_t *len_p, git_object_t *type_p, git_o
 	git_rawobj raw;
 	int error;
 
-	assert(backend && oid);
+	GIT_ASSERT_ARG(backend);
+	GIT_ASSERT_ARG(oid);
 
 	raw.len = 0;
 	raw.type = GIT_OBJECT_INVALID;
@@ -609,7 +612,8 @@ static int loose_backend__read(void **buffer_p, size_t *len_p, git_object_t *typ
 	git_rawobj raw;
 	int error = 0;
 
-	assert(backend && oid);
+	GIT_ASSERT_ARG(backend);
+	GIT_ASSERT_ARG(oid);
 
 	if (locate_object(&object_path, (loose_backend *)backend, oid) < 0) {
 		error = git_odb__error_notfound("no matching loose object",
@@ -636,7 +640,7 @@ static int loose_backend__read_prefix(
 {
 	int error = 0;
 
-	assert(len >= GIT_OID_MINPREFIXLEN && len <= GIT_OID_HEXSZ);
+	GIT_ASSERT_ARG(len >= GIT_OID_MINPREFIXLEN && len <= GIT_OID_HEXSZ);
 
 	if (len == GIT_OID_HEXSZ) {
 		/* We can fall back to regular read method */
@@ -647,7 +651,7 @@ static int loose_backend__read_prefix(
 		git_buf object_path = GIT_BUF_INIT;
 		git_rawobj raw;
 
-		assert(backend && short_oid);
+		GIT_ASSERT_ARG(backend && short_oid);
 
 		if ((error = locate_object_short_oid(&object_path, out_oid,
 				(loose_backend *)backend, short_oid, len)) == 0 &&
@@ -669,7 +673,8 @@ static int loose_backend__exists(git_odb_backend *backend, const git_oid *oid)
 	git_buf object_path = GIT_BUF_INIT;
 	int error;
 
-	assert(backend && oid);
+	GIT_ASSERT_ARG(backend);
+	GIT_ASSERT_ARG(oid);
 
 	error = locate_object(&object_path, (loose_backend *)backend, oid);
 
@@ -684,7 +689,10 @@ static int loose_backend__exists_prefix(
 	git_buf object_path = GIT_BUF_INIT;
 	int error;
 
-	assert(backend && out && short_id && len >= GIT_OID_MINPREFIXLEN);
+	GIT_ASSERT_ARG(backend);
+	GIT_ASSERT_ARG(out);
+	GIT_ASSERT_ARG(short_id);
+	GIT_ASSERT_ARG(len >= GIT_OID_MINPREFIXLEN);
 
 	error = locate_object_short_oid(
 		&object_path, out, (loose_backend *)backend, short_id, len);
@@ -759,7 +767,8 @@ static int loose_backend__foreach(git_odb_backend *_backend, git_odb_foreach_cb
 	struct foreach_state state;
 	loose_backend *backend = (loose_backend *) _backend;
 
-	assert(backend && cb);
+	GIT_ASSERT_ARG(backend);
+	GIT_ASSERT_ARG(cb);
 
 	objects_dir = backend->objects_dir;
 
@@ -833,7 +842,7 @@ static int loose_backend__writestream(git_odb_stream **stream_out, git_odb_backe
 	size_t hdrlen;
 	int error;
 
-	assert(_backend);
+	GIT_ASSERT_ARG(_backend);
 
 	backend = (loose_backend *)_backend;
 	*stream_out = NULL;
@@ -991,7 +1000,11 @@ static int loose_backend__readstream(
 	obj_hdr hdr;
 	int error = 0;
 
-	assert(stream_out && len_out && type_out && _backend && oid);
+	GIT_ASSERT_ARG(stream_out);
+	GIT_ASSERT_ARG(len_out);
+	GIT_ASSERT_ARG(type_out);
+	GIT_ASSERT_ARG(_backend);
+	GIT_ASSERT_ARG(oid);
 
 	backend = (loose_backend *)_backend;
 	*stream_out = NULL;
@@ -1108,11 +1121,7 @@ static int loose_backend__freshen(
 
 static void loose_backend__free(git_odb_backend *_backend)
 {
-	loose_backend *backend;
-	assert(_backend);
-	backend = (loose_backend *)_backend;
-
-	git__free(backend);
+	git__free(_backend);
 }
 
 int git_odb_backend_loose(
@@ -1126,7 +1135,8 @@ int git_odb_backend_loose(
 	loose_backend *backend;
 	size_t objects_dirlen, alloclen;
 
-	assert(backend_out && objects_dir);
+	GIT_ASSERT_ARG(backend_out);
+	GIT_ASSERT_ARG(objects_dir);
 
 	objects_dirlen = strlen(objects_dir);
 

data/vendor/libgit2/src/odb_mempack.c

@@ -156,7 +156,7 @@ int git_mempack_new(git_odb_backend **out)
 {
 	struct memory_packer_db *db;
 
-	assert(out);
+	GIT_ASSERT_ARG(out);
 
 	db = git__calloc(1, sizeof(struct memory_packer_db));
 	GIT_ERROR_CHECK_ALLOC(db);

data/vendor/libgit2/src/odb_pack.c

@@ -11,11 +11,12 @@
 #include "git2/repository.h"
 #include "git2/indexer.h"
 #include "git2/sys/odb_backend.h"
+#include "delta.h"
 #include "futils.h"
 #include "hash.h"
-#include "odb.h"
-#include "delta.h"
+#include "midx.h"
 #include "mwindow.h"
+#include "odb.h"
 #include "pack.h"
 
 #include "git2/odb_backend.h"
@@ -25,6 +26,8 @@
 
 struct pack_backend {
 	git_odb_backend parent;
+	git_midx_file *midx;
+	git_vector midx_packs;
 	git_vector packs;
 	struct git_pack_file *last_found;
 	char *pack_folder;
@@ -47,36 +50,43 @@ struct pack_writepack {
  *	Initialization of the Pack Backend
  *	--------------------------------------------------
  *
- *	# git_odb_backend_pack
- *	| Creates the pack backend structure, initializes the
- *	| callback pointers to our default read() and exist() methods,
- *	| and tries to preload all the known packfiles in the ODB.
+ * # git_odb_backend_pack
+ * | Creates the pack backend structure, initializes the
+ * | callback pointers to our default read() and exist() methods,
+ * | and tries to find the `pack` folder, if it exists. ODBs without a `pack`
+ * | folder are ignored altogether. If there is a `pack` folder, it tries to
+ * | preload all the known packfiles in the ODB.
  * |
- *	|-# packfile_load_all
- *	 | Tries to find the `pack` folder, if it exists. ODBs without
- *	 | a pack folder are ignored altogether. If there's a `pack` folder
- *	 | we run a `dirent` callback through every file in the pack folder
- *	 | to find our packfiles. The packfiles are then sorted according
- *	 | to a sorting callback.
- * 	 |
- *	 |-# packfile_load__cb
- *	 | | This callback is called from `dirent` with every single file
- *	 | | inside the pack folder. We find the packs by actually locating
- *	 | | their index (ends in ".idx"). From that index, we verify that
- *	 | | the corresponding packfile exists and is valid, and if so, we
- *	| | add it to the pack list.
- *	 | |
- *	 | |-# packfile_check
- *	 |		Make sure that there's a packfile to back this index, and store
- *	 |		some very basic information regarding the packfile itself,
- *	 |		such as the full path, the size, and the modification time.
- *	 |		We don't actually open the packfile to check for internal consistency.
- *	|
- *	|-# packfile_sort__cb
- *		Sort all the preloaded packs according to some specific criteria:
- *		we prioritize the "newer" packs because it's more likely they
- *		contain the objects we are looking for, and we prioritize local
- *		packs over remote ones.
+ * |-# pack_backend__refresh
+ *   | The `multi-pack-index` is loaded if it exists and is valid.
+ *   | Then we run a `dirent` callback through every file in the pack folder,
+ *   | even those present in `multi-pack-index`. The unindexed packfiles are
+ *   | then sorted according to a sorting callback.
+ *   |
+ *   |-# refresh_multi_pack_index
+ *   |   Detect the presence of the `multi-pack-index` file. If it needs to be
+ *   |   refreshed, frees the old copy and tries to load the new one, together
+ *   |   with all the packfiles it indexes. If the process fails, fall back to
+ *   |   the old behavior, as if the `multi-pack-index` file was not there.
+ *   |
+ *   |-# packfile_load__cb
+ *   | | This callback is called from `dirent` with every single file
+ *   | | inside the pack folder. We find the packs by actually locating
+ *   | | their index (ends in ".idx"). From that index, we verify that
+ *   | | the corresponding packfile exists and is valid, and if so, we
+ *   | | add it to the pack list.
+ *   | |
+ *   | # git_mwindow_get_pack
+ *   |   Make sure that there's a packfile to back this index, and store
+ *   |   some very basic information regarding the packfile itself,
+ *   |   such as the full path, the size, and the modification time.
+ *   |   We don't actually open the packfile to check for internal consistency.
+ *   |
+ *   |-# packfile_sort__cb
+ *       Sort all the preloaded packs according to some specific criteria:
+ *       we prioritize the "newer" packs because it's more likely they
+ *       contain the objects we are looking for, and we prioritize local
+ *       packs over remote ones.
  *
  *
  *
@@ -84,48 +94,66 @@ struct pack_writepack {
  *	A standard packed `exist` query for an OID
  *	--------------------------------------------------
  *
- * # pack_backend__exists
- * | Check if the given SHA1 oid exists in any of the packs
- * | that have been loaded for our ODB.
+ * # pack_backend__exists / pack_backend__exists_prefix
+ * | Check if the given SHA1 oid (or a SHA1 oid prefix) exists in any of the
+ * | packs that have been loaded for our ODB.
  * |
- * |-# pack_entry_find
- *	| Iterate through all the packs that have been preloaded
- *	| (starting by the pack where the latest object was found)
- *	| to try to find the OID in one of them.
- *	|
- *	|-# pack_entry_find1
- *		| Check the index of an individual pack to see if the SHA1
- *		| OID can be found. If we can find the offset to that SHA1
- *		| inside of the index, that means the object is contained
- *		| inside of the packfile and we can stop searching.
- *		| Before returning, we verify that the packfile behing the
- *		| index we are searching still exists on disk.
- *		|
- *		|-# pack_entry_find_offset
- *		| | Mmap the actual index file to disk if it hasn't been opened
- *		| | yet, and run a binary search through it to find the OID.
- *		| | See <http://book.git-scm.com/7_the_packfile.html> for specifics
- *		| | on the Packfile Index format and how do we find entries in it.
- *		| |
- *		| |-# pack_index_open
- *		|	| Guess the name of the index based on the full path to the
- *		|	| packfile, open it and verify its contents. Only if the index
- *		|	| has not been opened already.
- *		|	|
- *		|	|-# pack_index_check
- *		|		Mmap the index file and do a quick run through the header
- *		|		to guess the index version (right now we support v1 and v2),
- *		|		and to verify that the size of the index makes sense.
- *		|
- *		|-# packfile_open
- *			See `packfile_open` in Chapter 3
+ * |-# pack_entry_find / pack_entry_find_prefix
+ *   | If there is a multi-pack-index present, search the SHA1 oid in that
+ *   | index first. If it is not found there, iterate through all the unindexed
+ *   | packs that have been preloaded (starting by the pack where the latest
+ *   | object was found) to try to find the OID in one of them.
+ *   |
+ *   |-# git_midx_entry_find
+ *   |   Search for the SHA1 oid in the multi-pack-index. See
+ *   |   <https://github.com/git/git/blob/master/Documentation/technical/pack-format.txt>
+ *   |   for specifics on the multi-pack-index format and how do we find
+ *   |   entries in it.
+ *   |
+ *   |-# git_pack_entry_find
+ *     | Check the index of an individual unindexed pack to see if the SHA1
+ *     | OID can be found. If we can find the offset to that SHA1 inside of the
+ *     | index, that means the object is contained inside of the packfile and
+ *     | we can stop searching. Before returning, we verify that the
+ *     | packfile behing the index we are searching still exists on disk.
+ *     |
+ *     |-# pack_entry_find_offset
+ *       | Mmap the actual index file to disk if it hasn't been opened
+ *       | yet, and run a binary search through it to find the OID.
+ *       | See <https://github.com/git/git/blob/master/Documentation/technical/pack-format.txt>
+ *       | for specifics on the Packfile Index format and how do we find
+ *       | entries in it.
+ *       |
+ *       |-# pack_index_open
+ *         | Guess the name of the index based on the full path to the
+ *         | packfile, open it and verify its contents. Only if the index
+ *         | has not been opened already.
+ *         |
+ *         |-# pack_index_check
+ *             Mmap the index file and do a quick run through the header
+ *             to guess the index version (right now we support v1 and v2),
+ *             and to verify that the size of the index makes sense.
  *
  *
  *
  *	Chapter 3: The neverending story...
  *	A standard packed `lookup` query for an OID
  *	--------------------------------------------------
- *	TODO
+ *
+ * # pack_backend__read / pack_backend__read_prefix
+ * | Check if the given SHA1 oid (or a SHA1 oid prefix) exists in any of the
+ * | packs that have been loaded for our ODB. If it does, open the packfile and
+ * | read from it.
+ * |
+ * |-# git_packfile_unpack
+ *     Armed with a packfile and the offset within it, we can finally unpack
+ *     the object pointed at by the SHA1 oid. This involves mmapping part of
+ *     the `.pack` file, and uncompressing the object within it (if it is
+ *     stored in the undelfitied representation), or finding a base object and
+ *     applying some deltas to its uncompressed representation (if it is stored
+ *     in the deltified representation). See
+ *     <https://github.com/git/git/blob/master/Documentation/technical/pack-format.txt>
+ *     for specifics on the Packfile format and how do we read from it.
  *
  */
 
@@ -140,6 +168,8 @@ static int packfile_sort__cb(const void *a_, const void *b_);
 
 static int packfile_load__cb(void *_data, git_buf *path);
 
+static int packfile_byname_search_cmp(const void *path, const void *pack_entry);
+
 static int pack_entry_find(struct git_pack_entry *e,
 	struct pack_backend *backend, const git_oid *oid);
 
@@ -163,6 +193,14 @@ static int pack_entry_find_prefix(
  *
  ***********************************************************/
 
+static int packfile_byname_search_cmp(const void *path_, const void *p_)
+{
+	const git_buf *path = (const git_buf *)path_;
+	const struct git_pack_file *p = (const struct git_pack_file *)p_;
+
+	return strncmp(p->pack_name, git_buf_cstr(path), git_buf_len(path));
+}
+
 static int packfile_sort__cb(const void *a_, const void *b_)
 {
 	const struct git_pack_file *a = a_;
@@ -198,20 +236,20 @@ static int packfile_load__cb(void *data, git_buf *path)
 	struct pack_backend *backend = data;
 	struct git_pack_file *pack;
 	const char *path_str = git_buf_cstr(path);
-	size_t i, cmp_len = git_buf_len(path);
+	git_buf index_prefix = GIT_BUF_INIT;
+	size_t cmp_len = git_buf_len(path);
 	int error;
 
 	if (cmp_len <= strlen(".idx") || git__suffixcmp(path_str, ".idx") != 0)
 		return 0; /* not an index */
 
 	cmp_len -= strlen(".idx");
+	git_buf_attach_notowned(&index_prefix, path_str, cmp_len);
 
-	for (i = 0; i < backend->packs.length; ++i) {
-		struct git_pack_file *p = git_vector_get(&backend->packs, i);
-
-		if (strncmp(p->pack_name, path_str, cmp_len) == 0)
-			return 0;
-	}
+	if (git_vector_search2(NULL, &backend->midx_packs, packfile_byname_search_cmp, &index_prefix) == 0)
+		return 0;
+	if (git_vector_search2(NULL, &backend->packs, packfile_byname_search_cmp, &index_prefix) == 0)
+		return 0;
 
 	error = git_mwindow_get_pack(&pack, path->ptr);
 
@@ -228,22 +266,26 @@ static int packfile_load__cb(void *data, git_buf *path)
 
 }
 
-static int pack_entry_find_inner(
-	struct git_pack_entry *e,
-	struct pack_backend *backend,
-	const git_oid *oid,
-	struct git_pack_file *last_found)
+static int pack_entry_find(struct git_pack_entry *e, struct pack_backend *backend, const git_oid *oid)
 {
+	struct git_pack_file *last_found = backend->last_found, *p;
+	git_midx_entry midx_entry;
 	size_t i;
 
+	if (backend->midx &&
+		git_midx_entry_find(&midx_entry, backend->midx, oid, GIT_OID_HEXSZ) == 0 &&
+		midx_entry.pack_index < git_vector_length(&backend->midx_packs)) {
+		e->offset = midx_entry.offset;
+		git_oid_cpy(&e->sha1, &midx_entry.sha1);
+		e->p = git_vector_get(&backend->midx_packs, midx_entry.pack_index);
+		return 0;
+	}
+
 	if (last_found &&
 		git_pack_entry_find(e, last_found, oid, GIT_OID_HEXSZ) == 0)
 		return 0;
 
-	for (i = 0; i < backend->packs.length; ++i) {
-		struct git_pack_file *p;
-
-		p = git_vector_get(&backend->packs, i);
+	git_vector_foreach(&backend->packs, i, p) {
 		if (p == last_found)
 			continue;
 
@@ -253,20 +295,6 @@ static int pack_entry_find_inner(
 		}
 	}
 
-	return -1;
-}
-
-static int pack_entry_find(struct git_pack_entry *e, struct pack_backend *backend, const git_oid *oid)
-{
-	struct git_pack_file *last_found = backend->last_found;
-
-	if (backend->last_found &&
-		git_pack_entry_find(e, backend->last_found, oid, GIT_OID_HEXSZ) == 0)
-		return 0;
-
-	if (!pack_entry_find_inner(e, backend, oid, last_found))
-		return 0;
-
 	return git_odb__error_notfound(
 		"failed to find pack entry", oid, GIT_OID_HEXSZ);
 }
@@ -281,22 +309,35 @@ static int pack_entry_find_prefix(
 	size_t i;
 	git_oid found_full_oid = {{0}};
 	bool found = false;
-	struct git_pack_file *last_found = backend->last_found;
+	struct git_pack_file *last_found = backend->last_found, *p;
+	git_midx_entry midx_entry;
+
+	if (backend->midx) {
+		error = git_midx_entry_find(&midx_entry, backend->midx, short_oid, len);
+		if (error == GIT_EAMBIGUOUS)
+			return error;
+		if (!error && midx_entry.pack_index < git_vector_length(&backend->midx_packs)) {
+			e->offset = midx_entry.offset;
+			git_oid_cpy(&e->sha1, &midx_entry.sha1);
+			e->p = git_vector_get(&backend->midx_packs, midx_entry.pack_index);
+			git_oid_cpy(&found_full_oid, &e->sha1);
+			found = true;
+		}
+	}
 
 	if (last_found) {
 		error = git_pack_entry_find(e, last_found, short_oid, len);
 		if (error == GIT_EAMBIGUOUS)
 			return error;
 		if (!error) {
+			if (found && git_oid_cmp(&e->sha1, &found_full_oid))
+				return git_odb__error_ambiguous("found multiple pack entries");
 			git_oid_cpy(&found_full_oid, &e->sha1);
 			found = true;
 		}
 	}
 
-	for (i = 0; i < backend->packs.length; ++i) {
-		struct git_pack_file *p;
-
-		p = git_vector_get(&backend->packs, i);
+	git_vector_foreach(&backend->packs, i, p) {
 		if (p == last_found)
 			continue;
 
@@ -319,6 +360,139 @@ static int pack_entry_find_prefix(
 		return 0;
 }
 
+/***********************************************************
+ *
+ * MULTI-PACK-INDEX SUPPORT
+ *
+ * Functions needed to support the multi-pack-index.
+ *
+ ***********************************************************/
+
+/*
+ * Remove the multi-pack-index, and move all midx_packs to packs.
+ */
+static int remove_multi_pack_index(struct pack_backend *backend)
+{
+	size_t i, j = git_vector_length(&backend->packs);
+	struct pack_backend *p;
+	int error = git_vector_size_hint(
+			&backend->packs,
+			j + git_vector_length(&backend->midx_packs));
+	if (error < 0)
+		return error;
+
+	git_vector_foreach(&backend->midx_packs, i, p)
+		git_vector_set(NULL, &backend->packs, j++, p);
+	git_vector_clear(&backend->midx_packs);
+
+	git_midx_free(backend->midx);
+	backend->midx = NULL;
+
+	return 0;
+}
+
+/*
+ * Loads a single .pack file referred to by the multi-pack-index. These must
+ * match the order in which they are declared in the multi-pack-index file,
+ * since these files are referred to by their index.
+ */
+static int process_multi_pack_index_pack(
+		struct pack_backend *backend,
+		size_t i,
+		const char *packfile_name)
+{
+	int error;
+	struct git_pack_file *pack;
+	size_t found_position;
+	git_buf pack_path = GIT_BUF_INIT, index_prefix = GIT_BUF_INIT;
+
+	error = git_buf_joinpath(&pack_path, backend->pack_folder, packfile_name);
+	if (error < 0)
+		return error;
+
+	/* This is ensured by midx_parse_packfile_name() */
+	if (git_buf_len(&pack_path) <= strlen(".idx") || git__suffixcmp(git_buf_cstr(&pack_path), ".idx") != 0)
+		return git_odb__error_notfound("midx file contained a non-index", NULL, 0);
+
+	git_buf_attach_notowned(&index_prefix, git_buf_cstr(&pack_path), git_buf_len(&pack_path) - strlen(".idx"));
+
+	if (git_vector_search2(&found_position, &backend->packs, packfile_byname_search_cmp, &index_prefix) == 0) {
+		/* Pack was found in the packs list. Moving it to the midx_packs list. */
+		git_buf_dispose(&pack_path);
+		git_vector_set(NULL, &backend->midx_packs, i, git_vector_get(&backend->packs, found_position));
+		git_vector_remove(&backend->packs, found_position);
+		return 0;
+	}
+
+	/* Pack was not found. Allocate a new one. */
+	error = git_mwindow_get_pack(&pack, git_buf_cstr(&pack_path));
+	git_buf_dispose(&pack_path);
+	if (error < 0)
+		return error;
+
+	git_vector_set(NULL, &backend->midx_packs, i, pack);
+	return 0;
+}
+
+/*
+ * Reads the multi-pack-index. If this fails for whatever reason, the
+ * multi-pack-index object is freed, and all the packfiles that are related to
+ * it are moved to the unindexed packfiles vector.
+ */
+static int refresh_multi_pack_index(struct pack_backend *backend)
+{
+	int error;
+	git_buf midx_path = GIT_BUF_INIT;
+	const char *packfile_name;
+	size_t i;
+
+	error = git_buf_joinpath(&midx_path, backend->pack_folder, "multi-pack-index");
+	if (error < 0)
+		return error;
+
+	/*
+	 * Check whether the multi-pack-index has changed. If it has, close any
+	 * old multi-pack-index and move all the packfiles to the unindexed
+	 * packs. This is done to prevent losing any open packfiles in case
+	 * refreshing the new multi-pack-index fails, or the file is deleted.
+	 */
+	if (backend->midx) {
+		if (!git_midx_needs_refresh(backend->midx, git_buf_cstr(&midx_path))) {
+			git_buf_dispose(&midx_path);
+			return 0;
+		}
+		error = remove_multi_pack_index(backend);
+		if (error < 0) {
+			git_buf_dispose(&midx_path);
+			return error;
+		}
+	}
+
+	error = git_midx_open(&backend->midx, git_buf_cstr(&midx_path));
+	git_buf_dispose(&midx_path);
+	if (error < 0)
+		return error;
+
+	git_vector_resize_to(&backend->midx_packs, git_vector_length(&backend->midx->packfile_names));
+
+	git_vector_foreach(&backend->midx->packfile_names, i, packfile_name) {
+		error = process_multi_pack_index_pack(backend, i, packfile_name);
+		if (error < 0) {
+			/*
+			 * Something failed during reading multi-pack-index.
+			 * Restore the state of backend as if the
+			 * multi-pack-index was never there, and move all
+			 * packfiles that have been processed so far to the
+			 * unindexed packs.
+			 */
+			git_vector_resize_to(&backend->midx_packs, i);
+			remove_multi_pack_index(backend);
+			return error;
+		}
+	}
+
+	return 0;
+}
 
 /***********************************************************
  *
@@ -340,9 +514,16 @@ static int pack_backend__refresh(git_odb_backend *backend_)
 	if (p_stat(backend->pack_folder, &st) < 0 || !S_ISDIR(st.st_mode))
 		return git_odb__error_notfound("failed to refresh packfiles", NULL, 0);
 
-	git_buf_sets(&path, backend->pack_folder);
+	if (refresh_multi_pack_index(backend) < 0) {
+		/*
+		 * It is okay if this fails. We will just not use the
+		 * multi-pack-index in this case.
+		 */
+		git_error_clear();
+	}
 
 	/* reload all packs */
+	git_buf_sets(&path, backend->pack_folder);
 	error = git_path_direach(&path, 0, packfile_load__cb, backend);
 
 	git_buf_dispose(&path);
@@ -358,7 +539,10 @@ static int pack_backend__read_header(
 	struct git_pack_entry e;
 	int error;
 
-	assert(len_p && type_p && backend && oid);
+	GIT_ASSERT_ARG(len_p);
+	GIT_ASSERT_ARG(type_p);
+	GIT_ASSERT_ARG(backend);
+	GIT_ASSERT_ARG(oid);
 
 	if ((error = pack_entry_find(&e, (struct pack_backend *)backend, oid)) < 0)
 		return error;
@@ -469,13 +653,17 @@ static int pack_backend__foreach(git_odb_backend *_backend, git_odb_foreach_cb c
 	struct pack_backend *backend;
 	unsigned int i;
 
-	assert(_backend && cb);
+	GIT_ASSERT_ARG(_backend);
+	GIT_ASSERT_ARG(cb);
+
 	backend = (struct pack_backend *)_backend;
 
 	/* Make sure we know about the packfiles */
-	if ((error = pack_backend__refresh(_backend)) < 0)
+	if ((error = pack_backend__refresh(_backend)) != 0)
 		return error;
 
+	if (backend->midx && (error = git_midx_foreach_entry(backend->midx, cb, data)) != 0)
+		return error;
 	git_vector_foreach(&backend->packs, i, p) {
 		if ((error = git_pack_foreach_entry(p, cb, data)) != 0)
 			return error;
@@ -488,7 +676,7 @@ static int pack_backend__writepack_append(struct git_odb_writepack *_writepack,
 {
 	struct pack_writepack *writepack = (struct pack_writepack *)_writepack;
 
-	assert(writepack);
+	GIT_ASSERT_ARG(writepack);
 
 	return git_indexer_append(writepack->indexer, data, size, stats);
 }
@@ -497,16 +685,19 @@ static int pack_backend__writepack_commit(struct git_odb_writepack *_writepack,
 {
 	struct pack_writepack *writepack = (struct pack_writepack *)_writepack;
 
-	assert(writepack);
+	GIT_ASSERT_ARG(writepack);
 
 	return git_indexer_commit(writepack->indexer, stats);
 }
 
 static void pack_backend__writepack_free(struct git_odb_writepack *_writepack)
 {
-	struct pack_writepack *writepack = (struct pack_writepack *)_writepack;
+	struct pack_writepack *writepack;
 
-	assert(writepack);
+	if (!_writepack)
+		return;
+
+	writepack = (struct pack_writepack *)_writepack;
 
 	git_indexer_free(writepack->indexer);
 	git__free(writepack);
@@ -522,7 +713,8 @@ static int pack_backend__writepack(struct git_odb_writepack **out,
 	struct pack_backend *backend;
 	struct pack_writepack *writepack;
 
-	assert(out && _backend);
+	GIT_ASSERT_ARG(out);
+	GIT_ASSERT_ARG(_backend);
 
 	*out = NULL;
 
@@ -550,20 +742,99 @@ static int pack_backend__writepack(struct git_odb_writepack **out,
 	return 0;
 }
 
+static int get_idx_path(
+		git_buf *idx_path,
+		struct pack_backend *backend,
+		struct git_pack_file *p)
+{
+	size_t path_len;
+	int error;
+
+	error = git_path_prettify(idx_path, p->pack_name, backend->pack_folder);
+	if (error < 0)
+		return error;
+	path_len = git_buf_len(idx_path);
+	if (path_len <= strlen(".pack") || git__suffixcmp(git_buf_cstr(idx_path), ".pack") != 0)
+		return git_odb__error_notfound("packfile does not end in .pack", NULL, 0);
+	path_len -= strlen(".pack");
+	error = git_buf_splice(idx_path, path_len, strlen(".pack"), ".idx", strlen(".idx"));
+	if (error < 0)
+		return error;
+
+	return 0;
+}
+
+static int pack_backend__writemidx(git_odb_backend *_backend)
+{
+	struct pack_backend *backend;
+	git_midx_writer *w = NULL;
+	struct git_pack_file *p;
+	size_t i;
+	int error = 0;
+
+	GIT_ASSERT_ARG(_backend);
+
+	backend = (struct pack_backend *)_backend;
+
+	error = git_midx_writer_new(&w, backend->pack_folder);
+	if (error < 0)
+		return error;
+
+	git_vector_foreach(&backend->midx_packs, i, p) {
+		git_buf idx_path = GIT_BUF_INIT;
+		error = get_idx_path(&idx_path, backend, p);
+		if (error < 0)
+			goto cleanup;
+		error = git_midx_writer_add(w, git_buf_cstr(&idx_path));
+		git_buf_dispose(&idx_path);
+		if (error < 0)
+			goto cleanup;
+	}
+	git_vector_foreach(&backend->packs, i, p) {
+		git_buf idx_path = GIT_BUF_INIT;
+		error = get_idx_path(&idx_path, backend, p);
+		if (error < 0)
+			goto cleanup;
+		error = git_midx_writer_add(w, git_buf_cstr(&idx_path));
+		git_buf_dispose(&idx_path);
+		if (error < 0)
+			goto cleanup;
+	}
+
+	/*
+	 * Invalidate the previous midx before writing the new one.
+	 */
+	error = remove_multi_pack_index(backend);
+	if (error < 0)
+		goto cleanup;
+	error = git_midx_writer_commit(w);
+	if (error < 0)
+		goto cleanup;
+	error = refresh_multi_pack_index(backend);
+
+cleanup:
+	git_midx_writer_free(w);
+	return error;
+}
+
 static void pack_backend__free(git_odb_backend *_backend)
 {
 	struct pack_backend *backend;
+	struct git_pack_file *p;
 	size_t i;
 
-	assert(_backend);
+	if (!_backend)
+		return;
 
 	backend = (struct pack_backend *)_backend;
 
-	for (i = 0; i < backend->packs.length; ++i) {
-		struct git_pack_file *p = git_vector_get(&backend->packs, i);
+	git_vector_foreach(&backend->midx_packs, i, p)
+		git_mwindow_put_pack(p);
+	git_vector_foreach(&backend->packs, i, p)
 		git_mwindow_put_pack(p);
-	}
 
+	git_midx_free(backend->midx);
+	git_vector_free(&backend->midx_packs);
 	git_vector_free(&backend->packs);
 	git__free(backend->pack_folder);
 	git__free(backend);
@@ -574,7 +845,12 @@ static int pack_backend__alloc(struct pack_backend **out, size_t initial_size)
 	struct pack_backend *backend = git__calloc(1, sizeof(struct pack_backend));
 	GIT_ERROR_CHECK_ALLOC(backend);
 
+	if (git_vector_init(&backend->midx_packs, 0, NULL) < 0) {
+		git__free(backend);
+		return -1;
+	}
 	if (git_vector_init(&backend->packs, initial_size, packfile_sort__cb) < 0) {
+		git_vector_free(&backend->midx_packs);
 		git__free(backend);
 		return -1;
 	}
@@ -589,6 +865,7 @@ static int pack_backend__alloc(struct pack_backend **out, size_t initial_size)
 	backend->parent.refresh = &pack_backend__refresh;
 	backend->parent.foreach = &pack_backend__foreach;
 	backend->parent.writepack = &pack_backend__writepack;
+	backend->parent.writemidx = &pack_backend__writemidx;
 	backend->parent.freshen = &pack_backend__freshen;
 	backend->parent.free = &pack_backend__free;