rugged 0.19.0 → 0.28.0

data/vendor/libgit2/src/index.c

@@ -5,17 +5,21 @@
  * a Linking Exception. For full terms see the included COPYING file.
  */
 
+#include "index.h"
+
 #include <stddef.h>
 
-#include "common.h"
 #include "repository.h"
-#include "index.h"
 #include "tree.h"
 #include "tree-cache.h"
 #include "hash.h"
 #include "iterator.h"
 #include "pathspec.h"
 #include "ignore.h"
+#include "blob.h"
+#include "idxmap.h"
+#include "diff.h"
+#include "varint.h"
 
 #include "git2/odb.h"
 #include "git2/oid.h"
@@ -23,17 +27,43 @@
 #include "git2/config.h"
 #include "git2/sys/index.h"
 
-#define entry_size(type,len) ((offsetof(type, path) + (len) + 8) & ~7)
-#define short_entry_size(len) entry_size(struct entry_short, len)
-#define long_entry_size(len) entry_size(struct entry_long, len)
+#define INSERT_IN_MAP_EX(idx, map, e, err) do {				\
+		if ((idx)->ignore_case)					\
+			git_idxmap_icase_insert((git_idxmap_icase *) (map), (e), (e), (err)); \
+		else							\
+			git_idxmap_insert((map), (e), (e), (err));	\
+	} while (0)
+
+#define INSERT_IN_MAP(idx, e, err) INSERT_IN_MAP_EX(idx, (idx)->entries_map, e, err)
+
+#define LOOKUP_IN_MAP(p, idx, k) do {					\
+		if ((idx)->ignore_case)					\
+			(p) = git_idxmap_icase_lookup_index((git_idxmap_icase *) index->entries_map, (k)); \
+		else							\
+			(p) = git_idxmap_lookup_index(index->entries_map, (k)); \
+	} while (0)
+
+#define DELETE_IN_MAP(idx, e) do {					\
+		if ((idx)->ignore_case)					\
+			git_idxmap_icase_delete((git_idxmap_icase *) (idx)->entries_map, (e)); \
+		else							\
+			git_idxmap_delete((idx)->entries_map, (e));	\
+	} while (0)
+
+static int index_apply_to_wd_diff(git_index *index, int action, const git_strarray *paths,
+				  unsigned int flags,
+				  git_index_matched_path_cb cb, void *payload);
 
 #define minimal_entry_size (offsetof(struct entry_short, path))
 
 static const size_t INDEX_FOOTER_SIZE = GIT_OID_RAWSZ;
 static const size_t INDEX_HEADER_SIZE = 12;
 
-static const unsigned int INDEX_VERSION_NUMBER = 2;
+static const unsigned int INDEX_VERSION_NUMBER_DEFAULT = 2;
+static const unsigned int INDEX_VERSION_NUMBER_LB = 2;
 static const unsigned int INDEX_VERSION_NUMBER_EXT = 3;
+static const unsigned int INDEX_VERSION_NUMBER_COMP = 4;
+static const unsigned int INDEX_VERSION_NUMBER_UB = 4;
 
 static const unsigned int INDEX_HEADER_SIG = 0x44495243;
 static const char INDEX_EXT_TREECACHE_SIG[] = {'T', 'R', 'E', 'E'};
@@ -89,76 +119,101 @@ struct entry_long {
 
 struct entry_srch_key {
 	const char *path;
+	size_t pathlen;
 	int stage;
 };
 
+struct entry_internal {
+	git_index_entry entry;
+	size_t pathlen;
+	char path[GIT_FLEX_ARRAY];
+};
+
+struct reuc_entry_internal {
+	git_index_reuc_entry entry;
+	size_t pathlen;
+	char path[GIT_FLEX_ARRAY];
+};
+
+bool git_index__enforce_unsaved_safety = false;
+
 /* local declarations */
-static size_t read_extension(git_index *index, const char *buffer, size_t buffer_size);
-static size_t read_entry(git_index_entry *dest, const void *buffer, size_t buffer_size);
+static int read_extension(size_t *read_len, git_index *index, const char *buffer, size_t buffer_size);
 static int read_header(struct index_header *dest, const void *buffer);
 
 static int parse_index(git_index *index, const char *buffer, size_t buffer_size);
 static bool is_index_extended(git_index *index);
-static int write_index(git_index *index, git_filebuf *file);
-
-static int index_find(size_t *at_pos, git_index *index, const char *path, int stage);
+static int write_index(git_oid *checksum, git_index *index, git_filebuf *file);
 
 static void index_entry_free(git_index_entry *entry);
 static void index_entry_reuc_free(git_index_reuc_entry *reuc);
 
-static int index_srch(const void *key, const void *array_member)
+int git_index_entry_srch(const void *key, const void *array_member)
 {
 	const struct entry_srch_key *srch_key = key;
-	const git_index_entry *entry = array_member;
-	int ret;
-
-	ret = strcmp(srch_key->path, entry->path);
+	const struct entry_internal *entry = array_member;
+	int cmp;
+	size_t len1, len2, len;
+
+	len1 = srch_key->pathlen;
+	len2 = entry->pathlen;
+	len = len1 < len2 ? len1 : len2;
+
+	cmp = memcmp(srch_key->path, entry->path, len);
+	if (cmp)
+		return cmp;
+	if (len1 < len2)
+		return -1;
+	if (len1 > len2)
+		return 1;
 
-	if (ret == 0)
-		ret = srch_key->stage - GIT_IDXENTRY_STAGE(entry);
+	if (srch_key->stage != GIT_INDEX_STAGE_ANY)
+		return srch_key->stage - GIT_INDEX_ENTRY_STAGE(&entry->entry);
 
-	return ret;
+	return 0;
 }
 
-static int index_isrch(const void *key, const void *array_member)
+int git_index_entry_isrch(const void *key, const void *array_member)
 {
 	const struct entry_srch_key *srch_key = key;
-	const git_index_entry *entry = array_member;
-	int ret;
+	const struct entry_internal *entry = array_member;
+	int cmp;
+	size_t len1, len2, len;
 
-	ret = strcasecmp(srch_key->path, entry->path);
+	len1 = srch_key->pathlen;
+	len2 = entry->pathlen;
+	len = len1 < len2 ? len1 : len2;
 
-	if (ret == 0)
-		ret = srch_key->stage - GIT_IDXENTRY_STAGE(entry);
+	cmp = strncasecmp(srch_key->path, entry->path, len);
 
-	return ret;
-}
+	if (cmp)
+		return cmp;
+	if (len1 < len2)
+		return -1;
+	if (len1 > len2)
+		return 1;
 
-static int index_cmp_path(const void *a, const void *b)
-{
-	return strcmp((const char *)a, (const char *)b);
-}
+	if (srch_key->stage != GIT_INDEX_STAGE_ANY)
+		return srch_key->stage - GIT_INDEX_ENTRY_STAGE(&entry->entry);
 
-static int index_icmp_path(const void *a, const void *b)
-{
-	return strcasecmp((const char *)a, (const char *)b);
+	return 0;
 }
 
-static int index_srch_path(const void *path, const void *array_member)
+static int index_entry_srch_path(const void *path, const void *array_member)
 {
 	const git_index_entry *entry = array_member;
 
 	return strcmp((const char *)path, entry->path);
 }
 
-static int index_isrch_path(const void *path, const void *array_member)
+static int index_entry_isrch_path(const void *path, const void *array_member)
 {
 	const git_index_entry *entry = array_member;
 
 	return strcasecmp((const char *)path, entry->path);
 }
 
-static int index_cmp(const void *a, const void *b)
+int git_index_entry_cmp(const void *a, const void *b)
 {
 	int diff;
 	const git_index_entry *entry_a = a;
@@ -167,12 +222,12 @@ static int index_cmp(const void *a, const void *b)
 	diff = strcmp(entry_a->path, entry_b->path);
 
 	if (diff == 0)
-		diff = (GIT_IDXENTRY_STAGE(entry_a) - GIT_IDXENTRY_STAGE(entry_b));
+		diff = (GIT_INDEX_ENTRY_STAGE(entry_a) - GIT_INDEX_ENTRY_STAGE(entry_b));
 
 	return diff;
 }
 
-static int index_icmp(const void *a, const void *b)
+int git_index_entry_icmp(const void *a, const void *b)
 {
 	int diff;
 	const git_index_entry *entry_a = a;
@@ -181,7 +236,7 @@ static int index_icmp(const void *a, const void *b)
 	diff = strcasecmp(entry_a->path, entry_b->path);
 
 	if (diff == 0)
-		diff = (GIT_IDXENTRY_STAGE(entry_a) - GIT_IDXENTRY_STAGE(entry_b));
+		diff = (GIT_INDEX_ENTRY_STAGE(entry_a) - GIT_INDEX_ENTRY_STAGE(entry_b));
 
 	return diff;
 }
@@ -261,7 +316,21 @@ static int reuc_icmp(const void *a, const void *b)
 	return strcasecmp(info_a->path, info_b->path);
 }
 
-static unsigned int index_create_mode(unsigned int mode)
+static void index_entry_reuc_free(git_index_reuc_entry *reuc)
+{
+	git__free(reuc);
+}
+
+static void index_entry_free(git_index_entry *entry)
+{
+	if (!entry)
+		return;
+
+	memset(&entry->id, 0, sizeof(entry->id));
+	git__free(entry);
+}
+
+unsigned int git_index__create_mode(unsigned int mode)
 {
 	if (S_ISLNK(mode))
 		return S_IFLNK;
@@ -269,7 +338,7 @@ static unsigned int index_create_mode(unsigned int mode)
 	if (S_ISDIR(mode) || (mode & S_IFMT) == (S_IFLNK | S_IFDIR))
 		return (S_IFLNK | S_IFDIR);
 
-	return S_IFREG | ((mode & 0100) ? 0755 : 0644);
+	return S_IFREG | GIT_PERMS_CANONICAL(mode);
 }
 
 static unsigned int index_merge_mode(
@@ -281,24 +350,52 @@ static unsigned int index_merge_mode(
 
 	if (index->distrust_filemode && S_ISREG(mode))
 		return (existing && S_ISREG(existing->mode)) ?
-			existing->mode : index_create_mode(0666);
+			existing->mode : git_index__create_mode(0666);
+
+	return git_index__create_mode(mode);
+}
+
+GIT_INLINE(int) index_find_in_entries(
+	size_t *out, git_vector *entries, git_vector_cmp entry_srch,
+	const char *path, size_t path_len, int stage)
+{
+	struct entry_srch_key srch_key;
+	srch_key.path = path;
+	srch_key.pathlen = !path_len ? strlen(path) : path_len;
+	srch_key.stage = stage;
+	return git_vector_bsearch2(out, entries, entry_srch, &srch_key);
+}
 
-	return index_create_mode(mode);
+GIT_INLINE(int) index_find(
+	size_t *out, git_index *index,
+	const char *path, size_t path_len, int stage)
+{
+	git_vector_sort(&index->entries);
+
+	return index_find_in_entries(
+		out, &index->entries, index->entries_search, path, path_len, stage);
 }
 
 void git_index__set_ignore_case(git_index *index, bool ignore_case)
 {
 	index->ignore_case = ignore_case;
 
-	index->entries_cmp_path = ignore_case ? index_icmp_path : index_cmp_path;
-	index->entries_search = ignore_case ? index_isrch : index_srch;
-	index->entries_search_path = ignore_case ? index_isrch_path : index_srch_path;
+	if (ignore_case) {
+		index->entries_cmp_path    = git__strcasecmp_cb;
+		index->entries_search      = git_index_entry_isrch;
+		index->entries_search_path = index_entry_isrch_path;
+		index->reuc_search         = reuc_isrch;
+	} else {
+		index->entries_cmp_path    = git__strcmp_cb;
+		index->entries_search      = git_index_entry_srch;
+		index->entries_search_path = index_entry_srch_path;
+		index->reuc_search         = reuc_srch;
+	}
 
-	git_vector_set_cmp(&index->entries, ignore_case ? index_icmp : index_cmp);
+	git_vector_set_cmp(&index->entries,
+		ignore_case ? git_index_entry_icmp : git_index_entry_cmp);
 	git_vector_sort(&index->entries);
 
-	index->reuc_search = ignore_case ? reuc_isrch : reuc_srch;
-
 	git_vector_set_cmp(&index->reuc, ignore_case ? reuc_icmp : reuc_cmp);
 	git_vector_sort(&index->reuc);
 }
@@ -306,35 +403,50 @@ void git_index__set_ignore_case(git_index *index, bool ignore_case)
 int git_index_open(git_index **index_out, const char *index_path)
 {
 	git_index *index;
+	int error = -1;
 
 	assert(index_out);
 
 	index = git__calloc(1, sizeof(git_index));
-	GITERR_CHECK_ALLOC(index);
+	GIT_ERROR_CHECK_ALLOC(index);
+
+	git_pool_init(&index->tree_pool, 1);
 
 	if (index_path != NULL) {
 		index->index_file_path = git__strdup(index_path);
-		GITERR_CHECK_ALLOC(index->index_file_path);
+		if (!index->index_file_path)
+			goto fail;
 
 		/* Check if index file is stored on disk already */
 		if (git_path_exists(index->index_file_path) == true)
 			index->on_disk = 1;
 	}
 
-	if (git_vector_init(&index->entries, 32, index_cmp) < 0 ||
-		git_vector_init(&index->names, 32, conflict_name_cmp) < 0 ||
-		git_vector_init(&index->reuc, 32, reuc_cmp) < 0)
-		return -1;
+	if (git_vector_init(&index->entries, 32, git_index_entry_cmp) < 0 ||
+		git_idxmap_alloc(&index->entries_map) < 0 ||
+		git_vector_init(&index->names, 8, conflict_name_cmp) < 0 ||
+		git_vector_init(&index->reuc, 8, reuc_cmp) < 0 ||
+		git_vector_init(&index->deleted, 8, git_index_entry_cmp) < 0)
+		goto fail;
 
-	index->entries_cmp_path = index_cmp_path;
-	index->entries_search = index_srch;
-	index->entries_search_path = index_srch_path;
+	index->entries_cmp_path = git__strcmp_cb;
+	index->entries_search = git_index_entry_srch;
+	index->entries_search_path = index_entry_srch_path;
 	index->reuc_search = reuc_srch;
+	index->version = INDEX_VERSION_NUMBER_DEFAULT;
+
+	if (index_path != NULL && (error = git_index_read(index, true)) < 0)
+		goto fail;
 
 	*index_out = index;
 	GIT_REFCOUNT_INC(index);
 
-	return (index_path != NULL) ? git_index_read(index) : 0;
+	return 0;
+
+fail:
+	git_pool_clear(&index->tree_pool);
+	git_index_free(index);
+	return error;
 }
 
 int git_index_new(git_index **out)
@@ -344,10 +456,17 @@ int git_index_new(git_index **out)
 
 static void index_free(git_index *index)
 {
+	/* index iterators increment the refcount of the index, so if we
+	 * get here then there should be no outstanding iterators.
+	 */
+	assert(!git_atomic_get(&index->readers));
+
 	git_index_clear(index);
+	git_idxmap_free(index->entries_map);
 	git_vector_free(&index->entries);
 	git_vector_free(&index->names);
 	git_vector_free(&index->reuc);
+	git_vector_free(&index->deleted);
 
 	git__free(index->index_file_path);
 
@@ -363,54 +482,93 @@ void git_index_free(git_index *index)
 	GIT_REFCOUNT_DEC(index, index_free);
 }
 
-static void index_entries_free(git_vector *entries)
+/* call with locked index */
+static void index_free_deleted(git_index *index)
 {
+	int readers = (int)git_atomic_get(&index->readers);
 	size_t i;
 
-	for (i = 0; i < entries->length; ++i) {
-		git_index_entry *e = git_vector_get(entries, i);
-		git__free(e->path);
-		git__free(e);
+	if (readers > 0 || !index->deleted.length)
+		return;
+
+	for (i = 0; i < index->deleted.length; ++i) {
+		git_index_entry *ie = git__swap(index->deleted.contents[i], NULL);
+		index_entry_free(ie);
 	}
 
-	git_vector_clear(entries);
+	git_vector_clear(&index->deleted);
 }
 
-void git_index_clear(git_index *index)
+/* call with locked index */
+static int index_remove_entry(git_index *index, size_t pos)
 {
+	int error = 0;
+	git_index_entry *entry = git_vector_get(&index->entries, pos);
+
+	if (entry != NULL) {
+		git_tree_cache_invalidate_path(index->tree, entry->path);
+		DELETE_IN_MAP(index, entry);
+	}
+
+	error = git_vector_remove(&index->entries, pos);
+
+	if (!error) {
+		if (git_atomic_get(&index->readers) > 0) {
+			error = git_vector_insert(&index->deleted, entry);
+		} else {
+			index_entry_free(entry);
+		}
+
+		index->dirty = 1;
+	}
+
+	return error;
+}
+
+int git_index_clear(git_index *index)
+{
+	int error = 0;
+
 	assert(index);
 
-	index_entries_free(&index->entries);
+	index->dirty = 1;
+	index->tree = NULL;
+	git_pool_clear(&index->tree_pool);
+
+	git_idxmap_clear(index->entries_map);
+	while (!error && index->entries.length > 0)
+		error = index_remove_entry(index, index->entries.length - 1);
+	index_free_deleted(index);
+
 	git_index_reuc_clear(index);
 	git_index_name_clear(index);
 
 	git_futils_filestamp_set(&index->stamp, NULL);
 
-	git_tree_cache_free(index->tree);
-	index->tree = NULL;
+	return error;
 }
 
 static int create_index_error(int error, const char *msg)
 {
-	giterr_set(GITERR_INDEX, msg);
+	git_error_set_str(GIT_ERROR_INDEX, msg);
 	return error;
 }
 
-int git_index_set_caps(git_index *index, unsigned int caps)
+int git_index_set_caps(git_index *index, int caps)
 {
-	int old_ignore_case;
+	unsigned int old_ignore_case;
 
 	assert(index);
 
 	old_ignore_case = index->ignore_case;
 
-	if (caps == GIT_INDEXCAP_FROM_OWNER) {
+	if (caps == GIT_INDEX_CAPABILITY_FROM_OWNER) {
 		git_repository *repo = INDEX_OWNER(index);
 		int val;
 
 		if (!repo)
 			return create_index_error(
-				-1, "Cannot access repository to set index caps");
+				-1, "cannot access repository to set index caps");
 
 		if (!git_repository__cvar(&val, repo, GIT_CVAR_IGNORECASE))
 			index->ignore_case = (val != 0);
@@ -420,91 +578,240 @@ int git_index_set_caps(git_index *index, unsigned int caps)
 			index->no_symlinks = (val == 0);
 	}
 	else {
-		index->ignore_case = ((caps & GIT_INDEXCAP_IGNORE_CASE) != 0);
-		index->distrust_filemode = ((caps & GIT_INDEXCAP_NO_FILEMODE) != 0);
-		index->no_symlinks = ((caps & GIT_INDEXCAP_NO_SYMLINKS) != 0);
+		index->ignore_case = ((caps & GIT_INDEX_CAPABILITY_IGNORE_CASE) != 0);
+		index->distrust_filemode = ((caps & GIT_INDEX_CAPABILITY_NO_FILEMODE) != 0);
+		index->no_symlinks = ((caps & GIT_INDEX_CAPABILITY_NO_SYMLINKS) != 0);
 	}
 
 	if (old_ignore_case != index->ignore_case) {
-		git_index__set_ignore_case(index, index->ignore_case);
+		git_index__set_ignore_case(index, (bool)index->ignore_case);
 	}
 
 	return 0;
 }
 
-unsigned int git_index_caps(const git_index *index)
+int git_index_caps(const git_index *index)
+{
+	return ((index->ignore_case ? GIT_INDEX_CAPABILITY_IGNORE_CASE : 0) |
+			(index->distrust_filemode ? GIT_INDEX_CAPABILITY_NO_FILEMODE : 0) |
+			(index->no_symlinks ? GIT_INDEX_CAPABILITY_NO_SYMLINKS : 0));
+}
+
+const git_oid *git_index_checksum(git_index *index)
+{
+	return &index->checksum;
+}
+
+/**
+ * Returns 1 for changed, 0 for not changed and <0 for errors
+ */
+static int compare_checksum(git_index *index)
 {
-	return ((index->ignore_case ? GIT_INDEXCAP_IGNORE_CASE : 0) |
-			(index->distrust_filemode ? GIT_INDEXCAP_NO_FILEMODE : 0) |
-			(index->no_symlinks ? GIT_INDEXCAP_NO_SYMLINKS : 0));
+	int fd;
+	ssize_t bytes_read;
+	git_oid checksum = {{ 0 }};
+
+	if ((fd = p_open(index->index_file_path, O_RDONLY)) < 0)
+		return fd;
+
+	if (p_lseek(fd, -20, SEEK_END) < 0) {
+		p_close(fd);
+		git_error_set(GIT_ERROR_OS, "failed to seek to end of file");
+		return -1;
+	}
+
+	bytes_read = p_read(fd, &checksum, GIT_OID_RAWSZ);
+	p_close(fd);
+
+	if (bytes_read < 0)
+		return -1;
+
+	return !!git_oid_cmp(&checksum, &index->checksum);
 }
 
-int git_index_read(git_index *index)
+int git_index_read(git_index *index, int force)
 {
 	int error = 0, updated;
 	git_buf buffer = GIT_BUF_INIT;
-	git_futils_filestamp stamp = {0};
+	git_futils_filestamp stamp = index->stamp;
 
 	if (!index->index_file_path)
 		return create_index_error(-1,
-			"Failed to read index: The index is in-memory only");
+			"failed to read index: The index is in-memory only");
+
+	index->on_disk = git_path_exists(index->index_file_path);
+
+	if (!index->on_disk) {
+		if (force && (error = git_index_clear(index)) < 0)
+			return error;
 
-	if (!index->on_disk || git_path_exists(index->index_file_path) == false) {
-		git_index_clear(index);
-		index->on_disk = 0;
+		index->dirty = 0;
 		return 0;
 	}
 
-	updated = git_futils_filestamp_check(&stamp, index->index_file_path);
-	if (updated <= 0)
+	if ((updated = git_futils_filestamp_check(&stamp, index->index_file_path) < 0) ||
+	    ((updated = compare_checksum(index)) < 0)) {
+		git_error_set(
+			GIT_ERROR_INDEX,
+			"failed to read index: '%s' no longer exists",
+			index->index_file_path);
 		return updated;
+	}
+
+	if (!updated && !force)
+		return 0;
 
 	error = git_futils_readbuffer(&buffer, index->index_file_path);
 	if (error < 0)
 		return error;
 
-	git_index_clear(index);
-	error = parse_index(index, buffer.ptr, buffer.size);
+	index->tree = NULL;
+	git_pool_clear(&index->tree_pool);
+
+	error = git_index_clear(index);
 
 	if (!error)
+		error = parse_index(index, buffer.ptr, buffer.size);
+
+	if (!error) {
 		git_futils_filestamp_set(&index->stamp, &stamp);
+		index->dirty = 0;
+	}
 
-	git_buf_free(&buffer);
+	git_buf_dispose(&buffer);
 	return error;
 }
 
-int git_index_write(git_index *index)
+int git_index_read_safely(git_index *index)
+{
+	if (git_index__enforce_unsaved_safety && index->dirty) {
+		git_error_set(GIT_ERROR_INDEX,
+			"the index has unsaved changes that would be overwritten by this operation");
+		return GIT_EINDEXDIRTY;
+	}
+
+	return git_index_read(index, false);
+}
+
+int git_index__changed_relative_to(
+	git_index *index, const git_oid *checksum)
+{
+	/* attempt to update index (ignoring errors) */
+	if (git_index_read(index, false) < 0)
+		git_error_clear();
+
+	return !!git_oid_cmp(&index->checksum, checksum);
+}
+
+static bool is_racy_entry(git_index *index, const git_index_entry *entry)
+{
+	/* Git special-cases submodules in the check */
+	if (S_ISGITLINK(entry->mode))
+		return false;
+
+	return git_index_entry_newer_than_index(entry, index);
+}
+
+/*
+ * Force the next diff to take a look at those entries which have the
+ * same timestamp as the current index.
+ */
+static int truncate_racily_clean(git_index *index)
 {
-	git_filebuf file = GIT_FILEBUF_INIT;
+	size_t i;
 	int error;
+	git_index_entry *entry;
+	git_diff_options diff_opts = GIT_DIFF_OPTIONS_INIT;
+	git_diff *diff = NULL;
+	git_vector paths = GIT_VECTOR_INIT;
+	git_diff_delta *delta;
 
-	if (!index->index_file_path)
-		return create_index_error(-1,
-			"Failed to read index: The index is in-memory only");
+	/* Nothing to do if there's no repo to talk about */
+	if (!INDEX_OWNER(index))
+		return 0;
 
-	git_vector_sort(&index->entries);
-	git_vector_sort(&index->reuc);
+	/* If there's no workdir, we can't know where to even check */
+	if (!git_repository_workdir(INDEX_OWNER(index)))
+		return 0;
 
-	if ((error = git_filebuf_open(
-			 &file, index->index_file_path, GIT_FILEBUF_HASH_CONTENTS)) < 0)
-		return error;
+	diff_opts.flags |= GIT_DIFF_INCLUDE_TYPECHANGE | GIT_DIFF_IGNORE_SUBMODULES | GIT_DIFF_DISABLE_PATHSPEC_MATCH;
+	git_vector_foreach(&index->entries, i, entry) {
+		if ((entry->flags_extended & GIT_INDEX_ENTRY_UPTODATE) == 0 &&
+			is_racy_entry(index, entry))
+			git_vector_insert(&paths, (char *)entry->path);
+	}
+
+	if (paths.length == 0)
+		goto done;
 
-	if ((error = write_index(index, &file)) < 0) {
-		git_filebuf_cleanup(&file);
+	diff_opts.pathspec.count = paths.length;
+	diff_opts.pathspec.strings = (char **)paths.contents;
+
+	if ((error = git_diff_index_to_workdir(&diff, INDEX_OWNER(index), index, &diff_opts)) < 0)
 		return error;
+
+	git_vector_foreach(&diff->deltas, i, delta) {
+		entry = (git_index_entry *)git_index_get_bypath(index, delta->old_file.path, 0);
+
+		/* Ensure that we have a stage 0 for this file (ie, it's not a
+		 * conflict), otherwise smudging it is quite pointless.
+		 */
+		if (entry) {
+			entry->file_size = 0;
+			index->dirty = 1;
+		}
 	}
 
-	if ((error = git_filebuf_commit(&file, GIT_INDEX_FILE_MODE)) < 0)
-		return error;
+done:
+	git_diff_free(diff);
+	git_vector_free(&paths);
+	return 0;
+}
 
-	error = git_futils_filestamp_check(&index->stamp, index->index_file_path);
-	if (error < 0)
-		return error;
+unsigned git_index_version(git_index *index)
+{
+	assert(index);
+
+	return index->version;
+}
+
+int git_index_set_version(git_index *index, unsigned int version)
+{
+	assert(index);
+
+	if (version < INDEX_VERSION_NUMBER_LB ||
+	    version > INDEX_VERSION_NUMBER_UB) {
+		git_error_set(GIT_ERROR_INDEX, "invalid version number");
+		return -1;
+	}
+
+	index->version = version;
 
-	index->on_disk = 1;
 	return 0;
 }
 
+int git_index_write(git_index *index)
+{
+	git_indexwriter writer = GIT_INDEXWRITER_INIT;
+	int error;
+
+	truncate_racily_clean(index);
+
+	if ((error = git_indexwriter_init(&writer, index)) == 0 &&
+		(error = git_indexwriter_commit(&writer)) == 0)
+		index->dirty = 0;
+
+	git_indexwriter_cleanup(&writer);
+
+	return error;
+}
+
+const char * git_index_path(const git_index *index)
+{
+	assert(index);
+	return index->index_file_path;
+}
+
 int git_index_write_tree(git_oid *oid, git_index *index)
 {
 	git_repository *repo;
@@ -515,12 +822,13 @@ int git_index_write_tree(git_oid *oid, git_index *index)
 
 	if (repo == NULL)
 		return create_index_error(-1, "Failed to write tree. "
-		  "The index file is not backed up by an existing repository");
+		  "the index file is not backed up by an existing repository");
 
 	return git_tree__write_index(oid, index, repo);
 }
 
-int git_index_write_tree_to(git_oid *oid, git_index *index, git_repository *repo)
+int git_index_write_tree_to(
+	git_oid *oid, git_index *index, git_repository *repo)
 {
 	assert(oid && index && repo);
 	return git_tree__write_index(oid, index, repo);
@@ -543,99 +851,172 @@ const git_index_entry *git_index_get_byindex(
 const git_index_entry *git_index_get_bypath(
 	git_index *index, const char *path, int stage)
 {
+	git_index_entry key = {{ 0 }};
 	size_t pos;
 
 	assert(index);
 
-	git_vector_sort(&index->entries);
+	key.path = path;
+	GIT_INDEX_ENTRY_STAGE_SET(&key, stage);
 
-	if (index_find(&pos, index, path, stage) < 0) {
-		giterr_set(GITERR_INDEX, "Index does not contain %s", path);
-		return NULL;
-	}
+	LOOKUP_IN_MAP(pos, index, &key);
+
+	if (git_idxmap_valid_index(index->entries_map, pos))
+		return git_idxmap_value_at(index->entries_map, pos);
 
-	return git_index_get_byindex(index, pos);
+	git_error_set(GIT_ERROR_INDEX, "index does not contain '%s'", path);
+	return NULL;
 }
 
-void git_index_entry__init_from_stat(git_index_entry *entry, struct stat *st)
+void git_index_entry__init_from_stat(
+	git_index_entry *entry, struct stat *st, bool trust_mode)
 {
-	entry->ctime.seconds = (git_time_t)st->st_ctime;
-	entry->mtime.seconds = (git_time_t)st->st_mtime;
-	/* entry->mtime.nanoseconds = st->st_mtimensec; */
-	/* entry->ctime.nanoseconds = st->st_ctimensec; */
+	entry->ctime.seconds = (int32_t)st->st_ctime;
+	entry->mtime.seconds = (int32_t)st->st_mtime;
+#if defined(GIT_USE_NSEC)
+	entry->mtime.nanoseconds = st->st_mtime_nsec;
+	entry->ctime.nanoseconds = st->st_ctime_nsec;
+#endif
 	entry->dev  = st->st_rdev;
 	entry->ino  = st->st_ino;
-	entry->mode = index_create_mode(st->st_mode);
+	entry->mode = (!trust_mode && S_ISREG(st->st_mode)) ?
+		git_index__create_mode(0666) : git_index__create_mode(st->st_mode);
 	entry->uid  = st->st_uid;
 	entry->gid  = st->st_gid;
-	entry->file_size = st->st_size;
+	entry->file_size = (uint32_t)st->st_size;
 }
 
-int git_index_entry__cmp(const void *a, const void *b)
+static void index_entry_adjust_namemask(
+		git_index_entry *entry,
+		size_t path_length)
 {
-	const git_index_entry *entry_a = a;
-	const git_index_entry *entry_b = b;
+	entry->flags &= ~GIT_INDEX_ENTRY_NAMEMASK;
 
-	return strcmp(entry_a->path, entry_b->path);
+	if (path_length < GIT_INDEX_ENTRY_NAMEMASK)
+		entry->flags |= path_length & GIT_INDEX_ENTRY_NAMEMASK;
+	else
+		entry->flags |= GIT_INDEX_ENTRY_NAMEMASK;
 }
 
-int git_index_entry__cmp_icase(const void *a, const void *b)
+/* When `from_workdir` is true, we will validate the paths to avoid placing
+ * paths that are invalid for the working directory on the current filesystem
+ * (eg, on Windows, we will disallow `GIT~1`, `AUX`, `COM1`, etc).  This
+ * function will *always* prevent `.git` and directory traversal `../` from
+ * being added to the index.
+ */
+static int index_entry_create(
+	git_index_entry **out,
+	git_repository *repo,
+	const char *path,
+	struct stat *st,
+	bool from_workdir)
 {
-	const git_index_entry *entry_a = a;
-	const git_index_entry *entry_b = b;
+	size_t pathlen = strlen(path), alloclen;
+	struct entry_internal *entry;
+	unsigned int path_valid_flags = GIT_PATH_REJECT_INDEX_DEFAULTS;
+	uint16_t mode = 0;
+
+	/* always reject placing `.git` in the index and directory traversal.
+	 * when requested, disallow platform-specific filenames and upgrade to
+	 * the platform-specific `.git` tests (eg, `git~1`, etc).
+	 */
+	if (from_workdir)
+		path_valid_flags |= GIT_PATH_REJECT_WORKDIR_DEFAULTS;
+	if (st)
+		mode = st->st_mode;
 
-	return strcasecmp(entry_a->path, entry_b->path);
+	if (!git_path_isvalid(repo, path, mode, path_valid_flags)) {
+		git_error_set(GIT_ERROR_INDEX, "invalid path: '%s'", path);
+		return -1;
+	}
+
+	GIT_ERROR_CHECK_ALLOC_ADD(&alloclen, sizeof(struct entry_internal), pathlen);
+	GIT_ERROR_CHECK_ALLOC_ADD(&alloclen, alloclen, 1);
+	entry = git__calloc(1, alloclen);
+	GIT_ERROR_CHECK_ALLOC(entry);
+
+	entry->pathlen = pathlen;
+	memcpy(entry->path, path, pathlen);
+	entry->entry.path = entry->path;
+
+	*out = (git_index_entry *)entry;
+	return 0;
 }
 
-static int index_entry_init(git_index_entry **entry_out, git_index *index, const char *rel_path)
+static int index_entry_init(
+	git_index_entry **entry_out,
+	git_index *index,
+	const char *rel_path)
 {
+	int error = 0;
 	git_index_entry *entry = NULL;
+	git_buf path = GIT_BUF_INIT;
 	struct stat st;
 	git_oid oid;
-	const char *workdir;
-	git_buf full_path = GIT_BUF_INIT;
-	int error;
+	git_repository *repo;
 
 	if (INDEX_OWNER(index) == NULL)
 		return create_index_error(-1,
-			"Could not initialize index entry. "
+			"could not initialize index entry. "
 			"Index is not backed up by an existing repository.");
 
-	workdir = git_repository_workdir(INDEX_OWNER(index));
+	/*
+	 * FIXME: this is duplicated with the work in
+	 * git_blob__create_from_paths. It should accept an optional stat
+	 * structure so we can pass in the one we have to do here.
+	 */
+	repo = INDEX_OWNER(index);
+	if (git_repository__ensure_not_bare(repo, "create blob from file") < 0)
+		return GIT_EBAREREPO;
+
+	if (git_buf_joinpath(&path, git_repository_workdir(repo), rel_path) < 0)
+		return -1;
 
-	if (!workdir)
-		return create_index_error(GIT_EBAREREPO,
-			"Could not initialize index entry. Repository is bare");
+	error = git_path_lstat(path.ptr, &st);
+	git_buf_dispose(&path);
 
-	if ((error = git_buf_joinpath(&full_path, workdir, rel_path)) < 0)
+	if (error < 0)
 		return error;
 
-	if ((error = git_path_lstat(full_path.ptr, &st)) < 0) {
-		git_buf_free(&full_path);
+	if (index_entry_create(&entry, INDEX_OWNER(index), rel_path, &st, true) < 0)
+		return -1;
+
+	/* write the blob to disk and get the oid and stat info */
+	error = git_blob__create_from_paths(
+		&oid, &st, INDEX_OWNER(index), NULL, rel_path, 0, true);
+
+	if (error < 0) {
+		index_entry_free(entry);
 		return error;
 	}
 
-	git_buf_free(&full_path); /* done with full path */
+	entry->id = oid;
+	git_index_entry__init_from_stat(entry, &st, !index->distrust_filemode);
 
-	/* There is no need to validate the rel_path here, since it will be
-	 * immediately validated by the call to git_blob_create_fromfile.
-	 */
+	*entry_out = (git_index_entry *)entry;
+	return 0;
+}
 
-	/* write the blob to disk and get the oid */
-	if ((error = git_blob_create_fromworkdir(&oid, INDEX_OWNER(index), rel_path)) < 0)
-		return error;
+static git_index_reuc_entry *reuc_entry_alloc(const char *path)
+{
+	size_t pathlen = strlen(path),
+		structlen = sizeof(struct reuc_entry_internal),
+		alloclen;
+	struct reuc_entry_internal *entry;
 
-	entry = git__calloc(1, sizeof(git_index_entry));
-	GITERR_CHECK_ALLOC(entry);
+	if (GIT_ADD_SIZET_OVERFLOW(&alloclen, structlen, pathlen) ||
+		GIT_ADD_SIZET_OVERFLOW(&alloclen, alloclen, 1))
+		return NULL;
 
-	git_index_entry__init_from_stat(entry, &st);
+	entry = git__calloc(1, alloclen);
+	if (!entry)
+		return NULL;
 
-	entry->oid = oid;
-	entry->path = git__strdup(rel_path);
-	GITERR_CHECK_ALLOC(entry->path);
+	entry->pathlen = pathlen;
+	memcpy(entry->path, path, pathlen);
+	entry->entry.path = entry->path;
 
-	*entry_out = entry;
-	return 0;
+	return (git_index_reuc_entry *)entry;
 }
 
 static int index_entry_reuc_init(git_index_reuc_entry **reuc_out,
@@ -648,103 +1029,392 @@ static int index_entry_reuc_init(git_index_reuc_entry **reuc_out,
 
 	assert(reuc_out && path);
 
-	*reuc_out = NULL;
-
-	reuc = git__calloc(1, sizeof(git_index_reuc_entry));
-	GITERR_CHECK_ALLOC(reuc);
-
-	reuc->path = git__strdup(path);
-	if (reuc->path == NULL)
-		return -1;
+	*reuc_out = reuc = reuc_entry_alloc(path);
+	GIT_ERROR_CHECK_ALLOC(reuc);
 
-	if ((reuc->mode[0] = ancestor_mode) > 0)
+	if ((reuc->mode[0] = ancestor_mode) > 0) {
+		assert(ancestor_oid);
 		git_oid_cpy(&reuc->oid[0], ancestor_oid);
+	}
 
-	if ((reuc->mode[1] = our_mode) > 0)
+	if ((reuc->mode[1] = our_mode) > 0) {
+		assert(our_oid);
 		git_oid_cpy(&reuc->oid[1], our_oid);
+	}
 
-	if ((reuc->mode[2] = their_mode) > 0)
+	if ((reuc->mode[2] = their_mode) > 0) {
+		assert(their_oid);
 		git_oid_cpy(&reuc->oid[2], their_oid);
+	}
 
-	*reuc_out = reuc;
 	return 0;
 }
 
-static void index_entry_reuc_free(git_index_reuc_entry *reuc)
+static void index_entry_cpy(
+	git_index_entry *tgt,
+	const git_index_entry *src)
 {
-	if (!reuc)
-		return;
-
-	git__free(reuc->path);
-	git__free(reuc);
+	const char *tgt_path = tgt->path;
+	memcpy(tgt, src, sizeof(*tgt));
+	tgt->path = tgt_path;
 }
 
-static git_index_entry *index_entry_dup(const git_index_entry *source_entry)
+static int index_entry_dup(
+	git_index_entry **out,
+	git_index *index,
+	const git_index_entry *src)
 {
-	git_index_entry *entry;
-
-	entry = git__malloc(sizeof(git_index_entry));
-	if (!entry)
-		return NULL;
-
-	memcpy(entry, source_entry, sizeof(git_index_entry));
-
-	/* duplicate the path string so we own it */
-	entry->path = git__strdup(entry->path);
-	if (!entry->path)
-		return NULL;
+	if (index_entry_create(out, INDEX_OWNER(index), src->path, NULL, false) < 0)
+		return -1;
 
-	return entry;
+	index_entry_cpy(*out, src);
+	return 0;
 }
 
-static void index_entry_free(git_index_entry *entry)
+static void index_entry_cpy_nocache(
+	git_index_entry *tgt,
+	const git_index_entry *src)
 {
-	if (!entry)
-		return;
-	git__free(entry->path);
-	git__free(entry);
+	git_oid_cpy(&tgt->id, &src->id);
+	tgt->mode = src->mode;
+	tgt->flags = src->flags;
+	tgt->flags_extended = (src->flags_extended & GIT_INDEX_ENTRY_EXTENDED_FLAGS);
 }
 
-static int index_insert(git_index *index, git_index_entry *entry, int replace)
+static int index_entry_dup_nocache(
+	git_index_entry **out,
+	git_index *index,
+	const git_index_entry *src)
 {
-	size_t path_length, position;
-	git_index_entry **existing = NULL;
-
-	assert(index && entry && entry->path != NULL);
-
-	/* make sure that the path length flag is correct */
-	path_length = strlen(entry->path);
-
-	entry->flags &= ~GIT_IDXENTRY_NAMEMASK;
+	if (index_entry_create(out, INDEX_OWNER(index), src->path, NULL, false) < 0)
+		return -1;
 
-	if (path_length < GIT_IDXENTRY_NAMEMASK)
-		entry->flags |= path_length & GIT_IDXENTRY_NAMEMASK;
-	else
-		entry->flags |= GIT_IDXENTRY_NAMEMASK;
+	index_entry_cpy_nocache(*out, src);
+	return 0;
+}
 
-	/* look if an entry with this path already exists */
-	if (!index_find(&position, index, entry->path, GIT_IDXENTRY_STAGE(entry))) {
-		existing = (git_index_entry **)&index->entries.contents[position];
+static int has_file_name(git_index *index,
+	 const git_index_entry *entry, size_t pos, int ok_to_replace)
+{
+	size_t len = strlen(entry->path);
+	int stage = GIT_INDEX_ENTRY_STAGE(entry);
+	const char *name = entry->path;
 
-		/* update filemode to existing values if stat is not trusted */
-		entry->mode = index_merge_mode(index, *existing, entry->mode);
-	}
+	while (pos < index->entries.length) {
+		struct entry_internal *p = index->entries.contents[pos++];
 
-	/* if replacing is not requested or no existing entry exists, just
-	 * insert entry at the end; the index is no longer sorted
+		if (len >= p->pathlen)
+			break;
+		if (memcmp(name, p->path, len))
+			break;
+		if (GIT_INDEX_ENTRY_STAGE(&p->entry) != stage)
+			continue;
+		if (p->path[len] != '/')
+			continue;
+		if (!ok_to_replace)
+			return -1;
+
+		if (index_remove_entry(index, --pos) < 0)
+			break;
+	}
+	return 0;
+}
+
+/*
+ * Do we have another file with a pathname that is a proper
+ * subset of the name we're trying to add?
+ */
+static int has_dir_name(git_index *index,
+		const git_index_entry *entry, int ok_to_replace)
+{
+	int stage = GIT_INDEX_ENTRY_STAGE(entry);
+	const char *name = entry->path;
+	const char *slash = name + strlen(name);
+
+	for (;;) {
+		size_t len, pos;
+
+		for (;;) {
+			if (*--slash == '/')
+				break;
+			if (slash <= entry->path)
+				return 0;
+		}
+		len = slash - name;
+
+		if (!index_find(&pos, index, name, len, stage)) {
+			if (!ok_to_replace)
+				return -1;
+
+			if (index_remove_entry(index, pos) < 0)
+				break;
+			continue;
+		}
+
+		/*
+		 * Trivial optimization: if we find an entry that
+		 * already matches the sub-directory, then we know
+		 * we're ok, and we can exit.
+		 */
+		for (; pos < index->entries.length; ++pos) {
+			struct entry_internal *p = index->entries.contents[pos];
+
+			if (p->pathlen <= len ||
+			    p->path[len] != '/' ||
+			    memcmp(p->path, name, len))
+				break; /* not our subdirectory */
+
+			if (GIT_INDEX_ENTRY_STAGE(&p->entry) == stage)
+				return 0;
+		}
+	}
+
+	return 0;
+}
+
+static int check_file_directory_collision(git_index *index,
+		git_index_entry *entry, size_t pos, int ok_to_replace)
+{
+	if (has_file_name(index, entry, pos, ok_to_replace) < 0 ||
+	    has_dir_name(index, entry, ok_to_replace) < 0) {
+		git_error_set(GIT_ERROR_INDEX,
+			"'%s' appears as both a file and a directory", entry->path);
+		return -1;
+	}
+
+	return 0;
+}
+
+static int canonicalize_directory_path(
+	git_index *index,
+	git_index_entry *entry,
+	git_index_entry *existing)
+{
+	const git_index_entry *match, *best = NULL;
+	char *search, *sep;
+	size_t pos, search_len, best_len;
+
+	if (!index->ignore_case)
+		return 0;
+
+	/* item already exists in the index, simply re-use the existing case */
+	if (existing) {
+		memcpy((char *)entry->path, existing->path, strlen(existing->path));
+		return 0;
+	}
+
+	/* nothing to do */
+	if (strchr(entry->path, '/') == NULL)
+		return 0;
+
+	if ((search = git__strdup(entry->path)) == NULL)
+		return -1;
+
+	/* starting at the parent directory and descending to the root, find the
+	 * common parent directory.
 	 */
-	if (!replace || !existing)
-		return git_vector_insert(&index->entries, entry);
+	while (!best && (sep = strrchr(search, '/'))) {
+		sep[1] = '\0';
+
+		search_len = strlen(search);
 
-	/* exists, replace it (preserving name from existing entry) */
-	git__free(entry->path);
-	entry->path = (*existing)->path;
-	git__free(*existing);
-	*existing = entry;
+		git_vector_bsearch2(
+			&pos, &index->entries, index->entries_search_path, search);
+
+		while ((match = git_vector_get(&index->entries, pos))) {
+			if (GIT_INDEX_ENTRY_STAGE(match) != 0) {
+				/* conflicts do not contribute to canonical paths */
+			} else if (strncmp(search, match->path, search_len) == 0) {
+				/* prefer an exact match to the input filename */
+				best = match;
+				best_len = search_len;
+				break;
+			} else if (strncasecmp(search, match->path, search_len) == 0) {
+				/* continue walking, there may be a path with an exact
+				 * (case sensitive) match later in the index, but use this
+				 * as the best match until that happens.
+				 */
+				if (!best) {
+					best = match;
+					best_len = search_len;
+				}
+			} else {
+				break;
+			}
+
+			pos++;
+		}
 
+		sep[0] = '\0';
+	}
+
+	if (best)
+		memcpy((char *)entry->path, best->path, best_len);
+
+	git__free(search);
 	return 0;
 }
 
+static int index_no_dups(void **old, void *new)
+{
+	const git_index_entry *entry = new;
+	GIT_UNUSED(old);
+	git_error_set(GIT_ERROR_INDEX, "'%s' appears multiple times at stage %d",
+		entry->path, GIT_INDEX_ENTRY_STAGE(entry));
+	return GIT_EEXISTS;
+}
+
+static void index_existing_and_best(
+	git_index_entry **existing,
+	size_t *existing_position,
+	git_index_entry **best,
+	git_index *index,
+	const git_index_entry *entry)
+{
+	git_index_entry *e;
+	size_t pos;
+	int error;
+
+	error = index_find(&pos,
+		index, entry->path, 0, GIT_INDEX_ENTRY_STAGE(entry));
+
+	if (error == 0) {
+		*existing = index->entries.contents[pos];
+		*existing_position = pos;
+		*best = index->entries.contents[pos];
+		return;
+	}
+
+	*existing = NULL;
+	*existing_position = 0;
+	*best = NULL;
+
+	if (GIT_INDEX_ENTRY_STAGE(entry) == 0) {
+		for (; pos < index->entries.length; pos++) {
+			int (*strcomp)(const char *a, const char *b) =
+				index->ignore_case ? git__strcasecmp : git__strcmp;
+
+			e = index->entries.contents[pos];
+
+			if (strcomp(entry->path, e->path) != 0)
+				break;
+
+			if (GIT_INDEX_ENTRY_STAGE(e) == GIT_INDEX_STAGE_ANCESTOR) {
+				*best = e;
+				continue;
+			} else {
+				*best = e;
+				break;
+			}
+		}
+	}
+}
+
+/* index_insert takes ownership of the new entry - if it can't insert
+ * it, then it will return an error **and also free the entry**.  When
+ * it replaces an existing entry, it will update the entry_ptr with the
+ * actual entry in the index (and free the passed in one).
+ *
+ * trust_path is whether we use the given path, or whether (on case
+ * insensitive systems only) we try to canonicalize the given path to
+ * be within an existing directory.
+ *
+ * trust_mode is whether we trust the mode in entry_ptr.
+ *
+ * trust_id is whether we trust the id or it should be validated.
+ */
+static int index_insert(
+	git_index *index,
+	git_index_entry **entry_ptr,
+	int replace,
+	bool trust_path,
+	bool trust_mode,
+	bool trust_id)
+{
+	git_index_entry *existing, *best, *entry;
+	size_t path_length, position;
+	int error;
+
+	assert(index && entry_ptr);
+
+	entry = *entry_ptr;
+
+	/* Make sure that the path length flag is correct */
+	path_length = ((struct entry_internal *)entry)->pathlen;
+	index_entry_adjust_namemask(entry, path_length);
+
+	/* This entry is now up-to-date and should not be checked for raciness */
+	entry->flags_extended |= GIT_INDEX_ENTRY_UPTODATE;
+
+	git_vector_sort(&index->entries);
+
+	/*
+	 * Look if an entry with this path already exists, either staged, or (if
+	 * this entry is a regular staged item) as the "ours" side of a conflict.
+	 */
+	index_existing_and_best(&existing, &position, &best, index, entry);
+
+	/* Update the file mode */
+	entry->mode = trust_mode ?
+		git_index__create_mode(entry->mode) :
+		index_merge_mode(index, best, entry->mode);
+
+	/* Canonicalize the directory name */
+	if (!trust_path && (error = canonicalize_directory_path(index, entry, best)) < 0)
+		goto out;
+
+	/* Ensure that the given id exists (unless it's a submodule) */
+	if (!trust_id && INDEX_OWNER(index) &&
+	    (entry->mode & GIT_FILEMODE_COMMIT) != GIT_FILEMODE_COMMIT) {
+
+		if (!git_object__is_valid(INDEX_OWNER(index), &entry->id,
+					  git_object__type_from_filemode(entry->mode))) {
+			error = -1;
+			goto out;
+		}
+	}
+
+	/* Look for tree / blob name collisions, removing conflicts if requested */
+	if ((error = check_file_directory_collision(index, entry, position, replace)) < 0)
+		goto out;
+
+	/*
+	 * If we are replacing an existing item, overwrite the existing entry
+	 * and return it in place of the passed in one.
+	 */
+	if (existing) {
+		if (replace) {
+			index_entry_cpy(existing, entry);
+
+			if (trust_path)
+				memcpy((char *)existing->path, entry->path, strlen(entry->path));
+		}
+
+		index_entry_free(entry);
+		*entry_ptr = existing;
+	} else {
+		/*
+		 * If replace is not requested or no existing entry exists, insert
+		 * at the sorted position.  (Since we re-sort after each insert to
+		 * check for dups, this is actually cheaper in the long run.)
+		 */
+		if ((error = git_vector_insert_sorted(&index->entries, entry, index_no_dups)) < 0)
+			goto out;
+
+		INSERT_IN_MAP(index, entry, &error);
+	}
+
+	index->dirty = 1;
+
+out:
+	if (error < 0) {
+		index_entry_free(*entry_ptr);
+		*entry_ptr = NULL;
+	}
+
+	return error;
+}
+
 static int index_conflict_to_reuc(git_index *index, const char *path)
 {
 	const git_index_entry *conflict_entries[3];
@@ -760,9 +1430,9 @@ static int index_conflict_to_reuc(git_index *index, const char *path)
 	our_mode = conflict_entries[1] == NULL ? 0 : conflict_entries[1]->mode;
 	their_mode = conflict_entries[2] == NULL ? 0 : conflict_entries[2]->mode;
 
-	ancestor_oid = conflict_entries[0] == NULL ? NULL : &conflict_entries[0]->oid;
-	our_oid = conflict_entries[1] == NULL ? NULL : &conflict_entries[1]->oid;
-	their_oid = conflict_entries[2] == NULL ? NULL : &conflict_entries[2]->oid;
+	ancestor_oid = conflict_entries[0] == NULL ? NULL : &conflict_entries[0]->id;
+	our_oid = conflict_entries[1] == NULL ? NULL : &conflict_entries[1]->id;
+	their_oid = conflict_entries[2] == NULL ? NULL : &conflict_entries[2]->id;
 
 	if ((ret = git_index_reuc_add(index, path, ancestor_mode, ancestor_oid,
 		our_mode, our_oid, their_mode, their_oid)) >= 0)
@@ -771,6 +1441,101 @@ static int index_conflict_to_reuc(git_index *index, const char *path)
 	return ret;
 }
 
+GIT_INLINE(bool) is_file_or_link(const int filemode)
+{
+	return (filemode == GIT_FILEMODE_BLOB ||
+		filemode == GIT_FILEMODE_BLOB_EXECUTABLE ||
+		filemode == GIT_FILEMODE_LINK);
+}
+
+GIT_INLINE(bool) valid_filemode(const int filemode)
+{
+	return (is_file_or_link(filemode) || filemode == GIT_FILEMODE_COMMIT);
+}
+
+int git_index_add_frombuffer(
+    git_index *index, const git_index_entry *source_entry,
+    const void *buffer, size_t len)
+{
+	git_index_entry *entry = NULL;
+	int error = 0;
+	git_oid id;
+
+	assert(index && source_entry->path);
+
+	if (INDEX_OWNER(index) == NULL)
+		return create_index_error(-1,
+			"could not initialize index entry. "
+			"Index is not backed up by an existing repository.");
+
+	if (!is_file_or_link(source_entry->mode)) {
+		git_error_set(GIT_ERROR_INDEX, "invalid filemode");
+		return -1;
+	}
+
+	if (index_entry_dup(&entry, index, source_entry) < 0)
+		return -1;
+
+	error = git_blob_create_frombuffer(&id, INDEX_OWNER(index), buffer, len);
+	if (error < 0) {
+		index_entry_free(entry);
+		return error;
+	}
+
+	git_oid_cpy(&entry->id, &id);
+	entry->file_size = len;
+
+	if ((error = index_insert(index, &entry, 1, true, true, true)) < 0)
+		return error;
+
+	/* Adding implies conflict was resolved, move conflict entries to REUC */
+	if ((error = index_conflict_to_reuc(index, entry->path)) < 0 && error != GIT_ENOTFOUND)
+		return error;
+
+	git_tree_cache_invalidate_path(index->tree, entry->path);
+	return 0;
+}
+
+static int add_repo_as_submodule(git_index_entry **out, git_index *index, const char *path)
+{
+	git_repository *sub;
+	git_buf abspath = GIT_BUF_INIT;
+	git_repository *repo = INDEX_OWNER(index);
+	git_reference *head;
+	git_index_entry *entry;
+	struct stat st;
+	int error;
+
+	if ((error = git_buf_joinpath(&abspath, git_repository_workdir(repo), path)) < 0)
+		return error;
+
+	if ((error = p_stat(abspath.ptr, &st)) < 0) {
+		git_error_set(GIT_ERROR_OS, "failed to stat repository dir");
+		return -1;
+	}
+
+	if (index_entry_create(&entry, INDEX_OWNER(index), path, &st, true) < 0)
+		return -1;
+
+	git_index_entry__init_from_stat(entry, &st, !index->distrust_filemode);
+
+	if ((error = git_repository_open(&sub, abspath.ptr)) < 0)
+		return error;
+
+	if ((error = git_repository_head(&head, sub)) < 0)
+		return error;
+
+	git_oid_cpy(&entry->id, git_reference_target(head));
+	entry->mode = GIT_FILEMODE_COMMIT;
+
+	git_reference_free(head);
+	git_repository_free(sub);
+	git_buf_dispose(&abspath);
+
+	*out = entry;
+	return 0;
+}
+
 int git_index_add_bypath(git_index *index, const char *path)
 {
 	git_index_entry *entry = NULL;
@@ -778,20 +1543,50 @@ int git_index_add_bypath(git_index *index, const char *path)
 
 	assert(index && path);
 
-	if ((ret = index_entry_init(&entry, index, path)) < 0 ||
-		(ret = index_insert(index, entry, 1)) < 0)
-		goto on_error;
+	if ((ret = index_entry_init(&entry, index, path)) == 0)
+		ret = index_insert(index, &entry, 1, false, false, true);
+
+	/* If we were given a directory, let's see if it's a submodule */
+	if (ret < 0 && ret != GIT_EDIRECTORY)
+		return ret;
+
+	if (ret == GIT_EDIRECTORY) {
+		git_submodule *sm;
+		git_error_state err;
+
+		git_error_state_capture(&err, ret);
+
+		ret = git_submodule_lookup(&sm, INDEX_OWNER(index), path);
+		if (ret == GIT_ENOTFOUND)
+			return git_error_state_restore(&err);
+
+		git_error_state_free(&err);
+
+		/*
+		 * EEXISTS means that there is a repository at that path, but it's not known
+		 * as a submodule. We add its HEAD as an entry and don't register it.
+		 */
+		if (ret == GIT_EEXISTS) {
+			if ((ret = add_repo_as_submodule(&entry, index, path)) < 0)
+				return ret;
+
+			if ((ret = index_insert(index, &entry, 1, false, false, true)) < 0)
+				return ret;
+		} else if (ret < 0) {
+			return ret;
+		} else {
+			ret = git_submodule_add_to_index(sm, false);
+			git_submodule_free(sm);
+			return ret;
+		}
+	}
 
 	/* Adding implies conflict was resolved, move conflict entries to REUC */
 	if ((ret = index_conflict_to_reuc(index, path)) < 0 && ret != GIT_ENOTFOUND)
-		goto on_error;
+		return ret;
 
 	git_tree_cache_invalidate_path(index->tree, entry->path);
 	return 0;
-
-on_error:
-	index_entry_free(entry);
-	return ret;
 }
 
 int git_index_remove_bypath(git_index *index, const char *path)
@@ -806,22 +1601,68 @@ int git_index_remove_bypath(git_index *index, const char *path)
 		ret != GIT_ENOTFOUND))
 		return ret;
 
+	if (ret == GIT_ENOTFOUND)
+		git_error_clear();
+
 	return 0;
 }
 
+int git_index__fill(git_index *index, const git_vector *source_entries)
+{
+	const git_index_entry *source_entry = NULL;
+	size_t i;
+	int ret = 0;
+
+	assert(index);
+
+	if (!source_entries->length)
+		return 0;
+
+	git_vector_size_hint(&index->entries, source_entries->length);
+	git_idxmap_resize(index->entries_map, (size_t)(source_entries->length * 1.3));
+
+	git_vector_foreach(source_entries, i, source_entry) {
+		git_index_entry *entry = NULL;
+
+		if ((ret = index_entry_dup(&entry, index, source_entry)) < 0)
+			break;
+
+		index_entry_adjust_namemask(entry, ((struct entry_internal *)entry)->pathlen);
+		entry->flags_extended |= GIT_INDEX_ENTRY_UPTODATE;
+		entry->mode = git_index__create_mode(entry->mode);
+
+		if ((ret = git_vector_insert(&index->entries, entry)) < 0)
+			break;
+
+		INSERT_IN_MAP(index, entry, &ret);
+		if (ret < 0)
+			break;
+
+		index->dirty = 1;
+	}
+
+	if (!ret)
+		git_vector_sort(&index->entries);
+
+	return ret;
+}
+
+
 int git_index_add(git_index *index, const git_index_entry *source_entry)
 {
 	git_index_entry *entry = NULL;
 	int ret;
 
-	entry = index_entry_dup(source_entry);
-	if (entry == NULL)
+	assert(index && source_entry && source_entry->path);
+
+	if (!valid_filemode(source_entry->mode)) {
+		git_error_set(GIT_ERROR_INDEX, "invalid entry mode");
 		return -1;
+	}
 
-	if ((ret = index_insert(index, entry, 1)) < 0) {
-		index_entry_free(entry);
+	if ((ret = index_entry_dup(&entry, index, source_entry)) < 0 ||
+		(ret = index_insert(index, &entry, 1, true, true, false)) < 0)
 		return ret;
-	}
 
 	git_tree_cache_invalidate_path(index->tree, entry->path);
 	return 0;
@@ -829,26 +1670,22 @@ int git_index_add(git_index *index, const git_index_entry *source_entry)
 
 int git_index_remove(git_index *index, const char *path, int stage)
 {
-	size_t position;
 	int error;
-	git_index_entry *entry;
-
-	git_vector_sort(&index->entries);
+	size_t position;
+	git_index_entry remove_key = {{ 0 }};
 
-	if (index_find(&position, index, path, stage) < 0) {
-		giterr_set(GITERR_INDEX, "Index does not contain %s at stage %d",
-			path, stage);
-		return GIT_ENOTFOUND;
-	}
+	remove_key.path = path;
+	GIT_INDEX_ENTRY_STAGE_SET(&remove_key, stage);
 
-	entry = git_vector_get(&index->entries, position);
-	if (entry != NULL)
-		git_tree_cache_invalidate_path(index->tree, entry->path);
+	DELETE_IN_MAP(index, &remove_key);
 
-	error = git_vector_remove(&index->entries, position);
-
-	if (!error)
-		index_entry_free(entry);
+	if (index_find(&position, index, path, 0, stage) < 0) {
+		git_error_set(
+			GIT_ERROR_INDEX, "index does not contain %s at stage %d", path, stage);
+		error = GIT_ENOTFOUND;
+	} else {
+		error = index_remove_entry(index, position);
+	}
 
 	return error;
 }
@@ -860,47 +1697,52 @@ int git_index_remove_directory(git_index *index, const char *dir, int stage)
 	size_t pos;
 	git_index_entry *entry;
 
-	if (git_buf_sets(&pfx, dir) < 0 || git_path_to_dir(&pfx) < 0)
-		return -1;
+	if (!(error = git_buf_sets(&pfx, dir)) &&
+		!(error = git_path_to_dir(&pfx)))
+		index_find(&pos, index, pfx.ptr, pfx.size, GIT_INDEX_STAGE_ANY);
 
-	git_vector_sort(&index->entries);
-
-	pos = git_index__prefix_position(index, pfx.ptr);
-
-	while (1) {
+	while (!error) {
 		entry = git_vector_get(&index->entries, pos);
 		if (!entry || git__prefixcmp(entry->path, pfx.ptr) != 0)
 			break;
 
-		if (GIT_IDXENTRY_STAGE(entry) != stage) {
+		if (GIT_INDEX_ENTRY_STAGE(entry) != stage) {
 			++pos;
 			continue;
 		}
 
-		git_tree_cache_invalidate_path(index->tree, entry->path);
-
-		if ((error = git_vector_remove(&index->entries, pos)) < 0)
-			break;
-		index_entry_free(entry);
+		error = index_remove_entry(index, pos);
 
-		/* removed entry at 'pos' so we don't need to increment it */
+		/* removed entry at 'pos' so we don't need to increment */
 	}
 
-	git_buf_free(&pfx);
+	git_buf_dispose(&pfx);
 
 	return error;
 }
 
-static int index_find(size_t *at_pos, git_index *index, const char *path, int stage)
+int git_index_find_prefix(size_t *at_pos, git_index *index, const char *prefix)
 {
-	struct entry_srch_key srch_key;
+	int error = 0;
+	size_t pos;
+	const git_index_entry *entry;
 
-	assert(path);
+	index_find(&pos, index, prefix, strlen(prefix), GIT_INDEX_STAGE_ANY);
+	entry = git_vector_get(&index->entries, pos);
+	if (!entry || git__prefixcmp(entry->path, prefix) != 0)
+		error = GIT_ENOTFOUND;
 
-	srch_key.path = path;
-	srch_key.stage = stage;
+	if (!error && at_pos)
+		*at_pos = pos;
 
-	return git_vector_bsearch2(at_pos, &index->entries, index->entries_search, &srch_key);
+	return error;
+}
+
+int git_index__find_pos(
+	size_t *out, git_index *index, const char *path, size_t path_len, int stage)
+{
+	assert(index && path);
+	return index_find(out, index, path, path_len, stage);
 }
 
 int git_index_find(size_t *at_pos, git_index *index, const char *path)
@@ -909,21 +1751,20 @@ int git_index_find(size_t *at_pos, git_index *index, const char *path)
 
 	assert(index && path);
 
-	if (git_vector_bsearch2(&pos, &index->entries, index->entries_search_path, path) < 0) {
-		giterr_set(GITERR_INDEX, "Index does not contain %s", path);
+	if (git_vector_bsearch2(
+			&pos, &index->entries, index->entries_search_path, path) < 0) {
+		git_error_set(GIT_ERROR_INDEX, "index does not contain %s", path);
 		return GIT_ENOTFOUND;
 	}
 
 	/* Since our binary search only looked at path, we may be in the
 	 * middle of a list of stages.
 	 */
-	while (pos > 0) {
-		const git_index_entry *prev = git_vector_get(&index->entries, pos-1);
+	for (; pos > 0; --pos) {
+		const git_index_entry *prev = git_vector_get(&index->entries, pos - 1);
 
 		if (index->entries_cmp_path(prev->path, path) != 0)
 			break;
-
-		--pos;
 	}
 
 	if (at_pos)
@@ -932,21 +1773,6 @@ int git_index_find(size_t *at_pos, git_index *index, const char *path)
 	return 0;
 }
 
-size_t git_index__prefix_position(git_index *index, const char *path)
-{
-	struct entry_srch_key srch_key;
-	size_t pos;
-
-	srch_key.path = path;
-	srch_key.stage = 0;
-
-	git_vector_sort(&index->entries);
-	git_vector_bsearch2(
-		&pos, &index->entries, index->entries_search, &srch_key);
-
-	return pos;
-}
-
 int git_index_conflict_add(git_index *index,
 	const git_index_entry *ancestor_entry,
 	const git_index_entry *our_entry,
@@ -958,21 +1784,50 @@ int git_index_conflict_add(git_index *index,
 
 	assert (index);
 
-	if ((ancestor_entry != NULL && (entries[0] = index_entry_dup(ancestor_entry)) == NULL) ||
-		(our_entry != NULL && (entries[1] = index_entry_dup(our_entry)) == NULL) ||
-		(their_entry != NULL && (entries[2] = index_entry_dup(their_entry)) == NULL))
-		return -1;
+	if ((ancestor_entry &&
+			(ret = index_entry_dup(&entries[0], index, ancestor_entry)) < 0) ||
+		(our_entry &&
+			(ret = index_entry_dup(&entries[1], index, our_entry)) < 0) ||
+		(their_entry &&
+			(ret = index_entry_dup(&entries[2], index, their_entry)) < 0))
+		goto on_error;
 
+	/* Validate entries */
+	for (i = 0; i < 3; i++) {
+		if (entries[i] && !valid_filemode(entries[i]->mode)) {
+			git_error_set(GIT_ERROR_INDEX, "invalid filemode for stage %d entry",
+				i + 1);
+			ret = -1;
+			goto on_error;
+		}
+	}
+
+	/* Remove existing index entries for each path */
+	for (i = 0; i < 3; i++) {
+		if (entries[i] == NULL)
+			continue;
+
+		if ((ret = git_index_remove(index, entries[i]->path, 0)) != 0) {
+			if (ret != GIT_ENOTFOUND)
+				goto on_error;
+
+			git_error_clear();
+			ret = 0;
+		}
+	}
+
+	/* Add the conflict entries */
 	for (i = 0; i < 3; i++) {
 		if (entries[i] == NULL)
 			continue;
 
 		/* Make sure stage is correct */
-		entries[i]->flags = (entries[i]->flags & ~GIT_IDXENTRY_STAGEMASK) |
-			((i+1) << GIT_IDXENTRY_STAGESHIFT);
+		GIT_INDEX_ENTRY_STAGE_SET(entries[i], i + 1);
 
-		if ((ret = index_insert(index, entries[i], 1)) < 0)
+		if ((ret = index_insert(index, &entries[i], 1, true, true, false)) < 0)
 			goto on_error;
+
+		entries[i] = NULL; /* don't free if later entry fails */
 	}
 
 	return 0;
@@ -1010,7 +1865,7 @@ static int index_conflict__get_byindex(
 		if (path && index->entries_cmp_path(conflict_entry->path, path) != 0)
 			break;
 
-		stage = GIT_IDXENTRY_STAGE(conflict_entry);
+		stage = GIT_INDEX_ENTRY_STAGE(conflict_entry);
 		path = conflict_entry->path;
 
 		switch (stage) {
@@ -1062,56 +1917,43 @@ int git_index_conflict_get(
 	return 0;
 }
 
-int git_index_conflict_remove(git_index *index, const char *path)
+static int index_conflict_remove(git_index *index, const char *path)
 {
-	size_t pos, posmax;
+	size_t pos = 0;
 	git_index_entry *conflict_entry;
 	int error = 0;
 
-	assert(index && path);
-
-	if (git_index_find(&pos, index, path) < 0)
+	if (path != NULL && git_index_find(&pos, index, path) < 0)
 		return GIT_ENOTFOUND;
 
-	posmax = git_index_entrycount(index);
+	while ((conflict_entry = git_vector_get(&index->entries, pos)) != NULL) {
 
-	while (pos < posmax) {
-		conflict_entry = git_vector_get(&index->entries, pos);
-
-		if (index->entries_cmp_path(conflict_entry->path, path) != 0)
+		if (path != NULL &&
+			index->entries_cmp_path(conflict_entry->path, path) != 0)
 			break;
 
-		if (GIT_IDXENTRY_STAGE(conflict_entry) == 0) {
+		if (GIT_INDEX_ENTRY_STAGE(conflict_entry) == 0) {
 			pos++;
 			continue;
 		}
 
-		if ((error = git_vector_remove(&index->entries, pos)) < 0)
-			return error;
-
-		index_entry_free(conflict_entry);
-		posmax--;
+		if ((error = index_remove_entry(index, pos)) < 0)
+			break;
 	}
 
-	return 0;
+	return error;
 }
 
-static int index_conflicts_match(const git_vector *v, size_t idx)
+int git_index_conflict_remove(git_index *index, const char *path)
 {
-	git_index_entry *entry = git_vector_get(v, idx);
-
-	if (GIT_IDXENTRY_STAGE(entry) > 0) {
-		index_entry_free(entry);
-		return 1;
-	}
-
-	return 0;
+	assert(index && path);
+	return index_conflict_remove(index, path);
 }
 
-void git_index_conflict_cleanup(git_index *index)
+int git_index_conflict_cleanup(git_index *index)
 {
 	assert(index);
-	git_vector_remove_matching(&index->entries, index_conflicts_match);
+	return index_conflict_remove(index, NULL);
 }
 
 int git_index_has_conflicts(const git_index *index)
@@ -1122,13 +1964,58 @@ int git_index_has_conflicts(const git_index *index)
 	assert(index);
 
 	git_vector_foreach(&index->entries, i, entry) {
-		if (GIT_IDXENTRY_STAGE(entry) > 0)
+		if (GIT_INDEX_ENTRY_STAGE(entry) > 0)
 			return 1;
 	}
 
 	return 0;
 }
 
+int git_index_iterator_new(
+	git_index_iterator **iterator_out,
+	git_index *index)
+{
+	git_index_iterator *it;
+	int error;
+
+	assert(iterator_out && index);
+
+	it = git__calloc(1, sizeof(git_index_iterator));
+	GIT_ERROR_CHECK_ALLOC(it);
+
+	if ((error = git_index_snapshot_new(&it->snap, index)) < 0) {
+		git__free(it);
+		return error;
+	}
+
+	it->index = index;
+
+	*iterator_out = it;
+	return 0;
+}
+
+int git_index_iterator_next(
+	const git_index_entry **out,
+	git_index_iterator *it)
+{
+	assert(out && it);
+
+	if (it->cur >= git_vector_length(&it->snap))
+		return GIT_ITEROVER;
+
+	*out = (git_index_entry *)git_vector_get(&it->snap, it->cur++);
+	return 0;
+}
+
+void git_index_iterator_free(git_index_iterator *it)
+{
+	if (it == NULL)
+		return;
+
+	git_index_snapshot_release(&it->snap, it->index);
+	git__free(it);
+}
+
 int git_index_conflict_iterator_new(
 	git_index_conflict_iterator **iterator_out,
 	git_index *index)
@@ -1138,7 +2025,7 @@ int git_index_conflict_iterator_new(
 	assert(iterator_out && index);
 
 	it = git__calloc(1, sizeof(git_index_conflict_iterator));
-	GITERR_CHECK_ALLOC(it);
+	GIT_ERROR_CHECK_ALLOC(it);
 
 	it->index = index;
 
@@ -1164,7 +2051,7 @@ int git_index_conflict_next(
 	while (iterator->cur < iterator->index->entries.length) {
 		entry = git_index_get_byindex(iterator->index, iterator->cur);
 
-		if (git_index_entry_stage(entry) > 0) {
+		if (git_index_entry_is_conflict(entry)) {
 			if ((len = index_conflict__get_byindex(
 				ancestor_out,
 				our_out,
@@ -1191,10 +2078,10 @@ void git_index_conflict_iterator_free(git_index_conflict_iterator *iterator)
 	git__free(iterator);
 }
 
-unsigned int git_index_name_entrycount(git_index *index)
+size_t git_index_name_entrycount(git_index *index)
 {
 	assert(index);
-	return (unsigned int)index->names.length;
+	return index->names.length;
 }
 
 const git_index_name_entry *git_index_name_get_byindex(
@@ -1206,32 +2093,37 @@ const git_index_name_entry *git_index_name_get_byindex(
 	return git_vector_get(&index->names, n);
 }
 
+static void index_name_entry_free(git_index_name_entry *ne)
+{
+	if (!ne)
+		return;
+	git__free(ne->ancestor);
+	git__free(ne->ours);
+	git__free(ne->theirs);
+	git__free(ne);
+}
+
 int git_index_name_add(git_index *index,
 	const char *ancestor, const char *ours, const char *theirs)
 {
 	git_index_name_entry *conflict_name;
 
-	assert ((ancestor && ours) || (ancestor && theirs) || (ours && theirs));
+	assert((ancestor && ours) || (ancestor && theirs) || (ours && theirs));
 
 	conflict_name = git__calloc(1, sizeof(git_index_name_entry));
-	GITERR_CHECK_ALLOC(conflict_name);
-
-	if (ancestor) {
-		conflict_name->ancestor = git__strdup(ancestor);
-		GITERR_CHECK_ALLOC(conflict_name->ancestor);
-	}
-
-	if (ours) {
-		conflict_name->ours = git__strdup(ours);
-		GITERR_CHECK_ALLOC(conflict_name->ours);
-	}
+	GIT_ERROR_CHECK_ALLOC(conflict_name);
 
-	if (theirs) {
-		conflict_name->theirs = git__strdup(theirs);
-		GITERR_CHECK_ALLOC(conflict_name->theirs);
+	if ((ancestor && !(conflict_name->ancestor = git__strdup(ancestor))) ||
+		(ours     && !(conflict_name->ours     = git__strdup(ours))) ||
+		(theirs   && !(conflict_name->theirs   = git__strdup(theirs))) ||
+		git_vector_insert(&index->names, conflict_name) < 0)
+	{
+		index_name_entry_free(conflict_name);
+		return -1;
 	}
 
-	return git_vector_insert(&index->names, conflict_name);
+	index->dirty = 1;
+	return 0;
 }
 
 void git_index_name_clear(git_index *index)
@@ -1241,50 +2133,40 @@ void git_index_name_clear(git_index *index)
 
 	assert(index);
 
-	git_vector_foreach(&index->names, i, conflict_name) {
-		if (conflict_name->ancestor)
-			git__free(conflict_name->ancestor);
-
-		if (conflict_name->ours)
-			git__free(conflict_name->ours);
-
-		if (conflict_name->theirs)
-			git__free(conflict_name->theirs);
-
-		git__free(conflict_name);
-	}
+	git_vector_foreach(&index->names, i, conflict_name)
+		index_name_entry_free(conflict_name);
 
 	git_vector_clear(&index->names);
+
+	index->dirty = 1;
 }
 
-unsigned int git_index_reuc_entrycount(git_index *index)
+size_t git_index_reuc_entrycount(git_index *index)
 {
 	assert(index);
-	return (unsigned int)index->reuc.length;
+	return index->reuc.length;
+}
+
+static int index_reuc_on_dup(void **old, void *new)
+{
+	index_entry_reuc_free(*old);
+	*old = new;
+	return GIT_EEXISTS;
 }
 
 static int index_reuc_insert(
 	git_index *index,
-	git_index_reuc_entry *reuc,
-	int replace)
+	git_index_reuc_entry *reuc)
 {
-	git_index_reuc_entry **existing = NULL;
-	size_t position;
+	int res;
 
 	assert(index && reuc && reuc->path != NULL);
+	assert(git_vector_is_sorted(&index->reuc));
 
-	if (!git_index_reuc_find(&position, index, reuc->path))
-		existing = (git_index_reuc_entry **)&index->reuc.contents[position];
-
-	if (!replace || !existing)
-		return git_vector_insert(&index->reuc, reuc);
+	res = git_vector_insert_sorted(&index->reuc, reuc, &index_reuc_on_dup);
+	index->dirty = 1;
 
-	/* exists, replace it */
-	git__free((*existing)->path);
-	git__free(*existing);
-	*existing = reuc;
-
-	return 0;
+	return res == GIT_EEXISTS ? 0 : res;
 }
 
 int git_index_reuc_add(git_index *index, const char *path,
@@ -1297,15 +2179,13 @@ int git_index_reuc_add(git_index *index, const char *path,
 
 	assert(index && path);
 
-	if ((error = index_entry_reuc_init(&reuc, path, ancestor_mode, ancestor_oid, our_mode, our_oid, their_mode, their_oid)) < 0 ||
-		(error = index_reuc_insert(index, reuc, 1)) < 0)
-	{
+	if ((error = index_entry_reuc_init(&reuc, path, ancestor_mode,
+			ancestor_oid, our_mode, our_oid, their_mode, their_oid)) < 0 ||
+		(error = index_reuc_insert(index, reuc)) < 0)
 		index_entry_reuc_free(reuc);
-		return error;
-	}
 
 	return error;
-} 
+}
 
 int git_index_reuc_find(size_t *at_pos, git_index *index, const char *path)
 {
@@ -1321,7 +2201,7 @@ const git_index_reuc_entry *git_index_reuc_get_bypath(
 	if (!index->reuc.length)
 		return NULL;
 
-	git_vector_sort(&index->reuc);
+	assert(git_vector_is_sorted(&index->reuc));
 
 	if (git_index_reuc_find(&pos, index, path) < 0)
 		return NULL;
@@ -1333,8 +2213,8 @@ const git_index_reuc_entry *git_index_reuc_get_byindex(
 	git_index *index, size_t n)
 {
 	assert(index);
+	assert(git_vector_is_sorted(&index->reuc));
 
-	git_vector_sort(&index->reuc);
 	return git_vector_get(&index->reuc, n);
 }
 
@@ -1343,7 +2223,7 @@ int git_index_reuc_remove(git_index *index, size_t position)
 	int error;
 	git_index_reuc_entry *reuc;
 
-	git_vector_sort(&index->reuc);
+	assert(git_vector_is_sorted(&index->reuc));
 
 	reuc = git_vector_get(&index->reuc, position);
 	error = git_vector_remove(&index->reuc, position);
@@ -1351,27 +2231,27 @@ int git_index_reuc_remove(git_index *index, size_t position)
 	if (!error)
 		index_entry_reuc_free(reuc);
 
+	index->dirty = 1;
 	return error;
 }
 
 void git_index_reuc_clear(git_index *index)
 {
 	size_t i;
-	git_index_reuc_entry *reuc;
 
 	assert(index);
 
-	git_vector_foreach(&index->reuc, i, reuc) {
-		git__free(reuc->path);
-		git__free(reuc);
-	}
+	for (i = 0; i < index->reuc.length; ++i)
+		index_entry_reuc_free(git__swap(index->reuc.contents[i], NULL));
 
 	git_vector_clear(&index->reuc);
+
+	index->dirty = 1;
 }
 
 static int index_error_invalid(const char *message)
 {
-	giterr_set(GITERR_INDEX, "Invalid data in index - %s", message);
+	git_error_set(GIT_ERROR_INDEX, "invalid data in index - %s", message);
 	return -1;
 }
 
@@ -1389,34 +2269,34 @@ static int read_reuc(git_index *index, const char *buffer, size_t size)
 	while (size) {
 		git_index_reuc_entry *lost;
 
-		len = strlen(buffer) + 1;
+		len = p_strnlen(buffer, size) + 1;
 		if (size <= len)
 			return index_error_invalid("reading reuc entries");
 
-		lost = git__calloc(1, sizeof(git_index_reuc_entry));
-		GITERR_CHECK_ALLOC(lost);
-
-		/* read NUL-terminated pathname for entry */
-		lost->path = git__strdup(buffer);
-		GITERR_CHECK_ALLOC(lost->path);
+		lost = reuc_entry_alloc(buffer);
+		GIT_ERROR_CHECK_ALLOC(lost);
 
 		size -= len;
 		buffer += len;
 
 		/* read 3 ASCII octal numbers for stage entries */
 		for (i = 0; i < 3; i++) {
-			int tmp;
+			int64_t tmp;
 
-			if (git__strtol32(&tmp, buffer, &endptr, 8) < 0 ||
+			if (git__strntol64(&tmp, buffer, size, &endptr, 8) < 0 ||
 				!endptr || endptr == buffer || *endptr ||
-				(unsigned)tmp > UINT_MAX)
+				tmp < 0 || tmp > UINT32_MAX) {
+				index_entry_reuc_free(lost);
 				return index_error_invalid("reading reuc entry stage");
+			}
 
-			lost->mode[i] = tmp;
+			lost->mode[i] = (uint32_t)tmp;
 
 			len = (endptr + 1) - buffer;
-			if (size <= len)
+			if (size <= len) {
+				index_entry_reuc_free(lost);
 				return index_error_invalid("reading reuc entry stage");
+			}
 
 			size -= len;
 			buffer += len;
@@ -1426,8 +2306,10 @@ static int read_reuc(git_index *index, const char *buffer, size_t size)
 		for (i = 0; i < 3; i++) {
 			if (!lost->mode[i])
 				continue;
-			if (size < 20)
+			if (size < 20) {
+				index_entry_reuc_free(lost);
 				return index_error_invalid("reading reuc entry oid");
+			}
 
 			git_oid_fromraw(&lost->oid[i], (const unsigned char *) buffer);
 			size -= 20;
@@ -1440,7 +2322,7 @@ static int read_reuc(git_index *index, const char *buffer, size_t size)
 	}
 
 	/* entries are guaranteed to be sorted on-disk */
-	index->reuc.sorted = 1;
+	git_vector_set_sorted(&index->reuc, true);
 
 	return 0;
 }
@@ -1456,15 +2338,16 @@ static int read_conflict_names(git_index *index, const char *buffer, size_t size
 		return -1;
 
 #define read_conflict_name(ptr) \
-	len = strlen(buffer) + 1; \
-	if (size < len) \
-		return index_error_invalid("reading conflict name entries"); \
-	\
+	len = p_strnlen(buffer, size) + 1; \
+	if (size < len) { \
+		index_error_invalid("reading conflict name entries"); \
+		goto out_err; \
+	} \
 	if (len == 1) \
 		ptr = NULL; \
 	else { \
 		ptr = git__malloc(len); \
-		GITERR_CHECK_ALLOC(ptr); \
+		GIT_ERROR_CHECK_ALLOC(ptr); \
 		memcpy(ptr, buffer, len); \
 	} \
 	\
@@ -1473,84 +2356,157 @@ static int read_conflict_names(git_index *index, const char *buffer, size_t size
 
 	while (size) {
 		git_index_name_entry *conflict_name = git__calloc(1, sizeof(git_index_name_entry));
-		GITERR_CHECK_ALLOC(conflict_name);
+		GIT_ERROR_CHECK_ALLOC(conflict_name);
 
 		read_conflict_name(conflict_name->ancestor);
 		read_conflict_name(conflict_name->ours);
 		read_conflict_name(conflict_name->theirs);
 
 		if (git_vector_insert(&index->names, conflict_name) < 0)
-			return -1;
+			goto out_err;
+
+		continue;
+
+out_err:
+		git__free(conflict_name->ancestor);
+		git__free(conflict_name->ours);
+		git__free(conflict_name->theirs);
+		git__free(conflict_name);
+		return -1;
 	}
 
 #undef read_conflict_name
 
 	/* entries are guaranteed to be sorted on-disk */
-	index->names.sorted = 1;
+	git_vector_set_sorted(&index->names, true);
 
 	return 0;
 }
 
-static size_t read_entry(git_index_entry *dest, const void *buffer, size_t buffer_size)
+static size_t index_entry_size(size_t path_len, size_t varint_len, uint32_t flags)
+{
+	if (varint_len) {
+		if (flags & GIT_INDEX_ENTRY_EXTENDED)
+			return offsetof(struct entry_long, path) + path_len + 1 + varint_len;
+		else
+			return offsetof(struct entry_short, path) + path_len + 1 + varint_len;
+	} else {
+#define entry_size(type,len) ((offsetof(type, path) + (len) + 8) & ~7)
+		if (flags & GIT_INDEX_ENTRY_EXTENDED)
+			return entry_size(struct entry_long, path_len);
+		else
+			return entry_size(struct entry_short, path_len);
+#undef entry_size
+	}
+}
+
+static int read_entry(
+	git_index_entry **out,
+	size_t *out_size,
+	git_index *index,
+	const void *buffer,
+	size_t buffer_size,
+	const char *last)
 {
 	size_t path_length, entry_size;
-	uint16_t flags_raw;
 	const char *path_ptr;
-	const struct entry_short *source = buffer;
+	struct entry_short source;
+	git_index_entry entry = {{0}};
+	bool compressed = index->version >= INDEX_VERSION_NUMBER_COMP;
+	char *tmp_path = NULL;
 
 	if (INDEX_FOOTER_SIZE + minimal_entry_size > buffer_size)
-		return 0;
+		return -1;
 
-	memset(dest, 0x0, sizeof(git_index_entry));
-
-	dest->ctime.seconds = (git_time_t)ntohl(source->ctime.seconds);
-	dest->ctime.nanoseconds = ntohl(source->ctime.nanoseconds);
-	dest->mtime.seconds = (git_time_t)ntohl(source->mtime.seconds);
-	dest->mtime.nanoseconds = ntohl(source->mtime.nanoseconds);
-	dest->dev = ntohl(source->dev);
-	dest->ino = ntohl(source->ino);
-	dest->mode = ntohl(source->mode);
-	dest->uid = ntohl(source->uid);
-	dest->gid = ntohl(source->gid);
-	dest->file_size = ntohl(source->file_size);
-	git_oid_cpy(&dest->oid, &source->oid);
-	dest->flags = ntohs(source->flags);
-
-	if (dest->flags & GIT_IDXENTRY_EXTENDED) {
-		const struct entry_long *source_l = (const struct entry_long *)source;
-		path_ptr = source_l->path;
-
-		flags_raw = ntohs(source_l->flags_extended);
-		memcpy(&dest->flags_extended, &flags_raw, 2);
+	/* buffer is not guaranteed to be aligned */
+	memcpy(&source, buffer, sizeof(struct entry_short));
+
+	entry.ctime.seconds = (git_time_t)ntohl(source.ctime.seconds);
+	entry.ctime.nanoseconds = ntohl(source.ctime.nanoseconds);
+	entry.mtime.seconds = (git_time_t)ntohl(source.mtime.seconds);
+	entry.mtime.nanoseconds = ntohl(source.mtime.nanoseconds);
+	entry.dev = ntohl(source.dev);
+	entry.ino = ntohl(source.ino);
+	entry.mode = ntohl(source.mode);
+	entry.uid = ntohl(source.uid);
+	entry.gid = ntohl(source.gid);
+	entry.file_size = ntohl(source.file_size);
+	git_oid_cpy(&entry.id, &source.oid);
+	entry.flags = ntohs(source.flags);
+
+	if (entry.flags & GIT_INDEX_ENTRY_EXTENDED) {
+		uint16_t flags_raw;
+		size_t flags_offset;
+
+		flags_offset = offsetof(struct entry_long, flags_extended);
+		memcpy(&flags_raw, (const char *) buffer + flags_offset,
+			sizeof(flags_raw));
+		flags_raw = ntohs(flags_raw);
+
+		memcpy(&entry.flags_extended, &flags_raw, sizeof(flags_raw));
+		path_ptr = (const char *) buffer + offsetof(struct entry_long, path);
 	} else
-		path_ptr = source->path;
+		path_ptr = (const char *) buffer + offsetof(struct entry_short, path);
 
-	path_length = dest->flags & GIT_IDXENTRY_NAMEMASK;
+	if (!compressed) {
+		path_length = entry.flags & GIT_INDEX_ENTRY_NAMEMASK;
 
-	/* if this is a very long string, we must find its
-	 * real length without overflowing */
-	if (path_length == 0xFFF) {
-		const char *path_end;
+		/* if this is a very long string, we must find its
+		 * real length without overflowing */
+		if (path_length == 0xFFF) {
+			const char *path_end;
 
-		path_end = memchr(path_ptr, '\0', buffer_size);
-		if (path_end == NULL)
-			return 0;
+			path_end = memchr(path_ptr, '\0', buffer_size);
+			if (path_end == NULL)
+				return -1;
+
+			path_length = path_end - path_ptr;
+		}
+
+		entry_size = index_entry_size(path_length, 0, entry.flags);
+		entry.path = (char *)path_ptr;
+	} else {
+		size_t varint_len, last_len, prefix_len, suffix_len, path_len;
+		uintmax_t strip_len;
+
+		strip_len = git_decode_varint((const unsigned char *)path_ptr, &varint_len);
+		last_len = strlen(last);
+
+		if (varint_len == 0 || last_len < strip_len)
+			return index_error_invalid("incorrect prefix length");
+
+		prefix_len = last_len - (size_t)strip_len;
+		suffix_len = strlen(path_ptr + varint_len);
+
+		GIT_ERROR_CHECK_ALLOC_ADD(&path_len, prefix_len, suffix_len);
+		GIT_ERROR_CHECK_ALLOC_ADD(&path_len, path_len, 1);
+
+		if (path_len > GIT_PATH_MAX)
+			return index_error_invalid("unreasonable path length");
 
-		path_length = path_end - path_ptr;
+		tmp_path = git__malloc(path_len);
+		GIT_ERROR_CHECK_ALLOC(tmp_path);
+
+		memcpy(tmp_path, last, prefix_len);
+		memcpy(tmp_path + prefix_len, path_ptr + varint_len, suffix_len + 1);
+		entry_size = index_entry_size(suffix_len, varint_len, entry.flags);
+		entry.path = tmp_path;
 	}
 
-	if (dest->flags & GIT_IDXENTRY_EXTENDED)
-		entry_size = long_entry_size(path_length);
-	else
-		entry_size = short_entry_size(path_length);
+	if (entry_size == 0)
+		return -1;
 
 	if (INDEX_FOOTER_SIZE + entry_size > buffer_size)
-		return 0;
+		return -1;
 
-	dest->path = git__strdup(path_ptr);
-	assert(dest->path);
+	if (index_entry_dup(out, index, &entry) < 0) {
+		git__free(tmp_path);
+		return -1;
+	}
 
-	return entry_size;
+	git__free(tmp_path);
+	*out_size = entry_size;
+	return 0;
 }
 
 static int read_header(struct index_header *dest, const void *buffer)
@@ -1562,64 +2518,72 @@ static int read_header(struct index_header *dest, const void *buffer)
 		return index_error_invalid("incorrect header signature");
 
 	dest->version = ntohl(source->version);
-	if (dest->version != INDEX_VERSION_NUMBER_EXT &&
-		dest->version != INDEX_VERSION_NUMBER)
+	if (dest->version < INDEX_VERSION_NUMBER_LB ||
+		dest->version > INDEX_VERSION_NUMBER_UB)
 		return index_error_invalid("incorrect header version");
 
 	dest->entry_count = ntohl(source->entry_count);
 	return 0;
 }
 
-static size_t read_extension(git_index *index, const char *buffer, size_t buffer_size)
+static int read_extension(size_t *read_len, git_index *index, const char *buffer, size_t buffer_size)
 {
-	const struct index_extension *source;
 	struct index_extension dest;
 	size_t total_size;
 
-	source = (const struct index_extension *)(buffer);
-
-	memcpy(dest.signature, source->signature, 4);
-	dest.extension_size = ntohl(source->extension_size);
+	/* buffer is not guaranteed to be aligned */
+	memcpy(&dest, buffer, sizeof(struct index_extension));
+	dest.extension_size = ntohl(dest.extension_size);
 
 	total_size = dest.extension_size + sizeof(struct index_extension);
 
-	if (buffer_size < total_size ||
-		buffer_size - total_size < INDEX_FOOTER_SIZE)
-		return 0;
+	if (dest.extension_size > total_size ||
+		buffer_size < total_size ||
+		buffer_size - total_size < INDEX_FOOTER_SIZE) {
+		index_error_invalid("extension is truncated");
+		return -1;
+	}
 
 	/* optional extension */
 	if (dest.signature[0] >= 'A' && dest.signature[0] <= 'Z') {
 		/* tree cache */
 		if (memcmp(dest.signature, INDEX_EXT_TREECACHE_SIG, 4) == 0) {
-			if (git_tree_cache_read(&index->tree, buffer + 8, dest.extension_size) < 0)
-				return 0;
+			if (git_tree_cache_read(&index->tree, buffer + 8, dest.extension_size, &index->tree_pool) < 0)
+				return -1;
 		} else if (memcmp(dest.signature, INDEX_EXT_UNMERGED_SIG, 4) == 0) {
 			if (read_reuc(index, buffer + 8, dest.extension_size) < 0)
-				return 0;
+				return -1;
 		} else if (memcmp(dest.signature, INDEX_EXT_CONFLICT_NAME_SIG, 4) == 0) {
 			if (read_conflict_names(index, buffer + 8, dest.extension_size) < 0)
-				return 0;
+				return -1;
 		}
 		/* else, unsupported extension. We cannot parse this, but we can skip
 		 * it by returning `total_size */
 	} else {
 		/* we cannot handle non-ignorable extensions;
 		 * in fact they aren't even defined in the standard */
-		return 0;
+		git_error_set(GIT_ERROR_INDEX, "unsupported mandatory extension: '%.4s'", dest.signature);
+		return -1;
 	}
 
-	return total_size;
+	*read_len = total_size;
+
+	return 0;
 }
 
 static int parse_index(git_index *index, const char *buffer, size_t buffer_size)
 {
+	int error = 0;
 	unsigned int i;
 	struct index_header header = { 0 };
 	git_oid checksum_calculated, checksum_expected;
+	const char *last = NULL;
+	const char *empty = "";
 
 #define seek_forward(_increase) { \
-	if (_increase >= buffer_size) \
-		return index_error_invalid("ran out of data while parsing"); \
+	if (_increase >= buffer_size) { \
+		error = index_error_invalid("ran out of data while parsing"); \
+		goto done; } \
 	buffer += _increase; \
 	buffer_size -= _increase;\
 }
@@ -1632,65 +2596,95 @@ static int parse_index(git_index *index, const char *buffer, size_t buffer_size)
 	git_hash_buf(&checksum_calculated, buffer, buffer_size - INDEX_FOOTER_SIZE);
 
 	/* Parse header */
-	if (read_header(&header, buffer) < 0)
-		return -1;
+	if ((error = read_header(&header, buffer)) < 0)
+		return error;
+
+	index->version = header.version;
+	if (index->version >= INDEX_VERSION_NUMBER_COMP)
+		last = empty;
 
 	seek_forward(INDEX_HEADER_SIZE);
 
-	git_vector_clear(&index->entries);
+	assert(!index->entries.length);
+
+	if (index->ignore_case)
+		git_idxmap_icase_resize((git_idxmap_icase *) index->entries_map, header.entry_count);
+	else
+		git_idxmap_resize(index->entries_map, header.entry_count);
 
 	/* Parse all the entries */
 	for (i = 0; i < header.entry_count && buffer_size > INDEX_FOOTER_SIZE; ++i) {
+		git_index_entry *entry = NULL;
 		size_t entry_size;
-		git_index_entry *entry;
 
-		entry = git__malloc(sizeof(git_index_entry));
-		GITERR_CHECK_ALLOC(entry);
+		if ((error = read_entry(&entry, &entry_size, index, buffer, buffer_size, last)) < 0) {
+			error = index_error_invalid("invalid entry");
+			goto done;
+		}
+
+		if ((error = git_vector_insert(&index->entries, entry)) < 0) {
+			index_entry_free(entry);
+			goto done;
+		}
 
-		entry_size = read_entry(entry, buffer, buffer_size);
+		INSERT_IN_MAP(index, entry, &error);
 
-		/* 0 bytes read means an object corruption */
-		if (entry_size == 0)
-			return index_error_invalid("invalid entry");
+		if (error < 0) {
+			index_entry_free(entry);
+			goto done;
+		}
+		error = 0;
 
-		if (git_vector_insert(&index->entries, entry) < 0)
-			return -1;
+		if (index->version >= INDEX_VERSION_NUMBER_COMP)
+			last = entry->path;
 
 		seek_forward(entry_size);
 	}
 
-	if (i != header.entry_count)
-		return index_error_invalid("header entries changed while parsing");
+	if (i != header.entry_count) {
+		error = index_error_invalid("header entries changed while parsing");
+		goto done;
+	}
 
 	/* There's still space for some extensions! */
 	while (buffer_size > INDEX_FOOTER_SIZE) {
 		size_t extension_size;
 
-		extension_size = read_extension(index, buffer, buffer_size);
-
-		/* see if we have read any bytes from the extension */
-		if (extension_size == 0)
-			return index_error_invalid("extension is truncated");
+		if ((error = read_extension(&extension_size, index, buffer, buffer_size)) < 0) {
+			goto done;
+		}
 
 		seek_forward(extension_size);
 	}
 
-	if (buffer_size != INDEX_FOOTER_SIZE)
-		return index_error_invalid("buffer size does not match index footer size");
+	if (buffer_size != INDEX_FOOTER_SIZE) {
+		error = index_error_invalid(
+			"buffer size does not match index footer size");
+		goto done;
+	}
 
 	/* 160-bit SHA-1 over the content of the index file before this checksum. */
 	git_oid_fromraw(&checksum_expected, (const unsigned char *)buffer);
 
-	if (git_oid__cmp(&checksum_calculated, &checksum_expected) != 0)
-		return index_error_invalid("calculated checksum does not match expected");
+	if (git_oid__cmp(&checksum_calculated, &checksum_expected) != 0) {
+		error = index_error_invalid(
+			"calculated checksum does not match expected");
+		goto done;
+	}
+
+	git_oid_cpy(&index->checksum, &checksum_calculated);
 
 #undef seek_forward
 
-	/* Entries are stored case-sensitively on disk. */
-	index->entries.sorted = !index->ignore_case;
+	/* Entries are stored case-sensitively on disk, so re-sort now if
+	 * in-memory index is supposed to be case-insensitive
+	 */
+	git_vector_set_sorted(&index->entries, !index->ignore_case);
 	git_vector_sort(&index->entries);
 
-	return 0;
+	index->dirty = 0;
+done:
+	return error;
 }
 
 static bool is_index_extended(git_index *index)
@@ -1701,36 +2695,48 @@ static bool is_index_extended(git_index *index)
 	extended = 0;
 
 	git_vector_foreach(&index->entries, i, entry) {
-		entry->flags &= ~GIT_IDXENTRY_EXTENDED;
-		if (entry->flags_extended & GIT_IDXENTRY_EXTENDED_FLAGS) {
+		entry->flags &= ~GIT_INDEX_ENTRY_EXTENDED;
+		if (entry->flags_extended & GIT_INDEX_ENTRY_EXTENDED_FLAGS) {
 			extended++;
-			entry->flags |= GIT_IDXENTRY_EXTENDED;
+			entry->flags |= GIT_INDEX_ENTRY_EXTENDED;
 		}
 	}
 
 	return (extended > 0);
 }
 
-static int write_disk_entry(git_filebuf *file, git_index_entry *entry)
+static int write_disk_entry(git_filebuf *file, git_index_entry *entry, const char *last)
 {
 	void *mem = NULL;
-	struct entry_short *ondisk;
+	struct entry_short ondisk;
 	size_t path_len, disk_size;
+	int varint_len = 0;
 	char *path;
+	const char *path_start = entry->path;
+	size_t same_len = 0;
 
-	path_len = strlen(entry->path);
+	path_len = ((struct entry_internal *)entry)->pathlen;
 
-	if (entry->flags & GIT_IDXENTRY_EXTENDED)
-		disk_size = long_entry_size(path_len);
-	else
-		disk_size = short_entry_size(path_len);
+	if (last) {
+		const char *last_c = last;
+
+		while (*path_start == *last_c) {
+			if (!*path_start || !*last_c)
+				break;
+			++path_start;
+			++last_c;
+			++same_len;
+		}
+		path_len -= same_len;
+		varint_len = git_encode_varint(NULL, 0, same_len);
+	}
+
+	disk_size = index_entry_size(path_len, varint_len, entry->flags);
 
 	if (git_filebuf_reserve(file, &mem, disk_size) < 0)
 		return -1;
 
-	ondisk = (struct entry_short *)mem;
-
-	memset(ondisk, 0x0, disk_size);
+	memset(mem, 0x0, disk_size);
 
 	/**
 	 * Yes, we have to truncate.
@@ -1742,31 +2748,57 @@ static int write_disk_entry(git_filebuf *file, git_index_entry *entry)
 	 *
 	 * In 2038 I will be either too dead or too rich to care about this
 	 */
-	ondisk->ctime.seconds = htonl((uint32_t)entry->ctime.seconds);
-	ondisk->mtime.seconds = htonl((uint32_t)entry->mtime.seconds);
-	ondisk->ctime.nanoseconds = htonl(entry->ctime.nanoseconds);
-	ondisk->mtime.nanoseconds = htonl(entry->mtime.nanoseconds);
-	ondisk->dev = htonl(entry->dev);
-	ondisk->ino = htonl(entry->ino);
-	ondisk->mode = htonl(entry->mode);
-	ondisk->uid = htonl(entry->uid);
-	ondisk->gid = htonl(entry->gid);
-	ondisk->file_size = htonl((uint32_t)entry->file_size);
-
-	git_oid_cpy(&ondisk->oid, &entry->oid);
-
-	ondisk->flags = htons(entry->flags);
-
-	if (entry->flags & GIT_IDXENTRY_EXTENDED) {
-		struct entry_long *ondisk_ext;
-		ondisk_ext = (struct entry_long *)ondisk;
-		ondisk_ext->flags_extended = htons(entry->flags_extended);
-		path = ondisk_ext->path;
+	ondisk.ctime.seconds = htonl((uint32_t)entry->ctime.seconds);
+	ondisk.mtime.seconds = htonl((uint32_t)entry->mtime.seconds);
+	ondisk.ctime.nanoseconds = htonl(entry->ctime.nanoseconds);
+	ondisk.mtime.nanoseconds = htonl(entry->mtime.nanoseconds);
+	ondisk.dev = htonl(entry->dev);
+	ondisk.ino = htonl(entry->ino);
+	ondisk.mode = htonl(entry->mode);
+	ondisk.uid = htonl(entry->uid);
+	ondisk.gid = htonl(entry->gid);
+	ondisk.file_size = htonl((uint32_t)entry->file_size);
+
+	git_oid_cpy(&ondisk.oid, &entry->id);
+
+	ondisk.flags = htons(entry->flags);
+
+	if (entry->flags & GIT_INDEX_ENTRY_EXTENDED) {
+		struct entry_long ondisk_ext;
+		memcpy(&ondisk_ext, &ondisk, sizeof(struct entry_short));
+		ondisk_ext.flags_extended = htons(entry->flags_extended &
+			GIT_INDEX_ENTRY_EXTENDED_FLAGS);
+		memcpy(mem, &ondisk_ext, offsetof(struct entry_long, path));
+		path = ((struct entry_long*)mem)->path;
+		disk_size -= offsetof(struct entry_long, path);
+	} else {
+		memcpy(mem, &ondisk, offsetof(struct entry_short, path));
+		path = ((struct entry_short*)mem)->path;
+		disk_size -= offsetof(struct entry_short, path);
 	}
-	else
-		path = ondisk->path;
 
-	memcpy(path, entry->path, path_len);
+	if (last) {
+		varint_len = git_encode_varint((unsigned char *) path,
+					  disk_size, same_len);
+		assert(varint_len > 0);
+		path += varint_len;
+		disk_size -= varint_len;
+
+		/*
+		 * If using path compression, we are not allowed
+		 * to have additional trailing NULs.
+		 */
+		assert(disk_size == path_len + 1);
+	} else {
+		/*
+		 * If no path compression is used, we do have
+		 * NULs as padding. As such, simply assert that
+		 * we have enough space left to write the path.
+		 */
+		assert(disk_size > path_len);
+	}
+
+	memcpy(path, path_start, path_len + 1);
 
 	return 0;
 }
@@ -1775,21 +2807,29 @@ static int write_entries(git_index *index, git_filebuf *file)
 {
 	int error = 0;
 	size_t i;
-	git_vector case_sorted;
+	git_vector case_sorted, *entries;
 	git_index_entry *entry;
-	git_vector *out = &index->entries;
+	const char *last = NULL;
 
 	/* If index->entries is sorted case-insensitively, then we need
 	 * to re-sort it case-sensitively before writing */
 	if (index->ignore_case) {
-		git_vector_dup(&case_sorted, &index->entries, index_cmp);
+		git_vector_dup(&case_sorted, &index->entries, git_index_entry_cmp);
 		git_vector_sort(&case_sorted);
-		out = &case_sorted;
+		entries = &case_sorted;
+	} else {
+		entries = &index->entries;
 	}
 
-	git_vector_foreach(out, i, entry)
-		if ((error = write_disk_entry(file, entry)) < 0)
+	if (index->version >= INDEX_VERSION_NUMBER_COMP)
+		last = "";
+
+	git_vector_foreach(entries, i, entry) {
+		if ((error = write_disk_entry(file, entry, last)) < 0)
 			break;
+		if (index->version >= INDEX_VERSION_NUMBER_COMP)
+			last = entry->path;
+	}
 
 	if (index->ignore_case)
 		git_vector_free(&case_sorted);
@@ -1800,16 +2840,13 @@ static int write_entries(git_index *index, git_filebuf *file)
 static int write_extension(git_filebuf *file, struct index_extension *header, git_buf *data)
 {
 	struct index_extension ondisk;
-	int error = 0;
 
 	memset(&ondisk, 0x0, sizeof(struct index_extension));
 	memcpy(&ondisk, header, 4);
 	ondisk.extension_size = htonl(header->extension_size);
 
-	if ((error = git_filebuf_write(file, &ondisk, sizeof(struct index_extension))) == 0)
-		error = git_filebuf_write(file, data->ptr, data->size);
-
-	return error;
+	git_filebuf_write(file, &ondisk, sizeof(struct index_extension));
+	return git_filebuf_write(file, data->ptr, data->size);
 }
 
 static int create_name_extension_data(git_buf *name_buf, git_index_name_entry *conflict_name)
@@ -1861,7 +2898,7 @@ static int write_name_extension(git_index *index, git_filebuf *file)
 
 	error = write_extension(file, &extension, &name_buf);
 
-	git_buf_free(&name_buf);
+	git_buf_dispose(&name_buf);
 
 done:
 	return error;
@@ -1909,24 +2946,62 @@ static int write_reuc_extension(git_index *index, git_filebuf *file)
 
 	error = write_extension(file, &extension, &reuc_buf);
 
-	git_buf_free(&reuc_buf);
+	git_buf_dispose(&reuc_buf);
 
 done:
 	return error;
 }
 
-static int write_index(git_index *index, git_filebuf *file)
+static int write_tree_extension(git_index *index, git_filebuf *file)
+{
+	struct index_extension extension;
+	git_buf buf = GIT_BUF_INIT;
+	int error;
+
+	if (index->tree == NULL)
+		return 0;
+
+	if ((error = git_tree_cache_write(&buf, index->tree)) < 0)
+		return error;
+
+	memset(&extension, 0x0, sizeof(struct index_extension));
+	memcpy(&extension.signature, INDEX_EXT_TREECACHE_SIG, 4);
+	extension.extension_size = (uint32_t)buf.size;
+
+	error = write_extension(file, &extension, &buf);
+
+	git_buf_dispose(&buf);
+
+	return error;
+}
+
+static void clear_uptodate(git_index *index)
+{
+	git_index_entry *entry;
+	size_t i;
+
+	git_vector_foreach(&index->entries, i, entry)
+		entry->flags_extended &= ~GIT_INDEX_ENTRY_UPTODATE;
+}
+
+static int write_index(git_oid *checksum, git_index *index, git_filebuf *file)
 {
 	git_oid hash_final;
 	struct index_header header;
 	bool is_extended;
+	uint32_t index_version_number;
 
 	assert(index && file);
 
-	is_extended = is_index_extended(index);
+	if (index->version <= INDEX_VERSION_NUMBER_EXT)  {
+		is_extended = is_index_extended(index);
+		index_version_number = is_extended ? INDEX_VERSION_NUMBER_EXT : INDEX_VERSION_NUMBER_LB;
+	} else {
+		index_version_number = index->version;
+	}
 
 	header.signature = htonl(INDEX_HEADER_SIG);
-	header.version = htonl(is_extended ? INDEX_VERSION_NUMBER_EXT : INDEX_VERSION_NUMBER);
+	header.version = htonl(index_version_number);
 	header.entry_count = htonl((uint32_t)index->entries.length);
 
 	if (git_filebuf_write(file, &header, sizeof(struct index_header)) < 0)
@@ -1935,7 +3010,9 @@ static int write_index(git_index *index, git_filebuf *file)
 	if (write_entries(index, file) < 0)
 		return -1;
 
-	/* TODO: write tree cache extension */
+	/* write the tree cache extension */
+	if (index->tree != NULL && write_tree_extension(index, file) < 0)
+		return -1;
 
 	/* write the rename conflict extension */
 	if (index->names.length > 0 && write_name_extension(index, file) < 0)
@@ -1947,19 +3024,34 @@ static int write_index(git_index *index, git_filebuf *file)
 
 	/* get out the hash for all the contents we've appended to the file */
 	git_filebuf_hash(&hash_final, file);
+	git_oid_cpy(checksum, &hash_final);
 
 	/* write it at the end of the file */
-	return git_filebuf_write(file, hash_final.id, GIT_OID_RAWSZ);
+	if (git_filebuf_write(file, hash_final.id, GIT_OID_RAWSZ) < 0)
+		return -1;
+
+	/* file entries are no longer up to date */
+	clear_uptodate(index);
+
+	return 0;
 }
 
 int git_index_entry_stage(const git_index_entry *entry)
 {
-	return GIT_IDXENTRY_STAGE(entry);
+	return GIT_INDEX_ENTRY_STAGE(entry);
+}
+
+int git_index_entry_is_conflict(const git_index_entry *entry)
+{
+	return (GIT_INDEX_ENTRY_STAGE(entry) > 0);
 }
 
 typedef struct read_tree_data {
 	git_index *index;
 	git_vector *old_entries;
+	git_vector *new_entries;
+	git_vector_cmp entry_cmp;
+	git_tree_cache *tree;
 } read_tree_data;
 
 static int read_tree_cb(
@@ -1968,6 +3060,7 @@ static int read_tree_cb(
 	read_tree_data *data = payload;
 	git_index_entry *entry = NULL, *old_entry;
 	git_buf path = GIT_BUF_INIT;
+	size_t pos;
 
 	if (git_tree_entry__is_tree(tentry))
 		return 0;
@@ -1975,40 +3068,28 @@ static int read_tree_cb(
 	if (git_buf_joinpath(&path, root, tentry->filename) < 0)
 		return -1;
 
-	entry = git__calloc(1, sizeof(git_index_entry));
-	GITERR_CHECK_ALLOC(entry);
+	if (index_entry_create(&entry, INDEX_OWNER(data->index), path.ptr, NULL, false) < 0)
+		return -1;
 
 	entry->mode = tentry->attr;
-	entry->oid = tentry->oid;
+	git_oid_cpy(&entry->id, git_tree_entry_id(tentry));
 
 	/* look for corresponding old entry and copy data to new entry */
-	if (data->old_entries) {
-		size_t pos;
-		struct entry_srch_key skey;
-
-		skey.path = path.ptr;
-		skey.stage = 0;
-
-		if (!git_vector_bsearch2(
-				&pos, data->old_entries, data->index->entries_search, &skey) &&
-			(old_entry = git_vector_get(data->old_entries, pos)) != NULL &&
-			entry->mode == old_entry->mode &&
-			git_oid_equal(&entry->oid, &old_entry->oid))
-		{
-			memcpy(entry, old_entry, sizeof(*entry));
-			entry->flags_extended = 0;
-		}
+	if (data->old_entries != NULL &&
+		!index_find_in_entries(
+			&pos, data->old_entries, data->entry_cmp, path.ptr, 0, 0) &&
+		(old_entry = git_vector_get(data->old_entries, pos)) != NULL &&
+		entry->mode == old_entry->mode &&
+		git_oid_equal(&entry->id, &old_entry->id))
+	{
+		index_entry_cpy(entry, old_entry);
+		entry->flags_extended = 0;
 	}
 
-	if (path.size < GIT_IDXENTRY_NAMEMASK)
-		entry->flags = path.size & GIT_IDXENTRY_NAMEMASK;
-	else
-		entry->flags = GIT_IDXENTRY_NAMEMASK;
-
-	entry->path = git_buf_detach(&path);
-	git_buf_free(&path);
+	index_entry_adjust_namemask(entry, path.size);
+	git_buf_dispose(&path);
 
-	if (git_vector_insert(&data->index->entries, entry) < 0) {
+	if (git_vector_insert(data->new_entries, entry) < 0) {
 		index_entry_free(entry);
 		return -1;
 	}
@@ -2020,25 +3101,225 @@ int git_index_read_tree(git_index *index, const git_tree *tree)
 {
 	int error = 0;
 	git_vector entries = GIT_VECTOR_INIT;
+	git_idxmap *entries_map;
 	read_tree_data data;
+	size_t i;
+	git_index_entry *e;
+
+	if (git_idxmap_alloc(&entries_map) < 0)
+		return -1;
+
+	git_vector_set_cmp(&entries, index->entries._cmp); /* match sort */
+
+	data.index = index;
+	data.old_entries = &index->entries;
+	data.new_entries = &entries;
+	data.entry_cmp   = index->entries_search;
+
+	index->tree = NULL;
+	git_pool_clear(&index->tree_pool);
 
 	git_vector_sort(&index->entries);
 
-	git_vector_set_cmp(&entries, index->entries._cmp);
-	git_vector_swap(&entries, &index->entries);
+	if ((error = git_tree_walk(tree, GIT_TREEWALK_POST, read_tree_cb, &data)) < 0)
+		goto cleanup;
+
+	if (index->ignore_case)
+		git_idxmap_icase_resize((git_idxmap_icase *) entries_map, entries.length);
+	else
+		git_idxmap_resize(entries_map, entries.length);
 
-	git_index_clear(index);
+	git_vector_foreach(&entries, i, e) {
+		INSERT_IN_MAP_EX(index, entries_map, e, &error);
 
-	data.index = index;
-	data.old_entries = &entries;
+		if (error < 0) {
+			git_error_set(GIT_ERROR_INDEX, "failed to insert entry into map");
+			return error;
+		}
+	}
 
-	error = git_tree_walk(tree, GIT_TREEWALK_POST, read_tree_cb, &data);
+	error = 0;
 
-	index_entries_free(&entries);
+	git_vector_sort(&entries);
+
+	if ((error = git_index_clear(index)) < 0) {
+		/* well, this isn't good */;
+	} else {
+		git_vector_swap(&entries, &index->entries);
+		entries_map = git__swap(index->entries_map, entries_map);
+	}
+
+	index->dirty = 1;
+
+cleanup:
 	git_vector_free(&entries);
+	git_idxmap_free(entries_map);
+	if (error < 0)
+		return error;
 
-	git_vector_sort(&index->entries);
+	error = git_tree_cache_read_tree(&index->tree, tree, &index->tree_pool);
+
+	return error;
+}
+
+static int git_index_read_iterator(
+	git_index *index,
+	git_iterator *new_iterator,
+	size_t new_length_hint)
+{
+	git_vector new_entries = GIT_VECTOR_INIT,
+		remove_entries = GIT_VECTOR_INIT;
+	git_idxmap *new_entries_map = NULL;
+	git_iterator *index_iterator = NULL;
+	git_iterator_options opts = GIT_ITERATOR_OPTIONS_INIT;
+	const git_index_entry *old_entry, *new_entry;
+	git_index_entry *entry;
+	size_t i;
+	int error;
+
+	assert((new_iterator->flags & GIT_ITERATOR_DONT_IGNORE_CASE));
+
+	if ((error = git_vector_init(&new_entries, new_length_hint, index->entries._cmp)) < 0 ||
+		(error = git_vector_init(&remove_entries, index->entries.length, NULL)) < 0 ||
+		(error = git_idxmap_alloc(&new_entries_map)) < 0)
+		goto done;
+
+	if (index->ignore_case && new_length_hint)
+		git_idxmap_icase_resize((git_idxmap_icase *) new_entries_map, new_length_hint);
+	else if (new_length_hint)
+		git_idxmap_resize(new_entries_map, new_length_hint);
+
+	opts.flags = GIT_ITERATOR_DONT_IGNORE_CASE |
+		GIT_ITERATOR_INCLUDE_CONFLICTS;
+
+	if ((error = git_iterator_for_index(&index_iterator,
+			git_index_owner(index), index, &opts)) < 0 ||
+		((error = git_iterator_current(&old_entry, index_iterator)) < 0 &&
+			error != GIT_ITEROVER) ||
+		((error = git_iterator_current(&new_entry, new_iterator)) < 0 &&
+			error != GIT_ITEROVER))
+		goto done;
+
+	while (true) {
+		git_index_entry
+			*dup_entry = NULL,
+			*add_entry = NULL,
+			*remove_entry = NULL;
+		int diff;
+
+		error = 0;
+
+		if (old_entry && new_entry)
+			diff = git_index_entry_cmp(old_entry, new_entry);
+		else if (!old_entry && new_entry)
+			diff = 1;
+		else if (old_entry && !new_entry)
+			diff = -1;
+		else
+			break;
+
+		if (diff < 0) {
+			remove_entry = (git_index_entry *)old_entry;
+		} else if (diff > 0) {
+			dup_entry = (git_index_entry *)new_entry;
+		} else {
+			/* Path and stage are equal, if the OID is equal, keep it to
+			 * keep the stat cache data.
+			 */
+			if (git_oid_equal(&old_entry->id, &new_entry->id) &&
+				old_entry->mode == new_entry->mode) {
+				add_entry = (git_index_entry *)old_entry;
+			} else {
+				dup_entry = (git_index_entry *)new_entry;
+				remove_entry = (git_index_entry *)old_entry;
+			}
+		}
+
+		if (dup_entry) {
+			if ((error = index_entry_dup_nocache(&add_entry, index, dup_entry)) < 0)
+				goto done;
+
+			index_entry_adjust_namemask(add_entry,
+				((struct entry_internal *)add_entry)->pathlen);
+		}
+
+		/* invalidate this path in the tree cache if this is new (to
+		 * invalidate the parent trees)
+		 */
+		if (dup_entry && !remove_entry && index->tree)
+			git_tree_cache_invalidate_path(index->tree, dup_entry->path);
+
+		if (add_entry) {
+			if ((error = git_vector_insert(&new_entries, add_entry)) == 0)
+				INSERT_IN_MAP_EX(index, new_entries_map, add_entry, &error);
+		}
+
+		if (remove_entry && error >= 0)
+			error = git_vector_insert(&remove_entries, remove_entry);
+
+		if (error < 0) {
+			git_error_set(GIT_ERROR_INDEX, "failed to insert entry");
+			goto done;
+		}
+
+		if (diff <= 0) {
+			if ((error = git_iterator_advance(&old_entry, index_iterator)) < 0 &&
+				error != GIT_ITEROVER)
+				goto done;
+		}
+
+		if (diff >= 0) {
+			if ((error = git_iterator_advance(&new_entry, new_iterator)) < 0 &&
+				error != GIT_ITEROVER)
+				goto done;
+		}
+	}
+
+	git_index_name_clear(index);
+	git_index_reuc_clear(index);
+
+	git_vector_swap(&new_entries, &index->entries);
+	new_entries_map = git__swap(index->entries_map, new_entries_map);
+
+	git_vector_foreach(&remove_entries, i, entry) {
+		if (index->tree)
+			git_tree_cache_invalidate_path(index->tree, entry->path);
+
+		index_entry_free(entry);
+	}
+
+	clear_uptodate(index);
+
+	index->dirty = 1;
+	error = 0;
+
+done:
+	git_idxmap_free(new_entries_map);
+	git_vector_free(&new_entries);
+	git_vector_free(&remove_entries);
+	git_iterator_free(index_iterator);
+	return error;
+}
+
+int git_index_read_index(
+	git_index *index,
+	const git_index *new_index)
+{
+	git_iterator *new_iterator = NULL;
+	git_iterator_options opts = GIT_ITERATOR_OPTIONS_INIT;
+	int error;
 
+	opts.flags = GIT_ITERATOR_DONT_IGNORE_CASE |
+		GIT_ITERATOR_INCLUDE_CONFLICTS;
+
+	if ((error = git_iterator_for_index(&new_iterator,
+		git_index_owner(new_index), (git_index *)new_index, &opts)) < 0 ||
+		(error = git_index_read_iterator(index, new_iterator,
+		new_index->entries.length)) < 0)
+		goto done;
+
+done:
+	git_iterator_free(new_iterator);
 	return error;
 }
 
@@ -2047,6 +3328,13 @@ git_repository *git_index_owner(const git_index *index)
 	return INDEX_OWNER(index);
 }
 
+enum {
+	INDEX_ACTION_NONE = 0,
+	INDEX_ACTION_UPDATE = 1,
+	INDEX_ACTION_REMOVE = 2,
+	INDEX_ACTION_ADDALL = 3,
+};
+
 int git_index_add_all(
 	git_index *index,
 	const git_strarray *paths,
@@ -2057,30 +3345,16 @@ int git_index_add_all(
 	int error;
 	git_repository *repo;
 	git_iterator *wditer = NULL;
-	const git_index_entry *wd = NULL;
-	git_index_entry *entry;
-	git_pathspec_context ps;
-	const char *match;
-	size_t existing;
+	git_pathspec ps;
 	bool no_fnmatch = (flags & GIT_INDEX_ADD_DISABLE_PATHSPEC_MATCH) != 0;
-	int ignorecase;
-	git_oid blobid;
 
 	assert(index);
 
-	if (INDEX_OWNER(index) == NULL)
-		return create_index_error(-1,
-			"Could not add paths to index. "
-			"Index is not backed up by an existing repository.");
-
 	repo = INDEX_OWNER(index);
 	if ((error = git_repository__ensure_not_bare(repo, "index add all")) < 0)
 		return error;
 
-	if (git_repository__cvar(&ignorecase, repo, GIT_CVAR_IGNORECASE) < 0)
-		return -1;
-
-	if ((error = git_pathspec_context_init(&ps, paths)) < 0)
+	if ((error = git_pathspec__init(&ps, paths)) < 0)
 		return error;
 
 	/* optionally check that pathspec doesn't mention any ignored files */
@@ -2090,81 +3364,118 @@ int git_index_add_all(
 			repo, &ps.pathspec, no_fnmatch)) < 0)
 		goto cleanup;
 
-	if ((error = git_iterator_for_workdir(
-			&wditer, repo, 0, ps.prefix, ps.prefix)) < 0)
-		goto cleanup;
+	error = index_apply_to_wd_diff(index, INDEX_ACTION_ADDALL, paths, flags, cb, payload);
 
-	while (!(error = git_iterator_advance(&wd, wditer))) {
+	if (error)
+		git_error_set_after_callback(error);
 
-		/* check if path actually matches */
-		if (!git_pathspec_match_path(
-				&ps.pathspec, wd->path, no_fnmatch, ignorecase, &match))
-			continue;
+cleanup:
+	git_iterator_free(wditer);
+	git_pathspec__clear(&ps);
 
-		/* skip ignored items that are not already in the index */
-		if ((flags & GIT_INDEX_ADD_FORCE) == 0 &&
-			git_iterator_current_is_ignored(wditer) &&
-			index_find(&existing, index, wd->path, 0) < 0)
-			continue;
+	return error;
+}
 
-		/* issue notification callback if requested */
-		if (cb && (error = cb(wd->path, match, payload)) != 0) {
-			if (error > 0) /* return > 0 means skip this one */
-				continue;
-			if (error < 0) { /* return < 0 means abort */
-				giterr_clear();
-				error = GIT_EUSER;
-				break;
-			}
-		}
+struct foreach_diff_data {
+	git_index *index;
+	const git_pathspec *pathspec;
+	unsigned int flags;
+	git_index_matched_path_cb cb;
+	void *payload;
+};
 
-		/* TODO: Should we check if the file on disk is already an exact
-		 * match to the file in the index and skip this work if it is?
-		 */
+static int apply_each_file(const git_diff_delta *delta, float progress, void *payload)
+{
+	struct foreach_diff_data *data = payload;
+	const char *match, *path;
+	int error = 0;
 
-		/* write the blob to disk and get the oid */
-		if ((error = git_blob_create_fromworkdir(&blobid, repo, wd->path)) < 0)
-			break;
+	GIT_UNUSED(progress);
 
-		/* make the new entry to insert */
-		if ((entry = index_entry_dup(wd)) == NULL) {
-			error = -1;
-			break;
-		}
-		entry->oid = blobid;
+	path = delta->old_file.path;
 
-		/* add working directory item to index */
-		if ((error = index_insert(index, entry, 1)) < 0) {
-			index_entry_free(entry);
-			break;
-		}
+	/* We only want those which match the pathspecs */
+	if (!git_pathspec__match(
+		    &data->pathspec->pathspec, path, false, (bool)data->index->ignore_case,
+		    &match, NULL))
+		return 0;
 
-		git_tree_cache_invalidate_path(index->tree, wd->path);
+	if (data->cb)
+		error = data->cb(path, match, data->payload);
 
-		/* add implies conflict resolved, move conflict entries to REUC */
-		if ((error = index_conflict_to_reuc(index, wd->path)) < 0) {
-			if (error != GIT_ENOTFOUND)
-				break;
-			giterr_clear();
-		}
+	if (error > 0) /* skip this entry */
+		return 0;
+	if (error < 0) /* actual error */
+		return error;
+
+	/* If the workdir item does not exist, remove it from the index. */
+	if ((delta->new_file.flags & GIT_DIFF_FLAG_EXISTS) == 0)
+		error = git_index_remove_bypath(data->index, path);
+	else
+		error = git_index_add_bypath(data->index, delta->new_file.path);
+
+	return error;
+}
+
+static int index_apply_to_wd_diff(git_index *index, int action, const git_strarray *paths,
+				  unsigned int flags,
+				  git_index_matched_path_cb cb, void *payload)
+{
+	int error;
+	git_diff *diff;
+	git_pathspec ps;
+	git_repository *repo;
+	git_diff_options opts = GIT_DIFF_OPTIONS_INIT;
+	struct foreach_diff_data data = {
+		index,
+		NULL,
+		flags,
+		cb,
+		payload,
+	};
+
+	assert(index);
+	assert(action == INDEX_ACTION_UPDATE || action == INDEX_ACTION_ADDALL);
+
+	repo = INDEX_OWNER(index);
+
+	if (!repo) {
+		return create_index_error(-1,
+			"cannot run update; the index is not backed up by a repository.");
 	}
 
-	if (error == GIT_ITEROVER)
-		error = 0;
+	/*
+	 * We do the matching ourselves intead of passing the list to
+	 * diff because we want to tell the callback which one
+	 * matched, which we do not know if we ask diff to filter for us.
+	 */
+	if ((error = git_pathspec__init(&ps, paths)) < 0)
+		return error;
 
-cleanup:
-	git_iterator_free(wditer);
-	git_pathspec_context_free(&ps);
+	opts.flags = GIT_DIFF_INCLUDE_TYPECHANGE;
+	if (action == INDEX_ACTION_ADDALL) {
+		opts.flags |= GIT_DIFF_INCLUDE_UNTRACKED |
+			GIT_DIFF_RECURSE_UNTRACKED_DIRS;
+
+		if (flags == GIT_INDEX_ADD_FORCE)
+			opts.flags |= GIT_DIFF_INCLUDE_IGNORED;
+	}
+
+	if ((error = git_diff_index_to_workdir(&diff, repo, index, &opts)) < 0)
+		goto cleanup;
+
+	data.pathspec = &ps;
+	error = git_diff_foreach(diff, apply_each_file, NULL, NULL, NULL, &data);
+	git_diff_free(diff);
 
+	if (error) /* make sure error is set if callback stopped iteration */
+		git_error_set_after_callback(error);
+
+cleanup:
+	git_pathspec__clear(&ps);
 	return error;
 }
 
-enum {
-	INDEX_ACTION_NONE = 0,
-	INDEX_ACTION_UPDATE = 1,
-	INDEX_ACTION_REMOVE = 2,
-};
-
 static int index_apply_to_all(
 	git_index *index,
 	int action,
@@ -2174,13 +3485,13 @@ static int index_apply_to_all(
 {
 	int error = 0;
 	size_t i;
-	git_pathspec_context ps;
+	git_pathspec ps;
 	const char *match;
 	git_buf path = GIT_BUF_INIT;
 
 	assert(index);
 
-	if ((error = git_pathspec_context_init(&ps, paths)) < 0)
+	if ((error = git_pathspec__init(&ps, paths)) < 0)
 		return error;
 
 	git_vector_sort(&index->entries);
@@ -2189,8 +3500,9 @@ static int index_apply_to_all(
 		git_index_entry *entry = git_vector_get(&index->entries, i);
 
 		/* check if path actually matches */
-		if (!git_pathspec_match_path(
-				&ps.pathspec, entry->path, false, index->ignore_case, &match))
+		if (!git_pathspec__match(
+				&ps.pathspec, entry->path, false, (bool)index->ignore_case,
+				&match, NULL))
 			continue;
 
 		/* issue notification callback if requested */
@@ -2199,11 +3511,8 @@ static int index_apply_to_all(
 				error = 0;
 				continue;
 			}
-			if (error < 0) { /* return < 0 means abort */
-				giterr_clear();
-				error = GIT_EUSER;
+			if (error < 0)   /* return < 0 means abort */
 				break;
-			}
 		}
 
 		/* index manipulation may alter entry, so don't depend on it */
@@ -2217,7 +3526,7 @@ static int index_apply_to_all(
 			error = git_index_add_bypath(index, path.ptr);
 
 			if (error == GIT_ENOTFOUND) {
-				giterr_clear();
+				git_error_clear();
 
 				error = git_index_remove_bypath(index, path.ptr);
 
@@ -2230,14 +3539,14 @@ static int index_apply_to_all(
 				i--; /* back up foreach if we removed this */
 			break;
 		default:
-			giterr_set(GITERR_INVALID, "Unknown index action %d", action);
+			git_error_set(GIT_ERROR_INVALID, "unknown index action %d", action);
 			error = -1;
 			break;
 		}
 	}
 
-	git_buf_free(&path);
-	git_pathspec_context_free(&ps);
+	git_buf_dispose(&path);
+	git_pathspec__clear(&ps);
 
 	return error;
 }
@@ -2248,8 +3557,13 @@ int git_index_remove_all(
 	git_index_matched_path_cb cb,
 	void *payload)
 {
-	return index_apply_to_all(
+	int error = index_apply_to_all(
 		index, INDEX_ACTION_REMOVE, pathspec, cb, payload);
+
+	if (error) /* make sure error is set if callback stopped iteration */
+		git_error_set_after_callback(error);
+
+	return error;
 }
 
 int git_index_update_all(
@@ -2258,6 +3572,131 @@ int git_index_update_all(
 	git_index_matched_path_cb cb,
 	void *payload)
 {
-	return index_apply_to_all(
-		index, INDEX_ACTION_UPDATE, pathspec, cb, payload);
+	int error = index_apply_to_wd_diff(index, INDEX_ACTION_UPDATE, pathspec, 0, cb, payload);
+	if (error) /* make sure error is set if callback stopped iteration */
+		git_error_set_after_callback(error);
+
+	return error;
+}
+
+int git_index_snapshot_new(git_vector *snap, git_index *index)
+{
+	int error;
+
+	GIT_REFCOUNT_INC(index);
+
+	git_atomic_inc(&index->readers);
+	git_vector_sort(&index->entries);
+
+	error = git_vector_dup(snap, &index->entries, index->entries._cmp);
+
+	if (error < 0)
+		git_index_snapshot_release(snap, index);
+
+	return error;
+}
+
+void git_index_snapshot_release(git_vector *snap, git_index *index)
+{
+	git_vector_free(snap);
+
+	git_atomic_dec(&index->readers);
+
+	git_index_free(index);
+}
+
+int git_index_snapshot_find(
+	size_t *out, git_vector *entries, git_vector_cmp entry_srch,
+	const char *path, size_t path_len, int stage)
+{
+	return index_find_in_entries(out, entries, entry_srch, path, path_len, stage);
+}
+
+int git_indexwriter_init(
+	git_indexwriter *writer,
+	git_index *index)
+{
+	int error;
+
+	GIT_REFCOUNT_INC(index);
+
+	writer->index = index;
+
+	if (!index->index_file_path)
+		return create_index_error(-1,
+			"failed to write index: The index is in-memory only");
+
+	if ((error = git_filebuf_open(
+		&writer->file, index->index_file_path, GIT_FILEBUF_HASH_CONTENTS, GIT_INDEX_FILE_MODE)) < 0) {
+
+		if (error == GIT_ELOCKED)
+			git_error_set(GIT_ERROR_INDEX, "the index is locked; this might be due to a concurrent or crashed process");
+
+		return error;
+	}
+
+	writer->should_write = 1;
+
+	return 0;
+}
+
+int git_indexwriter_init_for_operation(
+	git_indexwriter *writer,
+	git_repository *repo,
+	unsigned int *checkout_strategy)
+{
+	git_index *index;
+	int error;
+
+	if ((error = git_repository_index__weakptr(&index, repo)) < 0 ||
+		(error = git_indexwriter_init(writer, index)) < 0)
+		return error;
+
+	writer->should_write = (*checkout_strategy & GIT_CHECKOUT_DONT_WRITE_INDEX) == 0;
+	*checkout_strategy |= GIT_CHECKOUT_DONT_WRITE_INDEX;
+
+	return 0;
+}
+
+int git_indexwriter_commit(git_indexwriter *writer)
+{
+	int error;
+	git_oid checksum = {{ 0 }};
+
+	if (!writer->should_write)
+		return 0;
+
+	git_vector_sort(&writer->index->entries);
+	git_vector_sort(&writer->index->reuc);
+
+	if ((error = write_index(&checksum, writer->index, &writer->file)) < 0) {
+		git_indexwriter_cleanup(writer);
+		return error;
+	}
+
+	if ((error = git_filebuf_commit(&writer->file)) < 0)
+		return error;
+
+	if ((error = git_futils_filestamp_check(
+		&writer->index->stamp, writer->index->index_file_path)) < 0) {
+		git_error_set(GIT_ERROR_OS, "could not read index timestamp");
+		return -1;
+	}
+
+	writer->index->dirty = 0;
+	writer->index->on_disk = 1;
+	git_oid_cpy(&writer->index->checksum, &checksum);
+
+	git_index_free(writer->index);
+	writer->index = NULL;
+
+	return 0;
+}
+
+void git_indexwriter_cleanup(git_indexwriter *writer)
+{
+	git_filebuf_cleanup(&writer->file);
+
+	git_index_free(writer->index);
+	writer->index = NULL;
 }