rugged 1.0.0 → 1.2.0

data/vendor/libgit2/src/midx.h

@@ -0,0 +1,110 @@
+/*
+ * Copyright (C) the libgit2 contributors. All rights reserved.
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+
+#ifndef INCLUDE_midx_h__
+#define INCLUDE_midx_h__
+
+#include "common.h"
+
+#include <ctype.h>
+
+#include "git2/sys/midx.h"
+
+#include "map.h"
+#include "mwindow.h"
+#include "odb.h"
+
+/*
+ * A multi-pack-index file.
+ *
+ * This file contains a merged index for multiple independent .pack files. This
+ * can help speed up locating objects without requiring a garbage collection
+ * cycle to create a single .pack file.
+ *
+ * Support for this feature was added in git 2.21, and requires the
+ * `core.multiPackIndex` config option to be set.
+ */
+typedef struct git_midx_file {
+	git_map index_map;
+
+	/* The table of Packfile Names. */
+	git_vector packfile_names;
+
+	/* The OID Fanout table. */
+	const uint32_t *oid_fanout;
+	/* The total number of objects in the index. */
+	uint32_t num_objects;
+
+	/* The OID Lookup table. */
+	git_oid *oid_lookup;
+
+	/* The Object Offsets table. Each entry has two 4-byte fields with the pack index and the offset. */
+	const unsigned char *object_offsets;
+
+	/* The Object Large Offsets table. */
+	const unsigned char *object_large_offsets;
+	/* The number of entries in the Object Large Offsets table. Each entry has an 8-byte with an offset */
+	size_t num_object_large_offsets;
+
+	/* The trailer of the file. Contains the SHA1-checksum of the whole file. */
+	git_oid checksum;
+
+	/* something like ".git/objects/pack/multi-pack-index". */
+	git_buf filename;
+} git_midx_file;
+
+/*
+ * An entry in the multi-pack-index file. Similar in purpose to git_pack_entry.
+ */
+typedef struct git_midx_entry {
+	/* The index within idx->packfile_names where the packfile name can be found. */
+	size_t pack_index;
+	/* The offset within the .pack file where the requested object is found. */
+	off64_t offset;
+	/* The SHA-1 hash of the requested object. */
+	git_oid sha1;
+} git_midx_entry;
+
+/*
+ * A writer for `multi-pack-index` files.
+ */
+struct git_midx_writer {
+	/*
+	 * The path of the directory where the .pack/.idx files are stored. The
+	 * `multi-pack-index` file will be written to the same directory.
+	 */
+	git_buf pack_dir;
+
+	/* The list of `git_pack_file`s. */
+	git_vector packs;
+};
+
+int git_midx_open(
+		git_midx_file **idx_out,
+		const char *path);
+bool git_midx_needs_refresh(
+		const git_midx_file *idx,
+		const char *path);
+int git_midx_entry_find(
+		git_midx_entry *e,
+		git_midx_file *idx,
+		const git_oid *short_oid,
+		size_t len);
+int git_midx_foreach_entry(
+		git_midx_file *idx,
+		git_odb_foreach_cb cb,
+		void *data);
+int git_midx_close(git_midx_file *idx);
+void git_midx_free(git_midx_file *idx);
+
+/* This is exposed for use in the fuzzers. */
+int git_midx_parse(
+		git_midx_file *idx,
+		const unsigned char *data,
+		size_t size);
+
+#endif

data/vendor/libgit2/src/mwindow.c

@@ -10,7 +10,7 @@
 #include "vector.h"
 #include "futils.h"
 #include "map.h"
-#include "global.h"
+#include "runtime.h"
 #include "strmap.h"
 #include "pack.h"
 
@@ -20,31 +20,45 @@
 		: 32 * 1024 * 1024)
 
 #define DEFAULT_MAPPED_LIMIT \
-	((1024 * 1024) * (sizeof(void*) >= 8 ? 8192ULL : 256UL))
+	((1024 * 1024) * (sizeof(void*) >= 8 ? UINT64_C(8192) : UINT64_C(256)))
+
+/* default is unlimited */
+#define DEFAULT_FILE_LIMIT 0
 
 size_t git_mwindow__window_size = DEFAULT_WINDOW_SIZE;
 size_t git_mwindow__mapped_limit = DEFAULT_MAPPED_LIMIT;
+size_t git_mwindow__file_limit = DEFAULT_FILE_LIMIT;
+
+/* Mutex to control access to `git_mwindow__mem_ctl` and `git__pack_cache`. */
+git_mutex git__mwindow_mutex;
 
-/* Whenever you want to read or modify this, grab git__mwindow_mutex */
-static git_mwindow_ctl mem_ctl;
+/* Whenever you want to read or modify this, grab `git__mwindow_mutex` */
+git_mwindow_ctl git_mwindow__mem_ctl;
 
 /* Global list of mwindow files, to open packs once across repos */
 git_strmap *git__pack_cache = NULL;
 
-static void git_mwindow_files_free(void)
+static void git_mwindow_global_shutdown(void)
 {
 	git_strmap *tmp = git__pack_cache;
 
+	git_mutex_free(&git__mwindow_mutex);
+
 	git__pack_cache = NULL;
 	git_strmap_free(tmp);
 }
 
 int git_mwindow_global_init(void)
 {
-	assert(!git__pack_cache);
+	int error;
+
+	GIT_ASSERT(!git__pack_cache);
 
-	git__on_shutdown(git_mwindow_files_free);
-	return git_strmap_new(&git__pack_cache);
+	if ((error = git_mutex_init(&git__mwindow_mutex)) < 0 ||
+	    (error = git_strmap_new(&git__pack_cache)) < 0)
+	    return error;
+
+	return git_runtime_shutdown_register(git_mwindow_global_shutdown);
 }
 
 int git_mwindow_get_pack(struct git_pack_file **out, const char *path)
@@ -65,7 +79,7 @@ int git_mwindow_get_pack(struct git_pack_file **out, const char *path)
 	git__free(packname);
 
 	if (pack != NULL) {
-		git_atomic_inc(&pack->refcount);
+		git_atomic32_inc(&pack->refcount);
 		git_mutex_unlock(&git__mwindow_mutex);
 		*out = pack;
 		return 0;
@@ -77,62 +91,51 @@ int git_mwindow_get_pack(struct git_pack_file **out, const char *path)
 		return error;
 	}
 
-	git_atomic_inc(&pack->refcount);
+	git_atomic32_inc(&pack->refcount);
 
 	error = git_strmap_set(git__pack_cache, pack->pack_name, pack);
 	git_mutex_unlock(&git__mwindow_mutex);
-
 	if (error < 0) {
-		git_packfile_free(pack);
-		return -1;
+		git_packfile_free(pack, false);
+		return error;
 	}
 
 	*out = pack;
 	return 0;
 }
 
-void git_mwindow_put_pack(struct git_pack_file *pack)
+int git_mwindow_put_pack(struct git_pack_file *pack)
 {
-	int count;
+	int count, error;
+	struct git_pack_file *pack_to_delete = NULL;
 
-	if (git_mutex_lock(&git__mwindow_mutex) < 0)
-		return;
+	if ((error = git_mutex_lock(&git__mwindow_mutex)) < 0)
+		return error;
 
 	/* put before get would be a corrupted state */
-	assert(git__pack_cache);
+	GIT_ASSERT(git__pack_cache);
 
 	/* if we cannot find it, the state is corrupted */
-	assert(git_strmap_exists(git__pack_cache, pack->pack_name));
+	GIT_ASSERT(git_strmap_exists(git__pack_cache, pack->pack_name));
 
-	count = git_atomic_dec(&pack->refcount);
+	count = git_atomic32_dec(&pack->refcount);
 	if (count == 0) {
 		git_strmap_delete(git__pack_cache, pack->pack_name);
-		git_packfile_free(pack);
+		pack_to_delete = pack;
 	}
-
 	git_mutex_unlock(&git__mwindow_mutex);
-	return;
-}
-
-void git_mwindow_free_all(git_mwindow_file *mwf)
-{
-	if (git_mutex_lock(&git__mwindow_mutex)) {
-		git_error_set(GIT_ERROR_THREAD, "unable to lock mwindow mutex");
-		return;
-	}
-
-	git_mwindow_free_all_locked(mwf);
+	git_packfile_free(pack_to_delete, false);
 
-	git_mutex_unlock(&git__mwindow_mutex);
+	return 0;
 }
 
 /*
  * Free all the windows in a sequence, typically because we're done
- * with the file
+ * with the file. Needs to hold the git__mwindow_mutex.
  */
-void git_mwindow_free_all_locked(git_mwindow_file *mwf)
+static int git_mwindow_free_all_locked(git_mwindow_file *mwf)
 {
-	git_mwindow_ctl *ctl = &mem_ctl;
+	git_mwindow_ctl *ctl = &git_mwindow__mem_ctl;
 	size_t i;
 
 	/*
@@ -152,7 +155,7 @@ void git_mwindow_free_all_locked(git_mwindow_file *mwf)
 
 	while (mwf->windows) {
 		git_mwindow *w = mwf->windows;
-		assert(w->inuse_cnt == 0);
+		GIT_ASSERT(w->inuse_cnt == 0);
 
 		ctl->mapped -= w->window_map.len;
 		ctl->open_windows--;
@@ -162,6 +165,24 @@ void git_mwindow_free_all_locked(git_mwindow_file *mwf)
 		mwf->windows = w->next;
 		git__free(w);
 	}
+
+	return 0;
+}
+
+int git_mwindow_free_all(git_mwindow_file *mwf)
+{
+	int error;
+
+	if (git_mutex_lock(&git__mwindow_mutex)) {
+		git_error_set(GIT_ERROR_THREAD, "unable to lock mwindow mutex");
+		return -1;
+	}
+
+	error = git_mwindow_free_all_locked(mwf);
+
+	git_mutex_unlock(&git__mwindow_mutex);
+
+	return error;
 }
 
 /*
@@ -174,92 +195,154 @@ int git_mwindow_contains(git_mwindow *win, off64_t offset)
 		&& offset <= (off64_t)(win_off + win->window_map.len);
 }
 
+#define GIT_MWINDOW__LRU -1
+#define GIT_MWINDOW__MRU 1
+
 /*
- * Find the least-recently-used window in a file
+ * Find the least- or most-recently-used window in a file that is not currently
+ * being used. The 'only_unused' flag controls whether the caller requires the
+ * file to only have unused windows. If '*out_window' is non-null, it is used as
+ * a starting point for the comparison.
+ *
+ * Returns whether such a window was found in the file.
  */
-static void git_mwindow_scan_lru(
-	git_mwindow_file *mwf,
-	git_mwindow **lru_w,
-	git_mwindow **lru_l)
+static bool git_mwindow_scan_recently_used(
+		git_mwindow_file *mwf,
+		git_mwindow **out_window,
+		git_mwindow **out_last,
+		bool only_unused,
+		int comparison_sign)
 {
-	git_mwindow *w, *w_l;
-
-	for (w_l = NULL, w = mwf->windows; w; w = w->next) {
-		if (!w->inuse_cnt) {
-			/*
-			 * If the current one is more recent than the last one,
-			 * store it in the output parameter. If lru_w is NULL,
-			 * it's the first loop, so store it as well.
-			 */
-			if (!*lru_w || w->last_used < (*lru_w)->last_used) {
-				*lru_w = w;
-				*lru_l = w_l;
-			}
+	git_mwindow *w, *w_last;
+	git_mwindow *lru_window = NULL, *lru_last = NULL;
+	bool found = false;
+
+	GIT_ASSERT_ARG(mwf);
+	GIT_ASSERT_ARG(out_window);
+
+	lru_window = *out_window;
+	if (out_last)
+		lru_last = *out_last;
+
+	for (w_last = NULL, w = mwf->windows; w; w_last = w, w = w->next) {
+		if (w->inuse_cnt) {
+			if (only_unused)
+				return false;
+			/* This window is currently being used. Skip it. */
+			continue;
+		}
+
+		/*
+		 * If the current one is more (or less) recent than the last one,
+		 * store it in the output parameter. If lru_window is NULL,
+		 * it's the first loop, so store it as well.
+		 */
+		if (!lru_window ||
+				(comparison_sign == GIT_MWINDOW__LRU && lru_window->last_used > w->last_used) ||
+				(comparison_sign == GIT_MWINDOW__MRU && lru_window->last_used < w->last_used)) {
+			lru_window = w;
+			lru_last = w_last;
+			found = true;
 		}
-		w_l = w;
 	}
+
+	if (!found)
+		return false;
+
+	*out_window = lru_window;
+	if (out_last)
+		*out_last = lru_last;
+	return true;
 }
 
 /*
- * Close the least recently used window. You should check to see if
- * the file descriptors need closing from time to time. Called under
- * lock from new_window.
+ * Close the least recently used window (that is currently not being used) out
+ * of all the files. Called under lock from new_window_locked.
  */
-static int git_mwindow_close_lru(git_mwindow_file *mwf)
+static int git_mwindow_close_lru_window_locked(void)
 {
-	git_mwindow_ctl *ctl = &mem_ctl;
+	git_mwindow_ctl *ctl = &git_mwindow__mem_ctl;
+	git_mwindow_file *cur;
 	size_t i;
-	git_mwindow *lru_w = NULL, *lru_l = NULL, **list = &mwf->windows;
+	git_mwindow *lru_window = NULL, *lru_last = NULL, **list = NULL;
 
-	/* FIXME: Does this give us any advantage? */
-	if(mwf->windows)
-		git_mwindow_scan_lru(mwf, &lru_w, &lru_l);
-
-	for (i = 0; i < ctl->windowfiles.length; ++i) {
-		git_mwindow *last = lru_w;
-		git_mwindow_file *cur = git_vector_get(&ctl->windowfiles, i);
-		git_mwindow_scan_lru(cur, &lru_w, &lru_l);
-		if (lru_w != last)
+	git_vector_foreach(&ctl->windowfiles, i, cur) {
+		if (git_mwindow_scan_recently_used(
+				cur, &lru_window, &lru_last, false, GIT_MWINDOW__LRU)) {
 			list = &cur->windows;
+		}
 	}
 
-	if (!lru_w) {
+	if (!lru_window) {
 		git_error_set(GIT_ERROR_OS, "failed to close memory window; couldn't find LRU");
 		return -1;
 	}
 
-	ctl->mapped -= lru_w->window_map.len;
-	git_futils_mmap_free(&lru_w->window_map);
+	ctl->mapped -= lru_window->window_map.len;
+	git_futils_mmap_free(&lru_window->window_map);
 
-	if (lru_l)
-		lru_l->next = lru_w->next;
+	if (lru_last)
+		lru_last->next = lru_window->next;
 	else
-		*list = lru_w->next;
+		*list = lru_window->next;
 
-	git__free(lru_w);
+	git__free(lru_window);
 	ctl->open_windows--;
 
 	return 0;
 }
 
+/*
+ * Finds the file that does not have any open windows AND whose
+ * most-recently-used window is the least-recently used one across all
+ * currently open files.
+ *
+ * Called under lock from new_window_locked.
+ */
+static int git_mwindow_find_lru_file_locked(git_mwindow_file **out)
+{
+	git_mwindow_ctl *ctl = &git_mwindow__mem_ctl;
+	git_mwindow_file *lru_file = NULL, *current_file = NULL;
+	git_mwindow *lru_window = NULL;
+	size_t i;
+
+	git_vector_foreach(&ctl->windowfiles, i, current_file) {
+		git_mwindow *mru_window = NULL;
+		if (!git_mwindow_scan_recently_used(
+				current_file, &mru_window, NULL, true, GIT_MWINDOW__MRU)) {
+			continue;
+		}
+		if (!lru_window || lru_window->last_used > mru_window->last_used) {
+			lru_window = mru_window;
+			lru_file = current_file;
+		}
+	}
+
+	if (!lru_file) {
+		git_error_set(GIT_ERROR_OS, "failed to close memory window file; couldn't find LRU");
+		return -1;
+	}
+
+	*out = lru_file;
+	return 0;
+}
+
 /* This gets called under lock from git_mwindow_open */
-static git_mwindow *new_window(
-	git_mwindow_file *mwf,
+static git_mwindow *new_window_locked(
 	git_file fd,
 	off64_t size,
 	off64_t offset)
 {
-	git_mwindow_ctl *ctl = &mem_ctl;
+	git_mwindow_ctl *ctl = &git_mwindow__mem_ctl;
 	size_t walign = git_mwindow__window_size / 2;
 	off64_t len;
 	git_mwindow *w;
 
-	w = git__malloc(sizeof(*w));
+	w = git__calloc(1, sizeof(*w));
 
 	if (w == NULL)
 		return NULL;
 
-	memset(w, 0x0, sizeof(*w));
 	w->offset = (offset / walign) * walign;
 
 	len = size - w->offset;
@@ -269,7 +352,7 @@ static git_mwindow *new_window(
 	ctl->mapped += (size_t)len;
 
 	while (git_mwindow__mapped_limit < ctl->mapped &&
-			git_mwindow_close_lru(mwf) == 0) /* nop */;
+			git_mwindow_close_lru_window_locked() == 0) /* nop */;
 
 	/*
 	 * We treat `mapped_limit` as a soft limit. If we can't find a
@@ -283,7 +366,7 @@ static git_mwindow *new_window(
 		 * we're below our soft limits, so free up what we can and try again.
 		 */
 
-		while (git_mwindow_close_lru(mwf) == 0)
+		while (git_mwindow_close_lru_window_locked() == 0)
 			/* nop */;
 
 		if (git_futils_mmap_ro(&w->window_map, fd, w->offset, (size_t)len) < 0) {
@@ -315,7 +398,7 @@ unsigned char *git_mwindow_open(
 	size_t extra,
 	unsigned int *left)
 {
-	git_mwindow_ctl *ctl = &mem_ctl;
+	git_mwindow_ctl *ctl = &git_mwindow__mem_ctl;
 	git_mwindow *w = *cursor;
 
 	if (git_mutex_lock(&git__mwindow_mutex)) {
@@ -339,7 +422,7 @@ unsigned char *git_mwindow_open(
 		 * one.
 		 */
 		if (!w) {
-			w = new_window(mwf, mwf->fd, mwf->size, offset);
+			w = new_window_locked(mwf->fd, mwf->size, offset);
 			if (w == NULL) {
 				git_mutex_unlock(&git__mwindow_mutex);
 				return NULL;
@@ -367,8 +450,11 @@ unsigned char *git_mwindow_open(
 
 int git_mwindow_file_register(git_mwindow_file *mwf)
 {
-	git_mwindow_ctl *ctl = &mem_ctl;
-	int ret;
+	git_vector closed_files = GIT_VECTOR_INIT;
+	git_mwindow_ctl *ctl = &git_mwindow__mem_ctl;
+	int error;
+	size_t i;
+	git_mwindow_file *closed_file = NULL;
 
 	if (git_mutex_lock(&git__mwindow_mutex)) {
 		git_error_set(GIT_ERROR_THREAD, "unable to lock mwindow mutex");
@@ -376,20 +462,53 @@ int git_mwindow_file_register(git_mwindow_file *mwf)
 	}
 
 	if (ctl->windowfiles.length == 0 &&
-	    git_vector_init(&ctl->windowfiles, 8, NULL) < 0) {
+	    (error = git_vector_init(&ctl->windowfiles, 8, NULL)) < 0) {
 		git_mutex_unlock(&git__mwindow_mutex);
-		return -1;
+		goto cleanup;
+	}
+
+	if (git_mwindow__file_limit) {
+		git_mwindow_file *lru_file;
+		while (git_mwindow__file_limit <= ctl->windowfiles.length &&
+				git_mwindow_find_lru_file_locked(&lru_file) == 0) {
+			if ((error = git_vector_insert(&closed_files, lru_file)) < 0) {
+				/*
+				 * Exceeding the file limit seems preferrable to being open to
+				 * data races that can end up corrupting the heap.
+				 */
+				break;
+			}
+			git_mwindow_free_all_locked(lru_file);
+		}
 	}
 
-	ret = git_vector_insert(&ctl->windowfiles, mwf);
+	error = git_vector_insert(&ctl->windowfiles, mwf);
 	git_mutex_unlock(&git__mwindow_mutex);
+	if (error < 0)
+		goto cleanup;
+
+	/*
+	 * Once we have released the global windowfiles lock, we can close each
+	 * individual file. Before doing so, acquire that file's lock to avoid
+	 * closing a file that is currently being used.
+	 */
+	git_vector_foreach(&closed_files, i, closed_file) {
+		error = git_mutex_lock(&closed_file->lock);
+		if (error < 0)
+			continue;
+		p_close(closed_file->fd);
+		closed_file->fd = -1;
+		git_mutex_unlock(&closed_file->lock);
+	}
 
-	return ret;
+cleanup:
+	git_vector_free(&closed_files);
+	return error;
 }
 
 void git_mwindow_file_deregister(git_mwindow_file *mwf)
 {
-	git_mwindow_ctl *ctl = &mem_ctl;
+	git_mwindow_ctl *ctl = &git_mwindow__mem_ctl;
 	git_mwindow_file *cur;
 	size_t i;