rugged 1.1.1 → 1.2.0

data/vendor/libgit2/src/midx.h

@@ -12,8 +12,11 @@
 
 #include <ctype.h>
 
+#include "git2/sys/midx.h"
+
 #include "map.h"
 #include "mwindow.h"
+#include "odb.h"
 
 /*
  * A multi-pack-index file.
@@ -49,6 +52,9 @@ typedef struct git_midx_file {
 
 	/* The trailer of the file. Contains the SHA1-checksum of the whole file. */
 	git_oid checksum;
+
+	/* something like ".git/objects/pack/multi-pack-index". */
+	git_buf filename;
 } git_midx_file;
 
 /*
@@ -63,15 +69,36 @@ typedef struct git_midx_entry {
 	git_oid sha1;
 } git_midx_entry;
 
+/*
+ * A writer for `multi-pack-index` files.
+ */
+struct git_midx_writer {
+	/*
+	 * The path of the directory where the .pack/.idx files are stored. The
+	 * `multi-pack-index` file will be written to the same directory.
+	 */
+	git_buf pack_dir;
+
+	/* The list of `git_pack_file`s. */
+	git_vector packs;
+};
+
 int git_midx_open(
 		git_midx_file **idx_out,
 		const char *path);
+bool git_midx_needs_refresh(
+		const git_midx_file *idx,
+		const char *path);
 int git_midx_entry_find(
 		git_midx_entry *e,
 		git_midx_file *idx,
 		const git_oid *short_oid,
 		size_t len);
-void git_midx_close(git_midx_file *idx);
+int git_midx_foreach_entry(
+		git_midx_file *idx,
+		git_odb_foreach_cb cb,
+		void *data);
+int git_midx_close(git_midx_file *idx);
 void git_midx_free(git_midx_file *idx);
 
 /* This is exposed for use in the fuzzers. */

data/vendor/libgit2/src/mwindow.c

@@ -10,7 +10,7 @@
 #include "vector.h"
 #include "futils.h"
 #include "map.h"
-#include "global.h"
+#include "runtime.h"
 #include "strmap.h"
 #include "pack.h"
 
@@ -20,7 +20,7 @@
 		: 32 * 1024 * 1024)
 
 #define DEFAULT_MAPPED_LIMIT \
-	((1024 * 1024) * (sizeof(void*) >= 8 ? 8192ULL : 256UL))
+	((1024 * 1024) * (sizeof(void*) >= 8 ? UINT64_C(8192) : UINT64_C(256)))
 
 /* default is unlimited */
 #define DEFAULT_FILE_LIMIT 0
@@ -29,26 +29,36 @@ size_t git_mwindow__window_size = DEFAULT_WINDOW_SIZE;
 size_t git_mwindow__mapped_limit = DEFAULT_MAPPED_LIMIT;
 size_t git_mwindow__file_limit = DEFAULT_FILE_LIMIT;
 
-/* Whenever you want to read or modify this, grab git__mwindow_mutex */
+/* Mutex to control access to `git_mwindow__mem_ctl` and `git__pack_cache`. */
+git_mutex git__mwindow_mutex;
+
+/* Whenever you want to read or modify this, grab `git__mwindow_mutex` */
 git_mwindow_ctl git_mwindow__mem_ctl;
 
 /* Global list of mwindow files, to open packs once across repos */
 git_strmap *git__pack_cache = NULL;
 
-static void git_mwindow_files_free(void)
+static void git_mwindow_global_shutdown(void)
 {
 	git_strmap *tmp = git__pack_cache;
 
+	git_mutex_free(&git__mwindow_mutex);
+
 	git__pack_cache = NULL;
 	git_strmap_free(tmp);
 }
 
 int git_mwindow_global_init(void)
 {
-	assert(!git__pack_cache);
+	int error;
 
-	git__on_shutdown(git_mwindow_files_free);
-	return git_strmap_new(&git__pack_cache);
+	GIT_ASSERT(!git__pack_cache);
+
+	if ((error = git_mutex_init(&git__mwindow_mutex)) < 0 ||
+	    (error = git_strmap_new(&git__pack_cache)) < 0)
+	    return error;
+
+	return git_runtime_shutdown_register(git_mwindow_global_shutdown);
 }
 
 int git_mwindow_get_pack(struct git_pack_file **out, const char *path)
@@ -69,7 +79,7 @@ int git_mwindow_get_pack(struct git_pack_file **out, const char *path)
 	git__free(packname);
 
 	if (pack != NULL) {
-		git_atomic_inc(&pack->refcount);
+		git_atomic32_inc(&pack->refcount);
 		git_mutex_unlock(&git__mwindow_mutex);
 		*out = pack;
 		return 0;
@@ -81,60 +91,49 @@ int git_mwindow_get_pack(struct git_pack_file **out, const char *path)
 		return error;
 	}
 
-	git_atomic_inc(&pack->refcount);
+	git_atomic32_inc(&pack->refcount);
 
 	error = git_strmap_set(git__pack_cache, pack->pack_name, pack);
 	git_mutex_unlock(&git__mwindow_mutex);
-
 	if (error < 0) {
-		git_packfile_free(pack);
-		return -1;
+		git_packfile_free(pack, false);
+		return error;
 	}
 
 	*out = pack;
 	return 0;
 }
 
-void git_mwindow_put_pack(struct git_pack_file *pack)
+int git_mwindow_put_pack(struct git_pack_file *pack)
 {
-	int count;
+	int count, error;
+	struct git_pack_file *pack_to_delete = NULL;
 
-	if (git_mutex_lock(&git__mwindow_mutex) < 0)
-		return;
+	if ((error = git_mutex_lock(&git__mwindow_mutex)) < 0)
+		return error;
 
 	/* put before get would be a corrupted state */
-	assert(git__pack_cache);
+	GIT_ASSERT(git__pack_cache);
 
 	/* if we cannot find it, the state is corrupted */
-	assert(git_strmap_exists(git__pack_cache, pack->pack_name));
+	GIT_ASSERT(git_strmap_exists(git__pack_cache, pack->pack_name));
 
-	count = git_atomic_dec(&pack->refcount);
+	count = git_atomic32_dec(&pack->refcount);
 	if (count == 0) {
 		git_strmap_delete(git__pack_cache, pack->pack_name);
-		git_packfile_free(pack);
+		pack_to_delete = pack;
 	}
-
 	git_mutex_unlock(&git__mwindow_mutex);
-	return;
-}
+	git_packfile_free(pack_to_delete, false);
 
-void git_mwindow_free_all(git_mwindow_file *mwf)
-{
-	if (git_mutex_lock(&git__mwindow_mutex)) {
-		git_error_set(GIT_ERROR_THREAD, "unable to lock mwindow mutex");
-		return;
-	}
-
-	git_mwindow_free_all_locked(mwf);
-
-	git_mutex_unlock(&git__mwindow_mutex);
+	return 0;
 }
 
 /*
  * Free all the windows in a sequence, typically because we're done
- * with the file
+ * with the file. Needs to hold the git__mwindow_mutex.
  */
-void git_mwindow_free_all_locked(git_mwindow_file *mwf)
+static int git_mwindow_free_all_locked(git_mwindow_file *mwf)
 {
 	git_mwindow_ctl *ctl = &git_mwindow__mem_ctl;
 	size_t i;
@@ -156,7 +155,7 @@ void git_mwindow_free_all_locked(git_mwindow_file *mwf)
 
 	while (mwf->windows) {
 		git_mwindow *w = mwf->windows;
-		assert(w->inuse_cnt == 0);
+		GIT_ASSERT(w->inuse_cnt == 0);
 
 		ctl->mapped -= w->window_map.len;
 		ctl->open_windows--;
@@ -166,6 +165,24 @@ void git_mwindow_free_all_locked(git_mwindow_file *mwf)
 		mwf->windows = w->next;
 		git__free(w);
 	}
+
+	return 0;
+}
+
+int git_mwindow_free_all(git_mwindow_file *mwf)
+{
+	int error;
+
+	if (git_mutex_lock(&git__mwindow_mutex)) {
+		git_error_set(GIT_ERROR_THREAD, "unable to lock mwindow mutex");
+		return -1;
+	}
+
+	error = git_mwindow_free_all_locked(mwf);
+
+	git_mutex_unlock(&git__mwindow_mutex);
+
+	return error;
 }
 
 /*
@@ -200,8 +217,8 @@ static bool git_mwindow_scan_recently_used(
 	git_mwindow *lru_window = NULL, *lru_last = NULL;
 	bool found = false;
 
-	assert(mwf);
-	assert(out_window);
+	GIT_ASSERT_ARG(mwf);
+	GIT_ASSERT_ARG(out_window);
 
 	lru_window = *out_window;
 	if (out_last)
@@ -240,9 +257,9 @@ static bool git_mwindow_scan_recently_used(
 
 /*
  * Close the least recently used window (that is currently not being used) out
- * of all the files. Called under lock from new_window.
+ * of all the files. Called under lock from new_window_locked.
  */
-static int git_mwindow_close_lru_window(void)
+static int git_mwindow_close_lru_window_locked(void)
 {
 	git_mwindow_ctl *ctl = &git_mwindow__mem_ctl;
 	git_mwindow_file *cur;
@@ -276,13 +293,13 @@ static int git_mwindow_close_lru_window(void)
 }
 
 /*
- * Close the file that does not have any open windows AND whose
+ * Finds the file that does not have any open windows AND whose
  * most-recently-used window is the least-recently used one across all
  * currently open files.
  *
- * Called under lock from new_window.
+ * Called under lock from new_window_locked.
  */
-static int git_mwindow_close_lru_file(void)
+static int git_mwindow_find_lru_file_locked(git_mwindow_file **out)
 {
 	git_mwindow_ctl *ctl = &git_mwindow__mem_ctl;
 	git_mwindow_file *lru_file = NULL, *current_file = NULL;
@@ -306,15 +323,12 @@ static int git_mwindow_close_lru_file(void)
 		return -1;
 	}
 
-	git_mwindow_free_all_locked(lru_file);
-	p_close(lru_file->fd);
-	lru_file->fd = -1;
-
+	*out = lru_file;
 	return 0;
 }
 
 /* This gets called under lock from git_mwindow_open */
-static git_mwindow *new_window(
+static git_mwindow *new_window_locked(
 	git_file fd,
 	off64_t size,
 	off64_t offset)
@@ -324,12 +338,11 @@ static git_mwindow *new_window(
 	off64_t len;
 	git_mwindow *w;
 
-	w = git__malloc(sizeof(*w));
+	w = git__calloc(1, sizeof(*w));
 
 	if (w == NULL)
 		return NULL;
 
-	memset(w, 0x0, sizeof(*w));
 	w->offset = (offset / walign) * walign;
 
 	len = size - w->offset;
@@ -339,7 +352,7 @@ static git_mwindow *new_window(
 	ctl->mapped += (size_t)len;
 
 	while (git_mwindow__mapped_limit < ctl->mapped &&
-			git_mwindow_close_lru_window() == 0) /* nop */;
+			git_mwindow_close_lru_window_locked() == 0) /* nop */;
 
 	/*
 	 * We treat `mapped_limit` as a soft limit. If we can't find a
@@ -353,7 +366,7 @@ static git_mwindow *new_window(
 		 * we're below our soft limits, so free up what we can and try again.
 		 */
 
-		while (git_mwindow_close_lru_window() == 0)
+		while (git_mwindow_close_lru_window_locked() == 0)
 			/* nop */;
 
 		if (git_futils_mmap_ro(&w->window_map, fd, w->offset, (size_t)len) < 0) {
@@ -409,7 +422,7 @@ unsigned char *git_mwindow_open(
 		 * one.
 		 */
 		if (!w) {
-			w = new_window(mwf->fd, mwf->size, offset);
+			w = new_window_locked(mwf->fd, mwf->size, offset);
 			if (w == NULL) {
 				git_mutex_unlock(&git__mwindow_mutex);
 				return NULL;
@@ -437,8 +450,11 @@ unsigned char *git_mwindow_open(
 
 int git_mwindow_file_register(git_mwindow_file *mwf)
 {
+	git_vector closed_files = GIT_VECTOR_INIT;
 	git_mwindow_ctl *ctl = &git_mwindow__mem_ctl;
-	int ret;
+	int error;
+	size_t i;
+	git_mwindow_file *closed_file = NULL;
 
 	if (git_mutex_lock(&git__mwindow_mutex)) {
 		git_error_set(GIT_ERROR_THREAD, "unable to lock mwindow mutex");
@@ -446,20 +462,48 @@ int git_mwindow_file_register(git_mwindow_file *mwf)
 	}
 
 	if (ctl->windowfiles.length == 0 &&
-	    git_vector_init(&ctl->windowfiles, 8, NULL) < 0) {
+	    (error = git_vector_init(&ctl->windowfiles, 8, NULL)) < 0) {
 		git_mutex_unlock(&git__mwindow_mutex);
-		return -1;
+		goto cleanup;
 	}
 
 	if (git_mwindow__file_limit) {
+		git_mwindow_file *lru_file;
 		while (git_mwindow__file_limit <= ctl->windowfiles.length &&
-				git_mwindow_close_lru_file() == 0) /* nop */;
+				git_mwindow_find_lru_file_locked(&lru_file) == 0) {
+			if ((error = git_vector_insert(&closed_files, lru_file)) < 0) {
+				/*
+				 * Exceeding the file limit seems preferrable to being open to
+				 * data races that can end up corrupting the heap.
+				 */
+				break;
+			}
+			git_mwindow_free_all_locked(lru_file);
+		}
 	}
 
-	ret = git_vector_insert(&ctl->windowfiles, mwf);
+	error = git_vector_insert(&ctl->windowfiles, mwf);
 	git_mutex_unlock(&git__mwindow_mutex);
+	if (error < 0)
+		goto cleanup;
+
+	/*
+	 * Once we have released the global windowfiles lock, we can close each
+	 * individual file. Before doing so, acquire that file's lock to avoid
+	 * closing a file that is currently being used.
+	 */
+	git_vector_foreach(&closed_files, i, closed_file) {
+		error = git_mutex_lock(&closed_file->lock);
+		if (error < 0)
+			continue;
+		p_close(closed_file->fd);
+		closed_file->fd = -1;
+		git_mutex_unlock(&closed_file->lock);
+	}
 
-	return ret;
+cleanup:
+	git_vector_free(&closed_files);
+	return error;
 }
 
 void git_mwindow_file_deregister(git_mwindow_file *mwf)

data/vendor/libgit2/src/mwindow.h

@@ -22,6 +22,7 @@ typedef struct git_mwindow {
 } git_mwindow;
 
 typedef struct git_mwindow_file {
+	git_mutex lock; /* protects updates to fd */
 	git_mwindow *windows;
 	int fd;
 	off64_t size;
@@ -38,8 +39,7 @@ typedef struct git_mwindow_ctl {
 } git_mwindow_ctl;
 
 int git_mwindow_contains(git_mwindow *win, off64_t offset);
-void git_mwindow_free_all(git_mwindow_file *mwf); /* locks */
-void git_mwindow_free_all_locked(git_mwindow_file *mwf); /* run under lock */
+int git_mwindow_free_all(git_mwindow_file *mwf); /* locks */
 unsigned char *git_mwindow_open(git_mwindow_file *mwf, git_mwindow **cursor, off64_t offset, size_t extra, unsigned int *left);
 int git_mwindow_file_register(git_mwindow_file *mwf);
 void git_mwindow_file_deregister(git_mwindow_file *mwf);
@@ -49,6 +49,6 @@ extern int git_mwindow_global_init(void);
 
 struct git_pack_file; /* just declaration to avoid cyclical includes */
 int git_mwindow_get_pack(struct git_pack_file **out, const char *path);
-void git_mwindow_put_pack(struct git_pack_file *pack);
+int git_mwindow_put_pack(struct git_pack_file *pack);
 
 #endif

data/vendor/libgit2/src/net.c

@@ -14,7 +14,7 @@
 #include "posix.h"
 #include "buffer.h"
 #include "http_parser.h"
-#include "global.h"
+#include "runtime.h"
 
 #define DEFAULT_PORT_HTTP  "80"
 #define DEFAULT_PORT_HTTPS "443"
@@ -35,6 +35,46 @@ static const char *default_port_for_scheme(const char *scheme)
 	return NULL;
 }
 
+int git_net_url_dup(git_net_url *out, git_net_url *in)
+{
+	if (in->scheme) {
+		out->scheme = git__strdup(in->scheme);
+		GIT_ERROR_CHECK_ALLOC(out->scheme);
+	}
+
+	if (in->host) {
+		out->host = git__strdup(in->host);
+		GIT_ERROR_CHECK_ALLOC(out->host);
+	}
+
+	if (in->port) {
+		out->port = git__strdup(in->port);
+		GIT_ERROR_CHECK_ALLOC(out->port);
+	}
+
+	if (in->path) {
+		out->path = git__strdup(in->path);
+		GIT_ERROR_CHECK_ALLOC(out->path);
+	}
+
+	if (in->query) {
+		out->query = git__strdup(in->query);
+		GIT_ERROR_CHECK_ALLOC(out->query);
+	}
+
+	if (in->username) {
+		out->username = git__strdup(in->username);
+		GIT_ERROR_CHECK_ALLOC(out->username);
+	}
+
+	if (in->password) {
+		out->password = git__strdup(in->password);
+		GIT_ERROR_CHECK_ALLOC(out->password);
+	}
+
+	return 0;
+}
+
 int git_net_url_parse(git_net_url *url, const char *given)
 {
 	struct http_parser_url u = {0};
@@ -281,7 +321,8 @@ int git_net_url_apply_redirect(
 	git_net_url tmp = GIT_NET_URL_INIT;
 	int error = 0;
 
-	assert(url && redirect_location);
+	GIT_ASSERT(url);
+	GIT_ASSERT(redirect_location);
 
 	if (redirect_location[0] == '/') {
 		git__free(url->path);
@@ -334,7 +375,7 @@ bool git_net_url_valid(git_net_url *url)
 	return (url->host && url->port && url->path);
 }
 
-int git_net_url_is_default_port(git_net_url *url)
+bool git_net_url_is_default_port(git_net_url *url)
 {
 	const char *default_port;
 
@@ -344,6 +385,11 @@ int git_net_url_is_default_port(git_net_url *url)
 		return false;
 }
 
+bool git_net_url_is_ipv6(git_net_url *url)
+{
+	return (strchr(url->host, ':') != NULL);
+}
+
 void git_net_url_swap(git_net_url *a, git_net_url *b)
 {
 	git_net_url tmp = GIT_NET_URL_INIT;
@@ -355,6 +401,10 @@ void git_net_url_swap(git_net_url *a, git_net_url *b)
 
 int git_net_url_fmt(git_buf *buf, git_net_url *url)
 {
+	GIT_ASSERT_ARG(url);
+	GIT_ASSERT_ARG(url->scheme);
+	GIT_ASSERT_ARG(url->host);
+
 	git_buf_puts(buf, url->scheme);
 	git_buf_puts(buf, "://");
 
@@ -398,6 +448,80 @@ int git_net_url_fmt_path(git_buf *buf, git_net_url *url)
 	return git_buf_oom(buf) ? -1 : 0;
 }
 
+static bool matches_pattern(
+	git_net_url *url,
+	const char *pattern,
+	size_t pattern_len)
+{
+	const char *domain, *port = NULL, *colon;
+	size_t host_len, domain_len, port_len = 0, wildcard = 0;
+
+	GIT_UNUSED(url);
+	GIT_UNUSED(pattern);
+
+	if (!pattern_len)
+		return false;
+	else if (pattern_len == 1 && pattern[0] == '*')
+		return true;
+	else if (pattern_len > 1 && pattern[0] == '*' && pattern[1] == '.')
+		wildcard = 2;
+	else if (pattern[0] == '.')
+		wildcard = 1;
+
+	domain = pattern + wildcard;
+	domain_len = pattern_len - wildcard;
+
+	if ((colon = memchr(domain, ':', domain_len)) != NULL) {
+		domain_len = colon - domain;
+		port = colon + 1;
+		port_len = pattern_len - wildcard - domain_len - 1;
+	}
+
+	/* A pattern's port *must* match if it's specified */
+	if (port_len && git__strlcmp(url->port, port, port_len) != 0)
+		return false;
+
+	/* No wildcard?  Host must match exactly. */
+	if (!wildcard)
+		return !git__strlcmp(url->host, domain, domain_len);
+
+	/* Wildcard: ensure there's (at least) a suffix match */
+	if ((host_len = strlen(url->host)) < domain_len ||
+	    memcmp(url->host + (host_len - domain_len), domain, domain_len))
+		return false;
+
+	/* The pattern is *.domain and the host is simply domain */
+	if (host_len == domain_len)
+		return true;
+
+	/* The pattern is *.domain and the host is foo.domain */
+	return (url->host[host_len - domain_len - 1] == '.');
+}
+
+bool git_net_url_matches_pattern(git_net_url *url, const char *pattern)
+{
+	return matches_pattern(url, pattern, strlen(pattern));
+}
+
+bool git_net_url_matches_pattern_list(
+	git_net_url *url,
+	const char *pattern_list)
+{
+	const char *pattern, *pattern_end, *sep;
+
+	for (pattern = pattern_list;
+	     pattern && *pattern;
+	     pattern = sep ? sep + 1 : NULL) {
+		sep = strchr(pattern, ',');
+		pattern_end = sep ? sep : strchr(pattern, '\0');
+
+		if (matches_pattern(url, pattern, (pattern_end - pattern)))
+			return true;
+	}
+
+	return false;
+}
+
 void git_net_url_dispose(git_net_url *url)
 {
 	if (url->username)