rugged 0.28.5 → 1.1.1

data/vendor/libgit2/src/midx.h

@@ -0,0 +1,83 @@
+/*
+ * Copyright (C) the libgit2 contributors. All rights reserved.
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+
+#ifndef INCLUDE_midx_h__
+#define INCLUDE_midx_h__
+
+#include "common.h"
+
+#include <ctype.h>
+
+#include "map.h"
+#include "mwindow.h"
+
+/*
+ * A multi-pack-index file.
+ *
+ * This file contains a merged index for multiple independent .pack files. This
+ * can help speed up locating objects without requiring a garbage collection
+ * cycle to create a single .pack file.
+ *
+ * Support for this feature was added in git 2.21, and requires the
+ * `core.multiPackIndex` config option to be set.
+ */
+typedef struct git_midx_file {
+	git_map index_map;
+
+	/* The table of Packfile Names. */
+	git_vector packfile_names;
+
+	/* The OID Fanout table. */
+	const uint32_t *oid_fanout;
+	/* The total number of objects in the index. */
+	uint32_t num_objects;
+
+	/* The OID Lookup table. */
+	git_oid *oid_lookup;
+
+	/* The Object Offsets table. Each entry has two 4-byte fields with the pack index and the offset. */
+	const unsigned char *object_offsets;
+
+	/* The Object Large Offsets table. */
+	const unsigned char *object_large_offsets;
+	/* The number of entries in the Object Large Offsets table. Each entry has an 8-byte with an offset */
+	size_t num_object_large_offsets;
+
+	/* The trailer of the file. Contains the SHA1-checksum of the whole file. */
+	git_oid checksum;
+} git_midx_file;
+
+/*
+ * An entry in the multi-pack-index file. Similar in purpose to git_pack_entry.
+ */
+typedef struct git_midx_entry {
+	/* The index within idx->packfile_names where the packfile name can be found. */
+	size_t pack_index;
+	/* The offset within the .pack file where the requested object is found. */
+	off64_t offset;
+	/* The SHA-1 hash of the requested object. */
+	git_oid sha1;
+} git_midx_entry;
+
+int git_midx_open(
+		git_midx_file **idx_out,
+		const char *path);
+int git_midx_entry_find(
+		git_midx_entry *e,
+		git_midx_file *idx,
+		const git_oid *short_oid,
+		size_t len);
+void git_midx_close(git_midx_file *idx);
+void git_midx_free(git_midx_file *idx);
+
+/* This is exposed for use in the fuzzers. */
+int git_midx_parse(
+		git_midx_file *idx,
+		const unsigned char *data,
+		size_t size);
+
+#endif

data/vendor/libgit2/src/mwindow.c

@@ -8,7 +8,7 @@
 #include "mwindow.h"
 
 #include "vector.h"
-#include "fileops.h"
+#include "futils.h"
 #include "map.h"
 #include "global.h"
 #include "strmap.h"
@@ -22,11 +22,15 @@
 #define DEFAULT_MAPPED_LIMIT \
 	((1024 * 1024) * (sizeof(void*) >= 8 ? 8192ULL : 256UL))
 
+/* default is unlimited */
+#define DEFAULT_FILE_LIMIT 0
+
 size_t git_mwindow__window_size = DEFAULT_WINDOW_SIZE;
 size_t git_mwindow__mapped_limit = DEFAULT_MAPPED_LIMIT;
+size_t git_mwindow__file_limit = DEFAULT_FILE_LIMIT;
 
 /* Whenever you want to read or modify this, grab git__mwindow_mutex */
-static git_mwindow_ctl mem_ctl;
+git_mwindow_ctl git_mwindow__mem_ctl;
 
 /* Global list of mwindow files, to open packs once across repos */
 git_strmap *git__pack_cache = NULL;
@@ -44,15 +48,14 @@ int git_mwindow_global_init(void)
 	assert(!git__pack_cache);
 
 	git__on_shutdown(git_mwindow_files_free);
-	return git_strmap_alloc(&git__pack_cache);
+	return git_strmap_new(&git__pack_cache);
 }
 
 int git_mwindow_get_pack(struct git_pack_file **out, const char *path)
 {
-	int error;
-	char *packname;
-	size_t pos;
 	struct git_pack_file *pack;
+	char *packname;
+	int error;
 
 	if ((error = git_packfile__name(&packname, path)) < 0)
 		return error;
@@ -62,13 +65,11 @@ int git_mwindow_get_pack(struct git_pack_file **out, const char *path)
 		return -1;
 	}
 
-	pos = git_strmap_lookup_index(git__pack_cache, packname);
+	pack = git_strmap_get(git__pack_cache, packname);
 	git__free(packname);
 
-	if (git_strmap_valid_index(git__pack_cache, pos)) {
-		pack = git_strmap_value_at(git__pack_cache, pos);
+	if (pack != NULL) {
 		git_atomic_inc(&pack->refcount);
-
 		git_mutex_unlock(&git__mwindow_mutex);
 		*out = pack;
 		return 0;
@@ -82,7 +83,7 @@ int git_mwindow_get_pack(struct git_pack_file **out, const char *path)
 
 	git_atomic_inc(&pack->refcount);
 
-	git_strmap_insert(git__pack_cache, pack->pack_name, pack, &error);
+	error = git_strmap_set(git__pack_cache, pack->pack_name, pack);
 	git_mutex_unlock(&git__mwindow_mutex);
 
 	if (error < 0) {
@@ -97,7 +98,6 @@ int git_mwindow_get_pack(struct git_pack_file **out, const char *path)
 void git_mwindow_put_pack(struct git_pack_file *pack)
 {
 	int count;
-	size_t pos;
 
 	if (git_mutex_lock(&git__mwindow_mutex) < 0)
 		return;
@@ -105,13 +105,12 @@ void git_mwindow_put_pack(struct git_pack_file *pack)
 	/* put before get would be a corrupted state */
 	assert(git__pack_cache);
 
-	pos = git_strmap_lookup_index(git__pack_cache, pack->pack_name);
 	/* if we cannot find it, the state is corrupted */
-	assert(git_strmap_valid_index(git__pack_cache, pos));
+	assert(git_strmap_exists(git__pack_cache, pack->pack_name));
 
 	count = git_atomic_dec(&pack->refcount);
 	if (count == 0) {
-		git_strmap_delete_at(git__pack_cache, pos);
+		git_strmap_delete(git__pack_cache, pack->pack_name);
 		git_packfile_free(pack);
 	}
 
@@ -137,7 +136,7 @@ void git_mwindow_free_all(git_mwindow_file *mwf)
  */
 void git_mwindow_free_all_locked(git_mwindow_file *mwf)
 {
-	git_mwindow_ctl *ctl = &mem_ctl;
+	git_mwindow_ctl *ctl = &git_mwindow__mem_ctl;
 	size_t i;
 
 	/*
@@ -172,91 +171,157 @@ void git_mwindow_free_all_locked(git_mwindow_file *mwf)
 /*
  * Check if a window 'win' contains the address 'offset'
  */
-int git_mwindow_contains(git_mwindow *win, git_off_t offset)
+int git_mwindow_contains(git_mwindow *win, off64_t offset)
 {
-	git_off_t win_off = win->offset;
+	off64_t win_off = win->offset;
 	return win_off <= offset
-		&& offset <= (git_off_t)(win_off + win->window_map.len);
+		&& offset <= (off64_t)(win_off + win->window_map.len);
 }
 
+#define GIT_MWINDOW__LRU -1
+#define GIT_MWINDOW__MRU 1
+
 /*
- * Find the least-recently-used window in a file
+ * Find the least- or most-recently-used window in a file that is not currently
+ * being used. The 'only_unused' flag controls whether the caller requires the
+ * file to only have unused windows. If '*out_window' is non-null, it is used as
+ * a starting point for the comparison.
+ *
+ * Returns whether such a window was found in the file.
  */
-static void git_mwindow_scan_lru(
-	git_mwindow_file *mwf,
-	git_mwindow **lru_w,
-	git_mwindow **lru_l)
+static bool git_mwindow_scan_recently_used(
+		git_mwindow_file *mwf,
+		git_mwindow **out_window,
+		git_mwindow **out_last,
+		bool only_unused,
+		int comparison_sign)
 {
-	git_mwindow *w, *w_l;
-
-	for (w_l = NULL, w = mwf->windows; w; w = w->next) {
-		if (!w->inuse_cnt) {
-			/*
-			 * If the current one is more recent than the last one,
-			 * store it in the output parameter. If lru_w is NULL,
-			 * it's the first loop, so store it as well.
-			 */
-			if (!*lru_w || w->last_used < (*lru_w)->last_used) {
-				*lru_w = w;
-				*lru_l = w_l;
-			}
+	git_mwindow *w, *w_last;
+	git_mwindow *lru_window = NULL, *lru_last = NULL;
+	bool found = false;
+
+	assert(mwf);
+	assert(out_window);
+
+	lru_window = *out_window;
+	if (out_last)
+		lru_last = *out_last;
+
+	for (w_last = NULL, w = mwf->windows; w; w_last = w, w = w->next) {
+		if (w->inuse_cnt) {
+			if (only_unused)
+				return false;
+			/* This window is currently being used. Skip it. */
+			continue;
+		}
+
+		/*
+		 * If the current one is more (or less) recent than the last one,
+		 * store it in the output parameter. If lru_window is NULL,
+		 * it's the first loop, so store it as well.
+		 */
+		if (!lru_window ||
+				(comparison_sign == GIT_MWINDOW__LRU && lru_window->last_used > w->last_used) ||
+				(comparison_sign == GIT_MWINDOW__MRU && lru_window->last_used < w->last_used)) {
+			lru_window = w;
+			lru_last = w_last;
+			found = true;
 		}
-		w_l = w;
 	}
+
+	if (!found)
+		return false;
+
+	*out_window = lru_window;
+	if (out_last)
+		*out_last = lru_last;
+	return true;
 }
 
 /*
- * Close the least recently used window. You should check to see if
- * the file descriptors need closing from time to time. Called under
- * lock from new_window.
+ * Close the least recently used window (that is currently not being used) out
+ * of all the files. Called under lock from new_window.
  */
-static int git_mwindow_close_lru(git_mwindow_file *mwf)
+static int git_mwindow_close_lru_window(void)
 {
-	git_mwindow_ctl *ctl = &mem_ctl;
+	git_mwindow_ctl *ctl = &git_mwindow__mem_ctl;
+	git_mwindow_file *cur;
 	size_t i;
-	git_mwindow *lru_w = NULL, *lru_l = NULL, **list = &mwf->windows;
-
-	/* FIXME: Does this give us any advantage? */
-	if(mwf->windows)
-		git_mwindow_scan_lru(mwf, &lru_w, &lru_l);
+	git_mwindow *lru_window = NULL, *lru_last = NULL, **list = NULL;
 
-	for (i = 0; i < ctl->windowfiles.length; ++i) {
-		git_mwindow *last = lru_w;
-		git_mwindow_file *cur = git_vector_get(&ctl->windowfiles, i);
-		git_mwindow_scan_lru(cur, &lru_w, &lru_l);
-		if (lru_w != last)
+	git_vector_foreach(&ctl->windowfiles, i, cur) {
+		if (git_mwindow_scan_recently_used(
+				cur, &lru_window, &lru_last, false, GIT_MWINDOW__LRU)) {
 			list = &cur->windows;
+		}
 	}
 
-	if (!lru_w) {
+	if (!lru_window) {
 		git_error_set(GIT_ERROR_OS, "failed to close memory window; couldn't find LRU");
 		return -1;
 	}
 
-	ctl->mapped -= lru_w->window_map.len;
-	git_futils_mmap_free(&lru_w->window_map);
+	ctl->mapped -= lru_window->window_map.len;
+	git_futils_mmap_free(&lru_window->window_map);
 
-	if (lru_l)
-		lru_l->next = lru_w->next;
+	if (lru_last)
+		lru_last->next = lru_window->next;
 	else
-		*list = lru_w->next;
+		*list = lru_window->next;
 
-	git__free(lru_w);
+	git__free(lru_window);
 	ctl->open_windows--;
 
 	return 0;
 }
 
+/*
+ * Close the file that does not have any open windows AND whose
+ * most-recently-used window is the least-recently used one across all
+ * currently open files.
+ *
+ * Called under lock from new_window.
+ */
+static int git_mwindow_close_lru_file(void)
+{
+	git_mwindow_ctl *ctl = &git_mwindow__mem_ctl;
+	git_mwindow_file *lru_file = NULL, *current_file = NULL;
+	git_mwindow *lru_window = NULL;
+	size_t i;
+
+	git_vector_foreach(&ctl->windowfiles, i, current_file) {
+		git_mwindow *mru_window = NULL;
+		if (!git_mwindow_scan_recently_used(
+				current_file, &mru_window, NULL, true, GIT_MWINDOW__MRU)) {
+			continue;
+		}
+		if (!lru_window || lru_window->last_used > mru_window->last_used) {
+			lru_window = mru_window;
+			lru_file = current_file;
+		}
+	}
+
+	if (!lru_file) {
+		git_error_set(GIT_ERROR_OS, "failed to close memory window file; couldn't find LRU");
+		return -1;
+	}
+
+	git_mwindow_free_all_locked(lru_file);
+	p_close(lru_file->fd);
+	lru_file->fd = -1;
+
+	return 0;
+}
+
 /* This gets called under lock from git_mwindow_open */
 static git_mwindow *new_window(
-	git_mwindow_file *mwf,
 	git_file fd,
-	git_off_t size,
-	git_off_t offset)
+	off64_t size,
+	off64_t offset)
 {
-	git_mwindow_ctl *ctl = &mem_ctl;
+	git_mwindow_ctl *ctl = &git_mwindow__mem_ctl;
 	size_t walign = git_mwindow__window_size / 2;
-	git_off_t len;
+	off64_t len;
 	git_mwindow *w;
 
 	w = git__malloc(sizeof(*w));
@@ -268,13 +333,13 @@ static git_mwindow *new_window(
 	w->offset = (offset / walign) * walign;
 
 	len = size - w->offset;
-	if (len > (git_off_t)git_mwindow__window_size)
-		len = (git_off_t)git_mwindow__window_size;
+	if (len > (off64_t)git_mwindow__window_size)
+		len = (off64_t)git_mwindow__window_size;
 
 	ctl->mapped += (size_t)len;
 
 	while (git_mwindow__mapped_limit < ctl->mapped &&
-			git_mwindow_close_lru(mwf) == 0) /* nop */;
+			git_mwindow_close_lru_window() == 0) /* nop */;
 
 	/*
 	 * We treat `mapped_limit` as a soft limit. If we can't find a
@@ -288,7 +353,7 @@ static git_mwindow *new_window(
 		 * we're below our soft limits, so free up what we can and try again.
 		 */
 
-		while (git_mwindow_close_lru(mwf) == 0)
+		while (git_mwindow_close_lru_window() == 0)
 			/* nop */;
 
 		if (git_futils_mmap_ro(&w->window_map, fd, w->offset, (size_t)len) < 0) {
@@ -316,11 +381,11 @@ static git_mwindow *new_window(
 unsigned char *git_mwindow_open(
 	git_mwindow_file *mwf,
 	git_mwindow **cursor,
-	git_off_t offset,
+	off64_t offset,
 	size_t extra,
 	unsigned int *left)
 {
-	git_mwindow_ctl *ctl = &mem_ctl;
+	git_mwindow_ctl *ctl = &git_mwindow__mem_ctl;
 	git_mwindow *w = *cursor;
 
 	if (git_mutex_lock(&git__mwindow_mutex)) {
@@ -344,7 +409,7 @@ unsigned char *git_mwindow_open(
 		 * one.
 		 */
 		if (!w) {
-			w = new_window(mwf, mwf->fd, mwf->size, offset);
+			w = new_window(mwf->fd, mwf->size, offset);
 			if (w == NULL) {
 				git_mutex_unlock(&git__mwindow_mutex);
 				return NULL;
@@ -372,7 +437,7 @@ unsigned char *git_mwindow_open(
 
 int git_mwindow_file_register(git_mwindow_file *mwf)
 {
-	git_mwindow_ctl *ctl = &mem_ctl;
+	git_mwindow_ctl *ctl = &git_mwindow__mem_ctl;
 	int ret;
 
 	if (git_mutex_lock(&git__mwindow_mutex)) {
@@ -386,6 +451,11 @@ int git_mwindow_file_register(git_mwindow_file *mwf)
 		return -1;
 	}
 
+	if (git_mwindow__file_limit) {
+		while (git_mwindow__file_limit <= ctl->windowfiles.length &&
+				git_mwindow_close_lru_file() == 0) /* nop */;
+	}
+
 	ret = git_vector_insert(&ctl->windowfiles, mwf);
 	git_mutex_unlock(&git__mwindow_mutex);
 
@@ -394,7 +464,7 @@ int git_mwindow_file_register(git_mwindow_file *mwf)
 
 void git_mwindow_file_deregister(git_mwindow_file *mwf)
 {
-	git_mwindow_ctl *ctl = &mem_ctl;
+	git_mwindow_ctl *ctl = &git_mwindow__mem_ctl;
 	git_mwindow_file *cur;
 	size_t i;