rugged 0.17.0.b7 → 0.18.0.b1

data/vendor/libgit2/src/pack.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2009-2012 the libgit2 contributors
+ * Copyright (C) the libgit2 contributors. All rights reserved.
  *
  * This file is part of libgit2, distributed under the GNU GPL v2 with
  * a Linking Exception. For full terms see the included COPYING file.
@@ -46,6 +46,134 @@ static int packfile_error(const char *message)
 	return -1;
 }
 
+/********************
+ * Delta base cache
+ ********************/
+
+static git_pack_cache_entry *new_cache_object(git_rawobj *source)
+{
+	git_pack_cache_entry *e = git__calloc(1, sizeof(git_pack_cache_entry));
+	if (!e)
+		return NULL;
+
+	memcpy(&e->raw, source, sizeof(git_rawobj));
+
+	return e;
+}
+
+static void free_cache_object(void *o)
+{
+	git_pack_cache_entry *e = (git_pack_cache_entry *)o;
+
+	if (e != NULL) {
+		assert(e->refcount.val == 0);
+		git__free(e->raw.data);
+		git__free(e);
+	}
+}
+
+static void cache_free(git_pack_cache *cache)
+{
+	khiter_t k;
+
+	if (cache->entries) {
+		for (k = kh_begin(cache->entries); k != kh_end(cache->entries); k++) {
+			if (kh_exist(cache->entries, k))
+				free_cache_object(kh_value(cache->entries, k));
+		}
+
+		git_offmap_free(cache->entries);
+		git_mutex_free(&cache->lock);
+	}
+}
+
+static int cache_init(git_pack_cache *cache)
+{
+	memset(cache, 0, sizeof(git_pack_cache));
+	cache->entries = git_offmap_alloc();
+	GITERR_CHECK_ALLOC(cache->entries);
+	cache->memory_limit = GIT_PACK_CACHE_MEMORY_LIMIT;
+	git_mutex_init(&cache->lock);
+
+	return 0;
+}
+
+static git_pack_cache_entry *cache_get(git_pack_cache *cache, git_off_t offset)
+{
+	khiter_t k;
+	git_pack_cache_entry *entry = NULL;
+
+	if (git_mutex_lock(&cache->lock) < 0)
+		return NULL;
+
+	k = kh_get(off, cache->entries, offset);
+	if (k != kh_end(cache->entries)) { /* found it */
+		entry = kh_value(cache->entries, k);
+		git_atomic_inc(&entry->refcount);
+		entry->last_usage = cache->use_ctr++;
+	}
+	git_mutex_unlock(&cache->lock);
+
+	return entry;
+}
+
+/* Run with the cache lock held */
+static void free_lowest_entry(git_pack_cache *cache)
+{
+	git_pack_cache_entry *entry;
+	khiter_t k;
+
+	for (k = kh_begin(cache->entries); k != kh_end(cache->entries); k++) {
+		if (!kh_exist(cache->entries, k))
+			continue;
+
+		entry = kh_value(cache->entries, k);
+
+		if (entry && entry->refcount.val == 0) {
+			cache->memory_used -= entry->raw.len;
+			kh_del(off, cache->entries, k);
+			free_cache_object(entry);
+		}
+	}
+}
+
+static int cache_add(git_pack_cache *cache, git_rawobj *base, git_off_t offset)
+{
+	git_pack_cache_entry *entry;
+	int error, exists = 0;
+	khiter_t k;
+
+	if (base->len > GIT_PACK_CACHE_SIZE_LIMIT)
+		return -1;
+
+	entry = new_cache_object(base);
+	if (entry) {
+		if (git_mutex_lock(&cache->lock) < 0) {
+			giterr_set(GITERR_OS, "failed to lock cache");
+			return -1;
+		}
+		/* Add it to the cache if nobody else has */
+		exists = kh_get(off, cache->entries, offset) != kh_end(cache->entries);
+		if (!exists) {
+			while (cache->memory_used + base->len > cache->memory_limit)
+				free_lowest_entry(cache);
+
+			k = kh_put(off, cache->entries, offset, &error);
+			assert(error != 0);
+			kh_value(cache->entries, k) = entry;
+			cache->memory_used += entry->raw.len;
+		}
+		git_mutex_unlock(&cache->lock);
+		/* Somebody beat us to adding it into the cache */
+		if (exists) {
+			git__free(entry);
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
 /***********************************************************
  *
  * PACK INDEX METHODS
@@ -277,6 +405,56 @@ int git_packfile_unpack_header(
 	return 0;
 }
 
+int git_packfile_resolve_header(
+		size_t *size_p,
+		git_otype *type_p,
+		struct git_pack_file *p,
+		git_off_t offset)
+{
+	git_mwindow *w_curs = NULL;
+	git_off_t curpos = offset;
+	size_t size;
+	git_otype type;
+	git_off_t base_offset;
+	int error;
+
+	error = git_packfile_unpack_header(&size, &type, &p->mwf, &w_curs, &curpos);
+	git_mwindow_close(&w_curs);
+	if (error < 0)
+		return error;
+
+	if (type == GIT_OBJ_OFS_DELTA || type == GIT_OBJ_REF_DELTA) {
+		size_t base_size;
+		git_rawobj delta;
+		base_offset = get_delta_base(p, &w_curs, &curpos, type, offset);
+		git_mwindow_close(&w_curs);
+		error = packfile_unpack_compressed(&delta, p, &w_curs, &curpos, size, type);
+		git_mwindow_close(&w_curs);
+		if (error < 0)
+			return error;
+		error = git__delta_read_header(delta.data, delta.len, &base_size, size_p);
+		git__free(delta.data);
+		if (error < 0)
+			return error;
+	} else
+		*size_p = size;
+
+	while (type == GIT_OBJ_OFS_DELTA || type == GIT_OBJ_REF_DELTA) {
+		curpos = base_offset;
+		error = git_packfile_unpack_header(&size, &type, &p->mwf, &w_curs, &curpos);
+		git_mwindow_close(&w_curs);
+		if (error < 0)
+			return error;
+		if (type != GIT_OBJ_OFS_DELTA && type != GIT_OBJ_REF_DELTA)
+			break;
+		base_offset = get_delta_base(p, &w_curs, &curpos, type, base_offset);
+		git_mwindow_close(&w_curs);
+	}
+	*type_p = type;
+
+	return error;
+}
+
 static int packfile_unpack_delta(
 		git_rawobj *obj,
 		struct git_pack_file *p,
@@ -286,9 +464,10 @@ static int packfile_unpack_delta(
 		git_otype delta_type,
 		git_off_t obj_offset)
 {
-	git_off_t base_offset;
+	git_off_t base_offset, base_key;
 	git_rawobj base, delta;
-	int error;
+	git_pack_cache_entry *cached = NULL;
+	int error, found_base = 0;
 
 	base_offset = get_delta_base(p, w_curs, curpos, delta_type, obj_offset);
 	git_mwindow_close(w_curs);
@@ -297,32 +476,49 @@ static int packfile_unpack_delta(
 	if (base_offset < 0) /* must actually be an error code */
 		return (int)base_offset;
 
-	error = git_packfile_unpack(&base, p, &base_offset);
+	if (!p->bases.entries && (cache_init(&p->bases) < 0))
+		return -1;
 
-	/*
-	 * TODO: git.git tries to load the base from other packfiles
-	 * or loose objects.
-	 *
-	 * We'll need to do this in order to support thin packs.
-	 */
-	if (error < 0)
-		return error;
+	base_key = base_offset; /* git_packfile_unpack modifies base_offset */
+	if ((cached = cache_get(&p->bases, base_offset)) != NULL) {
+		memcpy(&base, &cached->raw, sizeof(git_rawobj));
+		found_base = 1;
+	}
+
+	if (!cached) { /* have to inflate it */
+		error = git_packfile_unpack(&base, p, &base_offset);
+
+		/*
+		 * TODO: git.git tries to load the base from other packfiles
+		 * or loose objects.
+		 *
+		 * We'll need to do this in order to support thin packs.
+		 */
+		if (error < 0)
+			return error;
+	}
 
 	error = packfile_unpack_compressed(&delta, p, w_curs, curpos, delta_size, delta_type);
 	git_mwindow_close(w_curs);
+
 	if (error < 0) {
-		git__free(base.data);
+		if (!found_base)
+			git__free(base.data);
 		return error;
 	}
 
 	obj->type = base.type;
 	error = git__delta_apply(obj, base.data, base.len, delta.data, delta.len);
+	if (error < 0)
+		goto on_error;
 
-	git__free(base.data);
-	git__free(delta.data);
+	if (found_base)
+		git_atomic_dec(&cached->refcount);
+	else if (cache_add(&p->bases, &base, base_key) < 0)
+		git__free(base.data);
 
-	/* TODO: we might want to cache this shit. eventually */
-	//add_delta_base_cache(p, base_offset, base, base_size, *type);
+on_error:
+	git__free(delta.data);
 
 	return error; /* error set by git__delta_apply */
 }
@@ -391,6 +587,72 @@ static void use_git_free(void *opaq, void *ptr)
 	git__free(ptr);
 }
 
+int git_packfile_stream_open(git_packfile_stream *obj, struct git_pack_file *p, git_off_t curpos)
+{
+	int st;
+
+	memset(obj, 0, sizeof(git_packfile_stream));
+	obj->curpos = curpos;
+	obj->p = p;
+	obj->zstream.zalloc = use_git_alloc;
+	obj->zstream.zfree = use_git_free;
+	obj->zstream.next_in = Z_NULL;
+	obj->zstream.next_out = Z_NULL;
+	st = inflateInit(&obj->zstream);
+	if (st != Z_OK) {
+		git__free(obj);
+		giterr_set(GITERR_ZLIB, "Failed to inflate packfile");
+		return -1;
+	}
+
+	return 0;
+}
+
+ssize_t git_packfile_stream_read(git_packfile_stream *obj, void *buffer, size_t len)
+{
+	unsigned char *in;
+	size_t written;
+	int st;
+
+	if (obj->done)
+		return 0;
+
+	in = pack_window_open(obj->p, &obj->mw, obj->curpos, &obj->zstream.avail_in);
+	if (in == NULL)
+		return GIT_EBUFS;
+
+	obj->zstream.next_out = buffer;
+	obj->zstream.avail_out = (unsigned int)len;
+	obj->zstream.next_in = in;
+
+	st = inflate(&obj->zstream, Z_SYNC_FLUSH);
+	git_mwindow_close(&obj->mw);
+
+	obj->curpos += obj->zstream.next_in - in;
+	written = len - obj->zstream.avail_out;
+
+	if (st != Z_OK && st != Z_STREAM_END) {
+		giterr_set(GITERR_ZLIB, "Failed to inflate packfile");
+		return -1;
+	}
+
+	if (st == Z_STREAM_END)
+		obj->done = 1;
+
+
+	/* If we didn't write anything out but we're not done, we need more data */
+	if (!written && st != Z_STREAM_END)
+		return GIT_EBUFS;
+
+	return written;
+
+}
+
+void git_packfile_stream_free(git_packfile_stream *obj)
+{
+	inflateEnd(&obj->zstream);
+}
+
 int packfile_unpack_compressed(
 	git_rawobj *obj,
 	struct git_pack_file *p,
@@ -498,14 +760,14 @@ git_off_t get_delta_base(
 	} else if (type == GIT_OBJ_REF_DELTA) {
 		/* If we have the cooperative cache, search in it first */
 		if (p->has_cache) {
-			int pos;
-			struct git_pack_entry key;
+			khiter_t k;
+			git_oid oid;
 
-			git_oid_fromraw(&key.sha1, base_info);
-			pos = git_vector_bsearch(&p->cache, &key);
-			if (pos >= 0) {
+			git_oid_fromraw(&oid, base_info);
+			k = kh_get(oid, p->idx_cache, &oid);
+			if (k != kh_end(p->idx_cache)) {
 				*curpos += 20;
-				return ((struct git_pack_entry *)git_vector_get(&p->cache, pos))->offset;
+				return ((struct git_pack_entry *)kh_value(p->idx_cache, k))->offset;
 			}
 		}
 		/* The base entry _must_ be in the same pack */
@@ -533,11 +795,12 @@ static struct git_pack_file *packfile_alloc(size_t extra)
 }
 
 
-void packfile_free(struct git_pack_file *p)
+void git_packfile_free(struct git_pack_file *p)
 {
 	assert(p);
 
-	/* clear_delta_base_cache(); */
+	cache_free(&p->bases);
+
 	git_mwindow_free_all(&p->mwf);
 	git_mwindow_file_deregister(&p->mwf);
 
@@ -564,8 +827,10 @@ static int packfile_open(struct git_pack_file *p)
 
 	/* TODO: open with noatime */
 	p->mwf.fd = git_futils_open_ro(p->pack_name);
-	if (p->mwf.fd < 0)
-		return p->mwf.fd;
+	if (p->mwf.fd < 0) {
+		p->mwf.fd = -1;
+		return -1;
+	}
 
 	if (p_fstat(p->mwf.fd, &st) < 0 ||
 		git_mwindow_file_register(&p->mwf) < 0)
@@ -696,7 +961,7 @@ static int git__memcmp4(const void *a, const void *b) {
 
 int git_pack_foreach_entry(
 	struct git_pack_file *p,
-	int (*cb)(git_oid *oid, void *data),
+	git_odb_foreach_cb cb,
 	void *data)
 {
 	const unsigned char *index = p->index_map.data, *current;

data/vendor/libgit2/src/pack.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2009-2012 the libgit2 contributors
+ * Copyright (C) the libgit2 contributors. All rights reserved.
  *
  * This file is part of libgit2, distributed under the GNU GPL v2 with
  * a Linking Exception. For full terms see the included COPYING file.
@@ -8,12 +8,15 @@
 #ifndef INCLUDE_pack_h__
 #define INCLUDE_pack_h__
 
+#include <zlib.h>
+
 #include "git2/oid.h"
 
 #include "common.h"
 #include "map.h"
 #include "mwindow.h"
 #include "odb.h"
+#include "oidmap.h"
 
 #define GIT_PACK_FILE_MODE 0444
 
@@ -51,6 +54,28 @@ struct git_pack_idx_header {
 	uint32_t idx_version;
 };
 
+typedef struct git_pack_cache_entry {
+	size_t last_usage; /* enough? */
+	git_atomic refcount;
+	git_rawobj raw;
+} git_pack_cache_entry;
+
+#include "offmap.h"
+
+GIT__USE_OFFMAP;
+GIT__USE_OIDMAP;
+
+#define GIT_PACK_CACHE_MEMORY_LIMIT 16 * 1024 * 1024
+#define GIT_PACK_CACHE_SIZE_LIMIT 1024 * 1024 /* don't bother caching anything over 1MB */
+
+typedef struct {
+	size_t memory_used;
+	size_t memory_limit;
+	size_t use_ctr;
+	git_mutex lock;
+	git_offmap *entries;
+} git_pack_cache;
+
 struct git_pack_file {
 	git_mwindow_file mwf;
 	git_map index_map;
@@ -63,9 +88,11 @@ struct git_pack_file {
 	git_time_t mtime;
 	unsigned pack_local:1, pack_keep:1, has_cache:1;
 	git_oid sha1;
-	git_vector cache;
+	git_oidmap *idx_cache;
 	git_oid **oids;
 
+	git_pack_cache bases; /* delta base cache */
+
 	/* something like ".git/objects/pack/xxxxx.pack" */
 	char pack_name[GIT_FLEX_ARRAY]; /* more */
 };
@@ -76,6 +103,14 @@ struct git_pack_entry {
 	struct git_pack_file *p;
 };
 
+typedef struct git_packfile_stream {
+	git_off_t curpos;
+	int done;
+	z_stream zstream;
+	struct git_pack_file *p;
+	git_mwindow *mw;
+} git_packfile_stream;
+
 int git_packfile_unpack_header(
 		size_t *size_p,
 		git_otype *type_p,
@@ -83,6 +118,12 @@ int git_packfile_unpack_header(
 		git_mwindow **w_curs,
 		git_off_t *curpos);
 
+int git_packfile_resolve_header(
+		size_t *size_p,
+		git_otype *type_p,
+		struct git_pack_file *p,
+		git_off_t offset);
+
 int git_packfile_unpack(git_rawobj *obj, struct git_pack_file *p, git_off_t *obj_offset);
 int packfile_unpack_compressed(
 	git_rawobj *obj,
@@ -92,11 +133,15 @@ int packfile_unpack_compressed(
 	size_t size,
 	git_otype type);
 
+int git_packfile_stream_open(git_packfile_stream *obj, struct git_pack_file *p, git_off_t curpos);
+ssize_t git_packfile_stream_read(git_packfile_stream *obj, void *buffer, size_t len);
+void git_packfile_stream_free(git_packfile_stream *obj);
+
 git_off_t get_delta_base(struct git_pack_file *p, git_mwindow **w_curs,
 		git_off_t *curpos, git_otype type,
 		git_off_t delta_obj_offset);
 
-void packfile_free(struct git_pack_file *p);
+void git_packfile_free(struct git_pack_file *p);
 int git_packfile_check(struct git_pack_file **pack_out, const char *path);
 int git_pack_entry_find(
 		struct git_pack_entry *e,
@@ -105,7 +150,7 @@ int git_pack_entry_find(
 		size_t len);
 int git_pack_foreach_entry(
 		struct git_pack_file *p,
-		int (*cb)(git_oid *oid, void *data),
+		git_odb_foreach_cb cb,
 		void *data);
 
 #endif