rugged 0.19.0 → 0.28.0

data/vendor/libgit2/src/odb.c

@@ -5,28 +5,35 @@
  * a Linking Exception. For full terms see the included COPYING file.
  */
 
-#include "common.h"
+#include "odb.h"
+
 #include <zlib.h>
 #include "git2/object.h"
 #include "git2/sys/odb_backend.h"
 #include "fileops.h"
 #include "hash.h"
-#include "odb.h"
-#include "delta-apply.h"
+#include "delta.h"
 #include "filter.h"
 #include "repository.h"
+#include "blob.h"
 
 #include "git2/odb_backend.h"
 #include "git2/oid.h"
+#include "git2/oidarray.h"
 
 #define GIT_ALTERNATES_FILE "info/alternates"
 
-/* TODO: is this correct? */
-#define GIT_LOOSE_PRIORITY 2
-#define GIT_PACKED_PRIORITY 1
+/*
+ * We work under the assumption that most objects for long-running
+ * operations will be packed
+ */
+#define GIT_LOOSE_PRIORITY 1
+#define GIT_PACKED_PRIORITY 2
 
 #define GIT_ALTERNATES_MAX_DEPTH 5
 
+bool git_odb__strict_hash_verification = true;
+
 typedef struct
 {
 	git_odb_backend *backend;
@@ -45,40 +52,90 @@ static git_cache *odb_cache(git_odb *odb)
 	return &odb->own_cache;
 }
 
+static int odb_otype_fast(git_object_t *type_p, git_odb *db, const git_oid *id);
 static int load_alternates(git_odb *odb, const char *objects_dir, int alternate_depth);
+static int error_null_oid(int error, const char *message);
+
+static git_object_t odb_hardcoded_type(const git_oid *id)
+{
+	static git_oid empty_tree = {{ 0x4b, 0x82, 0x5d, 0xc6, 0x42, 0xcb, 0x6e, 0xb9, 0xa0, 0x60,
+					   0xe5, 0x4b, 0xf8, 0xd6, 0x92, 0x88, 0xfb, 0xee, 0x49, 0x04 }};
+
+	if (!git_oid_cmp(id, &empty_tree))
+		return GIT_OBJECT_TREE;
+
+	return GIT_OBJECT_INVALID;
+}
+
+static int odb_read_hardcoded(bool *found, git_rawobj *raw, const git_oid *id)
+{
+	git_object_t type;
 
-int git_odb__format_object_header(char *hdr, size_t n, size_t obj_len, git_otype obj_type)
+	*found = false;
+
+	if ((type = odb_hardcoded_type(id)) == GIT_OBJECT_INVALID)
+		return 0;
+
+	raw->type = type;
+	raw->len = 0;
+	raw->data = git__calloc(1, sizeof(uint8_t));
+	GIT_ERROR_CHECK_ALLOC(raw->data);
+
+	*found = true;
+	return 0;
+}
+
+int git_odb__format_object_header(
+	size_t *written,
+	char *hdr,
+	size_t hdr_size,
+	git_off_t obj_len,
+	git_object_t obj_type)
 {
 	const char *type_str = git_object_type2string(obj_type);
-	int len = p_snprintf(hdr, n, "%s %"PRIuZ, type_str, obj_len);
-	assert(len > 0 && len <= (int)n);
-	return len+1;
+	int hdr_max = (hdr_size > INT_MAX-2) ? (INT_MAX-2) : (int)hdr_size;
+	int len;
+
+	len = p_snprintf(hdr, hdr_max, "%s %"PRId64, type_str, (int64_t)obj_len);
+
+	if (len < 0 || len >= hdr_max) {
+		git_error_set(GIT_ERROR_OS, "object header creation failed");
+		return -1;
+	}
+
+	*written = (size_t)(len + 1);
+	return 0;
 }
 
 int git_odb__hashobj(git_oid *id, git_rawobj *obj)
 {
 	git_buf_vec vec[2];
 	char header[64];
-	int hdrlen;
+	size_t hdrlen;
+	int error;
 
 	assert(id && obj);
 
-	if (!git_object_typeisloose(obj->type))
+	if (!git_object_typeisloose(obj->type)) {
+		git_error_set(GIT_ERROR_INVALID, "invalid object type");
 		return -1;
+	}
 
-	if (!obj->data && obj->len != 0)
+	if (!obj->data && obj->len != 0) {
+		git_error_set(GIT_ERROR_INVALID, "invalid object");
 		return -1;
+	}
 
-	hdrlen = git_odb__format_object_header(header, sizeof(header), obj->len, obj->type);
+	if ((error = git_odb__format_object_header(&hdrlen,
+		header, sizeof(header), obj->len, obj->type)) < 0)
+		return error;
 
 	vec[0].data = header;
 	vec[0].len = hdrlen;
 	vec[1].data = obj->data;
 	vec[1].len = obj->len;
 
-	git_hash_vec(id, vec, 2);
-
-	return 0;
+	return git_hash_vec(id, vec, 2);
 }
 
 
@@ -119,11 +176,18 @@ size_t git_odb_object_size(git_odb_object *object)
 	return object->cached.size;
 }
 
-git_otype git_odb_object_type(git_odb_object *object)
+git_object_t git_odb_object_type(git_odb_object *object)
 {
 	return object->cached.type;
 }
 
+int git_odb_object_dup(git_odb_object **dest, git_odb_object *source)
+{
+	git_cached_obj_incref(source);
+	*dest = source;
+	return 0;
+}
+
 void git_odb_object_free(git_odb_object *object)
 {
 	if (object == NULL)
@@ -132,23 +196,25 @@ void git_odb_object_free(git_odb_object *object)
 	git_cached_obj_decref(object);
 }
 
-int git_odb__hashfd(git_oid *out, git_file fd, size_t size, git_otype type)
+int git_odb__hashfd(git_oid *out, git_file fd, size_t size, git_object_t type)
 {
-	int hdr_len;
-	char hdr[64], buffer[2048];
+	size_t hdr_len;
+	char hdr[64], buffer[FILEIO_BUFSIZE];
 	git_hash_ctx ctx;
 	ssize_t read_len = 0;
 	int error = 0;
 
 	if (!git_object_typeisloose(type)) {
-		giterr_set(GITERR_INVALID, "Invalid object type for hash");
+		git_error_set(GIT_ERROR_INVALID, "invalid object type for hash");
 		return -1;
 	}
 
 	if ((error = git_hash_ctx_init(&ctx)) < 0)
-		return -1;
+		return error;
 
-	hdr_len = git_odb__format_object_header(hdr, sizeof(hdr), size, type);
+	if ((error = git_odb__format_object_header(&hdr_len, hdr,
+		sizeof(hdr), size, type)) < 0)
+		goto done;
 
 	if ((error = git_hash_update(&ctx, hdr, hdr_len)) < 0)
 		goto done;
@@ -164,11 +230,10 @@ int git_odb__hashfd(git_oid *out, git_file fd, size_t size, git_otype type)
 	 * If size is not zero, the file was truncated after we originally
 	 * stat'd it, so we consider this a read failure too */
 	if (read_len < 0 || size > 0) {
-		giterr_set(GITERR_OS, "Error reading file for hashing");
+		git_error_set(GIT_ERROR_OS, "error reading file for hashing");
 		error = -1;
 
 		goto done;
-		return -1;
 	}
 
 	error = git_hash_final(out, &ctx);
@@ -179,28 +244,30 @@ done:
 }
 
 int git_odb__hashfd_filtered(
-	git_oid *out, git_file fd, size_t size, git_otype type, git_vector *filters)
+	git_oid *out, git_file fd, size_t size, git_object_t type, git_filter_list *fl)
 {
 	int error;
 	git_buf raw = GIT_BUF_INIT;
-	git_buf filtered = GIT_BUF_INIT;
 
-	if (!filters || !filters->length)
+	if (!fl)
 		return git_odb__hashfd(out, fd, size, type);
 
 	/* size of data is used in header, so we have to read the whole file
 	 * into memory to apply filters before beginning to calculate the hash
 	 */
 
-	if (!(error = git_futils_readbuffer_fd(&raw, fd, size)))
-		error = git_filters_apply(&filtered, &raw, filters);
+	if (!(error = git_futils_readbuffer_fd(&raw, fd, size))) {
+		git_buf post = GIT_BUF_INIT;
 
-	git_buf_free(&raw);
+		error = git_filter_list_apply_to_data(&post, fl, &raw);
 
-	if (!error)
-		error = git_odb_hash(out, filtered.ptr, filtered.size, type);
+		git_buf_dispose(&raw);
+
+		if (!error)
+			error = git_odb_hash(out, post.ptr, post.size, type);
 
-	git_buf_free(&filtered);
+		git_buf_dispose(&post);
+	}
 
 	return error;
 }
@@ -208,47 +275,50 @@ int git_odb__hashfd_filtered(
 int git_odb__hashlink(git_oid *out, const char *path)
 {
 	struct stat st;
-	git_off_t size;
+	int size;
 	int result;
 
 	if (git_path_lstat(path, &st) < 0)
 		return -1;
 
-	size = st.st_size;
-
-	if (!git__is_sizet(size)) {
-		giterr_set(GITERR_OS, "File size overflow for 32-bit systems");
+	if (!git__is_int(st.st_size) || (int)st.st_size < 0) {
+		git_error_set(GIT_ERROR_FILESYSTEM, "file size overflow for 32-bit systems");
 		return -1;
 	}
 
+	size = (int)st.st_size;
+
 	if (S_ISLNK(st.st_mode)) {
 		char *link_data;
-		ssize_t read_len;
+		int read_len;
+		size_t alloc_size;
 
-		link_data = git__malloc((size_t)(size + 1));
-		GITERR_CHECK_ALLOC(link_data);
+		GIT_ERROR_CHECK_ALLOC_ADD(&alloc_size, size, 1);
+		link_data = git__malloc(alloc_size);
+		GIT_ERROR_CHECK_ALLOC(link_data);
 
-		read_len = p_readlink(path, link_data, (size_t)size);
+		read_len = p_readlink(path, link_data, size);
 		link_data[size] = '\0';
-		if (read_len != (ssize_t)size) {
-			giterr_set(GITERR_OS, "Failed to read symlink data for '%s'", path);
+		if (read_len != size) {
+			git_error_set(GIT_ERROR_OS, "failed to read symlink data for '%s'", path);
+			git__free(link_data);
 			return -1;
 		}
 
-		result = git_odb_hash(out, link_data, (size_t)size, GIT_OBJ_BLOB);
+		result = git_odb_hash(out, link_data, size, GIT_OBJECT_BLOB);
 		git__free(link_data);
 	} else {
 		int fd = git_futils_open_ro(path);
 		if (fd < 0)
 			return -1;
-		result = git_odb__hashfd(out, fd, (size_t)size, GIT_OBJ_BLOB);
+		result = git_odb__hashfd(out, fd, size, GIT_OBJECT_BLOB);
 		p_close(fd);
 	}
 
 	return result;
 }
 
-int git_odb_hashfile(git_oid *out, const char *path, git_otype type)
+int git_odb_hashfile(git_oid *out, const char *path, git_object_t type)
 {
 	git_off_t size;
 	int result, fd = git_futils_open_ro(path);
@@ -256,7 +326,7 @@ int git_odb_hashfile(git_oid *out, const char *path, git_otype type)
 		return fd;
 
 	if ((size = git_futils_filesize(fd)) < 0 || !git__is_sizet(size)) {
-		giterr_set(GITERR_OS, "File size overflow for 32-bit systems");
+		git_error_set(GIT_ERROR_OS, "file size overflow for 32-bit systems");
 		p_close(fd);
 		return -1;
 	}
@@ -266,7 +336,7 @@ int git_odb_hashfile(git_oid *out, const char *path, git_otype type)
 	return result;
 }
 
-int git_odb_hash(git_oid *id, const void *data, size_t len, git_otype type)
+int git_odb_hash(git_oid *id, const void *data, size_t len, git_object_t type)
 {
 	git_rawobj raw;
 
@@ -287,21 +357,20 @@ typedef struct {
 	git_odb_stream stream;
 	char *buffer;
 	size_t size, written;
-	git_otype type;
+	git_object_t type;
 } fake_wstream;
 
-static int fake_wstream__fwrite(git_oid *oid, git_odb_stream *_stream)
+static int fake_wstream__fwrite(git_odb_stream *_stream, const git_oid *oid)
 {
 	fake_wstream *stream = (fake_wstream *)_stream;
-	return _stream->backend->write(oid, _stream->backend, stream->buffer, stream->size, stream->type);
+	return _stream->backend->write(_stream->backend, oid, stream->buffer, stream->size, stream->type);
 }
 
 static int fake_wstream__write(git_odb_stream *_stream, const char *data, size_t len)
 {
 	fake_wstream *stream = (fake_wstream *)_stream;
 
-	if (stream->written + len > stream->size)
-		return -1;
+	assert(stream->written + len <= stream->size);
 
 	memcpy(stream->buffer + stream->written, data, len);
 	stream->written += len;
@@ -316,16 +385,20 @@ static void fake_wstream__free(git_odb_stream *_stream)
 	git__free(stream);
 }
 
-static int init_fake_wstream(git_odb_stream **stream_p, git_odb_backend *backend, size_t size, git_otype type)
+static int init_fake_wstream(git_odb_stream **stream_p, git_odb_backend *backend, git_off_t size, git_object_t type)
 {
 	fake_wstream *stream;
+	size_t blobsize;
+
+	GIT_ERROR_CHECK_BLOBSIZE(size);
+	blobsize = (size_t)size;
 
 	stream = git__calloc(1, sizeof(fake_wstream));
-	GITERR_CHECK_ALLOC(stream);
+	GIT_ERROR_CHECK_ALLOC(stream);
 
-	stream->size = size;
+	stream->size = blobsize;
 	stream->type = type;
-	stream->buffer = git__malloc(size);
+	stream->buffer = git__malloc(blobsize);
 	if (stream->buffer == NULL) {
 		git__free(stream);
 		return -1;
@@ -355,16 +428,20 @@ static int backend_sort_cmp(const void *a, const void *b)
 	const backend_internal *backend_a = (const backend_internal *)(a);
 	const backend_internal *backend_b = (const backend_internal *)(b);
 
-	if (backend_a->is_alternate == backend_b->is_alternate)
-		return (backend_b->priority - backend_a->priority);
-
-	return backend_a->is_alternate ? 1 : -1;
+	if (backend_b->priority == backend_a->priority) {
+		if (backend_a->is_alternate)
+			return -1;
+		if (backend_b->is_alternate)
+			return 1;
+		return 0;
+	}
+	return (backend_b->priority - backend_a->priority);
 }
 
 int git_odb_new(git_odb **out)
 {
 	git_odb *db = git__calloc(1, sizeof(*db));
-	GITERR_CHECK_ALLOC(db);
+	GIT_ERROR_CHECK_ALLOC(db);
 
 	if (git_cache_init(&db->own_cache) < 0 ||
 		git_vector_init(&db->backends, 4, backend_sort_cmp) < 0) {
@@ -385,13 +462,13 @@ static int add_backend_internal(
 
 	assert(odb && backend);
 
-	GITERR_CHECK_VERSION(backend, GIT_ODB_BACKEND_VERSION, "git_odb_backend");
+	GIT_ERROR_CHECK_VERSION(backend, GIT_ODB_BACKEND_VERSION, "git_odb_backend");
 
 	/* Check if the backend is already owned by another ODB */
 	assert(!backend->odb || backend->odb == odb);
 
 	internal = git__malloc(sizeof(backend_internal));
-	GITERR_CHECK_ALLOC(internal);
+	GIT_ERROR_CHECK_ALLOC(internal);
 
 	internal->backend = backend;
 	internal->priority = priority;
@@ -426,8 +503,8 @@ size_t git_odb_num_backends(git_odb *odb)
 
 static int git_odb__error_unsupported_in_backend(const char *action)
 {
-	giterr_set(GITERR_ODB,
-		"Cannot %s - unsupported in the loaded odb backends", action);
+	git_error_set(GIT_ERROR_ODB,
+		"cannot %s - unsupported in the loaded odb backends", action);
 	return -1;
 }
 
@@ -436,7 +513,7 @@ int git_odb_get_backend(git_odb_backend **out, git_odb *odb, size_t pos)
 {
 	backend_internal *internal;
 
-	assert(odb && odb);
+	assert(out && odb);
 	internal = git_vector_get(&odb->backends, pos);
 
 	if (internal && internal->backend) {
@@ -444,11 +521,11 @@ int git_odb_get_backend(git_odb_backend **out, git_odb *odb, size_t pos)
 		return 0;
 	}
 
-	giterr_set(GITERR_ODB, "No ODB backend loaded at index " PRIuZ, pos);
+	git_error_set(GIT_ERROR_ODB, "no ODB backend loaded at index %" PRIuZ, pos);
 	return GIT_ENOTFOUND;
 }
 
-static int add_default_backends(
+int git_odb__add_default_backends(
 	git_odb *db, const char *objects_dir,
 	bool as_alternates, int alternate_depth)
 {
@@ -469,7 +546,7 @@ static int add_default_backends(
 		if (as_alternates)
 			return 0;
 
-		giterr_set(GITERR_ODB, "Failed to load object database in '%s'", objects_dir);
+		git_error_set(GIT_ERROR_ODB, "failed to load object database in '%s'", objects_dir);
 		return -1;
 	}
 
@@ -483,7 +560,7 @@ static int add_default_backends(
 #endif
 
 	/* add the loose object backend */
-	if (git_odb_backend_loose(&loose, objects_dir, -1, 0) < 0 ||
+	if (git_odb_backend_loose(&loose, objects_dir, -1, db->do_fsync, 0, 0) < 0 ||
 		add_backend_internal(db, loose, GIT_LOOSE_PRIORITY, as_alternates, inode) < 0)
 		return -1;
 
@@ -511,12 +588,12 @@ static int load_alternates(git_odb *odb, const char *objects_dir, int alternate_
 		return -1;
 
 	if (git_path_exists(alternates_path.ptr) == false) {
-		git_buf_free(&alternates_path);
+		git_buf_dispose(&alternates_path);
 		return 0;
 	}
 
 	if (git_futils_readbuffer(&alternates_buf, alternates_path.ptr) < 0) {
-		git_buf_free(&alternates_path);
+		git_buf_dispose(&alternates_path);
 		return -1;
 	}
 
@@ -538,19 +615,19 @@ static int load_alternates(git_odb *odb, const char *objects_dir, int alternate_
 			alternate = git_buf_cstr(&alternates_path);
 		}
 
-		if ((result = add_default_backends(odb, alternate, true, alternate_depth + 1)) < 0)
+		if ((result = git_odb__add_default_backends(odb, alternate, true, alternate_depth + 1)) < 0)
 			break;
 	}
 
-	git_buf_free(&alternates_path);
-	git_buf_free(&alternates_buf);
+	git_buf_dispose(&alternates_path);
+	git_buf_dispose(&alternates_buf);
 
 	return result;
 }
 
 int git_odb_add_disk_alternate(git_odb *odb, const char *path)
 {
-	return add_default_backends(odb, path, true, 0);
+	return git_odb__add_default_backends(odb, path, true, 0);
 }
 
 int git_odb_open(git_odb **out, const char *objects_dir)
@@ -564,7 +641,7 @@ int git_odb_open(git_odb **out, const char *objects_dir)
 	if (git_odb_new(&db) < 0)
 		return -1;
 
-	if (add_default_backends(db, objects_dir, 0, 0) < 0) {
+	if (git_odb__add_default_backends(db, objects_dir, 0, 0) < 0) {
 		git_odb_free(db);
 		return -1;
 	}
@@ -573,6 +650,24 @@ int git_odb_open(git_odb **out, const char *objects_dir)
 	return 0;
 }
 
+int git_odb__set_caps(git_odb *odb, int caps)
+{
+	if (caps == GIT_ODB_CAP_FROM_OWNER) {
+		git_repository *repo = odb->rc.owner;
+		int val;
+
+		if (!repo) {
+			git_error_set(GIT_ERROR_ODB, "cannot access repository to set odb caps");
+			return -1;
+		}
+
+		if (!git_repository__cvar(&val, repo, GIT_CVAR_FSYNCOBJECTFILES))
+			odb->do_fsync = !!val;
+	}
+
+	return 0;
+}
+
 static void odb_free(git_odb *db)
 {
 	size_t i;
@@ -581,8 +676,7 @@ static void odb_free(git_odb *db)
 		backend_internal *internal = git_vector_get(&db->backends, i);
 		git_odb_backend *backend = internal->backend;
 
-		if (backend->free) backend->free(backend);
-		else git__free(backend);
+		backend->free(backend);
 
 		git__free(internal);
 	}
@@ -602,43 +696,234 @@ void git_odb_free(git_odb *db)
 	GIT_REFCOUNT_DEC(db, odb_free);
 }
 
-int git_odb_exists(git_odb *db, const git_oid *id)
+static int odb_exists_1(
+	git_odb *db,
+	const git_oid *id,
+	bool only_refreshed)
 {
-	git_odb_object *object;
 	size_t i;
 	bool found = false;
-	bool refreshed = false;
+
+	for (i = 0; i < db->backends.length && !found; ++i) {
+		backend_internal *internal = git_vector_get(&db->backends, i);
+		git_odb_backend *b = internal->backend;
+
+		if (only_refreshed && !b->refresh)
+			continue;
+
+		if (b->exists != NULL)
+			found = (bool)b->exists(b, id);
+	}
+
+	return (int)found;
+}
+
+static int odb_freshen_1(
+	git_odb *db,
+	const git_oid *id,
+	bool only_refreshed)
+{
+	size_t i;
+	bool found = false;
+
+	for (i = 0; i < db->backends.length && !found; ++i) {
+		backend_internal *internal = git_vector_get(&db->backends, i);
+		git_odb_backend *b = internal->backend;
+
+		if (only_refreshed && !b->refresh)
+			continue;
+
+		if (b->freshen != NULL)
+			found = !b->freshen(b, id);
+		else if (b->exists != NULL)
+			found = b->exists(b, id);
+	}
+
+	return (int)found;
+}
+
+int git_odb__freshen(git_odb *db, const git_oid *id)
+{
+	assert(db && id);
+
+	if (odb_freshen_1(db, id, false))
+		return 1;
+
+	if (!git_odb_refresh(db))
+		return odb_freshen_1(db, id, true);
+
+	/* Failed to refresh, hence not found */
+	return 0;
+}
+
+int git_odb_exists(git_odb *db, const git_oid *id)
+{
+	git_odb_object *object;
 
 	assert(db && id);
 
+	if (git_oid_iszero(id))
+		return 0;
+
 	if ((object = git_cache_get_raw(odb_cache(db), id)) != NULL) {
 		git_odb_object_free(object);
-		return (int)true;
+		return 1;
 	}
 
-attempt_lookup:
-	for (i = 0; i < db->backends.length && !found; ++i) {
+	if (odb_exists_1(db, id, false))
+		return 1;
+
+	if (!git_odb_refresh(db))
+		return odb_exists_1(db, id, true);
+
+	/* Failed to refresh, hence not found */
+	return 0;
+}
+
+static int odb_exists_prefix_1(git_oid *out, git_odb *db,
+	const git_oid *key, size_t len, bool only_refreshed)
+{
+	size_t i;
+	int error = GIT_ENOTFOUND, num_found = 0;
+	git_oid last_found = {{0}}, found;
+
+	for (i = 0; i < db->backends.length; ++i) {
 		backend_internal *internal = git_vector_get(&db->backends, i);
 		git_odb_backend *b = internal->backend;
 
-		if (b->exists != NULL)
-			found = b->exists(b, id);
+		if (only_refreshed && !b->refresh)
+			continue;
+
+		if (!b->exists_prefix)
+			continue;
+
+		error = b->exists_prefix(&found, b, key, len);
+		if (error == GIT_ENOTFOUND || error == GIT_PASSTHROUGH)
+			continue;
+		if (error)
+			return error;
+
+		/* make sure found item doesn't introduce ambiguity */
+		if (num_found) {
+			if (git_oid__cmp(&last_found, &found))
+				return git_odb__error_ambiguous("multiple matches for prefix");
+		} else {
+			git_oid_cpy(&last_found, &found);
+			num_found++;
+		}
 	}
 
-	if (!found && !refreshed) {
-		if (git_odb_refresh(db) < 0) {
-			giterr_clear();
-			return (int)false;
+	if (!num_found)
+		return GIT_ENOTFOUND;
+
+	if (out)
+		git_oid_cpy(out, &last_found);
+
+	return 0;
+}
+
+int git_odb_exists_prefix(
+	git_oid *out, git_odb *db, const git_oid *short_id, size_t len)
+{
+	int error;
+	git_oid key = {{0}};
+
+	assert(db && short_id);
+
+	if (len < GIT_OID_MINPREFIXLEN)
+		return git_odb__error_ambiguous("prefix length too short");
+
+	if (len >= GIT_OID_HEXSZ) {
+		if (git_odb_exists(db, short_id)) {
+			if (out)
+				git_oid_cpy(out, short_id);
+			return 0;
+		} else {
+			return git_odb__error_notfound(
+				"no match for id prefix", short_id, len);
+		}
+	}
+
+	git_oid__cpy_prefix(&key, short_id, len);
+
+	error = odb_exists_prefix_1(out, db, &key, len, false);
+
+	if (error == GIT_ENOTFOUND && !git_odb_refresh(db))
+		error = odb_exists_prefix_1(out, db, &key, len, true);
+
+	if (error == GIT_ENOTFOUND)
+		return git_odb__error_notfound("no match for id prefix", &key, len);
+
+	return error;
+}
+
+int git_odb_expand_ids(
+	git_odb *db,
+	git_odb_expand_id *ids,
+	size_t count)
+{
+	size_t i;
+
+	assert(db && ids);
+
+	for (i = 0; i < count; i++) {
+		git_odb_expand_id *query = &ids[i];
+		int error = GIT_EAMBIGUOUS;
+
+		if (!query->type)
+			query->type = GIT_OBJECT_ANY;
+
+		/* if we have a short OID, expand it first */
+		if (query->length >= GIT_OID_MINPREFIXLEN && query->length < GIT_OID_HEXSZ) {
+			git_oid actual_id;
+
+			error = odb_exists_prefix_1(&actual_id, db, &query->id, query->length, false);
+			if (!error) {
+				git_oid_cpy(&query->id, &actual_id);
+				query->length = GIT_OID_HEXSZ;
+			}
 		}
 
-		refreshed = true;
-		goto attempt_lookup;
+		/*
+		 * now we ought to have a 40-char OID, either because we've expanded it
+		 * or because the user passed a full OID. Ensure its type is right.
+		 */
+		if (query->length >= GIT_OID_HEXSZ) {
+			git_object_t actual_type;
+
+			error = odb_otype_fast(&actual_type, db, &query->id);
+			if (!error) {
+				if (query->type != GIT_OBJECT_ANY && query->type != actual_type)
+					error = GIT_ENOTFOUND;
+				else
+					query->type = actual_type;
+			}
+		}
+
+		switch (error) {
+		/* no errors, so we've successfully expanded the OID */
+		case 0:
+			continue;
+
+		/* the object is missing or ambiguous */
+		case GIT_ENOTFOUND:
+		case GIT_EAMBIGUOUS:
+			memset(&query->id, 0, sizeof(git_oid));
+			query->length = 0;
+			query->type = 0;
+			break;
+
+		/* something went very wrong with the ODB; bail hard */
+		default:
+			return error;
+		}
 	}
 
-	return (int)found;
+	git_error_clear();
+	return 0;
 }
 
-int git_odb_read_header(size_t *len_p, git_otype *type_p, git_odb *db, const git_oid *id)
+int git_odb_read_header(size_t *len_p, git_object_t *type_p, git_odb *db, const git_oid *id)
 {
 	int error;
 	git_odb_object *object;
@@ -651,16 +936,63 @@ int git_odb_read_header(size_t *len_p, git_otype *type_p, git_odb *db, const git
 	return error;
 }
 
+static int odb_read_header_1(
+	size_t *len_p, git_object_t *type_p, git_odb *db,
+	const git_oid *id, bool only_refreshed)
+{
+	size_t i;
+	git_object_t ht;
+	bool passthrough = false;
+	int error;
+
+	if (!only_refreshed && (ht = odb_hardcoded_type(id)) != GIT_OBJECT_INVALID) {
+		*type_p = ht;
+		*len_p = 0;
+		return 0;
+	}
+
+	for (i = 0; i < db->backends.length; ++i) {
+		backend_internal *internal = git_vector_get(&db->backends, i);
+		git_odb_backend *b = internal->backend;
+
+		if (only_refreshed && !b->refresh)
+			continue;
+
+		if (!b->read_header) {
+			passthrough = true;
+			continue;
+		}
+
+		error = b->read_header(len_p, type_p, b, id);
+
+		switch (error) {
+		case GIT_PASSTHROUGH:
+			passthrough = true;
+			break;
+		case GIT_ENOTFOUND:
+			break;
+		default:
+			return error;
+		}
+	}
+
+	return passthrough ? GIT_PASSTHROUGH : GIT_ENOTFOUND;
+}
+
 int git_odb__read_header_or_object(
-	git_odb_object **out, size_t *len_p, git_otype *type_p,
+	git_odb_object **out, size_t *len_p, git_object_t *type_p,
 	git_odb *db, const git_oid *id)
 {
-	size_t i;
 	int error = GIT_ENOTFOUND;
 	git_odb_object *object;
 
 	assert(db && id && out && len_p && type_p);
 
+	*out = NULL;
+
+	if (git_oid_iszero(id))
+		return error_null_oid(GIT_ENOTFOUND, "cannot read object");
+
 	if ((object = git_cache_get_raw(odb_cache(db), id)) != NULL) {
 		*len_p = object->cached.size;
 		*type_p = object->cached.type;
@@ -668,147 +1000,256 @@ int git_odb__read_header_or_object(
 		return 0;
 	}
 
-	*out = NULL;
+	error = odb_read_header_1(len_p, type_p, db, id, false);
 
-	for (i = 0; i < db->backends.length && error < 0; ++i) {
-		backend_internal *internal = git_vector_get(&db->backends, i);
-		git_odb_backend *b = internal->backend;
+	if (error == GIT_ENOTFOUND && !git_odb_refresh(db))
+		error = odb_read_header_1(len_p, type_p, db, id, true);
 
-		if (b->read_header != NULL)
-			error = b->read_header(len_p, type_p, b, id);
-	}
+	if (error == GIT_ENOTFOUND)
+		return git_odb__error_notfound("cannot read header for", id, GIT_OID_HEXSZ);
 
-	if (!error || error == GIT_PASSTHROUGH)
+	/* we found the header; return early */
+	if (!error)
 		return 0;
 
-	/*
-	 * no backend could read only the header.
-	 * try reading the whole object and freeing the contents
-	 */
-	if ((error = git_odb_read(&object, db, id)) < 0)
-		return error; /* error already set - pass along */
-
-	*len_p = object->cached.size;
-	*type_p = object->cached.type;
-	*out = object;
+	if (error == GIT_PASSTHROUGH) {
+		/*
+		 * no backend has header-reading functionality
+		 * so try using `git_odb_read` instead
+		 */
+		error = git_odb_read(&object, db, id);
+		if (!error) {
+			*len_p = object->cached.size;
+			*type_p = object->cached.type;
+			*out = object;
+		}
+	}
 
-	return 0;
+	return error;
 }
 
-int git_odb_read(git_odb_object **out, git_odb *db, const git_oid *id)
+static int odb_read_1(git_odb_object **out, git_odb *db, const git_oid *id,
+		bool only_refreshed)
 {
-	size_t i, reads = 0;
-	int error;
-	bool refreshed = false;
+	size_t i;
 	git_rawobj raw;
 	git_odb_object *object;
+	git_oid hashed;
+	bool found = false;
+	int error = 0;
 
-	assert(out && db && id);
-
-	*out = git_cache_get_raw(odb_cache(db), id);
-	if (*out != NULL)
-		return 0;
-
-attempt_lookup:
-	error = GIT_ENOTFOUND;
+	if (!only_refreshed) {
+		if ((error = odb_read_hardcoded(&found, &raw, id)) < 0)
+			return error;
+	}
 
-	for (i = 0; i < db->backends.length && error < 0; ++i) {
+	for (i = 0; i < db->backends.length && !found; ++i) {
 		backend_internal *internal = git_vector_get(&db->backends, i);
 		git_odb_backend *b = internal->backend;
 
+		if (only_refreshed && !b->refresh)
+			continue;
+
 		if (b->read != NULL) {
-			++reads;
 			error = b->read(&raw.data, &raw.len, &raw.type, b, id);
+			if (error == GIT_PASSTHROUGH || error == GIT_ENOTFOUND)
+				continue;
+
+			if (error < 0)
+				return error;
+
+			found = true;
 		}
 	}
 
-	if (error == GIT_ENOTFOUND && !refreshed) {
-		if ((error = git_odb_refresh(db)) < 0)
-			return error;
+	if (!found)
+		return GIT_ENOTFOUND;
 
-		refreshed = true;
-		goto attempt_lookup;
-	}
+	if (git_odb__strict_hash_verification) {
+		if ((error = git_odb_hash(&hashed, raw.data, raw.len, raw.type)) < 0)
+			goto out;
 
-	if (error && error != GIT_PASSTHROUGH) {
-		if (!reads)
-			return git_odb__error_notfound("no match for id", id);
-		return error;
+		if (!git_oid_equal(id, &hashed)) {
+			error = git_odb__error_mismatch(id, &hashed);
+			goto out;
+		}
 	}
 
-	if ((object = odb_object__alloc(id, &raw)) == NULL)
-		return -1;
+	git_error_clear();
+	if ((object = odb_object__alloc(id, &raw)) == NULL) {
+		error = -1;
+		goto out;
+	}
 
 	*out = git_cache_store_raw(odb_cache(db), object);
-	return 0;
+
+out:
+	if (error)
+		git__free(raw.data);
+	return error;
 }
 
-int git_odb_read_prefix(
-	git_odb_object **out, git_odb *db, const git_oid *short_id, size_t len)
+int git_odb_read(git_odb_object **out, git_odb *db, const git_oid *id)
+{
+	int error;
+
+	assert(out && db && id);
+
+	if (git_oid_iszero(id))
+		return error_null_oid(GIT_ENOTFOUND, "cannot read object");
+
+	*out = git_cache_get_raw(odb_cache(db), id);
+	if (*out != NULL)
+		return 0;
+
+	error = odb_read_1(out, db, id, false);
+
+	if (error == GIT_ENOTFOUND && !git_odb_refresh(db))
+		error = odb_read_1(out, db, id, true);
+
+	if (error == GIT_ENOTFOUND)
+		return git_odb__error_notfound("no match for id", id, GIT_OID_HEXSZ);
+
+	return error;
+}
+
+static int odb_otype_fast(git_object_t *type_p, git_odb *db, const git_oid *id)
 {
-	size_t i;
-	int error = GIT_ENOTFOUND;
-	git_oid found_full_oid = {{0}};
-	git_rawobj raw;
-	void *data = NULL;
-	bool found = false, refreshed = false;
 	git_odb_object *object;
+	size_t _unused;
+	int error;
 
-	assert(out && db);
+	if (git_oid_iszero(id))
+		return error_null_oid(GIT_ENOTFOUND, "cannot get object type");
 
-	if (len < GIT_OID_MINPREFIXLEN)
-		return git_odb__error_ambiguous("prefix length too short");
+	if ((object = git_cache_get_raw(odb_cache(db), id)) != NULL) {
+		*type_p = object->cached.type;
+		return 0;
+	}
 
-	if (len > GIT_OID_HEXSZ)
-		len = GIT_OID_HEXSZ;
+	error = odb_read_header_1(&_unused, type_p, db, id, false);
 
-	if (len == GIT_OID_HEXSZ) {
-		*out = git_cache_get_raw(odb_cache(db), short_id);
-		if (*out != NULL)
-			return 0;
+	if (error == GIT_PASSTHROUGH) {
+		error = odb_read_1(&object, db, id, false);
+		if (!error)
+			*type_p = object->cached.type;
+		git_odb_object_free(object);
 	}
 
-attempt_lookup:
+	return error;
+}
+
+static int read_prefix_1(git_odb_object **out, git_odb *db,
+		const git_oid *key, size_t len, bool only_refreshed)
+{
+	size_t i;
+	int error = 0;
+	git_oid found_full_oid = {{0}};
+	git_rawobj raw = {0};
+	void *data = NULL;
+	bool found = false;
+	git_odb_object *object;
+
 	for (i = 0; i < db->backends.length; ++i) {
 		backend_internal *internal = git_vector_get(&db->backends, i);
 		git_odb_backend *b = internal->backend;
 
+		if (only_refreshed && !b->refresh)
+			continue;
+
 		if (b->read_prefix != NULL) {
 			git_oid full_oid;
-			error = b->read_prefix(&full_oid, &raw.data, &raw.len, &raw.type, b, short_id, len);
-			if (error == GIT_ENOTFOUND || error == GIT_PASSTHROUGH)
+			error = b->read_prefix(&full_oid, &raw.data, &raw.len, &raw.type, b, key, len);
+
+			if (error == GIT_ENOTFOUND || error == GIT_PASSTHROUGH) {
+				error = 0;
 				continue;
+			}
 
 			if (error)
-				return error;
+				goto out;
 
 			git__free(data);
 			data = raw.data;
 
-			if (found && git_oid__cmp(&full_oid, &found_full_oid))
-				return git_odb__error_ambiguous("multiple matches for prefix");
+			if (found && git_oid__cmp(&full_oid, &found_full_oid)) {
+				git_buf buf = GIT_BUF_INIT;
+
+				git_buf_printf(&buf, "multiple matches for prefix: %s",
+					git_oid_tostr_s(&full_oid));
+				git_buf_printf(&buf, " %s",
+					git_oid_tostr_s(&found_full_oid));
+
+				error = git_odb__error_ambiguous(buf.ptr);
+				git_buf_dispose(&buf);
+				goto out;
+			}
 
 			found_full_oid = full_oid;
 			found = true;
 		}
 	}
 
-	if (!found && !refreshed) {
-		if ((error = git_odb_refresh(db)) < 0)
-			return error;
+	if (!found)
+		return GIT_ENOTFOUND;
 
-		refreshed = true;
-		goto attempt_lookup;
-	}
+	if (git_odb__strict_hash_verification) {
+		git_oid hash;
 
-	if (!found)
-		return git_odb__error_notfound("no match for prefix", short_id);
+		if ((error = git_odb_hash(&hash, raw.data, raw.len, raw.type)) < 0)
+			goto out;
 
-	if ((object = odb_object__alloc(&found_full_oid, &raw)) == NULL)
-		return -1;
+		if (!git_oid_equal(&found_full_oid, &hash)) {
+			error = git_odb__error_mismatch(&found_full_oid, &hash);
+			goto out;
+		}
+	}
+
+	if ((object = odb_object__alloc(&found_full_oid, &raw)) == NULL) {
+		error = -1;
+		goto out;
+	}
 
 	*out = git_cache_store_raw(odb_cache(db), object);
-	return 0;
+
+out:
+	if (error)
+		git__free(raw.data);
+
+	return error;
+}
+
+int git_odb_read_prefix(
+	git_odb_object **out, git_odb *db, const git_oid *short_id, size_t len)
+{
+	git_oid key = {{0}};
+	int error;
+
+	assert(out && db);
+
+	if (len < GIT_OID_MINPREFIXLEN)
+		return git_odb__error_ambiguous("prefix length too short");
+
+	if (len > GIT_OID_HEXSZ)
+		len = GIT_OID_HEXSZ;
+
+	if (len == GIT_OID_HEXSZ) {
+		*out = git_cache_get_raw(odb_cache(db), short_id);
+		if (*out != NULL)
+			return 0;
+	}
+
+	git_oid__cpy_prefix(&key, short_id, len);
+
+	error = read_prefix_1(out, db, &key, len, false);
+
+	if (error == GIT_ENOTFOUND && !git_odb_refresh(db))
+		error = read_prefix_1(out, db, &key, len, true);
+
+	if (error == GIT_ENOTFOUND)
+		return git_odb__error_notfound("no match for prefix", &key, len);
+
+	return error;
 }
 
 int git_odb_foreach(git_odb *db, git_odb_foreach_cb cb, void *payload)
@@ -819,7 +1260,7 @@ int git_odb_foreach(git_odb *db, git_odb_foreach_cb cb, void *payload)
 	git_vector_foreach(&db->backends, i, internal) {
 		git_odb_backend *b = internal->backend;
 		int error = b->foreach(b, cb, payload);
-		if (error < 0)
+		if (error != 0)
 			return error;
 	}
 
@@ -827,7 +1268,7 @@ int git_odb_foreach(git_odb *db, git_odb_foreach_cb cb, void *payload)
 }
 
 int git_odb_write(
-	git_oid *oid, git_odb *db, const void *data, size_t len, git_otype type)
+	git_oid *oid, git_odb *db, const void *data, size_t len, git_object_t type)
 {
 	size_t i;
 	int error = GIT_ERROR;
@@ -836,7 +1277,11 @@ int git_odb_write(
 	assert(oid && db);
 
 	git_odb_hash(oid, data, len, type);
-	if (git_odb_exists(db, oid))
+
+	if (git_oid_iszero(oid))
+		return error_null_oid(GIT_EINVALID, "cannot write object");
+
+	if (git_odb__freshen(db, oid))
 		return 0;
 
 	for (i = 0; i < db->backends.length && error < 0; ++i) {
@@ -848,7 +1293,7 @@ int git_odb_write(
 			continue;
 
 		if (b->write != NULL)
-			error = b->write(oid, b, data, len, type);
+			error = b->write(b, oid, data, len, type);
 	}
 
 	if (!error || error == GIT_PASSTHROUGH)
@@ -862,17 +1307,31 @@ int git_odb_write(
 		return error;
 
 	stream->write(stream, data, len);
-	error = stream->finalize_write(oid, stream);
-	stream->free(stream);
+	error = stream->finalize_write(stream, oid);
+	git_odb_stream_free(stream);
 
 	return error;
 }
 
+static int hash_header(git_hash_ctx *ctx, git_off_t size, git_object_t type)
+{
+	char header[64];
+	size_t hdrlen;
+	int error;
+
+	 if ((error = git_odb__format_object_header(&hdrlen,
+		header, sizeof(header), size, type)) < 0)
+		return error;
+
+	return git_hash_update(ctx, header, hdrlen);
+}
+
 int git_odb_open_wstream(
-	git_odb_stream **stream, git_odb *db, size_t size, git_otype type)
+	git_odb_stream **stream, git_odb *db, git_off_t size, git_object_t type)
 {
 	size_t i, writes = 0;
 	int error = GIT_ERROR;
+	git_hash_ctx *ctx = NULL;
 
 	assert(stream && db);
 
@@ -893,15 +1352,93 @@ int git_odb_open_wstream(
 		}
 	}
 
-	if (error == GIT_PASSTHROUGH)
-		error = 0;
-	if (error < 0 && !writes)
-		error = git_odb__error_unsupported_in_backend("write object");
+	if (error < 0) {
+		if (error == GIT_PASSTHROUGH)
+			error = 0;
+		else if (!writes)
+			error = git_odb__error_unsupported_in_backend("write object");
+
+		goto done;
+	}
+
+	ctx = git__malloc(sizeof(git_hash_ctx));
+	GIT_ERROR_CHECK_ALLOC(ctx);
 
+	if ((error = git_hash_ctx_init(ctx)) < 0 ||
+		(error = hash_header(ctx, size, type)) < 0)
+		goto done;
+
+	(*stream)->hash_ctx = ctx;
+	(*stream)->declared_size = size;
+	(*stream)->received_bytes = 0;
+
+done:
+	if (error)
+		git__free(ctx);
 	return error;
 }
 
-int git_odb_open_rstream(git_odb_stream **stream, git_odb *db, const git_oid *oid)
+static int git_odb_stream__invalid_length(
+	const git_odb_stream *stream,
+	const char *action)
+{
+	git_error_set(GIT_ERROR_ODB,
+		"cannot %s - "
+		"Invalid length. %"PRId64" was expected. The "
+		"total size of the received chunks amounts to %"PRId64".",
+		action, stream->declared_size, stream->received_bytes);
+
+	return -1;
+}
+
+int git_odb_stream_write(git_odb_stream *stream, const char *buffer, size_t len)
+{
+	git_hash_update(stream->hash_ctx, buffer, len);
+
+	stream->received_bytes += len;
+
+	if (stream->received_bytes > stream->declared_size)
+		return git_odb_stream__invalid_length(stream,
+			"stream_write()");
+
+	return stream->write(stream, buffer, len);
+}
+
+int git_odb_stream_finalize_write(git_oid *out, git_odb_stream *stream)
+{
+	if (stream->received_bytes != stream->declared_size)
+		return git_odb_stream__invalid_length(stream,
+			"stream_finalize_write()");
+
+	git_hash_final(out, stream->hash_ctx);
+
+	if (git_odb__freshen(stream->backend->odb, out))
+		return 0;
+
+	return stream->finalize_write(stream, out);
+}
+
+int git_odb_stream_read(git_odb_stream *stream, char *buffer, size_t len)
+{
+	return stream->read(stream, buffer, len);
+}
+
+void git_odb_stream_free(git_odb_stream *stream)
+{
+	if (stream == NULL)
+		return;
+
+	git_hash_ctx_cleanup(stream->hash_ctx);
+	git__free(stream->hash_ctx);
+	stream->free(stream);
+}
+
+int git_odb_open_rstream(
+	git_odb_stream **stream,
+	size_t *len,
+	git_object_t *type,
+	git_odb *db,
+	const git_oid *oid)
 {
 	size_t i, reads = 0;
 	int error = GIT_ERROR;
@@ -914,7 +1451,7 @@ int git_odb_open_rstream(git_odb_stream **stream, git_odb *db, const git_oid *oi
 
 		if (b->readstream != NULL) {
 			++reads;
-			error = b->readstream(stream, b, oid);
+			error = b->readstream(stream, len, type, b, oid);
 		}
 	}
 
@@ -926,7 +1463,7 @@ int git_odb_open_rstream(git_odb_stream **stream, git_odb *db, const git_oid *oi
 	return error;
 }
 
-int git_odb_write_pack(struct git_odb_writepack **out, git_odb *db, git_transfer_progress_callback progress_cb, void *progress_payload)
+int git_odb_write_pack(struct git_odb_writepack **out, git_odb *db, git_transfer_progress_cb progress_cb, void *progress_payload)
 {
 	size_t i, writes = 0;
 	int error = GIT_ERROR;
@@ -943,7 +1480,7 @@ int git_odb_write_pack(struct git_odb_writepack **out, git_odb *db, git_transfer
 
 		if (b->writepack != NULL) {
 			++writes;
-			error = b->writepack(out, b, progress_cb, progress_payload);
+			error = b->writepack(out, b, db, progress_cb, progress_payload);
 		}
 	}
 
@@ -980,21 +1517,48 @@ int git_odb_refresh(struct git_odb *db)
 	return 0;
 }
 
-int git_odb__error_notfound(const char *message, const git_oid *oid)
+int git_odb__error_mismatch(const git_oid *expected, const git_oid *actual)
+{
+	char expected_oid[GIT_OID_HEXSZ + 1], actual_oid[GIT_OID_HEXSZ + 1];
+
+	git_oid_tostr(expected_oid, sizeof(expected_oid), expected);
+	git_oid_tostr(actual_oid, sizeof(actual_oid), actual);
+
+	git_error_set(GIT_ERROR_ODB, "object hash mismatch - expected %s but got %s",
+		expected_oid, actual_oid);
+
+	return GIT_EMISMATCH;
+}
+
+int git_odb__error_notfound(
+	const char *message, const git_oid *oid, size_t oid_len)
 {
 	if (oid != NULL) {
 		char oid_str[GIT_OID_HEXSZ + 1];
-		git_oid_tostr(oid_str, sizeof(oid_str), oid);
-		giterr_set(GITERR_ODB, "Object not found - %s (%s)", message, oid_str);
+		git_oid_tostr(oid_str, oid_len+1, oid);
+		git_error_set(GIT_ERROR_ODB, "object not found - %s (%.*s)",
+			message, (int) oid_len, oid_str);
 	} else
-		giterr_set(GITERR_ODB, "Object not found - %s", message);
+		git_error_set(GIT_ERROR_ODB, "object not found - %s", message);
 
 	return GIT_ENOTFOUND;
 }
 
+static int error_null_oid(int error, const char *message)
+{
+	git_error_set(GIT_ERROR_ODB, "odb: %s: null OID cannot exist", message);
+	return error;
+}
+
 int git_odb__error_ambiguous(const char *message)
 {
-	giterr_set(GITERR_ODB, "Ambiguous SHA1 prefix - %s", message);
+	git_error_set(GIT_ERROR_ODB, "ambiguous SHA1 prefix - %s", message);
 	return GIT_EAMBIGUOUS;
 }
 
+int git_odb_init_backend(git_odb_backend *backend, unsigned int version)
+{
+	GIT_INIT_STRUCTURE_FROM_TEMPLATE(
+		backend, version, git_odb_backend, GIT_ODB_BACKEND_INIT);
+	return 0;
+}