ruby-libstorj 0.0.0

data/ext/libstorj/src/uploader.h

@@ -0,0 +1,181 @@
+/**
+ * @file uploader.h
+ * @brief Storj upload methods and definitions.
+ *
+ * Structures and functions useful for uploading files.
+ */
+#ifndef STORJ_UPLOADER_H
+#define STORJ_UPLOADER_H
+#include "storj.h"
+#include "http.h"
+#include "utils.h"
+#include "crypto.h"
+#include "rs.h"
+
+#define STORJ_NULL -1
+#define STORJ_MAX_REPORT_TRIES 2
+#define STORJ_MAX_PUSH_FRAME_COUNT 6
+
+typedef enum {
+    CANCELED = 0,
+    AWAITING_PREPARE_FRAME = 1,
+    PREPARING_FRAME = 2,
+    AWAITING_PUSH_FRAME = 3,
+    PUSHING_FRAME = 4,
+    AWAITING_PUSH_SHARD = 5,
+    PUSHING_SHARD = 6,
+    COMPLETED_PUSH_SHARD = 7
+} storj_state_progress_t;
+
+typedef enum {
+    PREPARE_FRAME_LIMIT = 1,
+    PUSH_FRAME_LIMIT = 32,
+    PUSH_SHARD_LIMIT = 32
+} storj_state_progress_limits_t;
+
+typedef struct {
+    /* state should not be modified in worker threads */
+    storj_upload_state_t *upload_state;
+    int status_code;
+    int error_status;
+    shard_meta_t *shard_meta;
+    // Position in shard meta array
+    int shard_meta_index;
+    // Either parity file pointer or original file
+    FILE *shard_file;
+    storj_log_levels_t *log;
+} frame_builder_t;
+
+typedef struct {
+    int error_status;
+    /* state should not be modified in worker threads */
+    storj_upload_state_t *upload_state;
+} parity_shard_req_t;
+
+typedef struct {
+    int error_status;
+    /* state should not be modified in worker threads */
+    storj_upload_state_t *upload_state;
+} encrypt_file_req_t;
+
+typedef struct {
+    storj_http_options_t *http_options;
+    storj_bridge_options_t *options;
+    int status_code;
+    int error_status;
+    storj_log_levels_t *log;
+    int shard_index;
+    int shard_meta_index;
+    FILE *shard_file;
+    uv_async_t progress_handle;
+    uint64_t start;
+    uint64_t end;
+
+    /* state should not be modified in worker threads */
+    storj_upload_state_t *upload_state;
+    bool *canceled;
+} push_shard_request_t;
+
+typedef struct {
+    storj_http_options_t *http_options;
+    storj_bridge_options_t *options;
+    char *token;
+    const char *bucket_id;
+    char *bucket_op;
+    /* state should not be modified in worker threads */
+    storj_upload_state_t *upload_state;
+    int status_code;
+    int error_status;
+    storj_log_levels_t *log;
+} request_token_t;
+
+typedef struct {
+    storj_http_options_t *http_options;
+    storj_bridge_options_t *options;
+    /* state should not be modified in worker threads */
+    storj_upload_state_t *upload_state;
+    char *frame_id;
+    int status_code;
+    int error_status;
+
+    // Add shard to frame
+    int shard_meta_index;
+    farmer_pointer_t *farmer_pointer;
+
+    storj_log_levels_t *log;
+} frame_request_t;
+
+typedef struct {
+  storj_http_options_t *http_options;
+  storj_bridge_options_t *options;
+  /* state should not be modified in worker threads */
+  storj_upload_state_t *upload_state;
+  int status_code;
+  int error_status;
+  struct json_object *response;
+  storj_log_levels_t *log;
+} post_to_bucket_request_t;
+
+typedef struct {
+    uint32_t pointer_index;
+    storj_http_options_t *http_options;
+    storj_bridge_options_t *options;
+    int status_code;
+    storj_exchange_report_t *report;
+    /* state should not be modified in worker threads */
+    storj_upload_state_t *state;
+} shard_send_report_t;
+
+static farmer_pointer_t *farmer_pointer_new();
+static shard_meta_t *shard_meta_new();
+static uv_work_t *shard_meta_work_new(int index, storj_upload_state_t *state);
+static uv_work_t *frame_work_new(int *index, storj_upload_state_t *state);
+static uv_work_t *uv_work_new();
+static int prepare_encryption_key(storj_upload_state_t *state,
+                               char *pre_pass,
+                               int pre_pass_size,
+                               char *pre_salt,
+                               int pre_salt_size);
+
+static int check_in_progress(storj_upload_state_t *state, int status);
+char *create_tmp_name(storj_upload_state_t *state, char *extension);
+
+static void shard_meta_cleanup(shard_meta_t *shard_meta);
+static void pointer_cleanup(farmer_pointer_t *farmer_pointer);
+static void cleanup_state(storj_upload_state_t *state);
+static void free_encryption_ctx(storj_encryption_ctx_t *ctx);
+
+static void queue_next_work(storj_upload_state_t *state);
+
+static void queue_request_frame_id(storj_upload_state_t *state);
+static void queue_prepare_frame(storj_upload_state_t *state, int index);
+static void queue_push_frame(storj_upload_state_t *state, int index);
+static void queue_push_shard(storj_upload_state_t *state, int index);
+static void queue_create_bucket_entry(storj_upload_state_t *state);
+static void queue_send_exchange_report(storj_upload_state_t *state, int index);
+static void queue_create_encrypted_file(storj_upload_state_t *state);
+
+static void request_token(uv_work_t *work);
+static void request_frame_id(uv_work_t *work);
+static void prepare_frame(uv_work_t *work);
+static void push_frame(uv_work_t *work);
+static void push_shard(uv_work_t *work);
+static void create_bucket_entry(uv_work_t *work);
+static void send_exchange_report(uv_work_t *work);
+static void create_encrypted_file(uv_work_t *work);
+
+static void after_request_token(uv_work_t *work, int status);
+static void after_request_frame_id(uv_work_t *work, int status);
+static void after_prepare_frame(uv_work_t *work, int status);
+static void after_push_frame(uv_work_t *work, int status);
+static void after_push_shard(uv_work_t *work, int status);
+static void after_create_bucket_entry(uv_work_t *work, int status);
+static void after_send_exchange_report(uv_work_t *work, int status);
+static void after_create_encrypted_file(uv_work_t *work, int status);
+
+static void queue_verify_bucket_id(storj_upload_state_t *state);
+static void queue_verify_file_id(storj_upload_state_t *state);
+static void verify_bucket_id_callback(uv_work_t *work_req, int status);
+static void verify_file_id_callback(uv_work_t *work_req, int status);
+
+#endif /* STORJ_UPLOADER_H */

data/ext/libstorj/src/utils.c

@@ -0,0 +1,336 @@
+#include "utils.h"
+
+char *hex2str(size_t length, uint8_t *data)
+{
+    size_t encode_len = BASE16_ENCODE_LENGTH(length);
+    uint8_t *result = calloc(encode_len + 1, sizeof(uint8_t));
+    if (!result) {
+        return NULL;
+    }
+
+    base16_encode_update(result, length, data);
+
+    return (char *)result;
+}
+
+void print_int_array(uint8_t *array, unsigned length)
+{
+    printf("{");
+    for (int i = 0; i < length; i++) {
+        printf("%i", array[i]);
+        if (i != length - 1) {
+            printf(",");
+        }
+    }
+    printf("}\n");
+}
+
+uint8_t *str2hex(size_t length, char *data)
+{
+    char *result = calloc(BASE16_DECODE_LENGTH(length) + 1, sizeof(char));
+    if (!result) {
+        return NULL;
+    }
+
+    struct base16_decode_ctx *ctx = malloc(sizeof(struct base16_decode_ctx));
+    base16_decode_init(ctx);
+
+    size_t decode_len = 0;
+    if (!base16_decode_update(ctx, &decode_len, (uint8_t *)result,
+                              length, (uint8_t *)data)) {
+        free(result);
+        free(ctx);
+        return NULL;
+    }
+
+    if (!base16_decode_final(ctx)) {
+        free(result);
+        free(ctx);
+        return NULL;
+    }
+
+    free(ctx);
+    return (uint8_t *)result;
+}
+
+char *str_concat_many(int count, ...)
+{
+    int length = 1;
+
+    va_list args;
+    va_start(args, count);
+    for (int i = 0; i < count; i++) {
+        char *item = va_arg(args, char *);
+        length += strlen(item);
+    }
+    va_end(args);
+
+    char *combined = calloc(length, sizeof(char));
+    if (!combined) {
+        return NULL;
+    }
+
+    va_start(args, count);
+    for (int i = 0; i < count; i++) {
+        char *item = va_arg(args, char *);
+        strcat(combined, item);
+    }
+    va_end(args);
+
+    return combined;
+}
+
+void random_buffer(uint8_t *buf, size_t len)
+{
+    static FILE *frand = NULL;
+#ifdef _WIN32
+    HCRYPTPROV hProvider;
+    int ret = CryptAcquireContext(&hProvider, NULL, NULL, PROV_RSA_FULL, CRYPT_VERIFYCONTEXT);
+    assert(ret);
+    ret = CryptGenRandom(hProvider, len, buf);
+    assert(ret);
+    CryptReleaseContext(hProvider, 0);
+#else
+    if (!frand) {
+        frand = fopen("/dev/urandom", "r");
+    }
+    size_t len_read = fread(buf, 1, len, frand);
+    (void)len_read;
+    assert(len_read == len);
+#endif
+}
+
+uint64_t shard_size(int hops)
+{
+    return MIN_SHARD_SIZE * pow(2, hops);
+};
+
+uint64_t get_time_milliseconds() {
+#ifdef _WIN32
+
+    // Time between windows epoch and standard epoch
+    const int64_t time_to_epoch = 116444736000000000LL;
+
+    FILETIME ft;
+
+    GetSystemTimeAsFileTime(&ft);
+
+    LARGE_INTEGER li;
+    li.LowPart = ft.dwLowDateTime;
+    li.HighPart = ft.dwHighDateTime;
+    li.QuadPart -= time_to_epoch;
+    li.QuadPart /= 10000;
+
+    uint64_t milliseconds = li.QuadPart;
+#else
+    struct timeval t;
+    gettimeofday(&t, NULL);
+    uint64_t milliseconds = t.tv_sec * 1000LL + t.tv_usec / 1000;
+#endif
+
+    return milliseconds;
+}
+
+void memset_zero(void *v, size_t n)
+{
+#ifdef _WIN32
+    SecureZeroMemory(v, n);
+#else
+    volatile unsigned char *p = v;
+    while (n--) {
+        *p++ = 0;
+    }
+#endif
+}
+
+uint64_t determine_shard_size(uint64_t file_size, int accumulator)
+{
+    if (file_size <= 0) {
+        return 0;
+    }
+
+    accumulator = accumulator ? accumulator : 0;
+
+    // Determine hops back by accumulator
+    int hops = ((accumulator - SHARD_MULTIPLES_BACK) < 0 ) ?
+        0 : accumulator - SHARD_MULTIPLES_BACK;
+
+    uint64_t byte_multiple = shard_size(accumulator);
+    double check = (double) file_size / byte_multiple;
+
+    // Determine if bytemultiple is highest bytemultiple that is still <= size
+    if (check > 0 && check <= 1) {
+        while (hops > 0 && shard_size(hops) > MAX_SHARD_SIZE) {
+            hops = hops - 1 <= 0 ? 0 : hops - 1;
+        }
+
+        return shard_size(hops);
+    }
+
+    // Maximum of 2 ^ 41 * 8 * 1024 * 1024
+    if (accumulator > 41) {
+        return 0;
+    }
+
+    return determine_shard_size(file_size, ++accumulator);
+}
+
+#ifdef _WIN32
+ssize_t pread(int fd, void *buf, size_t count, uint64_t offset)
+{
+    long unsigned int read_bytes = 0;
+
+    OVERLAPPED overlapped;
+    memset(&overlapped, 0, sizeof(OVERLAPPED));
+
+    overlapped.OffsetHigh = (uint32_t)((offset & 0xFFFFFFFF00000000LL) >> 32);
+    overlapped.Offset = (uint32_t)(offset & 0xFFFFFFFFLL);
+
+    HANDLE file = (HANDLE)_get_osfhandle(fd);
+    SetLastError(0);
+    bool RF = ReadFile(file, buf, count, &read_bytes, &overlapped);
+
+     // For some reason it errors when it hits end of file so we don't want to check that
+    if ((RF == 0) && GetLastError() != ERROR_HANDLE_EOF) {
+        errno = GetLastError();
+        // printf ("Error reading file : %d\n", GetLastError());
+        return -1;
+    }
+
+    return read_bytes;
+}
+
+ssize_t pwrite(int fd, const void *buf, size_t count, uint64_t offset)
+{
+    long unsigned int written_bytes = 0;
+
+    OVERLAPPED overlapped;
+    memset(&overlapped, 0, sizeof(OVERLAPPED));
+
+    overlapped.OffsetHigh = (uint32_t)((offset & 0xFFFFFFFF00000000LL) >> 32);
+    overlapped.Offset = (uint32_t)(offset & 0xFFFFFFFFLL);
+
+    HANDLE file = (HANDLE)_get_osfhandle(fd);
+    SetLastError(0);
+    bool RF = WriteFile(file, buf, count, &written_bytes, &overlapped);
+    if ((RF == 0)) {
+        errno = GetLastError();
+        // printf ("Error reading file :%d\n", GetLastError());
+        return -1;
+    }
+
+    return written_bytes;
+}
+#endif
+
+int allocatefile(int fd, uint64_t length)
+{
+#ifdef _WIN32
+    HANDLE file = (HANDLE)_get_osfhandle(fd);
+    if (file == INVALID_HANDLE_VALUE) {
+        return EBADF;
+    }
+
+    int status = 0;
+
+    LARGE_INTEGER size;
+    size.HighPart = (uint32_t)((length & 0xFFFFFFFF00000000LL) >> 32);
+    size.LowPart = (uint32_t)(length & 0xFFFFFFFFLL);
+
+    if (!SetFilePointerEx(file, size, 0, FILE_BEGIN)) {
+        status = GetLastError();
+        goto win_finished;
+    }
+
+    if (!SetEndOfFile(file)) {
+        status = GetLastError();
+        goto win_finished;
+    }
+
+win_finished:
+
+    return status;
+#elif HAVE_POSIX_FALLOCATE
+    return posix_fallocate(fd, 0, length);
+#elif __unix__
+    if (fallocate(fd, 0, 0, length)) {
+        return errno;
+    }
+    return 0;
+#elif __linux__
+    if (fallocate(fd, 0, 0, length)) {
+        return errno;
+    }
+    return 0;
+#elif __APPLE__
+    fstore_t store = {F_ALLOCATECONTIG, F_PEOFPOSMODE, 0, length, 0};
+    // Try to get a continous chunk of disk space
+    int ret = fcntl(fd, F_PREALLOCATE, &store);
+    if (-1 == ret) {
+        // OK, perhaps we are too fragmented, allocate non-continuous
+        store.fst_flags = F_ALLOCATEALL;
+        ret = fcntl(fd, F_PREALLOCATE, &store);
+        if ( -1 == ret) {
+            return -1;
+        }
+    }
+    return ftruncate(fd, length);
+#else
+    return -1;
+#endif
+}
+
+
+int unmap_file(uint8_t *map, uint64_t filesize)
+{
+#ifdef _WIN32
+    if (!FlushViewOfFile(map, filesize)) {
+        return GetLastError();
+    }
+    if (!UnmapViewOfFile(map)) {
+        return GetLastError();
+    }
+#else
+    if (munmap(map, filesize)) {
+        return errno;
+    }
+#endif
+    return 0;
+}
+
+int map_file(int fd, uint64_t filesize, uint8_t **map, bool read_only)
+{
+    int status = 0;
+#ifdef _WIN32
+    HANDLE fh = (HANDLE)_get_osfhandle(fd);
+    if (fh == INVALID_HANDLE_VALUE) {
+        return EBADF;
+    }
+
+    int prot = read_only ? PAGE_READONLY : PAGE_READWRITE;
+
+    HANDLE mh = CreateFileMapping(fh, NULL, prot, 0, 0, NULL);
+    if (!mh) {
+        status = GetLastError();
+        goto win_finished;
+    }
+
+    prot = read_only ? FILE_MAP_READ : FILE_MAP_WRITE;
+
+    *map = MapViewOfFileEx(mh, prot, 0, 0, filesize, NULL);
+    if (!*map) {
+        status = GetLastError();
+        goto win_finished;
+    }
+
+win_finished:
+    CloseHandle(mh);
+#else
+    int prot = read_only ? PROT_READ : PROT_READ | PROT_WRITE;
+    *map = (uint8_t *)mmap(NULL, filesize, prot, MAP_SHARED, fd, 0);
+    if (*map == MAP_FAILED) {
+        status = errno;
+    }
+#endif
+    return status;
+}