tigerbeetle 0.0.40 → 0.17.9

data/{ext/tb_client/tigerbeetle/src/clients/rust/assets → src/ext/tigerbeetle}/tb_client.h

@@ -69,8 +69,8 @@ typedef struct tb_transfer_t {
     uint64_t timestamp;
 } tb_transfer_t;
 
-typedef enum TB_CREATE_ACCOUNT_RESULT {
-    TB_CREATE_ACCOUNT_OK = 0,
+typedef enum TB_CREATE_ACCOUNT_STATUS {
+    TB_CREATE_ACCOUNT_CREATED = 0xFFFFFFFF,
     TB_CREATE_ACCOUNT_LINKED_EVENT_FAILED = 1,
     TB_CREATE_ACCOUNT_LINKED_EVENT_CHAIN_OPEN = 2,
     TB_CREATE_ACCOUNT_IMPORTED_EVENT_EXPECTED = 22,
@@ -97,10 +97,10 @@ typedef enum TB_CREATE_ACCOUNT_RESULT {
     TB_CREATE_ACCOUNT_LEDGER_MUST_NOT_BE_ZERO = 13,
     TB_CREATE_ACCOUNT_CODE_MUST_NOT_BE_ZERO = 14,
     TB_CREATE_ACCOUNT_IMPORTED_EVENT_TIMESTAMP_MUST_NOT_REGRESS = 26,
-} TB_CREATE_ACCOUNT_RESULT;
+} TB_CREATE_ACCOUNT_STATUS;
 
-typedef enum TB_CREATE_TRANSFER_RESULT {
-    TB_CREATE_TRANSFER_OK = 0,
+typedef enum TB_CREATE_TRANSFER_STATUS {
+    TB_CREATE_TRANSFER_CREATED = 0xFFFFFFFF,
     TB_CREATE_TRANSFER_LINKED_EVENT_FAILED = 1,
     TB_CREATE_TRANSFER_LINKED_EVENT_CHAIN_OPEN = 2,
     TB_CREATE_TRANSFER_IMPORTED_EVENT_EXPECTED = 56,
@@ -168,17 +168,19 @@ typedef enum TB_CREATE_TRANSFER_RESULT {
     TB_CREATE_TRANSFER_OVERFLOWS_TIMEOUT = 53,
     TB_CREATE_TRANSFER_EXCEEDS_CREDITS = 54,
     TB_CREATE_TRANSFER_EXCEEDS_DEBITS = 55,
-} TB_CREATE_TRANSFER_RESULT;
+} TB_CREATE_TRANSFER_STATUS;
 
-typedef struct tb_create_accounts_result_t {
-    uint32_t index;
-    uint32_t result;
-} tb_create_accounts_result_t;
+typedef struct tb_create_account_result_t {
+    uint64_t timestamp;
+    uint32_t status;
+    uint32_t reserved;
+} tb_create_account_result_t;
 
-typedef struct tb_create_transfers_result_t {
-    uint32_t index;
-    uint32_t result;
-} tb_create_transfers_result_t;
+typedef struct tb_create_transfer_result_t {
+    uint64_t timestamp;
+    uint32_t status;
+    uint32_t reserved;
+} tb_create_transfer_result_t;
 
 typedef struct tb_account_filter_t {
     tb_uint128_t account_id;
@@ -248,14 +250,14 @@ typedef struct tb_packet_t {
 typedef enum TB_OPERATION {
     TB_OPERATION_PULSE = 128,
     TB_OPERATION_GET_CHANGE_EVENTS = 137,
-    TB_OPERATION_CREATE_ACCOUNTS = 138,
-    TB_OPERATION_CREATE_TRANSFERS = 139,
     TB_OPERATION_LOOKUP_ACCOUNTS = 140,
     TB_OPERATION_LOOKUP_TRANSFERS = 141,
     TB_OPERATION_GET_ACCOUNT_TRANSFERS = 142,
     TB_OPERATION_GET_ACCOUNT_BALANCES = 143,
     TB_OPERATION_QUERY_ACCOUNTS = 144,
     TB_OPERATION_QUERY_TRANSFERS = 145,
+    TB_OPERATION_CREATE_ACCOUNTS = 146,
+    TB_OPERATION_CREATE_TRANSFERS = 147,
 } TB_OPERATION;
 
 typedef enum TB_PACKET_STATUS {

data/src/ext/tigerbeetle/tigerbeetle.c

@@ -0,0 +1,310 @@
+// Ruby doesn't guarantee ABI stability, so we must use ruby.h present on the user's machine.
+// We don't want to run a Zig compiler there though, so we have to implement this thin wrapper in C.
+#include "rb_tb_gen.h"
+#include "ruby.h"
+#include "ruby/thread.h"
+#include "tb_client.h"
+#include <stdatomic.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+typedef enum {
+    REQ_PENDING = 0,
+    REQ_DONE = 1,
+    REQ_ABANDONED = 2,
+} rb_tb_request_state_t;
+
+typedef struct rb_tb_request {
+    // Set at init, read by callback.
+    TB_OPERATION operation;
+    uint8_t *send_buf;
+    tb_packet_t packet;
+    // Shared between Ruby thread and callback.
+    _Atomic rb_tb_request_state_t state;
+    // Written by callback, read by Ruby thread.
+    uint8_t status;
+    uint8_t *result;
+    uint32_t result_size;
+} rb_tb_request_t;
+
+static VALUE rb_mTigerBeetle;
+static VALUE rb_cRequest;
+static VALUE rb_eInitError;
+static VALUE rb_eClientClosedError;
+
+static void rb_tb_init_native_client(VALUE mTigerBeetle);
+
+void Init_tigerbeetle(void) {
+    rb_mTigerBeetle = rb_define_module("TigerBeetle");
+
+    rb_eInitError = rb_define_class_under(rb_mTigerBeetle, "InitError", rb_eStandardError);
+    rb_eClientClosedError =
+        rb_define_class_under(rb_mTigerBeetle, "ClientClosedError", rb_eStandardError);
+    rb_define_class_under(rb_mTigerBeetle, "PacketError", rb_eStandardError);
+
+    rb_tb_init_native_client(rb_mTigerBeetle);
+}
+
+static void rb_tb_request_free(void *ptr) {
+    rb_tb_request_t *req = (rb_tb_request_t *)ptr;
+    rb_tb_request_state_t expected = REQ_PENDING;
+    if (atomic_compare_exchange_strong(&req->state, &expected, REQ_ABANDONED)) {
+        // Callback hasn't fired yet. It will free req once it does.
+        return;
+    }
+    // Callback already fired (state == REQ_DONE), we own cleanup.
+    tb_assert(expected == REQ_DONE);
+    free(req->result);
+    free(req);
+}
+
+static size_t rb_tb_request_size(const void *ptr) {
+    (void)ptr;
+    return sizeof(rb_tb_request_t);
+}
+
+static const rb_data_type_t rb_tb_request_type = {
+    .wrap_struct_name = "TigerBeetle::Request",
+    .function =
+        {
+            .dmark = NULL,
+            .dfree = rb_tb_request_free,
+            .dsize = rb_tb_request_size,
+        },
+    .flags = RUBY_TYPED_FREE_IMMEDIATELY,
+};
+
+// Request is only ever created from C via TypedData_Wrap_Struct.
+static VALUE rb_tb_request_alloc(VALUE klass) {
+    (void)klass;
+    rb_raise(rb_eTypeError, "TigerBeetle::Request cannot be instantiated directly");
+    return Qnil; // unreachable
+}
+
+static VALUE rb_tb_request_result(VALUE self) {
+    rb_tb_request_t *req;
+    TypedData_Get_Struct(self, rb_tb_request_t, &rb_tb_request_type, req);
+
+    VALUE result = Qnil;
+    if (req->status == TB_PACKET_OK) {
+        if (req->result_size == 0) {
+            result = rb_ary_new();
+        } else {
+            if (!req->result) {
+                rb_raise(rb_eNoMemError, "failed to allocate result buffer");
+            }
+            result = rb_tb_deserialize(req->operation, req->result, req->result_size);
+        }
+    }
+
+    return rb_ary_new_from_args(2, RB_UINT2NUM(req->status), result);
+}
+
+static void rb_tb_write_completion(int fd, rb_tb_request_t *req) {
+    uint64_t request_id = (uint64_t)(uintptr_t)req;
+    ssize_t written = write(fd, &request_id, sizeof(request_id));
+    tb_assert(written == sizeof(request_id));
+}
+
+static void rb_tb_on_completion(
+    uintptr_t completion_ctx,
+    tb_packet_t *packet,
+    uint64_t timestamp,
+    const uint8_t *result,
+    uint32_t result_size
+) {
+    (void)timestamp;
+    rb_tb_request_t *req = (rb_tb_request_t *)packet->user_data;
+
+    free(req->send_buf);
+    req->send_buf = NULL;
+    req->status = packet->status;
+
+    if (packet->status == TB_PACKET_OK) {
+        req->result_size = result_size;
+        tb_assert(result != NULL);
+
+        if (result_size > 0) {
+            req->result = malloc(result_size);
+            if (req->result) {
+                memcpy(req->result, result, result_size);
+            }
+        }
+    }
+
+    rb_tb_request_state_t expected = REQ_PENDING;
+    if (!atomic_compare_exchange_strong(&req->state, &expected, REQ_DONE)) {
+        // dfree already ran (state == REQ_ABANDONED). We own cleanup.
+        tb_assert(expected == REQ_ABANDONED);
+        free(req->result);
+        free(req);
+        return;
+    }
+
+    rb_tb_write_completion((int)completion_ctx, req);
+}
+
+static void rb_tb_client_free(void *ptr) {
+    tb_client_deinit((tb_client_t *)ptr);
+    ruby_xfree(ptr);
+}
+
+static size_t rb_tb_client_size(const void *ptr) {
+    (void)ptr;
+    return sizeof(tb_client_t);
+}
+
+static const rb_data_type_t rb_tb_client_type = {
+    .wrap_struct_name = "TigerBeetle::NativeClient",
+    .function =
+        {
+            .dmark = NULL,
+            .dfree = rb_tb_client_free,
+            .dsize = rb_tb_client_size,
+        },
+    .flags = RUBY_TYPED_FREE_IMMEDIATELY,
+};
+
+static VALUE rb_tb_client_alloc(VALUE klass) {
+    tb_client_t *data = ruby_xmalloc(sizeof(tb_client_t));
+    memset(data, 0, sizeof(tb_client_t));
+    return TypedData_Wrap_Struct(klass, &rb_tb_client_type, data);
+}
+
+static VALUE rb_tb_client_initialize(
+    VALUE self, VALUE cluster_id_rb, VALUE addresses_rb, VALUE completion_fd_rb
+) {
+    tb_client_t *client;
+    TypedData_Get_Struct(self, tb_client_t, &rb_tb_client_type, client);
+
+    uint8_t cluster_id_bytes[16] = {0};
+    rb_tb_pack_u128(cluster_id_rb, cluster_id_bytes);
+
+    const char *addr = StringValueCStr(addresses_rb);
+    uint32_t addr_len = (uint32_t)RSTRING_LEN(addresses_rb);
+
+    int completion_fd = RB_NUM2INT(completion_fd_rb);
+
+    TB_INIT_STATUS status = tb_client_init(
+        client, cluster_id_bytes, addr, addr_len, (uintptr_t)completion_fd, rb_tb_on_completion
+    );
+
+    if (status != TB_INIT_SUCCESS) {
+        rb_raise(rb_eInitError, "Init error: %s", rb_tb_init_error_message(status));
+    }
+
+    return self;
+}
+
+static void *rb_tb_deinit_without_gvl(void *arg) {
+    tb_client_deinit((tb_client_t *)arg);
+    return NULL;
+}
+
+static VALUE rb_tb_client_close(VALUE self) {
+    tb_client_t *client;
+    TypedData_Get_Struct(self, tb_client_t, &rb_tb_client_type, client);
+
+    // deinit blocks until all in-flight callbacks complete. The GVL is released
+    // while waiting so other Ruby threads can run.
+    rb_thread_call_without_gvl(rb_tb_deinit_without_gvl, client, NULL, NULL);
+    return Qnil;
+}
+
+typedef struct rb_tb_serialize_context {
+    TB_OPERATION operation;
+    VALUE items_rb;
+    uint8_t *buf;
+    long count;
+} rb_tb_serialize_context_t;
+
+static VALUE rb_tb_serialize_protected(VALUE context_value) {
+    rb_tb_serialize_context_t *context = (rb_tb_serialize_context_t *)context_value;
+    rb_tb_serialize(context->operation, context->items_rb, context->buf, context->count);
+    return Qnil;
+}
+
+static VALUE rb_tb_client_submit(VALUE self, VALUE operation_rb, VALUE items_rb) {
+    tb_client_t *client;
+    TypedData_Get_Struct(self, tb_client_t, &rb_tb_client_type, client);
+
+    TB_OPERATION operation = (TB_OPERATION)RB_NUM2INT(operation_rb);
+    long count = RARRAY_LEN(items_rb);
+
+    rb_tb_request_t *req = malloc(sizeof(rb_tb_request_t));
+    if (!req) {
+        rb_raise(rb_eNoMemError, "failed to allocate request");
+    }
+    memset(req, 0, sizeof(rb_tb_request_t));
+    req->operation = operation;
+
+    if (count > 0) {
+        size_t event_size = rb_tb_event_size(operation);
+        long max_count = (long)(UINT32_MAX / event_size);
+        if (count > max_count) {
+            free(req);
+            rb_raise(rb_eArgError, "batch size exceeds maximum request size");
+        }
+        size_t send_size = event_size * (size_t)count;
+        req->send_buf = malloc(send_size);
+        if (!req->send_buf) {
+            free(req);
+            rb_raise(rb_eNoMemError, "failed to allocate send buffer");
+        }
+
+        rb_tb_serialize_context_t serialize_context = {
+            .operation = operation,
+            .items_rb = items_rb,
+            .buf = req->send_buf,
+            .count = count,
+        };
+        int serialize_state = 0;
+        rb_protect(rb_tb_serialize_protected, (VALUE)&serialize_context, &serialize_state);
+        if (serialize_state) {
+            free(req->send_buf);
+            free(req);
+            rb_jump_tag(serialize_state);
+        }
+
+        req->packet.data_size = (uint32_t)send_size;
+    }
+
+    req->packet.data = req->send_buf;
+    req->packet.operation = (uint8_t)operation;
+    req->packet.user_data = req;
+
+    TB_CLIENT_STATUS cs = tb_client_submit(client, &req->packet);
+    if (cs == TB_CLIENT_INVALID) {
+        free(req->send_buf);
+        free(req);
+        rb_raise(rb_eClientClosedError, "client is closed");
+    }
+
+    return TypedData_Wrap_Struct(rb_cRequest, &rb_tb_request_type, req);
+}
+
+static VALUE rb_tb_request_id(VALUE self) {
+    rb_tb_request_t *req;
+    TypedData_Get_Struct(self, rb_tb_request_t, &rb_tb_request_type, req);
+    return RB_ULL2NUM((unsigned long long)(uintptr_t)req);
+}
+
+static void rb_tb_init_native_client(VALUE mTigerBeetle) {
+    rb_define_const(mTigerBeetle, "PACKET_OK", RB_INT2NUM(TB_PACKET_OK));
+    rb_define_const(mTigerBeetle, "PACKET_CLIENT_SHUTDOWN", RB_INT2NUM(TB_PACKET_CLIENT_SHUTDOWN));
+
+    VALUE cNativeClient = rb_define_class_under(mTigerBeetle, "NativeClient", rb_cObject);
+    rb_define_alloc_func(cNativeClient, rb_tb_client_alloc);
+    rb_define_method(cNativeClient, "initialize", rb_tb_client_initialize, 3);
+    rb_define_method(cNativeClient, "submit", rb_tb_client_submit, 2);
+    rb_define_method(cNativeClient, "close", rb_tb_client_close, 0);
+
+    rb_cRequest = rb_define_class_under(mTigerBeetle, "Request", rb_cObject);
+    rb_define_alloc_func(rb_cRequest, rb_tb_request_alloc);
+    rb_define_method(rb_cRequest, "id", rb_tb_request_id, 0);
+    rb_define_method(rb_cRequest, "result", rb_tb_request_result, 0);
+}