extralite 2.15 → 3.0.0

data/ext/extralite/transform.c

@@ -0,0 +1,420 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "ruby.h"
+#include "extralite.h"
+
+/*
+ * Document-class: Extralite::Transform
+ *
+ * This class represents a specification for retrieving structured data as the
+ * result of a query. When querying data from multiple tables using joins,
+ * transforms allow you to retrieve the results as an object graph containing
+ * the different entities.
+ *
+ * For example, a posts table and a tags table may be joined to represent a
+ * many-to-many relationship:
+ *
+ *     select
+ *       posts.id, posts.content,
+ *       tags.id, tags.name
+ *     from posts
+ *     left outer join posts_tags on posts_tags.post_id = posts.id
+ *     left outer join tags on posts_tags.tag_id = tags.id
+ *
+ * Normally, the resulting data will be represented as an array of hashes, of
+ * the form:
+ *
+ *     [
+ *       { posts_id: 1, posts_content: "foo", tags_id: 1, tags_name: "blah" },
+ *       { posts_id: 2, posts_content: "bar", tags_id: 1, tags_name: "blah" },
+ *       { posts_id: 2, posts_content: "bar", tags_id: 2, tags_name: "bleh" },
+ *       { posts_id: 3, posts_content: "baz", tags_id: 2, tags_name: "blah" },
+ *       ...
+ *     ]
+ *
+ * Those results, while containing all the information that we requested, also
+ * contain a lot of duplication: the same post entity may appear in multiple
+ * rows, and the same tag entity may be repeated for different posts.
+ *
+ * With a properly configured transform, we can convert those flat rows with
+ * duplicate data into an object graph that represents the different entities
+ * (posts and tags), such that each post entity will also include the
+ * corresponding tags. Furthermore, we can eliminate duplication by using
+ * identity maps to create each entity only once, and reuse it if it repeats
+ * (such as in the case of tags):
+ *
+ *     [
+ *       { id: 1, content: "foo", tags: [{id: 1, name: "blah"}] },
+ *       { id: 2, content: "bar", tags: [{id: 1, name: "blah"}, {id: 2, name: "bleh"}] },
+ *       { id: 3, content: "baz", tags: [{id: 2, name: "bleh"}] }
+ *     ]
+ *
+ * The transform is expressed using the transform DSL:
+ *
+ *     transform = Extralite::Transform.new do
+ *       {
+ *         id:      integer.identity,
+ *         content: text,
+ *         tags:    [{
+ *           id:    integer.identity,
+ *           name:  text
+ *         }]
+ *       }
+ *     end
+ *
+ * To use the transform we can feed it into Database#query:
+ *
+ *     db.query(transform, sql) #=> [...]
+ *
+ * A transform may also be used with a prepared query:
+ *
+ *     q = db.prepare(transform, sql)
+ *     q.to_a #=> [...]
+ *
+ * Transforms can also be used for type coercion. If the type is 'auto' or not
+ * specified, the returned value will reflect the native type of the value in
+ * the database. The following types are supported:
+ *
+ * - auto: native database type
+ * - integer: 64-bit integer
+ * - float: floating point number
+ * - text: text/string value
+ * - bool: a boolean (after coercion to integer)
+ * - json: parsed JSON representation
+ * - proc: a custom proc for converting a value
+ *
+ * To use the proc type, specify the proc as the type, e.g.:
+ *
+ *     Extralite::Transform.new do
+ *       {
+ *         stamp: ->(s) { Time.at(s) },
+ *         value: float
+ *       }
+ *     end
+ */
+
+VALUE cTransform;
+VALUE mJSON;
+
+VALUE SYM_bool;
+VALUE SYM_columns;
+VALUE SYM_float;
+VALUE SYM_identity;
+VALUE SYM_integer;
+VALUE SYM_json;
+VALUE SYM_name;
+VALUE SYM_relation;
+VALUE SYM_text;
+VALUE SYM_type;
+
+static size_t Transform_size(const void *ptr) {
+  return sizeof(Transform_t);
+}
+
+static inline void transform_node_mark(struct transform_node *node) {
+  rb_gc_mark_movable(node->name);
+  if (node->type == TRANSFORM_T_PROC)
+    rb_gc_mark_movable(node->conversion_proc);
+
+  struct transform_node *cur = node->subnodes_head;
+  while (cur) {
+    transform_node_mark(cur);
+    cur = cur->next;
+  }
+
+  if (node->next) transform_node_mark(node->next);
+}
+
+static void Transform_mark(void *ptr) {
+  Transform_t *t = ptr;
+  if (t->root) transform_node_mark(t->root);
+}
+
+static inline void transform_node_compact(struct transform_node *node) {
+  if (node->flags & TRANSFORM_F_NAME)
+    node->name = rb_gc_location(node->name);
+
+  if (node->type == TRANSFORM_T_PROC)
+    node->conversion_proc = rb_gc_location(node->conversion_proc);
+
+  struct transform_node *cur = node->subnodes_head;
+  while (cur) {
+    transform_node_compact(cur);
+    cur = cur->next;
+  }
+
+  if (node->next) transform_node_compact(node->next);
+}
+
+static void Transform_compact(void *ptr) {
+  Transform_t *t = ptr;
+  if (t->root) transform_node_compact(t->root);
+}
+
+static inline void transform_node_free(struct transform_node *node) {
+  if (node->subnodes_head)
+    transform_node_free(node->subnodes_head);
+  if (node->next)
+    transform_node_free(node->next);
+
+  free(node);
+}
+
+static void Transform_free(void *ptr) {
+  Transform_t *t = ptr;
+  if (t->root) transform_node_free(t->root);
+  free(ptr);
+}
+
+static const rb_data_type_t Transform_type = {
+    "Transform",
+    {Transform_mark, Transform_free, Transform_size, Transform_compact},
+    0, 0, RUBY_TYPED_FREE_IMMEDIATELY
+};
+
+static VALUE Transform_allocate(VALUE klass) {
+  Transform_t *t = ALLOC(Transform_t);
+  t->root = NULL;
+  return TypedData_Wrap_Struct(klass, &Transform_type, t);
+}
+
+static inline Transform_t *self_to_transform(VALUE self) {
+  Transform_t *t;
+  TypedData_Get_Struct(self, Transform_t, &Transform_type, t);
+  return t;
+}
+
+struct transform_node *get_transform_root(VALUE obj) {
+  return self_to_transform(obj)->root;
+}
+
+inline struct transform_node *allocate_transform_node() {
+  struct transform_node *node = malloc(sizeof(struct transform_node));
+  memset(node, 0, sizeof(struct transform_node));
+  node->name = Qnil;
+  node->conversion_proc = Qnil;
+  return node;
+}
+
+struct transform_node *compile_transform_relation(VALUE spec, int *col_counter);
+
+enum transform_node_type get_node_type(VALUE col_hash, VALUE *proc) {
+  VALUE type = rb_hash_aref(col_hash, SYM_type);
+  if (NIL_P(type))          return TRANSFORM_T_AUTO;
+  if (type == SYM_integer)  return TRANSFORM_T_INTEGER;
+  if (type == SYM_float)    return TRANSFORM_T_FLOAT;
+  if (type == SYM_text)     return TRANSFORM_T_TEXT;
+  if (type == SYM_bool)     return TRANSFORM_T_BOOL;
+  if (type == SYM_json)     return TRANSFORM_T_JSON;
+  if (type == SYM_relation) return TRANSFORM_T_RELATION;
+
+  if (TYPE(type) == T_DATA) {
+    *proc = type;
+    return TRANSFORM_T_PROC;
+  }
+
+  rb_raise(cError, "Invalid column type");
+}
+
+struct transform_node *compile_transform_column(VALUE col, int *col_counter) {
+  // In the PORO spec, a multi-row (array) relation is expressed as:
+  //
+  //     spec = {
+  //       columns: {
+  //         **,
+  //         tags:     [{
+  //           type: :relation,
+  //           columns: {**}
+  //         }]
+  //       }
+  //     }
+  //
+  // (Note the literal array surrounding the tags entry.) So we check for it
+  // here, it's also checked in compile_transform_relation.
+  if (TYPE(col) == T_ARRAY) goto relation_node;
+  if (TYPE(col) != T_HASH)
+    rb_raise(cError, "Each column must be a hash");
+
+  VALUE proc = Qnil;
+  enum transform_node_type node_type = get_node_type(col, &proc);
+  if (node_type == TRANSFORM_T_RELATION) goto relation_node;
+
+  struct transform_node *node = allocate_transform_node();
+  node->type = node_type;
+  node->idx = *col_counter;
+
+  VALUE identity = rb_hash_aref(col, SYM_identity);
+  if (RTEST(identity)) node->flags |= TRANSFORM_F_IDENTITY;
+
+  if (node_type == TRANSFORM_T_PROC) node->conversion_proc = proc;
+
+  (*col_counter)++;
+  RB_GC_GUARD(proc);
+  return node;
+
+relation_node:
+  return compile_transform_relation(col, col_counter);
+}
+
+struct column_iterator_ctx {
+  VALUE spec;
+  int *col_counter;
+  struct transform_node *node;
+};
+
+int column_iterator(VALUE name, VALUE col, VALUE arg) {
+  struct column_iterator_ctx *ctx = (struct column_iterator_ctx *)arg;
+  struct transform_node *node = ctx->node;
+
+  int col_idx = *(ctx->col_counter);
+  struct transform_node *col_node = compile_transform_column(col, ctx->col_counter);
+  col_node->flags |= TRANSFORM_F_NAME;
+  col_node->name = name;
+
+  if (col_node->flags & TRANSFORM_F_IDENTITY) {
+    node->identity_node = col_node;
+    node->identity_idx = col_idx;
+  }
+
+  if (node->subnodes_tail) {
+    node->subnodes_tail->next = col_node;
+    node->subnodes_tail = col_node;
+  }
+  else {
+    node->subnodes_head = node->subnodes_tail = col_node;
+  }
+
+  return ST_CONTINUE;
+}
+
+struct transform_node *compile_transform_relation(VALUE spec, int *col_counter) {
+  struct transform_node *node = allocate_transform_node();
+  node->type = TRANSFORM_T_RELATION;
+
+  if (TYPE(spec) == T_ARRAY) {
+    node->flags |= TRANSFORM_F_ARRAY;
+    spec = rb_ary_entry(spec, 0);
+  }
+
+  VALUE val = rb_hash_aref(spec, SYM_columns);
+  if (TYPE(val) != T_HASH)
+    rb_raise(cError, "columns member must be a hash");
+
+  struct column_iterator_ctx ctx = { spec, col_counter, node };
+  rb_hash_foreach(val, column_iterator, (VALUE)&ctx);
+  return node;
+}
+
+VALUE Transform_initialize(VALUE self, VALUE spec) {
+  Transform_t *t = self_to_transform(self);
+  int col_counter = 0;
+  t->root = compile_transform_relation(spec, &col_counter);
+  return self;
+}
+
+VALUE node_type_to_value(enum transform_node_type type) {
+  switch (type) {
+    case TRANSFORM_T_AUTO:    return Qnil;
+    case TRANSFORM_T_INTEGER: return SYM_integer;
+    case TRANSFORM_T_FLOAT:   return SYM_float;
+    case TRANSFORM_T_TEXT:    return SYM_text;
+    case TRANSFORM_T_BOOL:    return SYM_bool;
+    case TRANSFORM_T_JSON:    return SYM_json;
+    default:
+      rb_raise(cError, "Invalid node type in node_type_to_value");
+  }
+}
+
+VALUE transform_node_to_obj(struct transform_node *node) {
+  // printf("transform_node_to_obj type: %d flags: %02x\n", node->type, node->flags);
+  // if (node->flags & TRANSFORM_F_NAME) INSPECT("  name", node->name);
+  VALUE hash = rb_hash_new();
+  if (node->type == TRANSFORM_T_RELATION) {
+    VALUE cols = rb_hash_new();
+
+    if (node->flags & TRANSFORM_F_NAME)
+      rb_hash_aset(hash, SYM_type, SYM_relation);
+    rb_hash_aset(hash, SYM_columns, cols);
+
+    struct transform_node *cur = node->subnodes_head;
+    while (cur) {
+      struct transform_node *next = cur->next;
+      VALUE val = transform_node_to_obj(cur);
+      rb_hash_aset(cols, cur->name, val);
+      RB_GC_GUARD(val);
+
+      cur = next;
+    }
+
+    if (node->flags & TRANSFORM_F_ARRAY) {
+      VALUE array = rb_ary_new();
+      rb_ary_push(array, hash);
+      return array;
+      RB_GC_GUARD(array);
+    }
+    else
+      return hash;
+    RB_GC_GUARD(cols);
+  }
+  else {
+    switch (node->type) {
+      case TRANSFORM_T_AUTO:
+        break;
+      case TRANSFORM_T_PROC:
+        rb_hash_aset(hash, SYM_type, node->conversion_proc);
+        break;
+      default:
+        VALUE v = node_type_to_value(node->type);
+        RB_GC_GUARD(v);
+        rb_hash_aset(hash, SYM_type, node_type_to_value(node->type));
+    }
+    if (node->flags & TRANSFORM_F_IDENTITY)
+      rb_hash_aset(hash, SYM_identity, Qtrue);
+    return hash;
+  }
+  RB_GC_GUARD(hash);
+}
+
+/* Returns the transform spec in literal form.
+ *
+ * @return [Hash] literal transform spec
+ */
+VALUE Transform_to_h(VALUE self) {
+  Transform_t *t = self_to_transform(self);
+  return transform_node_to_obj(t->root);
+}
+
+void Init_ExtraliteTransform(void) {
+  VALUE mExtralite = rb_define_module("Extralite");
+
+  cTransform = rb_define_class_under(mExtralite, "Transform", rb_cObject);
+  rb_define_alloc_func(cTransform, Transform_allocate);
+
+  rb_define_method(cTransform, "initialize",  Transform_initialize, 1);
+  rb_define_method(cTransform, "to_h",        Transform_to_h, 0);
+
+  SYM_bool      = ID2SYM(rb_intern_const("bool"));
+  SYM_columns   = ID2SYM(rb_intern_const("columns"));
+  SYM_float     = ID2SYM(rb_intern_const("float"));
+  SYM_identity  = ID2SYM(rb_intern_const("identity"));
+  SYM_integer   = ID2SYM(rb_intern_const("integer"));
+  SYM_json      = ID2SYM(rb_intern_const("json"));
+  SYM_name      = ID2SYM(rb_intern_const("name"));
+  SYM_relation  = ID2SYM(rb_intern_const("relation"));
+  SYM_text      = ID2SYM(rb_intern_const("text"));
+  SYM_type      = ID2SYM(rb_intern_const("type"));
+
+  rb_gc_register_mark_object(SYM_bool);
+  rb_gc_register_mark_object(SYM_columns);
+  rb_gc_register_mark_object(SYM_float);
+  rb_gc_register_mark_object(SYM_identity);
+  rb_gc_register_mark_object(SYM_integer);
+  rb_gc_register_mark_object(SYM_json);
+  rb_gc_register_mark_object(SYM_name);
+  rb_gc_register_mark_object(SYM_relation);
+  rb_gc_register_mark_object(SYM_text);
+  rb_gc_register_mark_object(SYM_type);
+
+  mJSON = Qnil;
+}

data/gemspec.rb

@@ -15,7 +15,7 @@ def common_spec(s)
   s.rdoc_options = ['--title', 'Extralite', '--main', 'README.md']
   s.extra_rdoc_files = ['README.md']
   s.require_paths = ['lib']
-  s.required_ruby_version = '>= 3.2'
+  s.required_ruby_version = '>= 3.4'
 
   s.add_development_dependency  'rake-compiler',        '1.3.1'
   s.add_development_dependency  'minitest'

data/lib/extralite/version.rb

@@ -1,4 +1,4 @@
 module Extralite
   # Extralite version
-  VERSION = '2.15'
+  VERSION = '3.0.0'
 end

data/lib/extralite.rb

@@ -236,7 +236,7 @@ module Extralite
     # @return [Any] the given block's return value
     def transaction(mode = :immediate)
       abort = false
-      execute "begin #{mode} transaction"    
+      execute "begin #{mode} transaction"
       yield self
     rescue => e
       abort = true
@@ -296,6 +296,10 @@ module Extralite
       raise Rollback
     end
 
+    def quote(str)
+      str.gsub("'", "''")
+    end
+
     private
 
     def pragma_set(values)
@@ -311,4 +315,101 @@ module Extralite
   class Query
     alias_method :execute_multi, :batch_execute
   end
+
+  class Transform
+    alias_method :orig_initialize, :initialize
+
+    # call-seq:
+    #   Extralite::Transform.new(literal_spec) -> transform
+    #   Extralite::Transform.new { dsl_spec } -> transform
+    #
+    # Initializes a new transform with with the given spec. The spec may be
+    # expressed as a hash containing the columns and any nested entities, or as a
+    # block with the transform DSL.
+    #
+    #     # literal transform spec
+    #     transform = Extralite::Transform.new(
+    #       columns: {
+    #         id: { type: :integer, identity: true },
+    #         content: { type: :text },
+    #         author: {
+    #           type: :relation,
+    #           columns: {
+    #             id: { type: :integer, identity: true },
+    #             name: { type: :text },
+    #           }
+    #         }
+    #       }
+    #     )
+    #
+    #     # DSL spec
+    #     transform = Extralite::Transform.new do
+    #       {
+    #         id:       integer.identity,
+    #         content:  text,
+    #         author: {
+    #           id:     integer.identity,
+    #           name:   text
+    #         }
+    #       }
+    #     end
+    #
+    # @param spec [Hash, nil] literal spec
+    # @param block [Proc, nil] DSL spec
+    # @return [void]
+    def initialize(spec = nil, &block)
+      return orig_initialize(spec) if spec
+      raise "No spec given" if !block
+
+      orig_initialize(dsl_to_spec(block))
+    end
+
+    private
+
+    class DSLContext
+      attr_reader :hash
+
+      def initialize(hash = {})
+        @hash = hash
+      end
+
+      def auto      = mutate(type: nil)
+      def integer   = mutate(type: :integer)
+      def float     = mutate(type: :float)
+      def text      = mutate(type: :text)
+      def bool      = mutate(type: :bool)
+      def json      = mutate(type: :json)
+      def identity  = mutate(identity: true)
+
+    private
+      def mutate(opts)
+        DSLContext.new(@hash.merge(opts))
+      end
+    end
+
+    def dsl_to_spec(block)
+      intermediate = DSLContext.new.instance_eval(&block)
+      dsl_intermediate_to_spec(intermediate)
+    end
+
+    def dsl_intermediate_to_spec(intermediate)
+      columns = intermediate.each_with_object({}) do |(k, v), h|
+        h[k] = dsl_translate_intermediate_value(v)
+      end
+      { type: :relation, columns: }
+    end
+
+    def dsl_translate_intermediate_value(v)
+      case v
+      when DSLContext
+        v.hash
+      when Hash
+        dsl_intermediate_to_spec(v)
+      when Array
+        [dsl_intermediate_to_spec(v[0])]
+      else
+        raise Extralite::Error, 'Invalid transform spec'
+      end
+    end
+  end
 end

data/test/perf_array.rb

@@ -4,7 +4,7 @@ require 'bundler/inline'
 
 gemfile do
   source 'https://rubygems.org'
-  gem 'sqlite3', '2.6.0'
+  gem 'sqlite3', '2.9.5'
   gem 'extralite', path: '..'
   gem 'benchmark-ips'
 end

data/test/perf_hash.rb

@@ -7,7 +7,7 @@ require 'bundler/inline'
 gemfile do
   source 'https://rubygems.org'
   gem 'extralite', path: '..'
-  gem 'sqlite3', '2.6.0'
+  gem 'sqlite3', '2.9.5'
   gem 'benchmark-ips'
 end
 

data/test/perf_hash_prepared.rb

@@ -5,7 +5,7 @@ require 'bundler/inline'
 gemfile do
   source 'https://rubygems.org'
   gem 'extralite', path: '..'
-  gem 'sqlite3', '2.6.0'
+  gem 'sqlite3', '2.9.5'
   gem 'benchmark-ips'
 end
 
@@ -18,7 +18,7 @@ puts "DB_PATH = #{DB_PATH.inspect}"
 def prepare_database(count)
   $sqlite3_db = SQLite3::Database.new(DB_PATH, results_as_hash: true)
   $extralite_db = Extralite::Database.new(DB_PATH, gvl_release_threshold: -1)
-    
+
   $extralite_db.query('create table if not exists foo ( a integer primary key, b text )')
   $extralite_db.query('delete from foo')
   $extralite_db.query('begin')

data/test/perf_splat.rb

@@ -4,7 +4,7 @@ require 'bundler/inline'
 
 gemfile do
   source 'https://rubygems.org'
-  gem 'sqlite3', '2.6.0'
+  gem 'sqlite3', '2.9.5'
   gem 'extralite', path: '..'
   gem 'benchmark-ips'
 end

data/test/perf_transform.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+# Run on Ruby 3.3 with YJIT enabled
+
+require 'bundler/inline'
+
+gemfile do
+  source 'https://rubygems.org'
+  gem 'extralite', path: '..'
+  gem 'benchmark-ips'
+end
+
+require 'benchmark/ips'
+require 'fileutils'
+
+DB_PATH = "/tmp/extralite_sqlite3_perf-#{Time.now.to_i}-#{rand(10000)}.db"
+puts "DB_PATH = #{DB_PATH.inspect}"
+
+$extralite_db = Extralite::Database.new(DB_PATH, gvl_release_threshold: -1)
+
+def prepare_database(count)
+  $extralite_db.query('create table if not exists foo ( a integer primary key, b text )')
+  $extralite_db.query('delete from foo')
+  $extralite_db.query('begin')
+  count.times { $extralite_db.query('insert into foo (b) values (?)', "hello#{rand(1000)}" )}
+  $extralite_db.query('commit')
+end
+
+def run_normal(count)
+  results = $extralite_db.query('select * from foo')
+  raise unless results.size == count
+end
+
+@transform = Extralite::Transform.new {
+  { a: integer, b: text }
+}
+
+def run_transform(count)
+  results = $extralite_db.query(@transform, 'select * from foo')
+  raise unless results.size == count
+end
+
+[10, 1000, 100000].each do |c|
+  puts "Record count: #{c}"
+  prepare_database(c)
+
+  bm = Benchmark.ips do |x|
+    x.config(:time => 5, :warmup => 2)
+
+    x.report("normal")    { run_normal(c) }
+    x.report("transform") { run_transform(c) }
+
+    x.compare!
+  end
+  puts;
+  bm.entries.each { |e| puts "#{e.label}: #{(e.ips * c).round.to_i} rows/s" }
+  puts;
+end