chalk_ruby 0.2.7 → 0.2.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f55160574645e2052ff5e688d77f508e85dc77ab01f6e97905e3367d85b11156
-  data.tar.gz: 63f3c23d00f5432082229ee40666316058714170e0f71a06e6a61d989b1d0d89
+  metadata.gz: bd2402b4edf2bce128c0f86f9b81ff089bed5a2cb9564fc5c1b30d4290b2d062
+  data.tar.gz: 38380294054c04a7743adf9b2e2080dcba1d73e9a8cb9f1c1ddca032bbb46883
 SHA512:
-  metadata.gz: f6e201a24d058a626add01241c3e065dba7f83f9824701727b36b5991268e845a8aecb3878105dd7f7a2269f9b334e8aaff01696519aab6335213801a842376c
-  data.tar.gz: 107156aececa9c7fdfcd98368f78492cb76c08845be3e49c21a9060275da2844bd96cbe28ab836b6d8a7ec9a2db0a1a5197dba7c8bd5ad4286a447eba1f5e9cf
+  metadata.gz: 99e17dd176aab3cbec5b2c65f1b1e44364978e5f2451f631c0b0c93832078cf73519c2a04982e3cb31ef69b93209550a9610c20326d3120ff361cdd6cedff754
+  data.tar.gz: 0db6e82923b3aaa2ced3401d5a40988b2a98b79ababf13ec50ab228234ab08ebe7c723015f86a8a1438488c5db3627bced34a4ed3bcff5a82ed1aa23a67950a6

data/lib/chalk_ruby/grpc_client.rb

@@ -181,31 +181,7 @@ module ChalkRuby
       output_data = nil
 
       if (!response.scalars_data.nil?) and response.scalars_data.length > 0
-        buffer = Arrow::Buffer.new(response.scalars_data)
-
-        # Create a buffer reader
-        buffer_reader = Arrow::BufferInputStream.new(buffer)
-
-        # Create an IPC reader from the buffer reader
-        reader = Arrow::FeatherFileReader.new(buffer_reader)
-
-        # Read the table
-
-
-        output_data = []
-
-        table = reader.read
-
-
-        field_names = table.schema.fields.map(&:name)
-        table.each_record do |r|
-          row = {}
-          field_names.each do |f|
-            row[f] = r[f]
-          end
-
-          output_data << row
-        end
+        output_data = arrow_table_to_array(response.scalars_data)
       end
 
       {
@@ -419,6 +395,47 @@ module ChalkRuby
 
       private
 
+    # Converts Arrow binary data to an array of hashes
+    # @param arrow_data [String] Binary Arrow data (IPC stream format)
+    # @return [Array<Hash>] Array of hashes with column name as keys and Ruby values
+    def arrow_table_to_array(arrow_data)
+      require 'arrow'
+      
+      buffer = Arrow::Buffer.new(arrow_data)
+      buffer_reader = Arrow::BufferInputStream.new(buffer)
+      
+      # Try IPC stream format first (which is what we expect from the query service)
+      begin
+        reader = Arrow::RecordBatchStreamReader.new(buffer_reader)
+        table = reader.read_all
+      rescue => e
+        # Fall back to feather format for backward compatibility
+        # buffer_reader.rewind
+        begin
+          reader = Arrow::FeatherFileReader.new(buffer_reader)
+          table = reader.read
+        rescue => e2
+          raise "Failed to parse Arrow data: #{e.message}, #{e2.message}"
+        end
+      end
+      
+      output_data = []
+      field_names = table.schema.fields.map(&:name)
+      
+      table.each_record do |r|
+        row = {}
+        field_names.each do |f|
+          value = r[f]
+          # Convert GLib::Bytes to Ruby String for binary and large string types
+          value = value.to_s if value.is_a?(GLib::Bytes)
+          row[f] = value
+        end
+        output_data << row
+      end
+      
+      output_data
+    end
+
     def to_feather(input_hash)
       require 'arrow'
 

data/lib/chalk_ruby/version.rb

@@ -1,3 +1,3 @@
 module ChalkRuby
-  VERSION = '0.2.7'.freeze
+  VERSION = '0.2.8'.freeze
 end

data/test/chalk_ruby/integration/arrow_conversion_test.rb

@@ -0,0 +1,286 @@
+require 'minitest/autorun'
+require 'chalk_ruby/grpc_client'
+require 'arrow'
+
+class ArrowConversionTest < Minitest::Test
+  def setup
+    @client = create_client
+  end
+
+  def test_arrow_string_conversion
+    assert_type_conversion('test_string', 'string', String)
+  end
+
+  def test_arrow_large_string_conversion
+    test_string = 'test_large_string'
+    # For large string types, we might get a GLib::Bytes object, which can be converted to a string
+    result = assert_type_conversion(test_string, 'large_string', [String, GLib::Bytes])
+    # Check the actual content
+    actual_value = result.is_a?(GLib::Bytes) ? result.to_s : result
+    assert_equal test_string, actual_value
+  end
+
+  def test_arrow_int8_conversion
+    assert_type_conversion(8, 'int8', Integer)
+  end
+
+  def test_arrow_int16_conversion
+    assert_type_conversion(16, 'int16', Integer)
+  end
+
+  def test_arrow_int32_conversion
+    assert_type_conversion(32, 'int32', Integer)
+  end
+
+  def test_arrow_int64_conversion
+    assert_type_conversion(64, 'int64', Integer)
+  end
+
+  def test_arrow_uint8_conversion
+    assert_type_conversion(8, 'uint8', Integer)
+  end
+
+  def test_arrow_uint16_conversion
+    assert_type_conversion(16, 'uint16', Integer)
+  end
+
+  def test_arrow_uint32_conversion
+    assert_type_conversion(32, 'uint32', Integer)
+  end
+
+  def test_arrow_uint64_conversion
+    assert_type_conversion(64, 'uint64', Integer)
+  end
+
+  def test_arrow_float_conversion
+    assert_type_conversion(3.14, 'float', Float)
+  end
+
+  def test_arrow_double_conversion
+    assert_type_conversion(3.14159, 'double', Float)
+  end
+
+  def test_arrow_boolean_conversion
+    assert_type_conversion(true, 'boolean', [TrueClass, FalseClass])
+  end
+
+  def test_arrow_timestamp_conversion
+    # Create timestamp as a Ruby Time object
+    timestamp = Time.now
+    # Pass timestamp seconds since epoch to Arrow
+    arrow_timestamp = timestamp.to_i
+    # Check that it converts back to a Time-like object
+    result = assert_type_conversion(arrow_timestamp, 'timestamp[s]', Time)
+    # Check that the timestamp values are approximately equal
+    assert_in_delta timestamp.to_i, result.to_i, 1
+  end
+
+  def test_arrow_date32_conversion
+    # Date represented as days since epoch
+    today = Date.today
+    days_since_epoch = (today - Date.new(1970, 1, 1)).to_i
+    result = assert_type_conversion(days_since_epoch, 'date32', Date)
+    assert_equal today, result
+  end
+
+  def test_arrow_binary_conversion
+    binary_data = "\x01\x02\x03\x04"
+    # For binary types, we might get a GLib::Bytes object, which can be converted to a string
+    result = assert_type_conversion(binary_data, 'binary', [String, GLib::Bytes])
+    # Check the actual content
+    actual_value = result.is_a?(GLib::Bytes) ? result.to_s : result
+    assert_equal binary_data, actual_value
+  end
+
+  def test_arrow_large_binary_conversion
+    large_binary_data = "\x01\x02\x03\x04" * 10
+    # For binary types, we might get a GLib::Bytes object, which can be converted to a string
+    result = assert_type_conversion(large_binary_data, 'large_binary', [String, GLib::Bytes])
+    # Check the actual content
+    actual_value = result.is_a?(GLib::Bytes) ? result.to_s : result
+    assert_equal large_binary_data, actual_value
+  end
+
+  def test_arrow_list_conversion
+    # Skip this test as it's difficult to create a list array with the current Arrow Ruby API
+    skip "Creating list arrays requires more complex approach with current Arrow Ruby API"
+  end
+
+  def test_arrow_struct_conversion
+    # Skip this test as it's difficult to create a struct array with the current Arrow Ruby API
+    skip "Creating struct arrays requires more complex approach with current Arrow Ruby API"
+  end
+
+  def test_arrow_null_conversion
+    # Test that null values are properly converted to nil
+    schema = Arrow::Schema.new([Arrow::Field.new("test_null", Arrow::Int32DataType.new)])
+    
+    # Create array with a null value
+    builder = Arrow::Int32ArrayBuilder.new
+    builder.append_null
+    array = builder.finish
+    
+    table = Arrow::Table.new(schema, [array])
+    
+    # Serialize to Arrow IPC streaming format
+    buffer = Arrow::ResizableBuffer.new(0)
+    output = Arrow::BufferOutputStream.new(buffer)
+    writer = Arrow::RecordBatchStreamWriter.new(output, table.schema)
+    writer.write_table(table)
+    writer.close
+    output.close
+    arrow_data = buffer.data.to_s
+    
+    result = @client.send(:arrow_table_to_array, arrow_data)
+    
+    assert_instance_of Array, result
+    assert_equal 1, result.length
+    assert_nil result[0]["test_null"]
+  end
+
+  private
+
+  def create_client
+    config = ChalkRuby::Config.new(
+      client_id: 'test',
+      client_secret: 'test',
+      environment: 'test',
+      query_server: 'test'
+    )
+    ChalkRuby::GrpcClient.new(config)
+  end
+
+  def assert_type_conversion(value, arrow_type_name, expected_ruby_type)
+    # Create schema with the specified arrow type
+    data_type = create_arrow_data_type(arrow_type_name)
+    schema = Arrow::Schema.new([Arrow::Field.new("test_field", data_type)])
+    
+    # Create array with the value
+    array = create_arrow_array(data_type, value)
+    
+    # Create table with schema and array
+    table = Arrow::Table.new(schema, [array])
+    
+    # Serialize table to Arrow IPC streaming format
+    buffer = Arrow::ResizableBuffer.new(0)
+    output = Arrow::BufferOutputStream.new(buffer)
+    writer = Arrow::RecordBatchStreamWriter.new(output, table.schema)
+    writer.write_table(table)
+    writer.close
+    output.close
+    arrow_data = buffer.data.to_s
+    
+    # Convert arrow data to array of hashes
+    result = @client.send(:arrow_table_to_array, arrow_data)
+    
+    # Check that the value was properly converted
+    assert_instance_of Array, result
+    assert_equal 1, result.length
+    
+    expected_types = Array(expected_ruby_type)
+    assert expected_types.any? { |type| result[0]["test_field"].is_a?(type) }, 
+           "Expected #{result[0]["test_field"].inspect} to be a #{expected_ruby_type}, but was #{result[0]["test_field"].class}"
+    
+    # Return the converted value for further assertions
+    result[0]["test_field"]
+  end
+
+  def create_arrow_data_type(type_name)
+    case type_name
+    when 'int8' then Arrow::Int8DataType.new
+    when 'int16' then Arrow::Int16DataType.new
+    when 'int32' then Arrow::Int32DataType.new
+    when 'int64' then Arrow::Int64DataType.new
+    when 'uint8' then Arrow::UInt8DataType.new
+    when 'uint16' then Arrow::UInt16DataType.new
+    when 'uint32' then Arrow::UInt32DataType.new
+    when 'uint64' then Arrow::UInt64DataType.new
+    when 'float' then Arrow::FloatDataType.new
+    when 'double' then Arrow::DoubleDataType.new
+    when 'string' then Arrow::StringDataType.new
+    when 'large_string' then Arrow::LargeStringDataType.new
+    when 'binary' then Arrow::BinaryDataType.new
+    when 'large_binary' then Arrow::LargeBinaryDataType.new
+    when 'boolean' then Arrow::BooleanDataType.new
+    when 'timestamp[s]' then Arrow::TimestampDataType.new(:second)
+    when 'date32' then Arrow::Date32DataType.new
+    else
+      raise "Unsupported arrow type: #{type_name}"
+    end
+  end
+
+  def create_arrow_array(data_type, value)
+    case data_type
+    when Arrow::Int8DataType
+      builder = Arrow::Int8ArrayBuilder.new
+      builder.append_value(value)
+      builder.finish
+    when Arrow::Int16DataType
+      builder = Arrow::Int16ArrayBuilder.new
+      builder.append_value(value)
+      builder.finish
+    when Arrow::Int32DataType
+      builder = Arrow::Int32ArrayBuilder.new
+      builder.append_value(value)
+      builder.finish
+    when Arrow::Int64DataType
+      builder = Arrow::Int64ArrayBuilder.new
+      builder.append_value(value)
+      builder.finish
+    when Arrow::UInt8DataType
+      builder = Arrow::UInt8ArrayBuilder.new
+      builder.append_value(value)
+      builder.finish
+    when Arrow::UInt16DataType
+      builder = Arrow::UInt16ArrayBuilder.new
+      builder.append_value(value)
+      builder.finish
+    when Arrow::UInt32DataType
+      builder = Arrow::UInt32ArrayBuilder.new
+      builder.append_value(value)
+      builder.finish
+    when Arrow::UInt64DataType
+      builder = Arrow::UInt64ArrayBuilder.new
+      builder.append_value(value)
+      builder.finish
+    when Arrow::FloatDataType
+      builder = Arrow::FloatArrayBuilder.new
+      builder.append_value(value)
+      builder.finish
+    when Arrow::DoubleDataType
+      builder = Arrow::DoubleArrayBuilder.new
+      builder.append_value(value)
+      builder.finish
+    when Arrow::StringDataType
+      builder = Arrow::StringArrayBuilder.new
+      builder.append_value(value)
+      builder.finish
+    when Arrow::LargeStringDataType
+      builder = Arrow::LargeStringArrayBuilder.new
+      builder.append_value(value)
+      builder.finish
+    when Arrow::BinaryDataType
+      builder = Arrow::BinaryArrayBuilder.new
+      builder.append_value(value)
+      builder.finish
+    when Arrow::LargeBinaryDataType
+      builder = Arrow::LargeBinaryArrayBuilder.new
+      builder.append_value(value)
+      builder.finish
+    when Arrow::BooleanDataType
+      builder = Arrow::BooleanArrayBuilder.new
+      builder.append_value(value)
+      builder.finish
+    when Arrow::TimestampDataType
+      builder = Arrow::TimestampArrayBuilder.new(data_type)
+      builder.append_value(value)
+      builder.finish
+    when Arrow::Date32DataType
+      builder = Arrow::Date32ArrayBuilder.new
+      builder.append_value(value)
+      builder.finish
+    else
+      raise "Unsupported arrow data type: #{data_type.class}"
+    end
+  end
+end

data/test/chalk_ruby/integration/client_test.rb

@@ -3,8 +3,8 @@ require 'rspec/autorun'
 require 'chalk_ruby/client'
 require 'chalk_ruby/error'
 
-CLIENT_ID     = ''
-CLIENT_SECRET = ''
+CLIENT_ID     = 'client-095f628e339a593e12a58559f5f8cd00'
+CLIENT_SECRET = 'secret-55601b885bcc0d558c4abede69480aaac1dba6e1679889fa1d5196ba04f7af89'
 
 RSpec.describe 'Online query' do
   it 'should accept valid queries' do

data/test/chalk_ruby/integration/grpc_client_test.rb

@@ -14,6 +14,8 @@ require 'chalk_ruby/protos/chalk/engine/v1/query_server_services_pb'
 require 'arrow'
 
 
+CLIENT_ID     = 'client-095f628e339a593e12a58559f5f8cd00'
+CLIENT_SECRET = 'secret-55601b885bcc0d558c4abede69480aaac1dba6e1679889fa1d5196ba04f7af89'
 
 
 RSpec.describe ChalkRuby::GrpcClient do
@@ -49,7 +51,7 @@ RSpec.describe ChalkRuby::GrpcClient do
     it 'can perform bulk queries' do
       response = client.query_bulk(
         input: { 'user.id': 1 },
-        output: %w(user.id user.socure_score)
+        output: %w(user.id user.socure_score user.full_name)
       )
 
       expect(response).not_to be_nil

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: chalk_ruby
 version: !ruby/object:Gem::Version
-  version: 0.2.7
+  version: 0.2.8
 platform: ruby
 authors:
 - Chalk AI, Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-03-08 00:00:00.000000000 Z
+date: 2025-03-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -361,6 +361,7 @@ files:
 - sig/chalk_ruby/http/response.rbs
 - sig/chalk_ruby/token.rbs
 - sig/chalk_ruby/versions.rbs
+- test/chalk_ruby/integration/arrow_conversion_test.rb
 - test/chalk_ruby/integration/client_test.rb
 - test/chalk_ruby/integration/grpc_client_test.rb
 - test/chalk_ruby/test_helper.rb
@@ -391,6 +392,7 @@ signing_key:
 specification_version: 4
 summary: A simple Ruby client for Chalk
 test_files:
+- test/chalk_ruby/integration/arrow_conversion_test.rb
 - test/chalk_ruby/integration/client_test.rb
 - test/chalk_ruby/integration/grpc_client_test.rb
 - test/chalk_ruby/test_helper.rb