chalk_ruby 0.3.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8e3e6e01afae6634b4b66b557f658573bf4ffd2b657c082652e45185ec48ec05
-  data.tar.gz: 58222edf6ee281ae1cfbaaeec09b628985cfe96ddce3299054f5318e61a1d036
+  metadata.gz: 5078b236fa2532b078ba26509838b574bcb72c624ac4a27eb41f7fb34b9fb617
+  data.tar.gz: af01a9060816db57ff9198129a8d3ac2d0c7cce683e0ea59da1f9f6ed7ead09a
 SHA512:
-  metadata.gz: 0e995a01c66b8086fff20b2b7d1304458d2370a73205b87b2c6c2860df0a03e93affda126d20cf8d018ed3fe32f0e612e6ef4b7cdef5d95d1b5b02456994eb5b
-  data.tar.gz: b00008371f1f5f5a2fbd3113a7e2db1cca4e58100ac5ba8385cfad08f065acb1476d645edc5f853ca5d8a67392cd4b3ce3d2aed8b85f979aebd618bedde48a68
+  metadata.gz: f0f5a7f1ca26cefac916b76bf8f83b77b94ecc64344985d7d5adf6fe77aaebc61b811b6c9f5cbc3dd1cc78dc7b1fe519a3d7b9e45656294c0bc021b42cafe9cf
+  data.tar.gz: 910e8c846f446f44bb556f34c430d4eec7e24482ff3e13f29ad6874c635f48f5539e0502c25dfb8a9d6d48227a538beedbcd598474d6bc3c059136746f9194b6

data/lib/chalk_ruby/grpc_client.rb

@@ -195,31 +195,7 @@ module ChalkRuby
       output_data = nil
 
       if (!response.scalars_data.nil?) and response.scalars_data.length > 0
-        buffer = Arrow::Buffer.new(response.scalars_data)
-
-        # Create a buffer reader
-        buffer_reader = Arrow::BufferInputStream.new(buffer)
-
-        # Create an IPC reader from the buffer reader
-        reader = Arrow::FeatherFileReader.new(buffer_reader)
-
-        # Read the table
-
-
-        output_data = []
-
-        table = reader.read
-
-
-        field_names = table.schema.fields.map(&:name)
-        table.each_record do |r|
-          row = {}
-          field_names.each do |f|
-            row[f] = r[f]
-          end
-
-          output_data << row
-        end
+        output_data = arrow_table_to_array(response.scalars_data)
       end
 
       {
@@ -436,6 +412,47 @@ module ChalkRuby
 
       private
 
+    # Converts Arrow binary data to an array of hashes
+    # @param arrow_data [String] Binary Arrow data (IPC stream format)
+    # @return [Array<Hash>] Array of hashes with column name as keys and Ruby values
+    def arrow_table_to_array(arrow_data)
+      require 'arrow'
+      
+      buffer = Arrow::Buffer.new(arrow_data)
+      buffer_reader = Arrow::BufferInputStream.new(buffer)
+      
+      # Try IPC stream format first (which is what we expect from the query service)
+      begin
+        reader = Arrow::RecordBatchStreamReader.new(buffer_reader)
+        table = reader.read_all
+      rescue => e
+        # Fall back to feather format for backward compatibility
+        # buffer_reader.rewind
+        begin
+          reader = Arrow::FeatherFileReader.new(buffer_reader)
+          table = reader.read
+        rescue => e2
+          raise "Failed to parse Arrow data: #{e.message}, #{e2.message}"
+        end
+      end
+      
+      output_data = []
+      field_names = table.schema.fields.map(&:name)
+      
+      table.each_record do |r|
+        row = {}
+        field_names.each do |f|
+          value = r[f]
+          # Convert GLib::Bytes to Ruby String for binary and large string types
+          value = value.to_s if value.is_a?(GLib::Bytes)
+          row[f] = value
+        end
+        output_data << row
+      end
+      
+      output_data
+    end
+
     def to_feather(input_hash)
       require 'arrow'
 

data/lib/chalk_ruby/version.rb

@@ -1,3 +1,3 @@
 module ChalkRuby
-  VERSION = '0.3.0'.freeze
+  VERSION = '0.3.1'.freeze
 end

data/test/chalk_ruby/integration/arrow_conversion_test.rb

@@ -0,0 +1,286 @@
+require 'minitest/autorun'
+require 'chalk_ruby/grpc_client'
+require 'arrow'
+
+class ArrowConversionTest < Minitest::Test
+  def setup
+    @client = create_client
+  end
+
+  def test_arrow_string_conversion
+    assert_type_conversion('test_string', 'string', String)
+  end
+
+  def test_arrow_large_string_conversion
+    test_string = 'test_large_string'
+    # For large string types, we might get a GLib::Bytes object, which can be converted to a string
+    result = assert_type_conversion(test_string, 'large_string', [String, GLib::Bytes])
+    # Check the actual content
+    actual_value = result.is_a?(GLib::Bytes) ? result.to_s : result
+    assert_equal test_string, actual_value
+  end
+
+  def test_arrow_int8_conversion
+    assert_type_conversion(8, 'int8', Integer)
+  end
+
+  def test_arrow_int16_conversion
+    assert_type_conversion(16, 'int16', Integer)
+  end
+
+  def test_arrow_int32_conversion
+    assert_type_conversion(32, 'int32', Integer)
+  end
+
+  def test_arrow_int64_conversion
+    assert_type_conversion(64, 'int64', Integer)
+  end
+
+  def test_arrow_uint8_conversion
+    assert_type_conversion(8, 'uint8', Integer)
+  end
+
+  def test_arrow_uint16_conversion
+    assert_type_conversion(16, 'uint16', Integer)
+  end
+
+  def test_arrow_uint32_conversion
+    assert_type_conversion(32, 'uint32', Integer)
+  end
+
+  def test_arrow_uint64_conversion
+    assert_type_conversion(64, 'uint64', Integer)
+  end
+
+  def test_arrow_float_conversion
+    assert_type_conversion(3.14, 'float', Float)
+  end
+
+  def test_arrow_double_conversion
+    assert_type_conversion(3.14159, 'double', Float)
+  end
+
+  def test_arrow_boolean_conversion
+    assert_type_conversion(true, 'boolean', [TrueClass, FalseClass])
+  end
+
+  def test_arrow_timestamp_conversion
+    # Create timestamp as a Ruby Time object
+    timestamp = Time.now
+    # Pass timestamp seconds since epoch to Arrow
+    arrow_timestamp = timestamp.to_i
+    # Check that it converts back to a Time-like object
+    result = assert_type_conversion(arrow_timestamp, 'timestamp[s]', Time)
+    # Check that the timestamp values are approximately equal
+    assert_in_delta timestamp.to_i, result.to_i, 1
+  end
+
+  def test_arrow_date32_conversion
+    # Date represented as days since epoch
+    today = Date.today
+    days_since_epoch = (today - Date.new(1970, 1, 1)).to_i
+    result = assert_type_conversion(days_since_epoch, 'date32', Date)
+    assert_equal today, result
+  end
+
+  def test_arrow_binary_conversion
+    binary_data = "\x01\x02\x03\x04"
+    # For binary types, we might get a GLib::Bytes object, which can be converted to a string
+    result = assert_type_conversion(binary_data, 'binary', [String, GLib::Bytes])
+    # Check the actual content
+    actual_value = result.is_a?(GLib::Bytes) ? result.to_s : result
+    assert_equal binary_data, actual_value
+  end
+
+  def test_arrow_large_binary_conversion
+    large_binary_data = "\x01\x02\x03\x04" * 10
+    # For binary types, we might get a GLib::Bytes object, which can be converted to a string
+    result = assert_type_conversion(large_binary_data, 'large_binary', [String, GLib::Bytes])
+    # Check the actual content
+    actual_value = result.is_a?(GLib::Bytes) ? result.to_s : result
+    assert_equal large_binary_data, actual_value
+  end
+
+  def test_arrow_list_conversion
+    # Skip this test as it's difficult to create a list array with the current Arrow Ruby API
+    skip "Creating list arrays requires more complex approach with current Arrow Ruby API"
+  end
+
+  def test_arrow_struct_conversion
+    # Skip this test as it's difficult to create a struct array with the current Arrow Ruby API
+    skip "Creating struct arrays requires more complex approach with current Arrow Ruby API"
+  end
+
+  def test_arrow_null_conversion
+    # Test that null values are properly converted to nil
+    schema = Arrow::Schema.new([Arrow::Field.new("test_null", Arrow::Int32DataType.new)])
+    
+    # Create array with a null value
+    builder = Arrow::Int32ArrayBuilder.new
+    builder.append_null
+    array = builder.finish
+    
+    table = Arrow::Table.new(schema, [array])
+    
+    # Serialize to Arrow IPC streaming format
+    buffer = Arrow::ResizableBuffer.new(0)
+    output = Arrow::BufferOutputStream.new(buffer)
+    writer = Arrow::RecordBatchStreamWriter.new(output, table.schema)
+    writer.write_table(table)
+    writer.close
+    output.close
+    arrow_data = buffer.data.to_s
+    
+    result = @client.send(:arrow_table_to_array, arrow_data)
+    
+    assert_instance_of Array, result
+    assert_equal 1, result.length
+    assert_nil result[0]["test_null"]
+  end
+
+  private
+
+  def create_client
+    config = ChalkRuby::Config.new(
+      client_id: 'test',
+      client_secret: 'test',
+      environment: 'test',
+      query_server: 'test'
+    )
+    ChalkRuby::GrpcClient.new(config)
+  end
+
+  def assert_type_conversion(value, arrow_type_name, expected_ruby_type)
+    # Create schema with the specified arrow type
+    data_type = create_arrow_data_type(arrow_type_name)
+    schema = Arrow::Schema.new([Arrow::Field.new("test_field", data_type)])
+    
+    # Create array with the value
+    array = create_arrow_array(data_type, value)
+    
+    # Create table with schema and array
+    table = Arrow::Table.new(schema, [array])
+    
+    # Serialize table to Arrow IPC streaming format
+    buffer = Arrow::ResizableBuffer.new(0)
+    output = Arrow::BufferOutputStream.new(buffer)
+    writer = Arrow::RecordBatchStreamWriter.new(output, table.schema)
+    writer.write_table(table)
+    writer.close
+    output.close
+    arrow_data = buffer.data.to_s
+    
+    # Convert arrow data to array of hashes
+    result = @client.send(:arrow_table_to_array, arrow_data)
+    
+    # Check that the value was properly converted
+    assert_instance_of Array, result
+    assert_equal 1, result.length
+    
+    expected_types = Array(expected_ruby_type)
+    assert expected_types.any? { |type| result[0]["test_field"].is_a?(type) }, 
+           "Expected #{result[0]["test_field"].inspect} to be a #{expected_ruby_type}, but was #{result[0]["test_field"].class}"
+    
+    # Return the converted value for further assertions
+    result[0]["test_field"]
+  end
+
+  def create_arrow_data_type(type_name)
+    case type_name
+    when 'int8' then Arrow::Int8DataType.new
+    when 'int16' then Arrow::Int16DataType.new
+    when 'int32' then Arrow::Int32DataType.new
+    when 'int64' then Arrow::Int64DataType.new
+    when 'uint8' then Arrow::UInt8DataType.new
+    when 'uint16' then Arrow::UInt16DataType.new
+    when 'uint32' then Arrow::UInt32DataType.new
+    when 'uint64' then Arrow::UInt64DataType.new
+    when 'float' then Arrow::FloatDataType.new
+    when 'double' then Arrow::DoubleDataType.new
+    when 'string' then Arrow::StringDataType.new
+    when 'large_string' then Arrow::LargeStringDataType.new
+    when 'binary' then Arrow::BinaryDataType.new
+    when 'large_binary' then Arrow::LargeBinaryDataType.new
+    when 'boolean' then Arrow::BooleanDataType.new
+    when 'timestamp[s]' then Arrow::TimestampDataType.new(:second)
+    when 'date32' then Arrow::Date32DataType.new
+    else
+      raise "Unsupported arrow type: #{type_name}"
+    end
+  end
+
+  def create_arrow_array(data_type, value)
+    case data_type
+    when Arrow::Int8DataType
+      builder = Arrow::Int8ArrayBuilder.new
+      builder.append_value(value)
+      builder.finish
+    when Arrow::Int16DataType
+      builder = Arrow::Int16ArrayBuilder.new
+      builder.append_value(value)
+      builder.finish
+    when Arrow::Int32DataType
+      builder = Arrow::Int32ArrayBuilder.new
+      builder.append_value(value)
+      builder.finish
+    when Arrow::Int64DataType
+      builder = Arrow::Int64ArrayBuilder.new
+      builder.append_value(value)
+      builder.finish
+    when Arrow::UInt8DataType
+      builder = Arrow::UInt8ArrayBuilder.new
+      builder.append_value(value)
+      builder.finish
+    when Arrow::UInt16DataType
+      builder = Arrow::UInt16ArrayBuilder.new
+      builder.append_value(value)
+      builder.finish
+    when Arrow::UInt32DataType
+      builder = Arrow::UInt32ArrayBuilder.new
+      builder.append_value(value)
+      builder.finish
+    when Arrow::UInt64DataType
+      builder = Arrow::UInt64ArrayBuilder.new
+      builder.append_value(value)
+      builder.finish
+    when Arrow::FloatDataType
+      builder = Arrow::FloatArrayBuilder.new
+      builder.append_value(value)
+      builder.finish
+    when Arrow::DoubleDataType
+      builder = Arrow::DoubleArrayBuilder.new
+      builder.append_value(value)
+      builder.finish
+    when Arrow::StringDataType
+      builder = Arrow::StringArrayBuilder.new
+      builder.append_value(value)
+      builder.finish
+    when Arrow::LargeStringDataType
+      builder = Arrow::LargeStringArrayBuilder.new
+      builder.append_value(value)
+      builder.finish
+    when Arrow::BinaryDataType
+      builder = Arrow::BinaryArrayBuilder.new
+      builder.append_value(value)
+      builder.finish
+    when Arrow::LargeBinaryDataType
+      builder = Arrow::LargeBinaryArrayBuilder.new
+      builder.append_value(value)
+      builder.finish
+    when Arrow::BooleanDataType
+      builder = Arrow::BooleanArrayBuilder.new
+      builder.append_value(value)
+      builder.finish
+    when Arrow::TimestampDataType
+      builder = Arrow::TimestampArrayBuilder.new(data_type)
+      builder.append_value(value)
+      builder.finish
+    when Arrow::Date32DataType
+      builder = Arrow::Date32ArrayBuilder.new
+      builder.append_value(value)
+      builder.finish
+    else
+      raise "Unsupported arrow data type: #{data_type.class}"
+    end
+  end
+end

data/test/chalk_ruby/integration/grpc_client_test.rb

@@ -1,18 +1,51 @@
-require 'rspec'
-require 'chalk_ruby'
+$LOAD_PATH.unshift File.expand_path('../../../../lib', __FILE__)
+# require 'rspec'
+# require 'chalk_ruby'
+# require 'chalk_ruby'
+
+require 'date'
+require 'rspec/autorun'
+require 'chalk_ruby/grpc_client'
+require 'chalk_ruby/grpc/auth_interceptor'
+require 'chalk_ruby/error'
+require 'chalk_ruby/protos/chalk/server/v1/auth_pb'
+require 'chalk_ruby/protos/chalk/server/v1/auth_services_pb'
+require 'chalk_ruby/protos/chalk/engine/v1/query_server_services_pb'
+require 'arrow'
+
+
+
 
 RSpec.describe ChalkRuby::GrpcClient do
-  describe '#query_bulk' do
+  describe '#query' do
     let(:client) do
-      ChalkRuby::GrpcClient.create(
-        ENV.fetch('CHALK_CLIENT_ID'),
-        ENV.fetch('CHALK_CLIENT_SECRET'),
-        ENV.fetch('CHALK_ENVIRONMENT', 'tmnmc9beyujew'),
-        ENV.fetch('CHALK_QUERY_SERVER', 'standard-gke.chalk-develop.gcp.chalk.ai'),
-        ENV.fetch('CHALK_API_SERVER', 'api.staging.chalk.ai:443')
+      ChalkRuby::GrpcClient.new(
+        ChalkRuby::Config.new(
+          query_server: "standard-gke.chalk-develop.gcp.chalk.ai",
+          api_server: "api.staging.chalk.ai:443",
+          client_id: CLIENT_ID,
+          client_secret: CLIENT_SECRET,
+          environment: "tmnmc9beyujew",
+          # api_timeout: 0.6, # seconds
+          # connect_timeout: 0.3, # seconds
+          # query_service_root_ca_path: "/Users/andrew/found_ca.pem" # path to the root ca for chalkai.internal.found.app,
+        )
       )
     end
 
+    # it 'can perform queries' do
+    #   response = client.query(
+    #     input: { 'business.id': 1 },
+    #     output: %w(business.id)
+    #   )
+    #
+    #   expect(response).not_to be_nil
+    #
+    #   puts response
+    #   # The response should be a OnlineQueryBulkResponse
+    #   # expect(response).to be_a(Chalk::Common::V1::OnlineQueryBulkResponse)
+    # end
+
     it 'can perform bulk queries' do
       response = client.query_bulk(
         input: { 'user.id': 1 },

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: chalk_ruby
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
 platform: ruby
 authors:
 - Chalk AI, Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-03-26 00:00:00.000000000 Z
+date: 2025-05-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -362,6 +362,7 @@ files:
 - sig/chalk_ruby/http/response.rbs
 - sig/chalk_ruby/token.rbs
 - sig/chalk_ruby/versions.rbs
+- test/chalk_ruby/integration/arrow_conversion_test.rb
 - test/chalk_ruby/integration/client_test.rb
 - test/chalk_ruby/integration/grpc_client_test.rb
 - test/chalk_ruby/test_helper.rb
@@ -392,6 +393,7 @@ signing_key:
 specification_version: 4
 summary: A simple Ruby client for Chalk
 test_files:
+- test/chalk_ruby/integration/arrow_conversion_test.rb
 - test/chalk_ruby/integration/client_test.rb
 - test/chalk_ruby/integration/grpc_client_test.rb
 - test/chalk_ruby/test_helper.rb