athena-udf 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9cc2a9e47dee420f0b11442039d1873aca9ad04ecf12775dbe8ea49df3153f02
-  data.tar.gz: 54d15b93ffc2fb58793aad53d03c69608e607c00d0f8da3f5210b47860a5aecc
+  metadata.gz: 9e5ff4cb6e2f166491b1bb18cc2be70b9de602852ff9535eb7b888de86d46569
+  data.tar.gz: df66a5f63ea2ac6a036d4f47befd0dc55ae0f8487de80a8d7d1da8d38606307d
 SHA512:
-  metadata.gz: b0f36011d582e681d575fe8d6f33a6cf9935519ed9bb21953dac980aa1c479c7d943992bbdd0e992f053f32eee6b228f473d772e4e698c3517370009fe49d3b6
-  data.tar.gz: 318b411aa1949b998c1a34176ca049cbccea541861a89931b1a3f71730ab45fec383e03549ec941f7a48305d4ad1e9c0de419d9a46b8e5469205467772df8598
+  metadata.gz: 18e43fce15698f0fc1dd0160cb0098887cf78085ca13774e72819cc89369b3fd466b6bf27a216e1ef1ff863caa412881f0fab4ba5df9af62b6cc0ddfa77f2e85
+  data.tar.gz: eba9d3b48c2adabe2c02c6a22686e0f8b7a4ee35f535738d9a72a7384948765c14a4d978e3061b5d43e75610bab7df8580dd8cbcb9a270bd025a7bb8f0392d5a

data/Dockerfile.dev

@@ -25,4 +25,8 @@ RUN gem update bundler
 
 WORKDIR /src
 
+COPY Gemfile Gemfile.lock athena-udf.gemspec /src
+COPY lib/athena-udf/version.rb /src/lib/athena-udf/version.rb
+RUN bundle install
+
 CMD ["/bin/bash"]

data/Dockerfile.example

@@ -1,20 +1,36 @@
-FROM public.ecr.aws/lambda/ruby:3.2
+# Can NOT install apache-arrow on the amazonlinux:2023-minimal image,
+# so install the ruby directly on amazonlinux:2023.
+FROM public.ecr.aws/amazonlinux/amazonlinux:2023 AS builder
+
+# Apache Arrow
+RUN dnf upgrade -y --releasever=latest \
+  && dnf install -y https://apache.jfrog.io/artifactory/arrow/amazon-linux/$(cut -d: -f6 /etc/system-release-cpe)/apache-arrow-release-latest.rpm \
+  && dnf install -y arrow-devel arrow-glib-devel arrow-dataset-devel arrow-dataset-glib-devel
+
+############
+
+FROM public.ecr.aws/lambda/ruby:3.3
+
+COPY --from=builder /usr/include/parquet/arrow/ /usr/include/parquet/arrow/
+COPY --from=builder /usr/include/arrow* /usr/include/
+COPY --from=builder /usr/share/gdb/auto-load/usr/lib64/libarrow.so* /usr/share/gdb/auto-load/usr/lib64/
+COPY --from=builder /usr/share/licenses/arrow* /usr/share/licenses/
+COPY --from=builder /usr/share/doc/arrow* /usr/share/doc/
+COPY --from=builder /usr/share/arrow* /usr/share/
+COPY --from=builder /usr/share/vala/vapi/arrow* /usr/share/vala/vapi/
+COPY --from=builder /usr/lib64/libarrow* /usr/lib64/
+COPY --from=builder /usr/lib64/pkgconfig/arrow* /usr/lib64/pkgconfig/
+COPY --from=builder /usr/lib64/libarrow* /usr/lib64/libarrow/
+COPY --from=builder /usr/lib64/cmake/Arrow/ /usr/lib64/cmake/Arrow/
 
 # General packages
-RUN yum update -y \
-  && yum install -y \
-    amazon-linux-extras \
+RUN dnf upgrade -y --releasever=latest \
+  && dnf install -y \
     gcc-c++ \
+    glib2-devel \
     make \
     git \
-  && amazon-linux-extras install -y epel \
-  && yum clean all
-
-# Apache Arrow
-RUN yum update -y \
-  && yum install -y https://apache.jfrog.io/artifactory/arrow/amazon-linux/2/apache-arrow-release-latest.rpm \
-  && yum install -y --enablerepo=epel arrow-devel arrow-glib-devel arrow-dataset-devel arrow-dataset-glib-devel \
-  && yum clean all
+  && dnf clean all
 
 # Update bundler
 RUN gem update bundler
@@ -29,3 +45,4 @@ RUN bundle config set --local without development \
 COPY . ${LAMBDA_TASK_ROOT}
 
 CMD ["example.SimpleVarcharUDF.lambda_handler"]
+

data/Gemfile.lock

@@ -1,10 +1,10 @@
 PATH
   remote: .
   specs:
-    athena-udf (0.1.3)
+    athena-udf (0.2.0)
       base64
       csv
-      red-arrow (~> 12.0.1)
+      red-arrow (< 20.0.0)
 
 GEM
   remote: https://rubygems.org/
@@ -12,29 +12,30 @@ GEM
     ast (2.4.2)
     base64 (0.2.0)
     benchmark (0.3.0)
-    bigdecimal (3.1.8)
-    csv (3.3.0)
-    diff-lcs (1.5.1)
+    bigdecimal (3.1.9)
+    csv (3.3.2)
+    diff-lcs (1.6.0)
     docile (1.4.1)
     extpp (0.1.1)
-    fiddle (1.1.2)
-    gem-release (2.2.2)
-    gio2 (4.2.2)
+    fiddle (1.1.6)
+    gem-release (2.2.4)
+    gio2 (4.2.7)
       fiddle
-      gobject-introspection (= 4.2.2)
-    glib2 (4.2.2)
+      gobject-introspection (= 4.2.7)
+    glib2 (4.2.7)
       native-package-installer (>= 1.0.3)
       pkg-config (>= 1.3.5)
-    gobject-introspection (4.2.2)
-      glib2 (= 4.2.2)
-    json (2.7.2)
-    language_server-protocol (3.17.0.3)
+    gobject-introspection (4.2.7)
+      glib2 (= 4.2.7)
+    json (2.10.2)
+    language_server-protocol (3.17.0.4)
+    lint_roller (1.1.0)
     native-package-installer (1.1.9)
     parallel (1.26.3)
-    parser (3.3.4.2)
+    parser (3.3.7.1)
       ast (~> 2.4.1)
       racc
-    pkg-config (1.5.6)
+    pkg-config (1.6.0)
     racc (1.8.1)
     rainbow (3.1.1)
     rake (13.2.1)
@@ -44,41 +45,44 @@ GEM
       gio2 (>= 3.5.0)
       native-package-installer
       pkg-config
-    regexp_parser (2.9.2)
+    regexp_parser (2.10.0)
     rspec (3.13.0)
       rspec-core (~> 3.13.0)
       rspec-expectations (~> 3.13.0)
       rspec-mocks (~> 3.13.0)
-    rspec-core (3.13.0)
+    rspec-core (3.13.3)
       rspec-support (~> 3.13.0)
-    rspec-expectations (3.13.1)
+    rspec-expectations (3.13.3)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
-    rspec-mocks (3.13.1)
+    rspec-mocks (3.13.2)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
-    rspec-support (3.13.1)
-    rubocop (1.66.0)
+    rspec-support (3.13.2)
+    rubocop (1.74.0)
       json (~> 2.3)
-      language_server-protocol (>= 3.17.0)
+      language_server-protocol (~> 3.17.0.2)
+      lint_roller (~> 1.1.0)
       parallel (~> 1.10)
       parser (>= 3.3.0.2)
       rainbow (>= 2.2.2, < 4.0)
-      regexp_parser (>= 2.4, < 3.0)
-      rubocop-ast (>= 1.32.1, < 2.0)
+      regexp_parser (>= 2.9.3, < 3.0)
+      rubocop-ast (>= 1.38.0, < 2.0)
       ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 2.4.0, < 3.0)
-    rubocop-ast (1.32.2)
+      unicode-display_width (>= 2.4.0, < 4.0)
+    rubocop-ast (1.38.1)
       parser (>= 3.3.1.0)
     ruby-progressbar (1.13.0)
     simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
-    simplecov-html (0.12.3)
+    simplecov-html (0.13.1)
     simplecov-lcov (0.8.0)
     simplecov_json_formatter (0.1.4)
-    unicode-display_width (2.5.0)
+    unicode-display_width (3.1.4)
+      unicode-emoji (~> 4.0, >= 4.0.4)
+    unicode-emoji (4.0.4)
 
 PLATFORMS
   arm64-darwin-23

data/README.md

@@ -31,7 +31,7 @@ Just make a subclass of `AthenaUDF::BaseUDF` and implement a concrete function l
 require "athena-udf"
 
 class SimpleVarcharUDF < AthenaUDF::BaseUDF
-  def self.handle_athena_record(_input_schema, _output_schema, record)
+  def handle_athena_record(_input_schema, _output_schema, record)
     [record[0].downcase]
   end
 end

data/lib/athena-udf/utils.rb

@@ -2,6 +2,9 @@
 
 module AthenaUDF
   module Utils
+    SEPARATOR = "\xFF\xFF\xFF\xFF".b
+    SEPARATOR_SIZE = SEPARATOR.bytesize
+
     def read_record_batches(schema_data, record_batch_data)
       buffer = Arrow::ResizableBuffer.new(schema_data.bytes.size + record_batch_data.bytes.size)
       Arrow::BufferOutputStream.open(buffer) do |output|
@@ -38,8 +41,8 @@ module AthenaUDF
         end
 
         bytes = buffer.data.to_s
-        start_index = get_record_batch_index(bytes)
-        bytes[4..start_index - 5]
+        last_index = bytes.index(SEPARATOR, SEPARATOR_SIZE)
+        bytes[SEPARATOR_SIZE...last_index]
       end
     end
 
@@ -51,26 +54,9 @@ module AthenaUDF
         end
 
         bytes = buffer.data.to_s
-        start_index = get_record_batch_index(bytes)
+        start_index = bytes.index(SEPARATOR, SEPARATOR_SIZE) + SEPARATOR_SIZE
         bytes[start_index..]
       end
     end
-
-    def get_record_batch_index(bytes)
-      size = bytes.size
-      found_count = 0
-      start_index = 0
-      0.upto(size - 4).each do |i|
-        has_ffff = bytes.slice(i, 4) == "\xFF\xFF\xFF\xFF".b
-
-        found_count += 1 if has_ffff
-        next unless found_count == 2
-
-        start_index = i + 4
-        break
-      end
-
-      start_index
-    end
   end
 end

data/lib/athena-udf/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module AthenaUDF
-  VERSION = '0.2.0'
+  VERSION = '0.2.1'
 end

data/scripts/benchmark.rb

@@ -3,19 +3,21 @@
 require 'benchmark'
 require 'athena_udf'
 
-Benchmark.bm 10 do |r|
+Benchmark.bm 20 do |r|
   include AthenaUDF::Utils
 
   instance = Class.new(AthenaUDF::BaseUDF) do
     def handle_athena_record(_input_schema, _output_schema, record)
-      record.to_a
+      [record[0]]
     end
   end.new(event: {}, context: {})
 
   input_schema_1 = Arrow::Schema.new("0": :string)
   input_schema_bytes_1 = get_schema_bytes(input_schema_1)
-  Arrow::Schema.new(0.upto(100).map { |n| [n.to_s, :string] }.to_h)
-  input_schema_bytes_100 = get_schema_bytes(input_schema_1)
+  input_schema_100 = Arrow::Schema.new(0.upto(100).map { |n| [n.to_s, :string] }.to_h)
+  input_schema_bytes_100 = get_schema_bytes(input_schema_100)
+  output_schema = Arrow::Schema.new("0": :string)
+  output_schema_bytes = get_schema_bytes(output_schema)
 
   input_table1_1 = Arrow::Table.new(input_schema_1, [['FooBar']])
   input_records_bytes_1_1 = get_record_batch_bytes(input_schema_1, input_table1_1.each_record_batch.first)
@@ -27,13 +29,15 @@ Benchmark.bm 10 do |r|
     },
     'methodName' => 'lower',
     'outputSchema' => {
-      'schema' => Base64.strict_encode64(input_schema_bytes_1),
+      'schema' => Base64.strict_encode64(output_schema_bytes),
     },
     'functionType' => 'SCALAR',
   }
 
   r.report '1 record 1 column' do
-    instance.handle_udf_request(event_1_1)
+    100.times do
+      instance.handle_udf_request(event_1_1)
+    end
   end
 
   input_table100_1 = Arrow::Table.new(input_schema_1, [['FooBar']] * 100)
@@ -46,17 +50,19 @@ Benchmark.bm 10 do |r|
     },
     'methodName' => 'lower',
     'outputSchema' => {
-      'schema' => Base64.strict_encode64(input_schema_bytes_1),
+      'schema' => Base64.strict_encode64(output_schema_bytes),
     },
     'functionType' => 'SCALAR',
   }
 
   r.report '100 records 1 column' do
-    instance.handle_udf_request(event_100)
+    100.times do
+      instance.handle_udf_request(event_100)
+    end
   end
 
-  input_table1_100 = Arrow::Table.new(input_schema_1, [['FooBar']])
-  input_records_bytes_1_100 = get_record_batch_bytes(input_schema_1, input_table1_100.each_record_batch.first)
+  input_table1_100 = Arrow::Table.new(input_schema_100, [['FooBar'] * 100])
+  input_records_bytes_1_100 = get_record_batch_bytes(input_schema_100, input_table1_100.each_record_batch.first)
   event_1_100 = {
     '@type' => 'UserDefinedFunctionRequest',
     'inputRecords' => {
@@ -65,17 +71,19 @@ Benchmark.bm 10 do |r|
     },
     'methodName' => 'lower',
     'outputSchema' => {
-      'schema' => Base64.strict_encode64(input_schema_bytes_100),
+      'schema' => Base64.strict_encode64(output_schema_bytes),
     },
     'functionType' => 'SCALAR',
   }
 
   r.report '1 record 100 column' do
-    instance.handle_udf_request(event_1_100)
+    100.times do
+      instance.handle_udf_request(event_1_100)
+    end
   end
 
-  input_table_100_100 = Arrow::Table.new(input_schema_1, [['FooBar']])
-  input_records_bytes_100_100 = get_record_batch_bytes(input_schema_1, input_table_100_100.each_record_batch.first)
+  input_table_100_100 = Arrow::Table.new(input_schema_100, [['FooBar'] * 100] * 100)
+  input_records_bytes_100_100 = get_record_batch_bytes(input_schema_100, input_table_100_100.each_record_batch.first)
   event_100_100 = {
     '@type' => 'UserDefinedFunctionRequest',
     'inputRecords' => {
@@ -84,12 +92,14 @@ Benchmark.bm 10 do |r|
     },
     'methodName' => 'lower',
     'outputSchema' => {
-      'schema' => Base64.strict_encode64(input_schema_bytes_100),
+      'schema' => Base64.strict_encode64(output_schema_bytes),
     },
     'functionType' => 'SCALAR',
   }
 
   r.report '100 record 100 column' do
-    instance.handle_udf_request(event_100_100)
+    100.times do
+      instance.handle_udf_request(event_100_100)
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: athena-udf
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Daisuke Taniwaki
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-09-07 00:00:00.000000000 Z
+date: 2025-03-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: base64
@@ -42,16 +42,22 @@ dependencies:
   name: red-arrow
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 12.0.0
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 12.0.1
+        version: 20.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 12.0.0
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 12.0.1
+        version: 20.0.0
 description: ''
 email:
 - daisuketaniwaki@gmail.com
@@ -96,7 +102,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.19
+rubygems_version: 3.5.11
 signing_key:
 specification_version: 4
 summary: Ruby-version Athena UDF