cel 0.2.3 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: da576b6bec07758040135b557e1e135952bee1e4e6bb4bc19768233b817bbe8f
-  data.tar.gz: bce3a0b0b7cd41113898fc7ebfaa1a5e3adcd94fc84b5791ef2bd9a6e33b4a60
+  metadata.gz: bcb8e069501a6c9cfdf6af67f83701111d67130d04a9208a05482d14ac1c519a
+  data.tar.gz: 0af115f5924dd63f926227fd916c2fe3f2b25bb6a68e682df1c9f5be36b1be6f
 SHA512:
-  metadata.gz: 88ad2004603260595e7995be27fd3f77d0e0de8a0c90e16569ce67c3018608530b1d0e2455e6a8a65949254818bdc698d3be8fea2c45866589eb153ddb3a7d4d
-  data.tar.gz: '028149b8a3f5b06c0ae5ebb02f3d09144ec42af89506971c3677f8671e8ccace55ee8b575fb1282f32a041872edee4e44f1fd9a8cb14e6889ba343794e60ae0e'
+  metadata.gz: d78064a96c11a026f1b6f16a3ebdfed6f3f528170975ab9c26b1e7438c7805419830418f32e7b6557bafeace936c35f1dc18f695a1294862780c2aff9e0bc335
+  data.tar.gz: 9897a6ed3a7708cefaf1b6bd24f86802b1c50709cbbb9bab1fd843e31e3dc1f1f049f4770afa0c34ed89c91b30442a95738b30eb52d91e64d99f1dbc880e0de9

data/CHANGELOG.md

@@ -1,5 +1,22 @@
 ## [Unreleased]
 
+## [0.3.0] - 2025-06-12
+
+### Features
+
+* Integration with `google-protobuf` (optional dependency) by evaluating messages to protobuf stubs, and making them an integral part of the CEL type set. Some examples:
+  * auto-conversion of protobuf wrapper types.
+  * enums support
+
+### Improvements
+
+* More correct parsing, checking and evaluation of CEL expressions, thanks to the issues identified by conformance tests.
+
+### Chore
+
+* Incorporation of the oficial cel-spec conformance tests into the test suite, to ensure feature correctness.
+* `bigdecimal` added as explicit dependency.
+
 ## [0.2.3] - 2023-09-19
 
 ### Bugfixes

data/README.md

@@ -107,9 +107,23 @@ env2 = environment(foo: -> (a, b) { a + b})
 env2.evaluate("foo(2, 2)") #=> 4
 ```
 
+## Spec Coverage
+
+`cel` is tested against the conformance suite from the [cel-spec repository](https://github.com/google/cel-spec/tree/master/conformance), and supports all features from the language except:
+
+* math extensions
+* string extensions
+* bindings extensions
+* block extensions
+* encoders extensions
+* comprehensions V2 API
+* optionals
+
+If this is something you're interested in (helping out), add a mention in the corresponding issue (or create one when non is available already).
+
 ## Supported Rubies
 
-All Rubies greater or equal to 2.5, and always latest JRuby and Truffleruby.
+All Rubies greater or equal to 2.7, and always latest JRuby and Truffleruby.
 
 ## Development
 
@@ -118,8 +132,16 @@ Clone the repo in your local machine, where you have `ruby` installed. Then you
 ```bash
 # install dev dependencies
 > bundle install
+# create protobuf stubs for tests
+> git clone --depth 1 https://github.com/google/cel-spec.git
+> git clone --depth 1 https://github.com/googleapis/googleapis.git
+> bundle exec rake build_test_protos
 # run tests
 > bundle exec rake test
+# build protobuf stubs for conformance tests
+> bundle exec rake build_conformance_protos
+# run conformance tests
+> bundle exec rake conformance
 ```
 
 To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).

data/lib/cel/ast/elements/protobuf.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "google/protobuf/descriptor_pb"
 require "google/protobuf/struct_pb"
 require "google/protobuf/wrappers_pb"
 require "google/protobuf/any_pb"
@@ -9,94 +10,287 @@ module Cel
   module Protobuf
     module_function
 
-    def convert_from_protobuf(msg)
+    def base_class
+      Google::Protobuf::MessageExts
+    end
+
+    def map_class
+      Google::Protobuf::Map
+    end
+
+    def timestamp_class
+      Google::Protobuf::Timestamp
+    end
+
+    def duration_class
+      Google::Protobuf::Duration
+    end
+
+    def enum_class
+      Enum
+    end
+
+    def try_convert_from_wrapper(msg)
       case msg
       when Google::Protobuf::Any
+        raise EvaluateError, "error conversion for empty Any" unless msg.type_name
 
-        any.unpack(type_msg).to_ruby
+        type_msg = Object.const_get(Cel.package_to_module_name(msg.type_name))
+        try_convert_from_wrapper(msg.unpack(type_msg))
       when Google::Protobuf::ListValue
         msg.to_a
       when Google::Protobuf::Struct
         msg.to_h
       when Google::Protobuf::Value
-        msg.to_ruby
+        msg.to_ruby(true)
+      when Google::Protobuf::BytesValue
+        Cel::Bytes.new(msg.value.bytes)
+      when Google::Protobuf::Int32Value,
+          Google::Protobuf::Int64Value
+        Cel::Number.new(:int, msg.value)
+      when Google::Protobuf::UInt32Value,
+        Google::Protobuf::UInt64Value
+        Cel::Number.new(:uint, msg.value)
+      when Google::Protobuf::DoubleValue,
+            Google::Protobuf::FloatValue
+        Cel::Number.new(:double, msg.value)
       when Google::Protobuf::BoolValue,
-           Google::Protobuf::BytesValue,
-           Google::Protobuf::DoubleValue,
-           Google::Protobuf::FloatValue,
-           Google::Protobuf::Int32Value,
-           Google::Protobuf::Int64Value,
-           Google::Protobuf::UInt32Value,
-           Google::Protobuf::UInt64Value,
            Google::Protobuf::NullValue,
            Google::Protobuf::StringValue
+        # conversion to cel type won't be ambiguous here
         msg.value
       when Google::Protobuf::Timestamp
         msg.to_time
       when Google::Protobuf::Duration
         Cel::Duration.new(seconds: msg.seconds, nanos: msg.nanos)
       else
-        raise Error, "#{msg.class}: protobuf to cel unsupported"
+        msg.extend(CelComparisonMode)
+        msg
+      end
+    end
+
+    def try_convert_from_wrapper_type(*)
+      TYPES[:map]
+    end
+
+    def convert_to_proto_type(type, package)
+      proto_type = Cel.package_to_module_name(type)
+
+      return unless proto_type
+
+      return Object.const_get(proto_type) if Object.const_defined?(proto_type)
+
+      return unless package
+
+      return unless package.const_defined?(proto_type)
+
+      package.const_get(proto_type)
+    end
+
+    def convert_to_cel_type(proto, package)
+      case proto
+      when Google::Protobuf::FieldDescriptor
+
+        cel_type =
+          case proto.type
+          when :int32, :int64, :sint32, :sint64, :sfixed32, :sfixed64, :enum
+            TYPES[:int]
+          when :uint32, :uint64, :fixed32, :fixed64
+            TYPES[:uint]
+          when :float, :double
+            TYPES[:double]
+          when :bool
+            TYPES[:bool]
+          when :string
+            TYPES[:string]
+          when :bytes
+            TYPES[:bytes]
+          when :message
+            proto_type = convert_to_proto_type(proto.submsg_name, package)
+
+            proto_type = PROTO_TO_CEL_TYPE[proto_type] if PROTO_TO_CEL_TYPE.key?(proto_type)
+            proto_type
+          end
+
+        if proto.label == :repeated
+          return TYPES[:list, cel_type] unless proto.subtype
+
+          case proto.subtype.options
+          when Google::Protobuf::EnumOptions
+            return TYPES[:list, cel_type]
+          when Google::Protobuf::MessageOptions
+            # protobuf doesn't have a way to identify whether this field descriptors is from a map,
+            # nor the types for key and values.
+            return TYPES[:map, cel_type] if proto.subtype.options.map_entry
+          end
+
+          return TYPES[:list, cel_type]
+        end
+
+        cel_type
+      else
+        PROTO_TO_CEL_TYPE[proto]
       end
     end
 
-    def convert_from_type(type, value)
-      case type
-      when "Any", "google.protobuf.Any"
-        type_url = value[Identifier.new("type_url")].value
-        _, type_msg = type_url.split("/", 2)
-        type_msg = const_get(type_msg.split(".").map(&:capitalize).join("::"))
-        encoded_msg = value[Identifier.new("value")].value.pack("C*")
-        any = Google::Protobuf::Any.new(type_url: type_url, value: encoded_msg)
-        value = Literal.to_cel_type(any.unpack(type_msg).to_ruby)
-      when "ListValue", "google.protobuf.ListValue"
-        value = value.nil? ? List.new([]) : value[Identifier.new("values")]
-      when "Struct", "google.protobuf.Struct"
-        value = value.nil? ? Map.new({}) : value[Identifier.new("fields")]
-      when "Value", "google.protobuf.Value"
-        return Null.new if value.nil?
-
-        key = value.keys.first
-
-        value = value.fetch(key, value)
-
-        value = case key
-                when "null_value"
-                  Null.new if value.zero?
-                when "number_value"
-                  value = -value.operands.first if value.is_a?(Operation) && value.op == "-" && value.unary?
-                  Number.new(:double, value)
-                else
-                  value
+    def convert_to_proto(proto_type, values, convert_value_to_proto: false)
+      return values.unpack(proto_type) if values.is_a?(Google::Protobuf::Any)
+
+      if proto_type == Google::Protobuf::Struct
+        values = values[:fields] if values.key?(:fields)
+
+        raise EvaluateError, "bad key type" unless values.all? { |k, _| k.is_a?(::String) }
+
+        proto_type.from_hash(values)
+      elsif proto_type == Google::Protobuf::ListValue
+        if values.is_a?(Hash) && values.key?(:values)
+          proto_type.from_a(values[:values])
+        else
+          proto_type.from_a(values)
+        end
+      elsif proto_type == Google::Protobuf::NullValue
+        nil
+      elsif proto_type == Google::Protobuf::Value
+        case values
+        when Google::Protobuf::Duration
+          proto_type.new(string_value: "#{values.seconds}#{if values.nanos.positive?
+                                                             ".#{values.nanos / 1_000_000_000.0}"
+                                                           end}s")
+        when Google::Protobuf::Timestamp
+          proto_type.new(string_value: values.to_time.utc.iso8601(9))
+        when Hash
+          if values.key?(:struct_value)
+
+            raise EvaluateError, "bad key type" unless values[:struct_value].all? { |k, _| k.is_a?(::String) }
+
+            values[:struct_value] = Google::Protobuf::Struct.from_hash(values[:struct_value])
+          elsif values.key?(:list_value)
+            values[:list_value] = Google::Protobuf::ListValue.from_a(values[:list_value])
+          elsif values.empty?
+            if convert_value_to_proto
+              values = { struct_value: Google::Protobuf::Struct.from_hash(values) }
+            else
+              values[:null_value] = Google::Protobuf::NullValue::NULL_VALUE
+            end
+          elsif values.any? { |k, _| proto_type.descriptor.lookup(k.to_s).nil? }
+            raise EvaluateError, "bad key type" unless values.all? { |k, _| k.is_a?(::String) }
+
+            values = { struct_value: Google::Protobuf::Struct.from_hash(values) }
+          end
+          proto_type.new(values)
+        when Google::Protobuf::BoolValue
+          proto_type.new(bool_value: values.value)
+        when Google::Protobuf::DoubleValue,
+             Google::Protobuf::FloatValue,
+             Google::Protobuf::Int32Value,
+             Google::Protobuf::UInt32Value
+          proto_type.new(number_value: values.value)
+        when Google::Protobuf::Int64Value,
+             Google::Protobuf::UInt64Value
+          if values.value >= MAX_INT - 1
+            proto_type.new(string_value: values.value.to_s)
+          else
+            proto_type.new(number_value: values.value)
+          end
+        when Google::Protobuf::StringValue
+          proto_type.new(string_value: values.value)
+        when Google::Protobuf::BytesValue
+          proto_type.new(string_value: [values.value].pack("m0"))
+        when Google::Protobuf::NullValue
+          proto_type.new(null_value: Google::Protobuf::NullValue::NULL_VALUE)
+        when Google::Protobuf::Struct
+          proto_type.new(struct_value: values.value)
+        when Google::Protobuf::ListValue
+          proto_type.new(list_value: values.value)
+        when Google::Protobuf::Empty
+          proto_type.from_ruby({})
+        when Google::Protobuf::FieldMask
+          proto_type.from_ruby(values.paths.join(","))
+        else
+          proto_type.from_ruby(values)
+        end
+      elsif proto_type == Google::Protobuf::Any
+        case values
+        when Array
+          proto_type.pack(Google::Protobuf::ListValue.from_a(values))
+        when Hash
+          # type_url: "", value: ""
+          # TODO: consider pattern matching
+          if convert_value_to_proto
+            proto_type.pack(Google::Protobuf::Struct.from_hash(values))
+          else
+            proto_type.new(values)
+          end
+        when base_class
+          proto_type.pack(values)
+        else
+          proto_type.pack(Google::Protobuf::Value.from_ruby(values))
+        end
+      elsif proto_type == Google::Protobuf::Timestamp
+        case values
+        when Time
+          proto_type.from_time(values)
+        when nil
+          nil
+        else
+          proto_type.new(values)
+        end
+      elsif proto_type == Google::Protobuf::Duration
+        case values
+        when Numeric
+          seconds, nanos = values.divmod(1)
+          nanos *= 1_000_000_000
+          proto_type.new(seconds: seconds, nanos: nanos)
+        when nil
+          nil
+        else
+          proto_type.new(values)
+        end
+      elsif values.is_a?(Hash)
+        values = values.to_h do |key, value|
+          field_descriptor = proto_type.descriptor.lookup(key.to_s)
+          if value.nil? && (NILLABLE_WRAPPERS.include?(field_descriptor.subtype.msgclass) ||
+                   proto_type.descriptor.enum_for(:each_oneof).one? do |oneof|
+                     oneof.enum_for(:each).include?(field_descriptor)
+                   end)
+
+            next([key, value])
+          end
+
+          next([key, value]) unless field_descriptor && field_descriptor.type != :enum
+
+          field_subtype = field_descriptor.subtype
+
+          next([key, value]) unless field_subtype
+
+          [
+            key,
+            # discard protobuf enums as well
+            value.is_a?(field_subtype.msgclass) ?
+            value :
+            convert_to_proto(field_subtype.msgclass, value, convert_value_to_proto: true),
+          ]
         end
-      when "BoolValue", "google.protobuf.BoolValue"
-        value = value.nil? ? Bool.new(false) : value[Identifier.new("value")]
-      when "BytesValue", "google.protobuf.BytesValue"
-        value = value[Identifier.new("value")]
-      when "DoubleValue", "google.protobuf.DoubleValue",
-           "FloatValue", "google.protobuf.FloatValue"
-        value = value.nil? ? Number.new(:double, 0.0) : value[Identifier.new("value")]
-        value.type = TYPES[:double]
-      when "Int32Value", "google.protobuf.Int32Value",
-           "Int64Value", "google.protobuf.Int64Value"
-        value = value.nil? ? Number.new(:int, 0) : value[Identifier.new("value")]
-      when "Uint32Value", "google.protobuf.UInt32Value",
-           "Uint64Value", "google.protobuf.UInt64Value"
-        value = value.nil? ? Number.new(:uint, 0) : value[Identifier.new("value")]
-      when "NullValue", "google.protobuf.NullValue"
-        value = Null.new
-      when "StringValue", "google.protobuf.StringValue"
-        value = value.nil? ? String.new(+"") : value[Identifier.new("value")]
-      when "Timestamp", "google.protobuf.Timestamp"
-        seconds = value.fetch(Identifier.new("seconds"), 0)
-        nanos = value.fetch(Identifier.new("nanos"), 0)
-        value = Timestamp.new(Time.at(seconds, nanos, :nanosecond))
-      when "Duration", "google.protobuf.Duration"
-        seconds = value.fetch(Identifier.new("seconds"), 0)
-        nanos = value.fetch(Identifier.new("nanos"), 0)
-        value = Duration.new(seconds: seconds, nanos: nanos)
+
+        # protobuf sets maps as abstract classes, and there's apparently no way to identify them
+        # regardless, since their type is "message". Checking for ruby class name is the best
+        # proxy I could come up with.
+        return values unless proto_type.name
+
+        proto_type.new(values)
+      elsif values.nil?
+        Google::Protobuf::Value.from_ruby(values)
+      else
+        proto_type.new(value: values)
+      end
+    end
+
+    def convert_to_enum(value, proto_type = nil)
+      if proto_type
+        Enum.new(value, proto_type)
+      else
+        Enum.new(value)
       end
-      value
     end
 
     def try_invoke_from(var, func, args)
@@ -129,5 +323,156 @@ module Cel
         Literal.to_cel_type(value)
       end
     end
+
+    # finds value for +attribute+ declared in the +proto+.
+    #
+    # if none is found, and if +attribute+ is another proto message, then return an instance of the type.
+    # if field is an enum,
+    def lookup(proto, attribute)
+      value = proto.public_send(attribute)
+
+      field = proto.class.descriptor.lookup(attribute.to_s)
+
+      case field.label
+      when :repeated
+        case value
+        when Google::Protobuf::Map
+          value.to_h { |k, v| [k, convert_from_proto_field_type(field, v)] } # rubocop:disable Style/HashTransformValues
+        else
+          value.map { |v| convert_from_proto_field_type(field, v) }
+        end
+      else
+        convert_from_proto_field_type(field, value)
+      end
+    end
+
+    def convert_from_proto_field_type(field, value)
+      if value.nil? && (field.type == :message)
+        subtype = field.subtype.msgclass
+
+        if NILLABLE_WRAPPERS.include?(subtype) ||
+           subtype == Google::Protobuf::Value
+          # if a wrapper type, ignore null
+        else
+          value = subtype.new
+        end
+      end
+
+      case field.type
+      when :message
+        value
+      when :enum
+        enum_type = const_get(Cel.package_to_module_name(field.submsg_name))
+
+        return Enum.new(value, enum_type) if value.is_a?(Integer)
+
+        enum_mod = Cel.package_to_module_name(field.submsg_name)
+
+        Enum.new(const_get(enum_mod).resolve(value), enum_type)
+      when :int32, :int64, :sint32, :sint64, :sfixed32, :sfixed64
+        return unless value
+
+        Number.new(:int, value)
+      when :uint32, :uint64, :fixed32, :fixed64
+        return unless value
+
+        Number.new(:uint, value)
+      when :float, :double
+        return unless value
+
+        Number.new(:double, value)
+      when :bool
+        return unless value
+
+        Bool.cast(value)
+      when :string
+        return unless value
+
+        String.new(value)
+      when :bytes
+        return unless value
+
+        Bytes.new(value)
+      end
+    end
+
+    module CelComparisonMode
+      def ==(other)
+        super || begin
+          return false unless other.is_a?(Protobuf.base_class)
+
+          a1 = self
+          a2 = other
+
+          if a1.is_a?(Google::Protobuf::Any) && !a1.type_url.empty?
+            a1 = a1.unpack(Object.const_get(Cel.package_to_module_name(a1.type_name)))
+            a1.extend(CelComparisonMode)
+          end
+
+          if a2.is_a?(Google::Protobuf::Any) && !a2.type_url.empty?
+            a2 = a2.unpack(Object.const_get(Cel.package_to_module_name(a2.type_name)))
+          end
+
+          return false unless a1.class == a2.class
+
+          if a1.is_a?(Google::Protobuf::Any) && a1.type_url.empty?
+            a2.is_a?(Google::Protobuf::Any) && a2.type_url.empty?
+            return a1.value == a2.value
+          end
+
+          a1.class.descriptor.each do |field|
+            f1 = a1.public_send(field.name)
+            f1.extend(CelComparisonMode) if f1.is_a?(Protobuf.base_class)
+            f2 = a2.public_send(field.name)
+            return false unless f1 == f2
+          end
+          true
+        end
+      end
+    end
+
+    class Enum < Cel::Number
+      # protocol buffer enum fields can accept any signed 32-bit number, values outside that range will raise an error.
+      MAX_ENUM_INT = (2**31)
+
+      def initialize(value, enum_type = nil)
+        super(:int, value)
+        @enum_type = enum_type
+        check_overflow(@value)
+      end
+
+      def enum_type
+        @enum_type || type
+      end
+
+      private
+
+      def check_overflow(value)
+        raise EvaluateError, "return error for overflow" unless (-(MAX_ENUM_INT - 1)...MAX_ENUM_INT).cover?(value)
+      end
+    end
+
+    PROTO_TO_CEL_TYPE = {
+      Google::Protobuf::Any => TYPES[:any],
+      Google::Protobuf::Value => TYPES[:any],
+      Google::Protobuf::ListValue => TYPES[:list],
+      Google::Protobuf::Struct => TYPES[:map],
+      Google::Protobuf::BoolValue => TYPES[:bool],
+      Google::Protobuf::BytesValue => TYPES[:bytes],
+      Google::Protobuf::DoubleValue => TYPES[:double],
+      Google::Protobuf::FloatValue => TYPES[:double],
+      Google::Protobuf::Int32Value => TYPES[:int],
+      Google::Protobuf::Int64Value => TYPES[:int],
+      Google::Protobuf::UInt32Value => TYPES[:uint],
+      Google::Protobuf::UInt64Value => TYPES[:uint],
+      Google::Protobuf::NullValue => TYPES[:null],
+      Google::Protobuf::StringValue => TYPES[:string],
+      Google::Protobuf::Timestamp => TYPES[:timestamp],
+      Google::Protobuf::Duration => TYPES[:duration],
+    }.freeze
+
+    NILLABLE_WRAPPERS = (PROTO_TO_CEL_TYPE.keys - [Google::Protobuf::Value, Google::Protobuf::ListValue,
+                                                   Google::Protobuf::Struct, Google::Protobuf::Duration,
+                                                   Google::Protobuf::Timestamp]).freeze
   end
 end