async-grpc-xds 0.0.1

data/proto/xds/type/matcher/v3/ip.proto

@@ -0,0 +1,53 @@
+syntax = "proto3";
+
+package xds.type.matcher.v3;
+
+import "xds/annotations/v3/status.proto";
+import "xds/core/v3/cidr.proto";
+import "xds/type/matcher/v3/matcher.proto";
+
+import "validate/validate.proto";
+
+option java_package = "com.github.xds.type.matcher.v3";
+option java_outer_classname = "IPMatcherProto";
+option java_multiple_files = true;
+option go_package = "github.com/cncf/xds/go/xds/type/matcher/v3";
+
+option (xds.annotations.v3.file_status).work_in_progress = true;
+
+// [#protodoc-title: IP matcher]
+
+// Matches a specific IP address against a set of possibly overlapping subnets using a trie.
+message IPMatcher {
+  // Specifies a list of IP address ranges and a match action.
+  message IPRangeMatcher {
+    // A non-empty set of CIDR ranges.
+    repeated core.v3.CidrRange ranges = 1 [(validate.rules).repeated = {min_items: 1}];
+
+    // Match action to apply when the IP address is within one of the CIDR ranges.
+    Matcher.OnMatch on_match = 2;
+
+    // Indicates whether this match option should be considered if there is a
+    // more specific matcher. Exclusive matchers are not selected whenever a
+    // more specific matcher exists (e.g. matcher with a longer prefix) even
+    // when the more specific matcher fails its nested match condition.
+    // Non-exclusive matchers are considered if the more specific matcher
+    // exists but its nested match condition does not entirely match.
+    // Non-exclusive matchers are selected in the order of their specificity
+    // first (longest prefix first), then the order of declaration next.
+    //
+    // For example, consider two range matchers: an exclusive matcher *X* on
+    // ``0.0.0.0/0`` and a matcher *Y* on ``192.0.0.0/2`` with a nested match
+    // condition *Z*. For the input IP ``192.168.0.1`` matcher *Y* is the most
+    // specific. If its nested match condition *Z* does not accept the input,
+    // then the less specific matcher *X* does not apply either despite the
+    // input being within the range, because matcher *X* is exclusive.
+    //
+    // The opposite is true if matcher *X* is not marked as exclusive. In that
+    // case matcher *X* always matches whenever matcher "*Y* rejects the input.
+    bool exclusive = 3;
+  }
+
+  // Match IP address by CIDR ranges.
+  repeated IPRangeMatcher range_matchers = 1;
+}

data/proto/xds/type/matcher/v3/matcher.proto

@@ -0,0 +1,144 @@
+syntax = "proto3";
+
+package xds.type.matcher.v3;
+
+import "xds/core/v3/extension.proto";
+import "xds/type/matcher/v3/string.proto";
+
+import "validate/validate.proto";
+
+option java_package = "com.github.xds.type.matcher.v3";
+option java_outer_classname = "MatcherProto";
+option java_multiple_files = true;
+option go_package = "github.com/cncf/xds/go/xds/type/matcher/v3";
+
+// [#protodoc-title: Unified Matcher API]
+
+// A matcher, which may traverse a matching tree in order to result in a match action.
+// During matching, the tree will be traversed until a match is found, or if no match
+// is found the action specified by the most specific on_no_match will be evaluated.
+// As an on_no_match might result in another matching tree being evaluated, this process
+// might repeat several times until the final OnMatch (or no match) is decided.
+message Matcher {
+  // What to do if a match is successful.
+  message OnMatch {
+    oneof on_match {
+      option (validate.required) = true;
+
+      // Nested matcher to evaluate.
+      // If the nested matcher does not match and does not specify
+      // on_no_match, then this matcher is considered not to have
+      // matched, even if a predicate at this level or above returned
+      // true.
+      Matcher matcher = 1;
+
+      // Protocol-specific action to take.
+      core.v3.TypedExtensionConfig action = 2;
+    }
+
+    // If true and the Matcher matches, the action will be taken but the caller
+    // will behave as if the Matcher did not match. A subsequent matcher or
+    // on_no_match action will be used instead.
+    // This field is not supported in all contexts in which the matcher API is
+    // used. If this field is set in a context in which it's not supported,
+    // the resource will be rejected.
+    bool keep_matching = 3;
+  }
+
+  // A linear list of field matchers.
+  // The field matchers are evaluated in order, and the first match
+  // wins.
+  message MatcherList {
+    // Predicate to determine if a match is successful.
+    message Predicate {
+      // Predicate for a single input field.
+      message SinglePredicate {
+        // Protocol-specific specification of input field to match on.
+        // [#extension-category: envoy.matching.common_inputs]
+        core.v3.TypedExtensionConfig input = 1 [(validate.rules).message = {required: true}];
+
+        oneof matcher {
+          option (validate.required) = true;
+
+          // Built-in string matcher.
+          type.matcher.v3.StringMatcher value_match = 2;
+
+          // Extension for custom matching logic.
+          // [#extension-category: envoy.matching.input_matchers]
+          core.v3.TypedExtensionConfig custom_match = 3;
+        }
+      }
+
+      // A list of two or more matchers. Used to allow using a list within a oneof.
+      message PredicateList {
+        repeated Predicate predicate = 1 [(validate.rules).repeated = {min_items: 2}];
+      }
+
+      oneof match_type {
+        option (validate.required) = true;
+
+        // A single predicate to evaluate.
+        SinglePredicate single_predicate = 1;
+
+        // A list of predicates to be OR-ed together.
+        PredicateList or_matcher = 2;
+
+        // A list of predicates to be AND-ed together.
+        PredicateList and_matcher = 3;
+
+        // The invert of a predicate
+        Predicate not_matcher = 4;
+      }
+    }
+
+    // An individual matcher.
+    message FieldMatcher {
+      // Determines if the match succeeds.
+      Predicate predicate = 1 [(validate.rules).message = {required: true}];
+
+      // What to do if the match succeeds.
+      OnMatch on_match = 2 [(validate.rules).message = {required: true}];
+    }
+
+    // A list of matchers. First match wins.
+    repeated FieldMatcher matchers = 1 [(validate.rules).repeated = {min_items: 1}];
+  }
+
+  message MatcherTree {
+    // A map of configured matchers. Used to allow using a map within a oneof.
+    message MatchMap {
+      map<string, OnMatch> map = 1 [(validate.rules).map = {min_pairs: 1}];
+    }
+
+    // Protocol-specific specification of input field to match on.
+    core.v3.TypedExtensionConfig input = 1 [(validate.rules).message = {required: true}];
+
+    // Exact or prefix match maps in which to look up the input value.
+    // If the lookup succeeds, the match is considered successful, and
+    // the corresponding OnMatch is used.
+    oneof tree_type {
+      option (validate.required) = true;
+
+      MatchMap exact_match_map = 2;
+
+      // Longest matching prefix wins.
+      MatchMap prefix_match_map = 3;
+
+      // Extension for custom matching logic.
+      core.v3.TypedExtensionConfig custom_match = 4;
+    }
+  }
+
+  oneof matcher_type {
+    // A linear list of matchers to evaluate.
+    MatcherList matcher_list = 1;
+
+    // A match tree to evaluate.
+    MatcherTree matcher_tree = 2;
+  }
+
+  // Optional OnMatch to use if no matcher above matched (e.g., if there are no matchers specified
+  // above, or if none of the matches specified above succeeded).
+  // If no matcher above matched and this field is not populated, the match will be considered unsuccessful.
+  OnMatch on_no_match = 3;
+}

data/proto/xds/type/matcher/v3/range.proto

@@ -0,0 +1,69 @@
+syntax = "proto3";
+
+package xds.type.matcher.v3;
+
+import "xds/type/v3/range.proto";
+import "xds/type/matcher/v3/matcher.proto";
+
+import "validate/validate.proto";
+
+option java_package = "com.github.xds.type.matcher.v3";
+option java_outer_classname = "RangeProto";
+option java_multiple_files = true;
+option go_package = "github.com/cncf/xds/go/xds/type/matcher/v3";
+
+// [#protodoc-title: Range matcher]
+
+// Specifies a set of ranges for matching an int64 number and the associated
+// match actions.
+message Int64RangeMatcher {
+  // Specifies a list of number ranges and a match action.
+  message RangeMatcher {
+    // A non-empty set of int64 ranges.
+    repeated xds.type.v3.Int64Range ranges = 1
+        [(validate.rules).repeated = { min_items: 1 }];
+
+    // Match action to apply when the input number is within one of the ranges.
+    Matcher.OnMatch on_match = 2;
+  }
+
+  // Match a number by a list of number ranges. If multiple ranges contain the
+  // input number, then the first action in this list is taken.
+  repeated RangeMatcher range_matchers = 1;
+}
+
+// Specifies a set of ranges for matching an int32 number and the associated
+// match actions.
+message Int32RangeMatcher {
+  // Specifies a list of number ranges and a match action.
+  message RangeMatcher {
+    // A non-empty set of int32 ranges.
+    repeated xds.type.v3.Int32Range ranges = 1
+        [(validate.rules).repeated = { min_items: 1 }];
+
+    // Match action to apply when the input number is within one of the ranges.
+    Matcher.OnMatch on_match = 2;
+  }
+
+  // Match a number by a list of number ranges. If multiple ranges contain the
+  // input number, then the first action in this list is taken.
+  repeated RangeMatcher range_matchers = 1;
+}
+
+// Specifies a set of ranges for matching a double number and the associated
+// match actions.
+message DoubleRangeMatcher {
+  // Specifies a list of number ranges and a match action.
+  message RangeMatcher {
+    // A non-empty set of double ranges.
+    repeated xds.type.v3.DoubleRange ranges = 1
+        [(validate.rules).repeated = { min_items: 1 }];
+
+    // Match action to apply when the input number is within one of the ranges.
+    Matcher.OnMatch on_match = 2;
+  }
+
+  // Match a number by a list of number ranges. If multiple ranges contain the
+  // input number, then the first action in this list is taken.
+  repeated RangeMatcher range_matchers = 1;
+}

data/proto/xds/type/matcher/v3/regex.proto

@@ -0,0 +1,46 @@
+syntax = "proto3";
+
+package xds.type.matcher.v3;
+
+import "validate/validate.proto";
+
+option java_package = "com.github.xds.type.matcher.v3";
+option java_outer_classname = "RegexProto";
+option java_multiple_files = true;
+option go_package = "github.com/cncf/xds/go/xds/type/matcher/v3";
+
+// [#protodoc-title: Regex matcher]
+
+// A regex matcher designed for safety when used with untrusted input.
+message RegexMatcher {
+  // Google's `RE2 <https://github.com/google/re2>`_ regex engine. The regex
+  // string must adhere to the documented `syntax
+  // <https://github.com/google/re2/wiki/Syntax>`_. The engine is designed to
+  // complete execution in linear time as well as limit the amount of memory
+  // used.
+  //
+  // Envoy supports program size checking via runtime. The runtime keys
+  // `re2.max_program_size.error_level` and `re2.max_program_size.warn_level`
+  // can be set to integers as the maximum program size or complexity that a
+  // compiled regex can have before an exception is thrown or a warning is
+  // logged, respectively. `re2.max_program_size.error_level` defaults to 100,
+  // and `re2.max_program_size.warn_level` has no default if unset (will not
+  // check/log a warning).
+  //
+  // Envoy emits two stats for tracking the program size of regexes: the
+  // histogram `re2.program_size`, which records the program size, and the
+  // counter `re2.exceeded_warn_level`, which is incremented each time the
+  // program size exceeds the warn level threshold.
+  message GoogleRE2 {}
+
+  oneof engine_type {
+    option (validate.required) = true;
+
+    // Google's RE2 regex engine.
+    GoogleRE2 google_re2 = 1 [ (validate.rules).message = {required : true} ];
+  }
+
+  // The regex match string. The string must be supported by the configured
+  // engine.
+  string regex = 2 [ (validate.rules).string = {min_len : 1} ];
+}

data/proto/xds/type/matcher/v3/string.proto

@@ -0,0 +1,71 @@
+syntax = "proto3";
+
+package xds.type.matcher.v3;
+
+import "xds/core/v3/extension.proto";
+import "xds/type/matcher/v3/regex.proto";
+
+import "validate/validate.proto";
+
+option java_package = "com.github.xds.type.matcher.v3";
+option java_outer_classname = "StringProto";
+option java_multiple_files = true;
+option go_package = "github.com/cncf/xds/go/xds/type/matcher/v3";
+
+// [#protodoc-title: String matcher]
+
+// Specifies the way to match a string.
+// [#next-free-field: 9]
+message StringMatcher {
+  oneof match_pattern {
+    option (validate.required) = true;
+
+    // The input string must match exactly the string specified here.
+    //
+    // Examples:
+    //
+    // * *abc* only matches the value *abc*.
+    string exact = 1;
+
+    // The input string must have the prefix specified here.
+    // Note: empty prefix is not allowed, please use regex instead.
+    //
+    // Examples:
+    //
+    // * *abc* matches the value *abc.xyz*
+    string prefix = 2 [(validate.rules).string = {min_len: 1}];
+
+    // The input string must have the suffix specified here.
+    // Note: empty prefix is not allowed, please use regex instead.
+    //
+    // Examples:
+    //
+    // * *abc* matches the value *xyz.abc*
+    string suffix = 3 [(validate.rules).string = {min_len: 1}];
+
+    // The input string must match the regular expression specified here.
+    RegexMatcher safe_regex = 5 [(validate.rules).message = {required: true}];
+
+    // The input string must have the substring specified here.
+    // Note: empty contains match is not allowed, please use regex instead.
+    //
+    // Examples:
+    //
+    // * *abc* matches the value *xyz.abc.def*
+    string contains = 7 [(validate.rules).string = {min_len: 1}];
+
+    // Use an extension as the matcher type.
+    // [#extension-category: envoy.string_matcher]
+    xds.core.v3.TypedExtensionConfig custom = 8;
+  }
+
+  // If true, indicates the exact/prefix/suffix matching should be case insensitive. This has no
+  // effect for the safe_regex match.
+  // For example, the matcher *data* will match both input string *Data* and *data* if set to true.
+  bool ignore_case = 6;
+}
+
+// Specifies a list of ways to match a string.
+message ListStringMatcher {
+  repeated StringMatcher patterns = 1 [(validate.rules).repeated = {min_items: 1}];
+}

data/proto/xds/type/v3/cel.proto

@@ -0,0 +1,77 @@
+syntax = "proto3";
+
+package xds.type.v3;
+
+import "google/api/expr/v1alpha1/checked.proto";
+import "google/api/expr/v1alpha1/syntax.proto";
+import "cel/expr/checked.proto";
+import "cel/expr/syntax.proto";
+import "google/protobuf/wrappers.proto";
+
+import "xds/annotations/v3/status.proto";
+
+import "validate/validate.proto";
+
+option java_package = "com.github.xds.type.v3";
+option java_outer_classname = "CelProto";
+option java_multiple_files = true;
+option go_package = "github.com/cncf/xds/go/xds/type/v3";
+
+option (xds.annotations.v3.file_status).work_in_progress = true;
+
+// [#protodoc-title: Common Expression Language (CEL)]
+
+// Either parsed or checked representation of the `Common Expression Language
+// <https://github.com/google/cel-spec>`_ (CEL) program.
+message CelExpression {
+  oneof expr_specifier {
+    // Parsed expression in abstract syntax tree (AST) form.
+    //
+    // Deprecated -- use ``cel_expr_parsed`` field instead.
+    // If ``cel_expr_parsed`` or ``cel_expr_checked`` is set, this field is not used.
+    google.api.expr.v1alpha1.ParsedExpr parsed_expr = 1  [deprecated = true];
+
+    // Parsed expression in abstract syntax tree (AST) form that has been successfully type checked.
+    //
+    // Deprecated -- use ``cel_expr_checked`` field instead.
+    // If ``cel_expr_parsed`` or ``cel_expr_checked`` is set, this field is not used.
+    google.api.expr.v1alpha1.CheckedExpr checked_expr = 2  [deprecated = true];
+  }
+
+  // Parsed expression in abstract syntax tree (AST) form.
+  //
+  // If ``cel_expr_checked`` is set, this field is not used.
+  cel.expr.ParsedExpr cel_expr_parsed = 3;
+
+  // Parsed expression in abstract syntax tree (AST) form that has been successfully type checked.
+  //
+  // If set, takes precedence over ``cel_expr_parsed``.
+  cel.expr.CheckedExpr cel_expr_checked = 4;
+
+  // Unparsed expression in string form. For example, ``request.headers['x-env'] == 'prod'`` will
+  // get ``x-env`` header value and compare it with ``prod``.
+  // Check the `Common Expression Language <https://github.com/google/cel-spec>`_ for more details.
+  //
+  // If set, takes precedence over ``cel_expr_parsed`` and ``cel_expr_checked``.
+  string cel_expr_string = 5;
+}
+
+// Extracts a string by evaluating a `Common Expression Language
+// <https://github.com/google/cel-spec>`_ (CEL) expression against the standardized set of
+// :ref:`HTTP attributes <arch_overview_attributes>`.
+//
+// .. attention::
+//
+//   Besides CEL evaluation raising an error explicitly, CEL program returning a type other than
+//   the ``string``, or not returning anything, are considered an error as well.
+//
+// [#comment:TODO(sergiitk): When implemented, add the extension tag.]
+message CelExtractString {
+  // The CEL expression used to extract a string from the CEL environment.
+  // the "subject string") that should be replaced.
+  CelExpression expr_extract = 1 [(validate.rules).message = {required: true}];
+
+  // If CEL expression evaluates to an error, this value is be returned to the caller.
+  // If not set, the error is propagated to the caller.
+  google.protobuf.StringValue default_value = 2;
+}

data/proto/xds/type/v3/range.proto

@@ -0,0 +1,40 @@
+syntax = "proto3";
+
+package xds.type.v3;
+
+option java_package = "com.github.xds.type.v3";
+option java_outer_classname = "RangeProto";
+option java_multiple_files = true;
+option go_package = "github.com/cncf/xds/go/xds/type/v3";
+
+// [#protodoc-title: Number range]
+
+// Specifies the int64 start and end of the range using half-open interval
+// semantics [start, end).
+message Int64Range {
+  // start of the range (inclusive)
+  int64 start = 1;
+
+  // end of the range (exclusive)
+  int64 end = 2;
+}
+
+// Specifies the int32 start and end of the range using half-open interval
+// semantics [start, end).
+message Int32Range {
+  // start of the range (inclusive)
+  int32 start = 1;
+
+  // end of the range (exclusive)
+  int32 end = 2;
+}
+
+// Specifies the double start and end of the range using half-open interval
+// semantics [start, end).
+message DoubleRange {
+  // start of the range (inclusive)
+  double start = 1;
+
+  // end of the range (exclusive)
+  double end = 2;
+}

data/proto/xds/type/v3/typed_struct.proto

@@ -0,0 +1,44 @@
+syntax = "proto3";
+
+package xds.type.v3;
+
+option java_outer_classname = "TypedStructProto";
+option java_multiple_files = true;
+option java_package = "com.github.xds.type.v3";
+option go_package = "github.com/cncf/xds/go/xds/type/v3";
+
+import "google/protobuf/struct.proto";
+
+// A TypedStruct contains an arbitrary JSON serialized protocol buffer message with a URL that
+// describes the type of the serialized message. This is very similar to google.protobuf.Any,
+// instead of having protocol buffer binary, this employs google.protobuf.Struct as value.
+//
+// This message is intended to be embedded inside Any, so it shouldn't be directly referred
+// from other UDPA messages.
+//
+// When packing an opaque extension config, packing the expected type into Any is preferred
+// wherever possible for its efficiency. TypedStruct should be used only if a proto descriptor
+// is not available, for example if:
+//
+// - A control plane sends opaque message that is originally from external source in human readable
+//   format such as JSON or YAML.
+// - The control plane doesn't have the knowledge of the protocol buffer schema hence it cannot
+//   serialize the message in protocol buffer binary format.
+// - The DPLB doesn't have have the knowledge of the protocol buffer schema its plugin or extension
+//   uses. This has to be indicated in the DPLB capability negotiation.
+//
+// When a DPLB receives a TypedStruct in Any, it should:
+// - Check if the type_url of the TypedStruct matches the type the extension expects.
+// - Convert value to the type described in type_url and perform validation.
+//
+// TODO(lizan): Figure out how TypeStruct should be used with DPLB extensions that doesn't link
+// protobuf descriptor with DPLB itself, (e.g. gRPC LB Plugin, Envoy WASM extensions).
+message TypedStruct {
+  // A URL that uniquely identifies the type of the serialize protocol buffer message.
+  // This has same semantics and format described in google.protobuf.Any:
+  // https://github.com/protocolbuffers/protobuf/blob/master/src/google/protobuf/any.proto
+  string type_url = 1;
+
+  // A JSON representation of the above specified type.
+  google.protobuf.Struct value = 2;
+}

data/readme.md

@@ -0,0 +1,37 @@
+# Async::GRPC::XDS
+
+xDS support for `async-grpc` clients.
+
+This gem contains the experimental xDS implementation extracted from `async-grpc`, including Envoy xDS protobuf definitions, ADS discovery, CDS/EDS resource handling, and client-side load balancing.
+
+[![Development Status](https://github.com/socketry/async-grpc-xds/workflows/Test/badge.svg)](https://github.com/socketry/async-grpc-xds/actions?workflow=Test)
+
+## Usage
+
+Please see the [project documentation](https://socketry.github.io/async-grpc-xds/) for more details.
+
+## Status
+
+This is an early implementation focused on ADS with CDS and EDS. LDS/RDS, full routing semantics, NACK handling, locality weighting, and delta xDS are not complete yet.
+
+## Testing
+
+The `xds/` directory contains a Docker Compose integration environment with a Go xDS control plane and Ruby gRPC backends.
+
+``` bash
+docker-compose -f xds/docker-compose.yaml up --exit-code-from tests
+```
+
+## Releases
+
+Please see the [project releases](https://socketry.github.io/async-grpc-xds/releases/index) for all releases.
+
+### v0.1.0
+
+  - Initial extraction from `async-grpc`.
+
+## See Also
+
+  - [async-grpc](https://github.com/socketry/async-grpc)
+  - [protocol-grpc](https://github.com/socketry/protocol-grpc)
+  - [async-http](https://github.com/socketry/async-http)

data/releases.md

@@ -0,0 +1,5 @@
+# Releases
+
+## v0.1.0
+
+  - Initial extraction from `async-grpc`.

data/xds/Dockerfile.backend

@@ -0,0 +1,24 @@
+FROM ruby:latest
+
+WORKDIR /app
+ENV BUNDLE_GEMFILE=/app/gems.rb \
+    BUNDLE_WITHOUT=maintenance
+
+# Install dependencies
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy gemfile, gemspec, and minimal lib files needed for gemspec.
+COPY gems.rb *.gemspec ./
+COPY lib/async/grpc/xds/version.rb lib/async/grpc/xds/version.rb
+RUN bundle install
+
+# Copy application code
+COPY . .
+
+# Expose port
+EXPOSE ${PORT:-50051}
+
+# Run gRPC server
+CMD bundle exec ruby xds/backend_server.rb

data/xds/Dockerfile.control-plane

@@ -0,0 +1,22 @@
+FROM golang:1.21-alpine AS builder
+
+# Install build dependencies
+RUN apk add --no-cache git
+
+WORKDIR /build
+
+COPY go.mod go.sum ./
+RUN go mod download
+
+COPY test_server.go ./
+RUN go build -o /xds-test-server test_server.go
+
+FROM alpine:latest
+
+RUN apk add --no-cache netcat-openbsd ca-certificates
+
+COPY --from=builder /xds-test-server /xds-test-server
+
+EXPOSE 18000 18001
+
+CMD ["/xds-test-server", "-port", "18000"]