spark-connect 0.3.0 → 0.4.0

data/lib/spark_connect/tvf.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module SparkConnect
+  # Built-in table-valued functions, returned by {SparkSession#tvf}. Mirrors
+  # PySpark's `spark.tvf` (`TableValuedFunction`). Arguments are {Column}s or
+  # column names; wrap literals with {Functions.lit}.
+  #
+  # @example
+  #   spark.tvf.explode(F.array(F.lit(1), F.lit(2))).show
+  #   spark.tvf.sql_keywords.limit(5).collect
+  class TableValuedFunction
+    Proto = SparkConnect::Proto
+
+    # @param session [SparkSession]
+    def initialize(session)
+      @session = session
+    end
+
+    # Same as {SparkSession#range}.
+    # @return [DataFrame]
+    def range(start, end_ = nil, step = 1, num_partitions = nil)
+      @session.range(start, end_, step, num_partitions)
+    end
+
+    %w[explode explode_outer inline inline_outer posexplode posexplode_outer variant_explode
+       variant_explode_outer].each do |name|
+      define_method(name) { |input| call(name, input) }
+    end
+
+    # @!method explode(collection)
+    #   One row per element of an array (or entry of a map). @return [DataFrame]
+    # @!method explode_outer(collection)
+    #   Like {#explode}, but a null/empty input yields one row of nulls. @return [DataFrame]
+    # @!method inline(input)
+    #   One row per struct of an array of structs. @return [DataFrame]
+    # @!method inline_outer(input)
+    #   Like {#inline}, keeping null/empty inputs. @return [DataFrame]
+    # @!method posexplode(collection)
+    #   Like {#explode} with the element position. @return [DataFrame]
+    # @!method posexplode_outer(collection)
+    #   Like {#posexplode}, keeping null/empty inputs. @return [DataFrame]
+    # @!method variant_explode(input)
+    #   One row per element/field of a VARIANT array/object. @return [DataFrame]
+    # @!method variant_explode_outer(input)
+    #   Like {#variant_explode}, keeping null/empty inputs. @return [DataFrame]
+
+    # The given fields of a JSON string, one column each.
+    # @return [DataFrame]
+    def json_tuple(input, *fields)
+      raise IllegalArgumentError, "json_tuple requires at least one field" if fields.empty?
+
+      call("json_tuple", input, *fields)
+    end
+
+    # Separate `fields` into `n` rows.
+    # @return [DataFrame]
+    def stack(n, *fields) = call("stack", n, *fields)
+
+    # @return [DataFrame] the collations supported by the server.
+    def collations = call("collations")
+
+    # @return [DataFrame] the SQL keywords of the server.
+    def sql_keywords = call("sql_keywords")
+
+    # @return [DataFrame] the Python worker logs (when enabled on the server).
+    def python_worker_logs = call("python_worker_logs")
+
+    private
+
+    def call(name, *args)
+      tvf = Proto::UnresolvedTableValuedFunction.new(
+        function_name: name, arguments: args.map { |a| Functions._col(a).to_expr }
+      )
+      relation = PlanBuilder.relation(@session, unresolved_table_valued_function: tvf)
+      DataFrame.new(@session, PlanBuilder.with_subquery_references(@session, relation))
+    end
+  end
+end

data/lib/spark_connect/types.rb

@@ -149,6 +149,104 @@ module SparkConnect
       def to_proto = Types.wrap(timestamp_ntz: Proto::DataType::TimestampNTZ.new)
     end
 
+    # Shared behaviour of the nanosecond-capable timestamp types (Spark 4.3+,
+    # a preview behind `spark.sql.timestampNanosTypes.enabled`).
+    class TimestampNanosBase < DataType
+      MIN_PRECISION = 7
+      MAX_PRECISION = 9
+
+      # @return [Integer] fractional-second digits, 7..9 (9 = nanoseconds).
+      attr_reader :precision
+
+      def initialize(precision = MAX_PRECISION)
+        super()
+        unless precision.is_a?(Integer) && precision.between?(MIN_PRECISION, MAX_PRECISION)
+          raise IllegalArgumentError, "#{self.class.name.split('::').last} precision must be 7, 8, or 9, got #{precision.inspect}"
+        end
+
+        @precision = precision
+      end
+
+      def simple_string = "#{type_name}(#{precision})"
+      def json_value = simple_string
+      def ==(other) = other.instance_of?(self.class) && other.precision == precision
+      def hash = [self.class, precision].hash
+    end
+
+    # TIMESTAMP_NTZ(p): a zone-less local date-time with nanosecond-capable precision.
+    class TimestampNTZNanosType < TimestampNanosBase
+      def type_name = "timestamp_ntz"
+      def to_proto = Types.wrap(timestamp_ntz_nanos: Proto::DataType::TimestampNTZNanos.new(precision: precision))
+    end
+
+    # TIMESTAMP_LTZ(p): an instant with nanosecond-capable precision.
+    class TimestampLTZNanosType < TimestampNanosBase
+      def type_name = "timestamp_ltz"
+      def to_proto = Types.wrap(timestamp_ltz_nanos: Proto::DataType::TimestampLTZNanos.new(precision: precision))
+    end
+
+    # TIME(p): a time of day without a date or time zone (Spark 4.1+).
+    class TimeType < DataType
+      # @return [Integer] fractional-second digits, 0..6.
+      attr_reader :precision
+
+      def initialize(precision = 6)
+        super()
+        @precision = precision
+      end
+
+      def simple_string = "time(#{precision})"
+      def type_name = "time"
+      def json_value = simple_string
+      def to_proto = Types.wrap(time: Proto::DataType::Time.new(precision: precision))
+      def ==(other) = other.is_a?(TimeType) && other.precision == precision
+      def hash = [TimeType, precision].hash
+    end
+
+    # Shared behaviour of the spatial types (Spark 4.1+). An SRID of
+    # {MIXED_SRID} (`"ANY"`) allows values of any reference system.
+    class SpatialBase < DataType
+      MIXED_SRID = -1
+
+      # @return [Integer]
+      attr_reader :srid
+
+      def initialize(srid)
+        super()
+        @srid = srid.to_s.casecmp?("any") ? MIXED_SRID : Integer(srid)
+      end
+
+      def simple_string = srid == MIXED_SRID ? "#{type_name}(any)" : "#{type_name}(#{srid})"
+      def ==(other) = other.instance_of?(self.class) && other.srid == srid
+      def hash = [self.class, srid].hash
+
+      private
+
+      # The coordinate reference system id used by Spark's JSON schema form.
+      def crs
+        case srid
+        when MIXED_SRID then "SRID:ANY"
+        when 4326 then "OGC:CRS84"
+        when 0 then "SRID:0"
+        else "EPSG:#{srid}"
+        end
+      end
+    end
+
+    # GEOMETRY(srid): a planar (Cartesian) spatial value.
+    class GeometryType < SpatialBase
+      def type_name = "geometry"
+      def json_value = "geometry(#{crs})"
+      def to_proto = Types.wrap(geometry: Proto::DataType::Geometry.new(srid: srid))
+    end
+
+    # GEOGRAPHY(srid): a geodesic (spherical) spatial value.
+    class GeographyType < SpatialBase
+      def type_name = "geography"
+      def json_value = "geography(#{crs}, SPHERICAL)"
+      def to_proto = Types.wrap(geography: Proto::DataType::Geography.new(srid: srid))
+    end
+
     class VariantType < DataType
       def type_name = "variant"
       def to_proto = Types.wrap(variant: Proto::DataType::Variant.new)
@@ -254,6 +352,33 @@ module SparkConnect
       def to_proto = Types.wrap(calendar_interval: Proto::DataType::CalendarInterval.new)
     end
 
+    # A user-defined type, described by its class name(s) and the SQL type it
+    # is stored as. Values decode as their `sql_type`.
+    class UserDefinedType < DataType
+      attr_reader :jvm_class, :python_class, :sql_type
+
+      def initialize(sql_type, jvm_class: nil, python_class: nil)
+        super()
+        @sql_type = sql_type
+        @jvm_class = jvm_class
+        @python_class = python_class
+      end
+
+      def simple_string = sql_type ? sql_type.simple_string : "udt"
+      def type_name = "udt"
+
+      def to_proto
+        udt = Proto::DataType::UDT.new(type: "udt")
+        udt.jvm_class = jvm_class if jvm_class
+        udt.python_class = python_class if python_class
+        udt.sql_type = sql_type.to_proto if sql_type
+        Types.wrap(udt: udt)
+      end
+
+      def ==(other) = other.is_a?(UserDefinedType) && other.sql_type == sql_type && other.jvm_class == jvm_class
+      def hash = [UserDefinedType, sql_type, jvm_class].hash
+    end
+
     # An array type. `element_type` is the type of every element; `contains_null`
     # indicates whether the array may contain `null` values.
     class ArrayType < DataType
@@ -440,6 +565,21 @@ module SparkConnect
     def date = DateType.new
     def timestamp = TimestampType.new
     def timestamp_ntz = TimestampNTZType.new
+    # @param precision [Integer] fractional-second digits, 7..9 (Spark 4.3 preview).
+    # @return [TimestampNTZNanosType]
+    def timestamp_ntz_nanos(precision = 9) = TimestampNTZNanosType.new(precision)
+    # @param precision [Integer] fractional-second digits, 7..9 (Spark 4.3 preview).
+    # @return [TimestampLTZNanosType]
+    def timestamp_ltz_nanos(precision = 9) = TimestampLTZNanosType.new(precision)
+    # @param precision [Integer] fractional-second digits, 0..6 (Spark 4.1+).
+    # @return [TimeType]
+    def time(precision = 6) = TimeType.new(precision)
+    # @param srid [Integer, String] spatial reference id, or `"ANY"` (Spark 4.1+).
+    # @return [GeometryType]
+    def geometry(srid = 4326) = GeometryType.new(srid)
+    # @param srid [Integer, String] spatial reference id, or `"ANY"` (Spark 4.1+).
+    # @return [GeographyType]
+    def geography(srid = 4326) = GeographyType.new(srid)
     def variant = VariantType.new
     def decimal(precision = 10, scale = 0) = DecimalType.new(precision, scale)
     def array(element_type, contains_null: true) = ArrayType.new(element_type, contains_null: contains_null)
@@ -468,6 +608,15 @@ module SparkConnect
       when :date then DateType.new
       when :timestamp then TimestampType.new
       when :timestamp_ntz then TimestampNTZType.new
+      when :timestamp_ntz_nanos then TimestampNTZNanosType.new(sub.has_precision? ? sub.precision : 9)
+      when :timestamp_ltz_nanos then TimestampLTZNanosType.new(sub.has_precision? ? sub.precision : 9)
+      when :time then TimeType.new(sub.has_precision? ? sub.precision : 6)
+      when :geometry then GeometryType.new(sub.srid)
+      when :geography then GeographyType.new(sub.srid)
+      when :udt
+        UserDefinedType.new(sub.has_sql_type? ? from_proto(sub.sql_type) : nil,
+                            jvm_class: sub.has_jvm_class? ? sub.jvm_class : nil,
+                            python_class: sub.has_python_class? ? sub.python_class : nil)
       when :variant then VariantType.new
       when :calendar_interval then CalendarIntervalType.new
       when :day_time_interval then DayTimeIntervalType.new(sub.start_field || 0, sub.end_field || 3)

data/lib/spark_connect/version.rb

@@ -2,10 +2,10 @@
 
 module SparkConnect
   # The released version of the spark-connect gem.
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
 
   # The Apache Spark version whose Spark Connect protocol definitions this
   # client is generated against. The client aims to be wire-compatible with
   # Spark Connect servers of this major/minor line and newer.
-  SPARK_VERSION = "4.1.0"
+  SPARK_VERSION = "4.3.0"
 end

data/lib/spark_connect/writer.rb

@@ -31,6 +31,8 @@ module SparkConnect
       @sort_columns = []
       @bucket_cols = nil
       @num_buckets = nil
+      @clustering_columns = []
+      @schema_evolution = false
     end
 
     # @return [self] set the output format.
@@ -40,25 +42,48 @@ module SparkConnect
     end
 
     # @return [self] set the save mode (`:append`, `:overwrite`, `:ignore`,
-    #   `:error`).
+    #   `:error`). `nil` keeps the current mode.
     def mode(save_mode)
+      return self if save_mode.nil?
+
       @mode = SAVE_MODES.fetch(save_mode.to_s.downcase.to_sym) do
         raise IllegalArgumentError, "Unknown save mode: #{save_mode}"
       end
       self
     end
 
-    # @return [self] set a write option.
+    # @return [self] set a write option (a `nil` value is ignored).
     def option(key, value)
-      @options[key.to_s] = value.to_s
+      @options[key.to_s] = value.to_s unless value.nil?
       self
     end
 
     # @return [self] set multiple write options.
     def options(opts)
-      opts.each { |k, v| @options[k.to_s] = v.to_s }
+      opts.each { |k, v| option(k, v) }
+      self
+    end
+
+    # @return [self] cluster the output (e.g. liquid clustering) by these columns.
+    def cluster_by(*cols)
+      cols = cols.flatten.map(&:to_s)
+      raise IllegalArgumentError, "cluster_by needs one or more clustering columns" if cols.empty?
+
+      @clustering_columns = cols
+      self
+    end
+    alias clusterBy cluster_by
+
+    # Enable automatic schema evolution for this write (Spark 4.2+); the target
+    # table must declare the `AUTOMATIC_SCHEMA_EVOLUTION` capability. This is
+    # the Scala API's `withSchemaEvolution` (PySpark exposes it only on
+    # {MergeIntoWriter}).
+    # @return [self]
+    def with_schema_evolution
+      @schema_evolution = true
       self
     end
+    alias withSchemaEvolution with_schema_evolution
 
     # @return [self] partition the output by these columns.
     def partition_by(*cols)
@@ -115,13 +140,27 @@ module SparkConnect
     def csv(path) = format("csv").save(path)
     def orc(path) = format("orc").save(path)
     def text(path) = format("text").save(path)
+    # @return [void] convenience for `format("xml").save(path)` (set `option("rowTag", ...)`).
+    def xml(path) = format("xml").save(path)
+
+    # Write to a JDBC table.
+    #
+    # @param url [String]
+    # @param table [String]
+    # @param mode [Symbol, String, nil] the save mode.
+    # @param properties [Hash] connection properties (`user`, `password`, ...).
+    # @return [void]
+    def jdbc(url, table, mode = nil, properties = {})
+      format("jdbc").mode(mode).options(properties.transform_keys(&:to_s)).options("url" => url, "dbtable" => table).save
+    end
 
     private
 
     def base_operation
       op = WO.new(
         input: @df.relation, mode: @mode, options: @options,
-        partitioning_columns: @partitioning_columns, sort_column_names: @sort_columns
+        partitioning_columns: @partitioning_columns, sort_column_names: @sort_columns,
+        clustering_columns: @clustering_columns, with_schema_evolution: @schema_evolution
       )
       op.source = @source if @source
       op.bucket_by = WO::BucketBy.new(bucket_column_names: @bucket_cols, num_buckets: @num_buckets) if @num_buckets
@@ -153,6 +192,8 @@ module SparkConnect
       @options = {}
       @table_properties = {}
       @partitioning = []
+      @clustering_columns = []
+      @schema_evolution = false
     end
 
     # @return [self] set the table provider/format.
@@ -161,9 +202,15 @@ module SparkConnect
       self
     end
 
-    # @return [self] set a write option.
+    # @return [self] set a write option (a `nil` value is ignored).
     def option(key, value)
-      @options[key.to_s] = value.to_s
+      @options[key.to_s] = value.to_s unless value.nil?
+      self
+    end
+
+    # @return [self] set multiple write options.
+    def options(opts)
+      opts.each { |k, v| option(k, v) }
       self
     end
 
@@ -172,12 +219,32 @@ module SparkConnect
       @table_properties[key.to_s] = value.to_s
       self
     end
+    alias tableProperty table_property
 
-    # @return [self] partition by the given expressions/columns.
-    def partition_by(*cols)
+    # Partition the table by columns or partition transforms (`F.years("ts")`,
+    # `F.bucket(4, "id")`, ...).
+    # @return [self]
+    def partitioned_by(*cols)
       @partitioning = cols.flatten.map { |c| (c.is_a?(Column) ? c : Functions.col(c.to_s)).to_expr }
       self
     end
+    alias partition_by partitioned_by
+    alias partitionedBy partitioned_by
+
+    # @return [self] cluster the table by these columns.
+    def cluster_by(*cols)
+      @clustering_columns = cols.flatten.map(&:to_s)
+      self
+    end
+    alias clusterBy cluster_by
+
+    # Enable automatic schema evolution for this write (Spark 4.2+).
+    # @return [self]
+    def with_schema_evolution
+      @schema_evolution = true
+      self
+    end
+    alias withSchemaEvolution with_schema_evolution
 
     # Create the table. @return [void]
     def create = run(:MODE_CREATE)
@@ -185,19 +252,22 @@ module SparkConnect
     def replace = run(:MODE_REPLACE)
     # Create or replace the table. @return [void]
     def create_or_replace = run(:MODE_CREATE_OR_REPLACE)
+    alias createOrReplace create_or_replace
     # Append rows. @return [void]
     def append = run(:MODE_APPEND)
     # Overwrite rows matching `condition`. @return [void]
     def overwrite(condition) = run(:MODE_OVERWRITE, overwrite_condition: Column.to_col(condition).to_expr)
     # Dynamically overwrite partitions. @return [void]
     def overwrite_partitions = run(:MODE_OVERWRITE_PARTITIONS)
+    alias overwritePartitions overwrite_partitions
 
     private
 
     def run(mode, overwrite_condition: nil)
       op = WO2.new(
         input: @df.relation, table_name: @table, mode: mode,
-        options: @options, table_properties: @table_properties, partitioning_columns: @partitioning
+        options: @options, table_properties: @table_properties, partitioning_columns: @partitioning,
+        clustering_columns: @clustering_columns, with_schema_evolution: @schema_evolution
       )
       op.provider = @provider if @provider
       op.overwrite_condition = overwrite_condition if overwrite_condition

data/lib/spark_connect.rb

@@ -9,6 +9,7 @@ require_relative "spark_connect/types"
 require_relative "spark_connect/row"
 require_relative "spark_connect/plan"
 require_relative "spark_connect/column"
+require_relative "spark_connect/storage_level"
 require_relative "spark_connect/window"
 require_relative "spark_connect/functions"
 require_relative "spark_connect/arrow"
@@ -25,6 +26,9 @@ require_relative "spark_connect/streaming"
 require_relative "spark_connect/pipelines"
 require_relative "spark_connect/catalog"
 require_relative "spark_connect/data_frame"
+require_relative "spark_connect/table_arg"
+require_relative "spark_connect/merge"
+require_relative "spark_connect/tvf"
 require_relative "spark_connect/session"
 
 # spark-connect is a pure-Ruby client for {https://spark.apache.org/docs/latest/spark-connect-overview.html

data/proto/spark/connect/base.proto

@@ -491,6 +491,12 @@ message ExecutePlanResponse {
     repeated Expression.Literal values = 2;
     repeated string keys = 3;
     int64 plan_id = 4;
+    // (Optional) The index of the root error in errors.
+    // The field will not be set if there are no errors.
+    optional int32 root_error_idx = 5;
+    // A list of errors that occurred while collecting the observed metrics.
+    // If the length is 0, it means no errors occurred.
+    repeated FetchErrorDetailsResponse.Error errors = 6;
   }
 
   message ResultComplete {
@@ -1213,11 +1219,94 @@ message CloneSessionResponse {
 
   // Session id of the new cloned session.
   string new_session_id = 3;
-  
+
   // Server-side session ID of the new cloned session.
   string new_server_side_session_id = 4;
 }
 
+// Next ID: 6
+message GetStatusRequest {
+  // (Required)
+  //
+  // The session_id specifies a Spark session for a user identified by user_context.user_id.
+  // The id should be an UUID string of the format `00112233-4455-6677-8899-aabbccddeeff`
+  string session_id = 1;
+
+  // (Required)
+  //
+  // user_context.user_id and session_id both identify a unique remote spark session on the
+  // server side.
+  UserContext user_context = 2;
+
+  // (Optional)
+  //
+  // Provides optional information about the client sending the request. This field
+  // can be used for language or version specific information and is only intended for
+  // logging purposes and will not be interpreted by the server.
+  optional string client_type = 3;
+
+  // (Optional)
+  //
+  // Server-side generated idempotency key from the previous responses (if any). Server
+  // can use this to validate that the server side session has not changed.
+  optional string client_observed_server_side_session_id = 4;
+
+  // (Optional)
+  //
+  // Get status of operations in the session.
+  optional OperationStatusRequest operation_status = 5;
+
+  // Extension point for custom status request types.
+  repeated google.protobuf.Any extensions = 999;
+
+  message OperationStatusRequest {
+    // Get status of operations with these operation_ids.
+    // If unset or empty, returns status of all operations in the session.
+    repeated string operation_ids = 1;
+
+    // Extension point for custom operation-level status requests.
+    repeated google.protobuf.Any extensions = 999;
+  }
+}
+
+// Next ID: 4
+message GetStatusResponse {
+  // Session id of the session for which the status was requested.
+  string session_id = 1;
+
+  // Server-side generated idempotency key that the client can use to assert that the server side
+  // session has not changed.
+  string server_side_session_id = 2;
+
+  // Status information about requested operations.
+  repeated OperationStatus operation_statuses = 3;
+
+  // Extension point for custom status response types.
+  repeated google.protobuf.Any extensions = 999;
+
+  // Status information for a single operation.
+  message OperationStatus {
+    // The operation_id of the operation.
+    string operation_id = 1;
+
+    // The current status of the operation.
+    OperationState state = 2;
+
+    // Extension point for custom operation-level status fields.
+    repeated google.protobuf.Any extensions = 999;
+
+    enum OperationState {
+      OPERATION_STATE_UNSPECIFIED = 0;
+      OPERATION_STATE_UNKNOWN = 1;
+      OPERATION_STATE_RUNNING = 2;
+      OPERATION_STATE_TERMINATING = 3;
+      OPERATION_STATE_SUCCEEDED = 4;
+      OPERATION_STATE_FAILED = 5;
+      OPERATION_STATE_CANCELLED = 6;
+    }
+  }
+}
+
 // Main interface for the SparkConnect service.
 service SparkConnectService {
 
@@ -1272,4 +1361,7 @@ service SparkConnectService {
   // The request can optionally specify a custom session ID for the cloned session (must be
   // a valid UUID). If not provided, a new UUID will be generated automatically.
   rpc CloneSession(CloneSessionRequest) returns (CloneSessionResponse) {}
+
+  // Get status information of different types.
+  rpc GetStatus(GetStatusRequest) returns (GetStatusResponse) {}
 }

data/proto/spark/connect/catalog.proto

@@ -55,6 +55,16 @@ message Catalog {
     CurrentCatalog current_catalog = 24;
     SetCurrentCatalog set_current_catalog = 25;
     ListCatalogs list_catalogs = 26;
+    DropTable drop_table = 27;
+    DropView drop_view = 28;
+    CreateDatabase create_database = 29;
+    DropDatabase drop_database = 30;
+    ListPartitions list_partitions = 31;
+    ListViews list_views = 32;
+    GetTableProperties get_table_properties = 33;
+    GetCreateTableString get_create_table_string = 34;
+    TruncateTable truncate_table = 35;
+    AnalyzeTable analyze_table = 36;
   }
 }
 
@@ -241,3 +251,74 @@ message ListCatalogs {
   // (Optional) The pattern that the catalog name needs to match
   optional string pattern = 1;
 }
+
+// See `spark.catalog.dropTable`
+message DropTable {
+  // (Required)
+  string table_name = 1;
+  bool if_exists = 2;
+  bool purge = 3;
+}
+
+// See `spark.catalog.dropView`
+message DropView {
+  // (Required)
+  string view_name = 1;
+  bool if_exists = 2;
+}
+
+// See `spark.catalog.createDatabase`
+message CreateDatabase {
+  // (Required)
+  string db_name = 1;
+  bool if_not_exists = 2;
+  map<string, string> properties = 3;
+}
+
+// See `spark.catalog.dropDatabase`
+message DropDatabase {
+  // (Required)
+  string db_name = 1;
+  bool if_exists = 2;
+  bool cascade = 3;
+}
+
+// See `spark.catalog.listPartitions`
+message ListPartitions {
+  // (Required)
+  string table_name = 1;
+}
+
+// See `spark.catalog.listViews`
+message ListViews {
+  // (Optional)
+  optional string db_name = 1;
+  // (Optional) The pattern that the view name needs to match
+  optional string pattern = 2;
+}
+
+// See `spark.catalog.getTableProperties`
+message GetTableProperties {
+  // (Required)
+  string table_name = 1;
+}
+
+// See `spark.catalog.getCreateTableString`
+message GetCreateTableString {
+  // (Required)
+  string table_name = 1;
+  bool as_serde = 2;
+}
+
+// See `spark.catalog.truncateTable`
+message TruncateTable {
+  // (Required)
+  string table_name = 1;
+}
+
+// See `spark.catalog.analyzeTable`
+message AnalyzeTable {
+  // (Required)
+  string table_name = 1;
+  bool no_scan = 2;
+}

data/proto/spark/connect/commands.proto

@@ -144,6 +144,9 @@ message WriteOperation {
   // (Optional) Columns used for clustering the table.
   repeated string clustering_columns = 10;
 
+  // (Optional) Whether schema evolution is enabled for the write.
+  bool with_schema_evolution = 11;
+
   message SaveTable {
     // (Required) The table name.
     string table_name = 1;
@@ -211,6 +214,9 @@ message WriteOperationV2 {
 
   // (Optional) Columns used for clustering the table.
   repeated string clustering_columns = 9;
+
+  // (Optional) Whether schema evolution is enabled for the write.
+  bool with_schema_evolution = 10;
 }
 
 // Starts write stream operation as streaming query. Query ID and Run ID of the streaming
@@ -232,6 +238,7 @@ message WriteStreamOperationStart {
     bool available_now = 6;
     bool once = 7;
     string continuous_checkpoint_interval = 8;
+    string real_time_batch_duration = 100;
   }
 
   string output_mode = 9;