spark-connect 0.3.0 → 0.4.0

data/proto/spark/connect/expressions.proto

@@ -205,6 +205,11 @@ message Expression {
 
       SpecializedArray specialized_array = 25;
       Time time = 26;
+
+      // Nanosecond-capable timestamp literals (precision 7..9). NTZ and LTZ are distinct
+      // arms so the literal kind is self-describing.
+      TimestampNTZNanos timestamp_ntz_nanos = 29;
+      TimestampLTZNanos timestamp_ltz_nanos = 30;
     }
 
     // Reserved for Geometry and Geography.
@@ -289,6 +294,29 @@ message Expression {
       // The precision of this time, if omitted, uses the default value of MICROS_PRECISION.
       optional int32 precision = 2;
     }
+
+    // A TIMESTAMP_NTZ literal with nanosecond-capable precision. The physical value is carried
+    // as microseconds since the UNIX epoch plus the extra nanoseconds within that microsecond,
+    // because a single int64 of nanoseconds cannot span the supported year range.
+    message TimestampNTZNanos {
+      // Microseconds since the UNIX epoch (without timezone information).
+      int64 epoch_micros = 1;
+      // Additional nanoseconds within epoch_micros, in [0, 999].
+      int32 nanos_within_micro = 2;
+      // Number of fractional-second digits (7, 8, or 9). If omitted, defaults to 9 (nanoseconds).
+      optional int32 precision = 3;
+    }
+
+    // A TIMESTAMP_LTZ literal with nanosecond-capable precision. See TimestampNTZNanos for the
+    // rationale behind the two-component physical value.
+    message TimestampLTZNanos {
+      // Microseconds since the UNIX epoch.
+      int64 epoch_micros = 1;
+      // Additional nanoseconds within epoch_micros, in [0, 999].
+      int32 nanos_within_micro = 2;
+      // Number of fractional-second digits (7, 8, or 9). If omitted, defaults to 9 (nanoseconds).
+      optional int32 precision = 3;
+    }
   }
 
   // An unresolved attribute that is not explicitly bound to a specific column, but the column

data/proto/spark/connect/pipelines.proto

@@ -22,6 +22,7 @@ package spark.connect;
 import "google/protobuf/any.proto";
 import "google/protobuf/timestamp.proto";
 import "spark/connect/common.proto";
+import "spark/connect/expressions.proto";
 import "spark/connect/relations.proto";
 import "spark/connect/types.proto";
 
@@ -40,6 +41,8 @@ message PipelineCommand {
     DefineSqlGraphElements define_sql_graph_elements = 6;
     GetQueryFunctionExecutionSignalStream get_query_function_execution_signal_stream = 7;
     DefineFlowQueryFunctionResult define_flow_query_function_result = 8;
+    ExecuteOutputFlows execute_output_flows = 9;
+
     // Reserved field for protocol extensions.
     // Used to support forward-compatibility by carrying additional command types
     // that are not yet defined in this version of the proto. During planning, the
@@ -142,6 +145,7 @@ message PipelineCommand {
 
     oneof details {
       WriteRelationFlowDetails relation_flow_details = 7;
+      AutoCdcFlowDetails auto_cdc_flow_details = 10;
       google.protobuf.Any extension = 999;
     }
 
@@ -152,6 +156,55 @@ message PipelineCommand {
       optional spark.connect.Relation relation = 1;
     }
 
+    // Details for Auto CDC flows.
+    message AutoCdcFlowDetails {
+      // The name of the CDC source to stream from.
+      optional string source = 1;
+
+      // Column(s) that uniquely identify a row in source and target data.
+      repeated Expression keys = 2;
+
+      // Expression to order the source data.
+      optional Expression sequence_by = 3;
+
+      // Delete condition for the merged operation.
+      optional Expression apply_as_deletes = 6;
+
+      // Truncate condition for the merged operation.
+      optional Expression apply_as_truncates = 7;
+
+      // Columns included in the output table.
+      repeated Expression column_list = 8;
+
+      // Columns excluded from the output table.
+      repeated Expression except_column_list = 9;
+
+      // SCD Type for target table.
+      SCDType stored_as_scd_type = 10;
+
+      // SCD2 only. Columns whose value change opens a new history record. When empty, every
+      // eligible selected user column is tracked.
+      repeated Expression track_history_column_list = 11;
+
+      // SCD2 only. Columns excluded from history tracking. Mutually exclusive with
+      // track_history_column_list.
+      repeated Expression track_history_except_column_list = 12;
+
+      // Subset of columns to ignore null in updates.
+      repeated Expression ignore_null_updates_column_list = 14;
+
+      // Subset of columns excluded from ignoring null in updates.
+      repeated Expression ignore_null_updates_except_column_list = 15;
+
+    }
+
+    // SCD Type for Auto CDC target tables.
+    enum SCDType {
+      SCD_TYPE_UNSPECIFIED = 0;
+      SCD_TYPE_1 = 1;
+      SCD_TYPE_2 = 2;
+    }
+
     // If true, define the flow as a one-time flow, such as for backfill.
     // Set to true changes the flow in two ways:
     //   - The flow is run one time by default. If the pipeline is ran with a full refresh,
@@ -165,6 +218,25 @@ message PipelineCommand {
     }
   }
 
+  // Request to execute all flows for a single output (dataset or sink) remotely.
+  message ExecuteOutputFlows {
+
+    // The output (table or materialized view or sink) definition.
+    optional DefineOutput define_output = 1;
+
+    // The flows to execute for this table.
+    repeated DefineFlow define_flows = 2;
+
+    // Whether to perform a full refresh instead of an incremental update.
+    optional bool full_refresh = 3;
+
+    // Storage location for pipeline checkpoints and metadata.
+    optional string storage = 4;
+
+    // Reserved field for protocol extensions.
+    repeated google.protobuf.Any extension = 999;
+  }
+
   // Resolves all datasets and flows and start a pipeline update. Should be called after all
   // graph elements are registered.
   message StartRun {
@@ -213,8 +285,13 @@ message PipelineCommand {
   // Request from the client to update the flow function evaluation result
   // for a previously un-analyzed flow.
   message DefineFlowQueryFunctionResult {
-    // The fully qualified name of the flow being updated.
-    optional string flow_name = 1;
+    // (Deprecated) The fully qualified name of the flow being updated.
+    //
+    // This field is deprecated since Spark 4.2+. Use flow_identifier field instead.
+    optional string flow_name = 1 [deprecated = true];
+
+    // The fully qualified identifier of the flow being updated.
+    optional ResolvedIdentifier flow_identifier = 4;
 
     // The ID of the graph this flow belongs to.
     optional string dataflow_graph_id = 2;
@@ -290,7 +367,13 @@ message SourceCodeLocation {
 // A signal from the server to the client to execute the query function for one or more flows, and
 // to register their results with the server.
 message PipelineQueryFunctionExecutionSignal {
-  repeated string flow_names = 1;
+  // (Deprecated) The name of flows that are ready to be re-evaluated.
+  //
+  // This field is deprecated since Spark 4.2+. Use flow_identifiers field instead.
+  repeated string flow_names = 1 [deprecated = true];
+
+  // The identifier of flows that are ready to be re-evaluated
+  repeated ResolvedIdentifier flow_identifiers = 2;
 }
 
 // Metadata providing context about the pipeline during Spark Connect query analysis.
@@ -299,8 +382,12 @@ message PipelineAnalysisContext {
   optional string dataflow_graph_id = 1;
   // The path of the top-level pipeline file determined at runtime during pipeline initialization.
   optional string definition_path = 2;
-  // The name of the Flow involved in this analysis
-  optional string flow_name = 3;
+  // (Deprecated) The name of the Flow involved in this analysis
+  //
+  // This field is deprecated since Spark 4.2+. Use flow_identifier field instead.
+  optional string flow_name = 3 [deprecated = true];
+  // The identifier of the Flow involved in this analysis
+  optional ResolvedIdentifier flow_identifier = 4;
 
   // Reserved field for protocol extensions.
   repeated google.protobuf.Any extension = 999;

data/proto/spark/connect/relations.proto

@@ -81,6 +81,9 @@ message Relation {
     UnresolvedTableValuedFunction unresolved_table_valued_function = 43;
     LateralJoin lateral_join = 44;
     ChunkedCachedLocalRelation chunked_cached_local_relation = 45;
+    RelationChanges relation_changes = 46;
+    NearestByJoin nearest_by_join = 47;
+    Zip zip = 48;
 
     // NA functions
     NAFill fill_na = 90;
@@ -256,9 +259,33 @@ message Read {
     //
     // This is only supported by the JDBC data source.
     repeated string predicates = 5;
+
+    // (Optional) A user-provided name for the streaming source.
+    // This name is used in checkpoint metadata and enables stable checkpoint locations
+    // for source evolution.
+    optional string source_name = 6;
   }
 }
 
+// Reads Change Data Capture (CDC) changes for a named table.
+//
+// This corresponds to the `DataFrameReader.changes()` or `DataStreamReader.changes()` API.
+// CDC-specific options (startingVersion, endingVersion, startingTimestamp, endingTimestamp,
+// deduplicationMode, computeUpdates, etc.) are passed in the options map.
+message RelationChanges {
+  // (Required) Unparsed identifier for the table.
+  string unparsed_identifier = 1;
+
+  // Options for the CDC query. The map key is case insensitive.
+  // Supported keys include: startingVersion, endingVersion, startingTimestamp,
+  // endingTimestamp, deduplicationMode, computeUpdates, startingBoundInclusive,
+  // endingBoundInclusive.
+  map<string, string> options = 2;
+
+  // (Optional) Indicates if this is a streaming CDC read.
+  bool is_streaming = 3;
+}
+
 // Projection of a bag of expressions for a given input relation.
 //
 // The input relation must be specified.
@@ -1184,12 +1211,13 @@ message Parse {
   // (Optional) DataType representing the schema. If not set, Spark will infer the schema.
   optional DataType schema = 3;
 
-  // Options for the csv/json parser. The map key is case insensitive.
+  // Options for the csv/json/xml parser. The map key is case insensitive.
   map<string, string> options = 4;
   enum ParseFormat {
     PARSE_FORMAT_UNSPECIFIED = 0;
     PARSE_FORMAT_CSV = 1;
     PARSE_FORMAT_JSON = 2;
+    PARSE_FORMAT_XML = 3;
   }
 }
 
@@ -1250,3 +1278,45 @@ message LateralJoin {
   // (Required) The join type.
   Join.JoinType join_type = 4;
 }
+
+// Relation of type [[NearestByJoin]].
+//
+// For each row on the left side, returns up to `num_results` rows from the right side ranked
+// by `ranking_expression`.
+message NearestByJoin {
+  // (Required) Left (query) input relation.
+  Relation left = 1;
+
+  // (Required) Right (base) input relation.
+  Relation right = 2;
+
+  // (Required) Scalar expression used to rank candidate rows on the right side.
+  Expression ranking_expression = 3;
+
+  // (Required) Maximum number of matches per left row. Must be between 1 and 100000.
+  int32 num_results = 4;
+
+  // The following three fields use `string` (not typed enums) for parity with `AsOfJoin`,
+  // which models analogous fields the same way. Validation happens server-side at planning time.
+
+  // (Required) The join type. Must be one of: "inner", "leftouter".
+  string join_type = 5;
+
+  // (Required) Search algorithm contract. Must be one of: "approx", "exact".
+  string mode = 6;
+
+  // (Required) Ranking direction. Must be one of: "distance", "similarity".
+  string direction = 7;
+}
+
+// Relation of type [[Zip]].
+//
+// Combines the columns of two DataFrames side-by-side. Both DataFrames must produce the same
+// canonicalized plan after stripping outer Project chains.
+message Zip {
+  // (Required) Left input relation.
+  Relation left = 1;
+
+  // (Required) Right input relation.
+  Relation right = 2;
+}

data/proto/spark/connect/types.proto

@@ -76,6 +76,11 @@ message DataType {
     Unparsed unparsed = 24;
 
     Time time = 28;
+
+    // Nanosecond-capable timestamp types (precision 7..9). NTZ and LTZ are distinct kinds
+    // even though their physical value is identical, mirroring timestamp vs timestamp_ntz.
+    TimestampNTZNanos timestamp_ntz_nanos = 29;
+    TimestampLTZNanos timestamp_ltz_nanos = 30;
   }
 
   message Boolean {
@@ -136,6 +141,20 @@ message DataType {
     uint32 type_variation_reference = 2;
   }
 
+  // Timestamp without time zone with nanosecond-capable fractional-second precision.
+  message TimestampNTZNanos {
+    // Number of fractional-second digits. Valid values are 7, 8, and 9. If omitted, defaults to 9.
+    optional int32 precision = 1;
+    uint32 type_variation_reference = 2;
+  }
+
+  // Timestamp with local time zone with nanosecond-capable fractional-second precision.
+  message TimestampLTZNanos {
+    // Number of fractional-second digits. Valid values are 7, 8, and 9. If omitted, defaults to 9.
+    optional int32 precision = 1;
+    uint32 type_variation_reference = 2;
+  }
+
   message CalendarInterval {
     uint32 type_variation_reference = 1;
   }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: spark-connect
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Hyukjin Kwon
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-06-15 00:00:00.000000000 Z
+date: 2026-11-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: google-protobuf
@@ -86,6 +86,7 @@ files:
 - lib/spark_connect/errors.rb
 - lib/spark_connect/functions.rb
 - lib/spark_connect/grouped_data.rb
+- lib/spark_connect/merge.rb
 - lib/spark_connect/na_functions.rb
 - lib/spark_connect/observation.rb
 - lib/spark_connect/pipelines.rb
@@ -106,7 +107,10 @@ files:
 - lib/spark_connect/row.rb
 - lib/spark_connect/session.rb
 - lib/spark_connect/stat_functions.rb
+- lib/spark_connect/storage_level.rb
 - lib/spark_connect/streaming.rb
+- lib/spark_connect/table_arg.rb
+- lib/spark_connect/tvf.rb
 - lib/spark_connect/types.rb
 - lib/spark_connect/version.rb
 - lib/spark_connect/window.rb