impala 0.1.0

data/thrift/Partitions.thrift

@@ -0,0 +1,41 @@
+// Copyright 2012 Cloudera Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+namespace cpp impala
+namespace java com.cloudera.impala.thrift
+namespace rb Impala.Protocol
+
+include "Exprs.thrift"
+
+enum TPartitionType {
+  UNPARTITIONED,
+
+  // round-robin partitioning
+  RANDOM,
+
+  // unordered partitioning on a set of exprs
+  HASH_PARTITIONED,
+
+  // ordered partitioning on a list of exprs
+  RANGE_PARTITIONED
+}
+
+// Specification of how a single logical data stream is partitioned.
+// This leaves out the parameters that determine the physical partitioning (for hash
+// partitioning, the number of partitions; for range partitioning, the partitions'
+// boundaries), which need to be specified by the enclosing structure/context.
+struct TDataPartition {
+  1: required TPartitionType type
+  2: optional list<Exprs.TExpr> partitioning_exprs
+}

data/thrift/PlanNodes.thrift

@@ -0,0 +1,184 @@
+// Copyright 2012 Cloudera Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//
+// This file contains all structs, enums, etc., that together make up
+// a plan tree. All information recorded in struct TPlan and below is independent
+// of the execution parameters of any one of the backends on which it is running
+// (those are recorded in TPlanFragmentExecParams).
+
+namespace cpp impala
+namespace java com.cloudera.impala.thrift
+namespace rb Impala.Protocol
+
+include "Exprs.thrift"
+include "Types.thrift"
+
+enum TPlanNodeType {
+  HDFS_SCAN_NODE,
+  HBASE_SCAN_NODE,
+  HASH_JOIN_NODE,
+  AGGREGATION_NODE,
+  SORT_NODE,
+  EXCHANGE_NODE,
+  MERGE_NODE
+}
+
+// The information contained in subclasses of ScanNode captured in two separate
+// Thrift structs:
+// - TScanRange: the data range that's covered by the scan (which varies with the
+//   particular partition of the plan fragment of which the scan node is a part)
+// - T<subclass>: all other operational parameters that are the same across
+//   all plan fragments
+
+// Specification of subsection of a single hdfs file.
+struct THdfsFileSplit {
+  // file path
+  1: required string path
+
+  // starting offset
+  2: required i64 offset
+
+  // length of split
+  3: required i64 length
+
+  // ID of partition in parent THdfsScanNode. Meaningful only
+  // in the context of a single THdfsScanNode, may not be unique elsewhere.
+  4: required i64 partition_id
+}
+
+// key range for single THBaseScanNode
+// TODO: does 'binary' have an advantage over string? strings can
+// already store binary data
+struct THBaseKeyRange {
+  // inclusive
+  1: optional string startKey
+
+  // exclusive
+  2: optional string stopKey
+}
+
+// Specification of an individual data range which is held in its entirety
+// by a storage server
+struct TScanRange {
+  // one of these must be set for every TScanRange2
+  1: optional THdfsFileSplit hdfs_file_split
+  2: optional THBaseKeyRange hbase_key_range
+}
+
+struct THdfsScanNode {
+  1: required Types.TTupleId tuple_id
+}
+
+struct THBaseFilter {
+  1: required string family
+  2: required string qualifier
+  // Ordinal number into enum HBase CompareFilter.CompareOp.
+  // We don't use TExprOperator because the op is interpreted by an HBase Filter, and
+  // not the c++ expr eval.
+  3: required i32 op_ordinal
+  4: required string filter_constant
+}
+
+struct THBaseScanNode {
+  1: required Types.TTupleId tuple_id
+
+  // TODO: remove this, we already have THBaseTable.tableName
+  2: required string table_name
+
+  3: optional list<THBaseFilter> filters
+}
+
+struct TEqJoinCondition {
+  // left-hand side of "<a> = <b>"
+  1: required Exprs.TExpr left;
+  // right-hand side of "<a> = <b>"
+  2: required Exprs.TExpr right;
+}
+
+enum TJoinOp {
+  INNER_JOIN,
+  LEFT_OUTER_JOIN,
+  LEFT_SEMI_JOIN,
+  RIGHT_OUTER_JOIN,
+  FULL_OUTER_JOIN
+}
+
+struct THashJoinNode {
+  1: required TJoinOp join_op
+
+  // anything from the ON, USING or WHERE clauses that's an equi-join predicate
+  2: required list<TEqJoinCondition> eq_join_conjuncts
+
+  // anything from the ON or USING clauses (but *not* the WHERE clause) that's not an
+  // equi-join predicate
+  3: optional list<Exprs.TExpr> other_join_conjuncts
+}
+
+struct TAggregationNode {
+  1: optional list<Exprs.TExpr> grouping_exprs
+  2: required list<Exprs.TExpr> aggregate_exprs
+  3: required Types.TTupleId agg_tuple_id
+
+  // Set to true if this aggregation function requires finalization to complete after all
+  // rows have been aggregated, and this node is not an intermediate node.
+  4: required bool need_finalize
+}
+
+struct TSortNode {
+  1: required list<Exprs.TExpr> ordering_exprs
+  2: required list<bool> is_asc_order
+  // Indicates whether the backend service should use topn vs. sorting
+  3: required bool use_top_n;
+}
+
+struct TMergeNode {
+  // List or expr lists materialized by this node.
+  // There is one list of exprs per query stmt feeding into this merge node.
+  1: required list<list<Exprs.TExpr>> result_expr_lists
+  // Separate list of expr lists coming from a constant select stmts.
+  2: required list<list<Exprs.TExpr>> const_expr_lists
+}
+
+// This is essentially a union of all messages corresponding to subclasses
+// of PlanNode.
+struct TPlanNode {
+  // node id, needed to reassemble tree structure
+  1: required Types.TPlanNodeId node_id
+  2: required TPlanNodeType node_type
+  3: required i32 num_children
+  4: required i64 limit
+  5: required list<Types.TTupleId> row_tuples
+
+  // nullable_tuples[i] is true if row_tuples[i] is nullable
+  6: required list<bool> nullable_tuples
+  7: optional list<Exprs.TExpr> conjuncts
+
+  // Produce data in compact format.
+  8: required bool compact_data
+
+  // one field per PlanNode subclass
+  9: optional THdfsScanNode hdfs_scan_node
+  10: optional THBaseScanNode hbase_scan_node
+  11: optional THashJoinNode hash_join_node
+  12: optional TAggregationNode agg_node
+  13: optional TSortNode sort_node
+  14: optional TMergeNode merge_node
+}
+
+// A flattened representation of a tree of PlanNodes, obtained by depth-first
+// traversal.
+struct TPlan {
+  1: required list<TPlanNode> nodes
+}

data/thrift/Planner.thrift

@@ -0,0 +1,72 @@
+// Copyright 2012 Cloudera Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//
+// This file contains structures produced by the planner.
+
+namespace cpp impala
+namespace java com.cloudera.impala.thrift
+namespace rb Impala.Protocol
+
+include "Types.thrift"
+include "Exprs.thrift"
+include "DataSinks.thrift"
+include "PlanNodes.thrift"
+include "Partitions.thrift"
+
+// TPlanFragment encapsulates info needed to execute a particular
+// plan fragment, including how to produce and how to partition its output.
+// It leaves out node-specific parameters neede for the actual execution.
+struct TPlanFragment {
+  // no plan or descriptor table: query without From clause
+  2: optional PlanNodes.TPlan plan
+
+  // exprs that produce values for slots of output tuple (one expr per slot);
+  // if not set, plan fragment materializes full rows of plan_tree
+  4: optional list<Exprs.TExpr> output_exprs
+
+  // Specifies the destination of this plan fragment's output rows.
+  // For example, the destination could be a stream sink which forwards
+  // the data to a remote plan fragment, or a sink which writes to a table (for
+  // insert stmts).
+  5: optional DataSinks.TDataSink output_sink
+
+  // Partitioning of the data created by all instances of this plan fragment;
+  // partitioning.type has the following meaning:
+  // - UNPARTITIONED: there is only one instance of the plan fragment
+  // - RANDOM: a particular output row is randomly assigned to any of the instances
+  // - HASH_PARTITIONED: output row r is produced by
+  //   hash_value(partitioning.partitioning_exprs(r)) % #partitions
+  // - RANGE_PARTITIONING: currently not supported
+  // This is distinct from the partitioning of each plan fragment's
+  // output, which is specified by output_sink.output_partitioning.
+  6: required Partitions.TDataPartition partition
+}
+
+// location information for a single scan range
+struct TScanRangeLocation {
+  1: required Types.THostPort server
+
+  // disk volume identifier of a particular scan range at 'server';
+  // -1 indicates an unknown volume id;
+  // only set for TScanRange.hdfs_file_split
+  2: optional i32 volume_id = -1
+}
+
+// A single scan range plus the hosts that serve it
+struct TScanRangeLocations {
+  1: required PlanNodes.TScanRange scan_range
+  // non-empty list
+  2: list<TScanRangeLocation> locations
+}

data/thrift/RuntimeProfile.thrift

@@ -0,0 +1,58 @@
+// Copyright 2012 Cloudera Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+namespace cpp impala
+namespace java com.cloudera.impala.thrift
+namespace rb Impala.Protocol
+
+// Counter data types.
+enum TCounterType {
+  UNIT,
+  UNIT_PER_SECOND,
+  TIME_MS,
+  CPU_TICKS,
+  BYTES
+  BYTES_PER_SECOND,
+}
+
+// Counter data
+struct TCounter {
+  1: required string name
+  2: required TCounterType type
+  3: required i64 value
+}
+
+// A single runtime profile
+struct TRuntimeProfileNode {
+  1: required string name
+  2: required i32 num_children
+  3: required list<TCounter> counters
+  // TODO: should we make metadata a serializable struct?  We only use it to
+  // store the node id right now so this is sufficient.
+  4: required i64 metadata
+
+  // indicates whether the child will be printed with extra indentation;
+  // corresponds to indent param of RuntimeProfile::AddChild()
+  5: required bool indent
+
+  // map of key,value info strings that capture any kind of additional information
+  // about the profiled object
+  6: required map<string, string> info_strings
+}
+
+// A flattened tree of runtime profiles, obtained by an
+// in-order traversal
+struct TRuntimeProfileTree {
+  1: required list<TRuntimeProfileNode> nodes
+}

data/thrift/StateStoreService.thrift

@@ -0,0 +1,121 @@
+// Copyright 2012 Cloudera Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+namespace cpp impala
+namespace java com.cloudera.impala.thrift
+namespace rb Impala.Protocol
+
+include "StatestoreTypes.thrift"
+include "Status.thrift"
+include "Types.thrift"
+
+enum StateStoreServiceVersion {
+  V1
+}
+
+struct TRegisterServiceRequest {
+  1: required StateStoreServiceVersion protocol_version =
+      StateStoreServiceVersion.V1
+
+  // Address where the StateStoreSubscriberService is running. Required in V1.
+  2: optional Types.THostPort subscriber_address
+
+  // Service running on the node. Multiple services can be registered using multiple
+  // RegisterService() calls from the same subscriber. Currently, we assume that at most
+  // one instance of a particular service will be registered with each subscriber.
+  // Required in V1.
+  3: optional string service_id
+  4: optional Types.THostPort service_address
+}
+
+struct TRegisterServiceResponse {
+  // Required in V1.
+  1: optional Status.TStatus status
+}
+
+struct TUnregisterServiceRequest {
+  1: required StateStoreServiceVersion protocol_version =
+      StateStoreServiceVersion.V1
+
+  // Address of the subscriber. Required in V1.
+  2: optional Types.THostPort subscriber_address
+
+  // Service that should be unregistered. Required in V1.
+  3: optional string service_id
+}
+
+struct TUnregisterServiceResponse {
+  // Required in V1.
+  1: optional Status.TStatus status
+}
+
+struct TRegisterSubscriptionRequest {
+  1: required StateStoreServiceVersion protocol_version =
+      StateStoreServiceVersion.V1
+
+  // Address where the StateStoreSubscriberService is running. Required in V1.
+  2: optional Types.THostPort subscriber_address
+
+  // Services for which updates should be pushed to the given subscriber. Required in V1.
+  3: optional set<string> services
+
+  // The subscription ID to use for this request. All IDs are local to the
+  // subscriber, so as long as a single subscriber does not reuse IDs there is no
+  // danger of collision.
+  // Required in V1
+  4: optional string subscription_id;
+}
+
+struct TRegisterSubscriptionResponse {
+  // Required in V1.
+  1: optional Status.TStatus status
+}
+
+struct TUnregisterSubscriptionRequest {
+  1: required StateStoreServiceVersion protocol_version =
+      StateStoreServiceVersion.V1
+
+  // Address of the subscriber. Required in V1.
+  2: optional Types.THostPort subscriber_address
+
+  // Identifier for the subscription that should be unregistered. Required in V1.
+  3: optional string subscription_id
+}
+
+struct TUnregisterSubscriptionResponse {
+  // Required in V1.
+  1: optional Status.TStatus status
+}
+
+// A repository and distribution mechanism for global system state. Stored state is not
+// made persistent and is considered soft-state (i.e., it needs to be re-supplied
+// when a StateStore restarts). Updates to the global state are distributed to
+// subscribers asynchronously and with an arbitrary (but typically reasonably small)
+// delay.
+service StateStoreService {
+  // Registers an instance of a service.
+  TRegisterServiceResponse RegisterService(1: TRegisterServiceRequest request);
+
+  // Unregisters an instance of a service.
+  TUnregisterServiceResponse UnregisterService(1: TUnregisterServiceRequest request);
+
+  // Registers to receive updates for a set of services.
+  TRegisterSubscriptionResponse RegisterSubscription(
+      1: TRegisterSubscriptionRequest request);
+
+  // Unregisters the given subscription. A subscriber will be updated at most one more
+  // time after unregistering.
+  TUnregisterSubscriptionResponse UnregisterSubscription(
+      1: TUnregisterSubscriptionRequest request);
+}