impala 0.1.6 → 0.2.0

data/thrift/PlanNodes.thrift

@@ -1,184 +0,0 @@
-// Copyright 2012 Cloudera Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//
-// This file contains all structs, enums, etc., that together make up
-// a plan tree. All information recorded in struct TPlan and below is independent
-// of the execution parameters of any one of the backends on which it is running
-// (those are recorded in TPlanFragmentExecParams).
-
-namespace cpp impala
-namespace java com.cloudera.impala.thrift
-namespace rb Impala.Protocol
-
-include "Exprs.thrift"
-include "Types.thrift"
-
-enum TPlanNodeType {
-  HDFS_SCAN_NODE,
-  HBASE_SCAN_NODE,
-  HASH_JOIN_NODE,
-  AGGREGATION_NODE,
-  SORT_NODE,
-  EXCHANGE_NODE,
-  MERGE_NODE
-}
-
-// The information contained in subclasses of ScanNode captured in two separate
-// Thrift structs:
-// - TScanRange: the data range that's covered by the scan (which varies with the
-//   particular partition of the plan fragment of which the scan node is a part)
-// - T<subclass>: all other operational parameters that are the same across
-//   all plan fragments
-
-// Specification of subsection of a single hdfs file.
-struct THdfsFileSplit {
-  // file path
-  1: required string path
-
-  // starting offset
-  2: required i64 offset
-
-  // length of split
-  3: required i64 length
-
-  // ID of partition in parent THdfsScanNode. Meaningful only
-  // in the context of a single THdfsScanNode, may not be unique elsewhere.
-  4: required i64 partition_id
-}
-
-// key range for single THBaseScanNode
-// TODO: does 'binary' have an advantage over string? strings can
-// already store binary data
-struct THBaseKeyRange {
-  // inclusive
-  1: optional string startKey
-
-  // exclusive
-  2: optional string stopKey
-}
-
-// Specification of an individual data range which is held in its entirety
-// by a storage server
-struct TScanRange {
-  // one of these must be set for every TScanRange2
-  1: optional THdfsFileSplit hdfs_file_split
-  2: optional THBaseKeyRange hbase_key_range
-}
-
-struct THdfsScanNode {
-  1: required Types.TTupleId tuple_id
-}
-
-struct THBaseFilter {
-  1: required string family
-  2: required string qualifier
-  // Ordinal number into enum HBase CompareFilter.CompareOp.
-  // We don't use TExprOperator because the op is interpreted by an HBase Filter, and
-  // not the c++ expr eval.
-  3: required i32 op_ordinal
-  4: required string filter_constant
-}
-
-struct THBaseScanNode {
-  1: required Types.TTupleId tuple_id
-
-  // TODO: remove this, we already have THBaseTable.tableName
-  2: required string table_name
-
-  3: optional list<THBaseFilter> filters
-}
-
-struct TEqJoinCondition {
-  // left-hand side of "<a> = <b>"
-  1: required Exprs.TExpr left;
-  // right-hand side of "<a> = <b>"
-  2: required Exprs.TExpr right;
-}
-
-enum TJoinOp {
-  INNER_JOIN,
-  LEFT_OUTER_JOIN,
-  LEFT_SEMI_JOIN,
-  RIGHT_OUTER_JOIN,
-  FULL_OUTER_JOIN
-}
-
-struct THashJoinNode {
-  1: required TJoinOp join_op
-
-  // anything from the ON, USING or WHERE clauses that's an equi-join predicate
-  2: required list<TEqJoinCondition> eq_join_conjuncts
-
-  // anything from the ON or USING clauses (but *not* the WHERE clause) that's not an
-  // equi-join predicate
-  3: optional list<Exprs.TExpr> other_join_conjuncts
-}
-
-struct TAggregationNode {
-  1: optional list<Exprs.TExpr> grouping_exprs
-  2: required list<Exprs.TExpr> aggregate_exprs
-  3: required Types.TTupleId agg_tuple_id
-
-  // Set to true if this aggregation function requires finalization to complete after all
-  // rows have been aggregated, and this node is not an intermediate node.
-  4: required bool need_finalize
-}
-
-struct TSortNode {
-  1: required list<Exprs.TExpr> ordering_exprs
-  2: required list<bool> is_asc_order
-  // Indicates whether the backend service should use topn vs. sorting
-  3: required bool use_top_n;
-}
-
-struct TMergeNode {
-  // List or expr lists materialized by this node.
-  // There is one list of exprs per query stmt feeding into this merge node.
-  1: required list<list<Exprs.TExpr>> result_expr_lists
-  // Separate list of expr lists coming from a constant select stmts.
-  2: required list<list<Exprs.TExpr>> const_expr_lists
-}
-
-// This is essentially a union of all messages corresponding to subclasses
-// of PlanNode.
-struct TPlanNode {
-  // node id, needed to reassemble tree structure
-  1: required Types.TPlanNodeId node_id
-  2: required TPlanNodeType node_type
-  3: required i32 num_children
-  4: required i64 limit
-  5: required list<Types.TTupleId> row_tuples
-
-  // nullable_tuples[i] is true if row_tuples[i] is nullable
-  6: required list<bool> nullable_tuples
-  7: optional list<Exprs.TExpr> conjuncts
-
-  // Produce data in compact format.
-  8: required bool compact_data
-
-  // one field per PlanNode subclass
-  9: optional THdfsScanNode hdfs_scan_node
-  10: optional THBaseScanNode hbase_scan_node
-  11: optional THashJoinNode hash_join_node
-  12: optional TAggregationNode agg_node
-  13: optional TSortNode sort_node
-  14: optional TMergeNode merge_node
-}
-
-// A flattened representation of a tree of PlanNodes, obtained by depth-first
-// traversal.
-struct TPlan {
-  1: required list<TPlanNode> nodes
-}

data/thrift/Planner.thrift

@@ -1,72 +0,0 @@
-// Copyright 2012 Cloudera Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//
-// This file contains structures produced by the planner.
-
-namespace cpp impala
-namespace java com.cloudera.impala.thrift
-namespace rb Impala.Protocol
-
-include "Types.thrift"
-include "Exprs.thrift"
-include "DataSinks.thrift"
-include "PlanNodes.thrift"
-include "Partitions.thrift"
-
-// TPlanFragment encapsulates info needed to execute a particular
-// plan fragment, including how to produce and how to partition its output.
-// It leaves out node-specific parameters neede for the actual execution.
-struct TPlanFragment {
-  // no plan or descriptor table: query without From clause
-  2: optional PlanNodes.TPlan plan
-
-  // exprs that produce values for slots of output tuple (one expr per slot);
-  // if not set, plan fragment materializes full rows of plan_tree
-  4: optional list<Exprs.TExpr> output_exprs
-
-  // Specifies the destination of this plan fragment's output rows.
-  // For example, the destination could be a stream sink which forwards
-  // the data to a remote plan fragment, or a sink which writes to a table (for
-  // insert stmts).
-  5: optional DataSinks.TDataSink output_sink
-
-  // Partitioning of the data created by all instances of this plan fragment;
-  // partitioning.type has the following meaning:
-  // - UNPARTITIONED: there is only one instance of the plan fragment
-  // - RANDOM: a particular output row is randomly assigned to any of the instances
-  // - HASH_PARTITIONED: output row r is produced by
-  //   hash_value(partitioning.partitioning_exprs(r)) % #partitions
-  // - RANGE_PARTITIONING: currently not supported
-  // This is distinct from the partitioning of each plan fragment's
-  // output, which is specified by output_sink.output_partitioning.
-  6: required Partitions.TDataPartition partition
-}
-
-// location information for a single scan range
-struct TScanRangeLocation {
-  1: required Types.THostPort server
-
-  // disk volume identifier of a particular scan range at 'server';
-  // -1 indicates an unknown volume id;
-  // only set for TScanRange.hdfs_file_split
-  2: optional i32 volume_id = -1
-}
-
-// A single scan range plus the hosts that serve it
-struct TScanRangeLocations {
-  1: required PlanNodes.TScanRange scan_range
-  // non-empty list
-  2: list<TScanRangeLocation> locations
-}

data/thrift/RuntimeProfile.thrift

@@ -1,58 +0,0 @@
-// Copyright 2012 Cloudera Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-namespace cpp impala
-namespace java com.cloudera.impala.thrift
-namespace rb Impala.Protocol
-
-// Counter data types.
-enum TCounterType {
-  UNIT,
-  UNIT_PER_SECOND,
-  TIME_MS,
-  CPU_TICKS,
-  BYTES
-  BYTES_PER_SECOND,
-}
-
-// Counter data
-struct TCounter {
-  1: required string name
-  2: required TCounterType type
-  3: required i64 value
-}
-
-// A single runtime profile
-struct TRuntimeProfileNode {
-  1: required string name
-  2: required i32 num_children
-  3: required list<TCounter> counters
-  // TODO: should we make metadata a serializable struct?  We only use it to
-  // store the node id right now so this is sufficient.
-  4: required i64 metadata
-
-  // indicates whether the child will be printed with extra indentation;
-  // corresponds to indent param of RuntimeProfile::AddChild()
-  5: required bool indent
-
-  // map of key,value info strings that capture any kind of additional information
-  // about the profiled object
-  6: required map<string, string> info_strings
-}
-
-// A flattened tree of runtime profiles, obtained by an
-// in-order traversal
-struct TRuntimeProfileTree {
-  1: required list<TRuntimeProfileNode> nodes
-}

data/thrift/StateStoreService.thrift

@@ -1,121 +0,0 @@
-// Copyright 2012 Cloudera Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-namespace cpp impala
-namespace java com.cloudera.impala.thrift
-namespace rb Impala.Protocol
-
-include "StatestoreTypes.thrift"
-include "Status.thrift"
-include "Types.thrift"
-
-enum StateStoreServiceVersion {
-  V1
-}
-
-struct TRegisterServiceRequest {
-  1: required StateStoreServiceVersion protocol_version =
-      StateStoreServiceVersion.V1
-
-  // Address where the StateStoreSubscriberService is running. Required in V1.
-  2: optional Types.THostPort subscriber_address
-
-  // Service running on the node. Multiple services can be registered using multiple
-  // RegisterService() calls from the same subscriber. Currently, we assume that at most
-  // one instance of a particular service will be registered with each subscriber.
-  // Required in V1.
-  3: optional string service_id
-  4: optional Types.THostPort service_address
-}
-
-struct TRegisterServiceResponse {
-  // Required in V1.
-  1: optional Status.TStatus status
-}
-
-struct TUnregisterServiceRequest {
-  1: required StateStoreServiceVersion protocol_version =
-      StateStoreServiceVersion.V1
-
-  // Address of the subscriber. Required in V1.
-  2: optional Types.THostPort subscriber_address
-
-  // Service that should be unregistered. Required in V1.
-  3: optional string service_id
-}
-
-struct TUnregisterServiceResponse {
-  // Required in V1.
-  1: optional Status.TStatus status
-}
-
-struct TRegisterSubscriptionRequest {
-  1: required StateStoreServiceVersion protocol_version =
-      StateStoreServiceVersion.V1
-
-  // Address where the StateStoreSubscriberService is running. Required in V1.
-  2: optional Types.THostPort subscriber_address
-
-  // Services for which updates should be pushed to the given subscriber. Required in V1.
-  3: optional set<string> services
-
-  // The subscription ID to use for this request. All IDs are local to the
-  // subscriber, so as long as a single subscriber does not reuse IDs there is no
-  // danger of collision.
-  // Required in V1
-  4: optional string subscription_id;
-}
-
-struct TRegisterSubscriptionResponse {
-  // Required in V1.
-  1: optional Status.TStatus status
-}
-
-struct TUnregisterSubscriptionRequest {
-  1: required StateStoreServiceVersion protocol_version =
-      StateStoreServiceVersion.V1
-
-  // Address of the subscriber. Required in V1.
-  2: optional Types.THostPort subscriber_address
-
-  // Identifier for the subscription that should be unregistered. Required in V1.
-  3: optional string subscription_id
-}
-
-struct TUnregisterSubscriptionResponse {
-  // Required in V1.
-  1: optional Status.TStatus status
-}
-
-// A repository and distribution mechanism for global system state. Stored state is not
-// made persistent and is considered soft-state (i.e., it needs to be re-supplied
-// when a StateStore restarts). Updates to the global state are distributed to
-// subscribers asynchronously and with an arbitrary (but typically reasonably small)
-// delay.
-service StateStoreService {
-  // Registers an instance of a service.
-  TRegisterServiceResponse RegisterService(1: TRegisterServiceRequest request);
-
-  // Unregisters an instance of a service.
-  TUnregisterServiceResponse UnregisterService(1: TUnregisterServiceRequest request);
-
-  // Registers to receive updates for a set of services.
-  TRegisterSubscriptionResponse RegisterSubscription(
-      1: TRegisterSubscriptionRequest request);
-
-  // Unregisters the given subscription. A subscriber will be updated at most one more
-  // time after unregistering.
-  TUnregisterSubscriptionResponse UnregisterSubscription(
-      1: TUnregisterSubscriptionRequest request);
-}

data/thrift/StateStoreSubscriberService.thrift

@@ -1,64 +0,0 @@
-// Copyright 2012 Cloudera Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-namespace cpp impala
-namespace java com.cloudera.impala.thrift
-namespace rb Impala.Protocol
-
-include "StatestoreTypes.thrift"
-include "Status.thrift"
-include "Types.thrift"
-
-enum StateStoreSubscriberServiceVersion {
-  V1
-}
-
-struct TUpdateStateRequest {
-  1: required StateStoreSubscriberServiceVersion protocolVersion =
-      StateStoreSubscriberServiceVersion.V1
-
-  // Membership information for each service that the subscriber subscribed to.
-  // Required in V1.
-  2: optional list<StatestoreTypes.TServiceMembership> service_memberships
-
-  // Object updates for each service that the subscriber subscribed to.  Required in V1.
-  3: optional list<StatestoreTypes.TVersionedObject> updated_objects
-
-  // Objects that have been deleted, for each service that the subscriber has subscribed
-  // to. Required in V1.
-  4: optional list<string> deleted_object_keys
-}
-
-struct TUpdateStateResponse {
-  // Required in V1.
-  1: optional Status.TStatus status
-
-  // For each service running on the subscriber, the object updates.  Required in V1,
-  // but not yet implemented.
-  2: optional list<StatestoreTypes.TVersionedObject> updated_objects
-
-  // Objects that have been deleted, for each service that the subscriber has subscribed
-  // to. Required in V1, but not yet implemented.
-  3: optional list<string> deleted_object_keys
-}
-
-// The StateStoreSubscriber runs on all servers that need to connect to the StateStore,
-// and provides an interface between the StateStore and services that need to either push
-// updates to or receive updates from the StateStore.
-service StateStoreSubscriberService {
-  // The UpdateState() call serves two purposes: first, it allows the StateStore to push
-  // new updates to the subscriber; and second, it allows the StateStore to collect new
-  // updates from services running local with the StateStoreSubscriber.
-  TUpdateStateResponse UpdateState(1: TUpdateStateRequest request);
-}