impala 0.1.6 → 0.2.0

data/thrift/Descriptors.thrift

@@ -1,115 +0,0 @@
-// Copyright 2012 Cloudera Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-namespace cpp impala
-namespace java com.cloudera.impala.thrift
-namespace rb Impala.Protocol
-
-include "Types.thrift"
-include "Exprs.thrift"
-
-struct TSlotDescriptor {
-  1: required Types.TSlotId id
-  2: required Types.TTupleId parent
-  3: required Types.TPrimitiveType slotType
-  4: required i32 columnPos   // in originating table
-  5: required i32 byteOffset  // into tuple
-  6: required i32 nullIndicatorByte
-  7: required i32 nullIndicatorBit
-  9: required i32 slotIdx
-  10: required bool isMaterialized
-}
-
-enum TTableType {
-  HDFS_TABLE,
-  HBASE_TABLE
-}
-
-enum THdfsFileFormat {
-  TEXT,
-  LZO_TEXT,
-  RC_FILE,
-  SEQUENCE_FILE,
-  TREVNI
-}
-
-enum THdfsCompression {
-  NONE,
-  DEFAULT,
-  GZIP,
-  BZIP2,
-  SNAPPY,
-  SNAPPY_BLOCKED // Used by sequence and rc files but not stored in the metadata.
-}
-
-struct THdfsPartition {
-  1: required byte lineDelim
-  2: required byte fieldDelim
-  3: required byte collectionDelim
-  4: required byte mapKeyDelim
-  5: required byte escapeChar
-  6: required THdfsFileFormat fileFormat
-  7: list<Exprs.TExpr> partitionKeyExprs
-  8: required i32 blockSize
-  9: required THdfsCompression compression
-}
-
-struct THdfsTable {
-  1: required string hdfsBaseDir
-
-  // Partition keys are the same as clustering columns in
-  // TTableDescriptor, so there should be an equal number of each.
-  2: required list<string> partitionKeyNames
-  3: required string nullPartitionKeyValue
-
-  // map from partition id to partition metadata
-  4: required map<i64, THdfsPartition> partitions
-}
-
-struct THBaseTable {
-  1: required string tableName
-  2: required list<string> families
-  3: required list<string> qualifiers
-}
-
-// "Union" of all table types.
-struct TTableDescriptor {
-  1: required Types.TTableId id
-  2: required TTableType tableType
-  3: required i32 numCols
-  4: required i32 numClusteringCols
-  5: optional THdfsTable hdfsTable
-  6: optional THBaseTable hbaseTable
-
-  // Unqualified name of table
-  7: required string tableName;
-
-  // Name of the database that the table belongs to
-  8: required string dbName;
-}
-
-struct TTupleDescriptor {
-  1: required Types.TTupleId id
-  2: required i32 byteSize
-  3: required i32 numNullBytes
-  4: optional Types.TTableId tableId
-}
-
-struct TDescriptorTable {
-  1: optional list<TSlotDescriptor> slotDescriptors;
-  2: required list<TTupleDescriptor> tupleDescriptors;
-
-  // all table descriptors referenced by tupleDescriptors
-  3: optional list<TTableDescriptor> tableDescriptors;
-}

data/thrift/Exprs.thrift

@@ -1,134 +0,0 @@
-// Copyright 2012 Cloudera Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-namespace cpp impala
-namespace java com.cloudera.impala.thrift
-namespace rb Impala.Protocol
-
-include "Types.thrift"
-include "Opcodes.thrift"
-
-enum TExprNodeType {
-  AGG_EXPR,
-  ARITHMETIC_EXPR,
-  BINARY_PRED,
-  BOOL_LITERAL,
-  CASE_EXPR,
-  CAST_EXPR,
-  COMPOUND_PRED,
-  DATE_LITERAL,
-  FLOAT_LITERAL,
-  FUNCTION_CALL,
-  INT_LITERAL,
-  IN_PRED,
-  IS_NULL_PRED,
-  LIKE_PRED,
-  LITERAL_PRED,
-  NULL_LITERAL,
-  SLOT_REF,
-  STRING_LITERAL,
-}
-
-enum TAggregationOp {
-  INVALID,
-  COUNT,
-  MAX,
-  DISTINCT_PC,
-  MERGE_PC,
-  DISTINCT_PCSA,
-  MERGE_PCSA,
-  MIN,
-  SUM,
-}
-
-struct TAggregateExpr {
-  1: required bool is_star
-  2: required bool is_distinct
-  3: required TAggregationOp op
-}
-
-struct TBoolLiteral {
-  1: required bool value
-}
-
-struct TCaseExpr {
-  1: required bool has_case_expr
-  2: required bool has_else_expr
-}
-
-struct TDateLiteral {
-  1: required Types.TTimestamp value
-}
-
-struct TFloatLiteral {
-  1: required double value
-}
-
-struct TIntLiteral {
-  1: required i64 value
-}
-
-struct TInPredicate {
-  1: required bool is_not_in
-}
-
-struct TIsNullPredicate {
-  1: required bool is_not_null
-}
-
-struct TLikePredicate {
-  1: required string escape_char;
-}
-
-struct TLiteralPredicate {
-  1: required bool value
-  2: required bool is_null
-}
-
-struct TSlotRef {
-  1: required Types.TSlotId slot_id
-}
-
-struct TStringLiteral {
-  1: required string value;
-}
-
-// This is essentially a union over the subclasses of Expr.
-struct TExprNode {
-  1: required TExprNodeType node_type
-  2: required Types.TPrimitiveType type
-  3: optional Opcodes.TExprOpcode opcode
-  4: required i32 num_children
-
-  5: optional TAggregateExpr agg_expr
-  6: optional TBoolLiteral bool_literal
-  7: optional TCaseExpr case_expr
-  8: optional TDateLiteral date_literal
-  9: optional TFloatLiteral float_literal
-  10: optional TIntLiteral int_literal
-  11: optional TInPredicate in_predicate
-  12: optional TIsNullPredicate is_null_pred
-  13: optional TLikePredicate like_pred
-  14: optional TLiteralPredicate literal_pred
-  15: optional TSlotRef slot_ref
-  16: optional TStringLiteral string_literal
-}
-
-// A flattened representation of a tree of Expr nodes, obtained by depth-first
-// traversal.
-struct TExpr {
-  1: required list<TExprNode> nodes
-}
-
-

data/thrift/Frontend.thrift

@@ -1,193 +0,0 @@
-// Copyright 2012 Cloudera Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-namespace cpp impala
-namespace java com.cloudera.impala.thrift
-namespace rb Impala.Protocol
-
-include "Types.thrift"
-include "ImpalaInternalService.thrift"
-include "PlanNodes.thrift"
-include "Planner.thrift"
-include "Descriptors.thrift"
-
-// These are supporting structs for JniFrontend.java, which serves as the glue
-// between our C++ execution environment and the Java frontend.
-
-// Arguments to getTableNames, which returns a list of tables that match an
-// optional pattern.
-struct TGetTablesParams {
-  // If not set, match tables in all DBs
-  1: optional string db
-
-  // If not set, match every table
-  2: optional string pattern
-}
-
-// getTableNames returns a list of unqualified table names
-struct TGetTablesResult {
-  1: list<string> tables
-}
-
-// Arguments to getDbNames, which returns a list of dbs that match an optional
-// pattern
-struct TGetDbsParams {
-  // If not set, match every database
-  1: optional string pattern
-}
-
-// getDbNames returns a list of database names
-struct TGetDbsResult {
-  1: list<string> dbs
-}
-
-struct TColumnDesc {
-  1: required string columnName
-  2: required Types.TPrimitiveType columnType
-}
-
-// Arguments to DescribeTable, which returns a list of column descriptors for a
-// given table
-struct TDescribeTableParams {
-  1: optional string db
-  2: required string table_name
-}
-
-// Results of a call to describeTable()
-struct TDescribeTableResult {
-  1: required list<TColumnDesc> columns
-}
-
-// Per-client session state
-struct TSessionState {
-  // The default database, changed by USE <database> queries.
-  1: required string database
-}
-
-struct TClientRequest {
-  // select stmt to be executed
-  1: required string stmt
-
-  // query options
-  2: required ImpalaInternalService.TQueryOptions queryOptions
-
-  // session state
-  3: required TSessionState sessionState;
-}
-
-struct TResultSetMetadata {
-  1: required list<TColumnDesc> columnDescs
-}
-
-// Describes a set of changes to make to the metastore
-struct TCatalogUpdate {
-  // Unqualified name of the table to change
-  1: required string target_table;
-
-  // Database that the table belongs to
-  2: required string db_name;
-
-  // List of partitions that are new and need to be created. May
-  // include the root partition (represented by the empty string).
-  3: required set<string> created_partitions;
-}
-
-// Metadata required to finalize a query - that is, to clean up after the query is done.
-// Only relevant for INSERT queries.
-struct TFinalizeParams {
-  // True if the INSERT query was OVERWRITE, rather than INTO
-  1: required bool is_overwrite;
-
-  // The base directory in hdfs of the table targeted by this INSERT
-  2: required string hdfs_base_dir;
-
-  // The target table name
-  3: required string table_name;
-
-  // The target table database
-  4: required string table_db;
-}
-
-// Result of call to ImpalaPlanService/JniFrontend.CreateQueryRequest()
-struct TQueryExecRequest {
-  // global descriptor tbl for all fragments
-  1: optional Descriptors.TDescriptorTable desc_tbl
-
-  // fragments[i] may consume the output of fragments[j > i];
-  // fragments[0] is the root fragment and also the coordinator fragment, if
-  // it is unpartitioned.
-  2: required list<Planner.TPlanFragment> fragments
-
-  // Specifies the destination fragment of the output of each fragment.
-  // parent_fragment_idx.size() == fragments.size() - 1 and
-  // fragments[i] sends its output to fragments[dest_fragment_idx[i-1]]
-  3: optional list<i32> dest_fragment_idx
-
-  // A map from scan node ids to a list of scan range locations.
-  // The node ids refer to scan nodes in fragments[].plan_tree
-  4: optional map<Types.TPlanNodeId, list<Planner.TScanRangeLocations>>
-      per_node_scan_ranges
-
-  // Metadata of the query result set (only for select)
-  5: optional TResultSetMetadata result_set_metadata
-
-  // Set if the query needs finalization after it executes
-  6: optional TFinalizeParams finalize_params
-
-  7: required ImpalaInternalService.TQueryGlobals query_globals
-}
-
-enum TDdlType {
-  SHOW_TABLES,
-  SHOW_DBS,
-  USE,
-  DESCRIBE
-}
-
-struct TDdlExecRequest {
-  1: required TDdlType ddl_type
-
-  // Used for USE and DESCRIBE
-  2: optional string database;
-
-  // Table name (not fully-qualified) for DESCRIBE
-  3: optional string describe_table;
-
-  // Patterns to match table names against for SHOW
-  4: optional string show_pattern;
-}
-
-// Result of call to createExecRequest()
-struct TExecRequest {
-  1: required Types.TStmtType stmt_type;
-
-  2: optional string sql_stmt;
-
-  // Globally unique id for this request. Assigned by the planner.
-  3: required Types.TUniqueId request_id
-
-  // Copied from the corresponding TClientRequest
-  4: required ImpalaInternalService.TQueryOptions query_options;
-
-  // TQueryExecRequest for the backend
-  // Set iff stmt_type is QUERY or DML
-  5: optional TQueryExecRequest query_exec_request
-
-  // Set iff stmt_type is DDL
-  6: optional TDdlExecRequest ddl_exec_request
-
-  // Metadata of the query result set (not set for DML)
-  7: optional TResultSetMetadata result_set_metadata
-}
-

data/thrift/ImpalaInternalService.thrift

@@ -1,265 +0,0 @@
-// Copyright 2012 Cloudera Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//
-// This file contains the details of the protocol between coordinators and backends.
-
-namespace cpp impala
-namespace java com.cloudera.impala.thrift
-namespace rb Impala.Protocol
-
-include "Status.thrift"
-include "Types.thrift"
-include "Exprs.thrift"
-include "Descriptors.thrift"
-include "PlanNodes.thrift"
-include "Planner.thrift"
-include "DataSinks.thrift"
-include "Data.thrift"
-include "RuntimeProfile.thrift"
-include "ImpalaService.thrift"
-include "JavaConstants.thrift"
-
-// Query options that correspond to ImpalaService.ImpalaQueryOptions
-// TODO: make all of these optional, otherwise it will be impossible to
-// retire options and do rolling upgrades between releases
-struct TQueryOptions {
-  1: required bool abort_on_error = 0
-  2: required i32 max_errors = 0
-  3: required bool disable_codegen = 0
-  4: required i32 batch_size = 0
-
-  // return_as_ascii is not listed in ImpalaService.ImpalaQueryOptions because Beeswax
-  // should only return ascii. This option is only for internal testing.
-  // if true, return query results in ASCII format (TColumnValue.stringVal),
-  // otherwise return results in their native format (each TColumnValue
-  // uses the field corresponding to the column's native type).
-  5: required bool return_as_ascii = 1
-
-  6: required i32 num_nodes = JavaConstants.NUM_NODES_ALL
-  7: required i64 max_scan_range_length = 0
-  8: required i32 num_scanner_threads = 0
-  9: required i32 max_io_buffers = 0
-  10: required bool allow_unsupported_formats = 0
-  11: optional i64 default_order_by_limit = -1
-}
-
-// A scan range plus the parameters needed to execute that scan.
-struct TScanRangeParams {
-  1: required PlanNodes.TScanRange scan_range
-  2: optional i32 volume_id = -1
-}
-
-// Specification of one output destination of a plan fragment
-struct TPlanFragmentDestination {
-  // the globally unique fragment instance id
-  1: required Types.TUniqueId fragment_instance_id
-
-  // ... which is being executed on this server
-  2: required Types.THostPort server
-}
-
-// Parameters for a single execution instance of a particular TPlanFragment
-// TODO: for range partitioning, we also need to specify the range boundaries
-struct TPlanFragmentExecParams {
-  // a globally unique id assigned to the entire query
-  1: required Types.TUniqueId query_id
-
-  // a globally unique id assigned to this particular execution instance of
-  // a TPlanFragment
-  2: required Types.TUniqueId fragment_instance_id
-
-  // initial scan ranges for each scan node in TPlanFragment.plan_tree
-  3: required map<Types.TPlanNodeId, list<TScanRangeParams>> per_node_scan_ranges
-
-  // number of senders for ExchangeNodes contained in TPlanFragment.plan_tree;
-  // needed to create a DataStreamRecvr
-  4: required map<Types.TPlanNodeId, i32> per_exch_num_senders
-
-  // Output destinations, one per output partition.
-  // The partitioning of the output is specified by
-  // TPlanFragment.output_sink.output_partition.
-  // The number of output partitions is destinations.size().
-  5: list<TPlanFragmentDestination> destinations
-}
-
-// Global query parameters assigned by the coordinator.
-struct TQueryGlobals {
-  // String containing a timestamp set as the current time.
-  1: required string now_string
-}
-
-
-// Service Protocol Details
-
-enum ImpalaInternalServiceVersion {
-  V1
-}
-
-
-// ExecPlanFragment
-
-struct TExecPlanFragmentParams {
-  1: required ImpalaInternalServiceVersion protocol_version
-
-  // required in V1
-  2: optional Planner.TPlanFragment fragment
-
-  // required in V1
-  3: optional Descriptors.TDescriptorTable desc_tbl
-
-  // required in V1
-  4: optional TPlanFragmentExecParams params
-
-  // Initiating coordinator.
-  // TODO: determine whether we can get this somehow via the Thrift rpc mechanism.
-  // required in V1
-  5: optional Types.THostPort coord
-
-  // backend number assigned by coord to identify backend
-  // required in V1
-  6: optional i32 backend_num
-
-  // Global query parameters assigned by coordinator.
-  // required in V1
-  7: optional TQueryGlobals query_globals
-
-  // options for the query
-  // required in V1
-  8: optional TQueryOptions query_options
-}
-
-struct TExecPlanFragmentResult {
-  // required in V1
-  1: optional Status.TStatus status
-}
-
-
-// ReportExecStatus
-
-// The results of an INSERT query, sent to the coordinator as part of
-// TReportExecStatusParams
-struct TInsertExecStatus {
-  // Number of rows appended by an INSERT, per-partition.
-  // The keys represent partitions to create, coded as k1=v1/k2=v2/k3=v3..., with the
-  // root in an unpartitioned table being the empty string.
-  // The target table name is recorded in the corresponding TQueryExecRequest
-  1: optional map<string, i64> num_appended_rows
-
-  // A map from temporary absolute file path to final absolute destination. The
-  // coordinator performs these updates after the query completes.
-  2: required map<string, string> files_to_move;
-}
-
-struct TReportExecStatusParams {
-  1: required ImpalaInternalServiceVersion protocol_version
-
-  // required in V1
-  2: optional Types.TUniqueId query_id
-
-  // passed into ExecPlanFragment() as TExecPlanFragmentParams.backend_num
-  // required in V1
-  3: optional i32 backend_num
-
-  // required in V1
-  4: optional Types.TUniqueId fragment_instance_id
-
-  // Status of fragment execution; any error status means it's done.
-  // required in V1
-  5: optional Status.TStatus status
-
-  // If true, fragment finished executing.
-  // required in V1
-  6: optional bool done
-
-  // cumulative profile
-  // required in V1
-  7: optional RuntimeProfile.TRuntimeProfileTree profile
-
-  // Cumulative structural changes made by a table sink
-  // optional in V1
-  8: optional TInsertExecStatus insert_exec_status;
-
-  // New errors that have not been reported to the coordinator
-  // optional in V1
-  9: optional list<string> error_log
-}
-
-struct TReportExecStatusResult {
-  // required in V1
-  1: optional Status.TStatus status
-}
-
-
-// CancelPlanFragment
-
-struct TCancelPlanFragmentParams {
-  1: required ImpalaInternalServiceVersion protocol_version
-
-  // required in V1
-  2: optional Types.TUniqueId fragment_instance_id
-}
-
-struct TCancelPlanFragmentResult {
-  // required in V1
-  1: optional Status.TStatus status
-}
-
-
-// TransmitData
-
-struct TTransmitDataParams {
-  1: required ImpalaInternalServiceVersion protocol_version
-
-  // required in V1
-  2: optional Types.TUniqueId dest_fragment_instance_id
-
-  // for debugging purposes; currently ignored
-  //3: optional Types.TUniqueId src_fragment_instance_id
-
-  // required in V1
-  4: optional Types.TPlanNodeId dest_node_id
-
-  // required in V1
-  5: optional Data.TRowBatch row_batch
-
-  // if set to true, indicates that no more row batches will be sent
-  // for this dest_node_id
-  6: optional bool eos
-}
-
-struct TTransmitDataResult {
-  // required in V1
-  1: optional Status.TStatus status
-}
-
-
-service ImpalaInternalService {
-  // Called by coord to start asynchronous execution of plan fragment in backend.
-  // Returns as soon as all incoming data streams have been set up.
-  TExecPlanFragmentResult ExecPlanFragment(1:TExecPlanFragmentParams params);
-
-  // Periodically called by backend to report status of plan fragment execution
-  // back to coord; also called when execution is finished, for whatever reason.
-  TReportExecStatusResult ReportExecStatus(1:TReportExecStatusParams params);
-
-  // Called by coord to cancel execution of a single plan fragment, which this
-  // coordinator initiated with a prior call to ExecPlanFragment.
-  // Cancellation is asynchronous.
-  TCancelPlanFragmentResult CancelPlanFragment(1:TCancelPlanFragmentParams params);
-
-  // Called by sender to transmit single row batch. Returns error indication
-  // if params.fragmentId or params.destNodeId are unknown or if data couldn't be read.
-  TTransmitDataResult TransmitData(1:TTransmitDataParams params);
-}