impala 0.1.0

data/thrift/ImpalaPlanService.thrift

@@ -0,0 +1,44 @@
+// Copyright 2012 Cloudera Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+namespace cpp impala
+namespace java com.cloudera.impala.thrift
+namespace rb Impala.Protocol
+
+include "Frontend.thrift"
+
+exception TImpalaPlanServiceException {
+  1: string msg;
+}
+
+// We're running the Impala frontend as a service from which the backend
+// test driver can get plans to run.
+service ImpalaPlanService {
+  Frontend.TExecRequest CreateExecRequest(
+      1:Frontend.TClientRequest clientRequest) throws (1:TImpalaPlanServiceException e);
+
+  // Force planservice to reload table metadata, in case it has changed due to e.g. an
+  // insert
+  void RefreshMetadata();
+
+  string GetExplainString(1:Frontend.TClientRequest queryRequest)
+      throws (1:TImpalaPlanServiceException e);
+
+  // Makes changes to the metastore - usually creates partitions as a result of an INSERT
+  // statement
+  void UpdateMetastore(1:Frontend.TCatalogUpdate update)
+      throws (1:TImpalaPlanServiceException e);
+
+  void ShutdownServer();
+}

data/thrift/ImpalaService.thrift

@@ -0,0 +1,105 @@
+// Copyright 2012 Cloudera Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+namespace cpp impala
+namespace java com.cloudera.impala.thrift
+namespace rb Impala.Protocol
+
+include "Status.thrift"
+include "beeswax.thrift"
+
+// ImpalaService accepts query execution options through beeswax.Query.configuration in
+// key:value form. For example, the list of strings could be:
+//     "num_nodes:1", "abort_on_error:false"
+// The valid keys are listed in this enum. They map to TQueryOptions.
+// Note: If you add an option or change the default, you also need to update:
+// - JavaConstants.DEFAULT_QUERY_OPTIONS
+// - ImpalaInternalService.thrift: TQueryOptions
+// - ImpaladClientExecutor.getBeeswaxQueryConfigurations()
+// - ImpalaServer::QueryToTClientRequest()
+// - ImpalaServer::InitializeConfigVariables()
+enum TImpalaQueryOptions {
+  // if true, abort execution on the first error
+  ABORT_ON_ERROR,
+
+  // maximum # of errors to be reported; Unspecified or 0 indicates backend default
+  MAX_ERRORS,
+
+  // if true, disable llvm codegen
+  DISABLE_CODEGEN,
+
+  // batch size to be used by backend; Unspecified or a size of 0 indicates backend
+  // default
+  BATCH_SIZE,
+
+  // specifies the degree of parallelism with which to execute the query;
+  // 1: single-node execution
+  // NUM_NODES_ALL: executes on all nodes that contain relevant data
+  // NUM_NODES_ALL_RACKS: executes on one node per rack that holds relevant data
+  // > 1: executes on at most that many nodes at any point in time (ie, there can be
+  //      more nodes than numNodes with plan fragments for this query, but at most
+  //      numNodes would be active at any point in time)
+  // Constants (NUM_NODES_ALL, NUM_NODES_ALL_RACKS) are defined in JavaConstants.thrift.
+  NUM_NODES,
+
+  // maximum length of the scan range; only applicable to HDFS scan range; Unspecified or
+  // a length of 0 indicates backend default;
+  MAX_SCAN_RANGE_LENGTH,
+
+  // Maximum number of io buffers (per disk)
+  MAX_IO_BUFFERS,
+
+  // Number of scanner threads.
+  NUM_SCANNER_THREADS,
+
+  // If true, Impala will try to execute on file formats that are not fully supported yet
+  ALLOW_UNSUPPORTED_FORMATS,
+
+  // if set and > -1, specifies the default limit applied to a top-level SELECT statement
+  // with an ORDER BY but without a LIMIT clause (ie, if the SELECT statement also has
+  // a LIMIT clause, this default is ignored)
+  DEFAULT_ORDER_BY_LIMIT,
+}
+
+// The summary of an insert.
+struct TInsertResult {
+  // Number of appended rows per modified partition. Only applies to HDFS tables.
+  // The keys represent partitions to create, coded as k1=v1/k2=v2/k3=v3..., with the
+  // root in an unpartitioned table being the empty string.
+  1: required map<string, i64> rows_appended
+}
+
+// For all rpc that return a TStatus as part of their result type,
+// if the status_code field is set to anything other than OK, the contents
+// of the remainder of the result type is undefined (typically not set)
+service ImpalaService extends beeswax.BeeswaxService {
+  // Cancel execution of query. Returns RUNTIME_ERROR if query_id
+  // unknown.
+  // This terminates all threads running on behalf of this query at
+  // all nodes that were involved in the execution.
+  // Throws BeeswaxException if the query handle is invalid (this doesn't
+  // necessarily indicate an error: the query might have finished).
+  Status.TStatus Cancel(1:beeswax.QueryHandle query_id)
+      throws(1:beeswax.BeeswaxException error);
+
+  // Invalidates all catalog metadata, forcing a reload
+  Status.TStatus ResetCatalog();
+
+  // Closes the query handle and return the result summary of the insert.
+  TInsertResult CloseInsert(1:beeswax.QueryHandle handle)
+      throws(1:beeswax.QueryNotFoundException error, 2:beeswax.BeeswaxException error2);
+
+  // Client calls this RPC to verify that the server is an ImpalaService.
+  void PingImpalaService();
+}

data/thrift/JavaConstants.thrift

@@ -0,0 +1,60 @@
+// Copyright 2012 Cloudera Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Centralised store of constants that are shared between C++ and Java
+// - Thrift compiles all constants into Constants.java, which means it
+// gets overwritten if two separate files have constants.
+// This is fixed in Thrift 0.9 - see THRIFT-1090
+
+namespace cpp impala
+namespace java com.cloudera.impala.thrift
+namespace rb Impala.Protocol
+
+include "Descriptors.thrift"
+include "ImpalaService.thrift" // For TImpalaQueryOptions
+
+// constants for TQueryOptions.num_nodes
+const i32 NUM_NODES_ALL = 0
+const i32 NUM_NODES_ALL_RACKS = -1
+
+// constants for TPlanNodeId
+const i32 INVALID_PLAN_NODE_ID = -1
+
+// Constant default partition ID, must be < 0 to avoid collisions
+const i64 DEFAULT_PARTITION_ID = -1;
+
+// Mapping from names defined by Trevni to the enum.
+// We permit gzip and bzip2 in addtion.
+const map<string, Descriptors.THdfsCompression> COMPRESSION_MAP = {
+  "": Descriptors.THdfsCompression.NONE,
+  "none": Descriptors.THdfsCompression.NONE,
+  "deflate": Descriptors.THdfsCompression.DEFAULT,
+  "gzip": Descriptors.THdfsCompression.GZIP,
+  "bzip2": Descriptors.THdfsCompression.BZIP2,
+  "snappy": Descriptors.THdfsCompression.SNAPPY
+}
+
+// Default values for each query option in ImpalaService.TImpalaQueryOptions
+const map<ImpalaService.TImpalaQueryOptions, string> DEFAULT_QUERY_OPTIONS = {
+  ImpalaService.TImpalaQueryOptions.ABORT_ON_ERROR : "false",
+  ImpalaService.TImpalaQueryOptions.MAX_ERRORS : "0",
+  ImpalaService.TImpalaQueryOptions.DISABLE_CODEGEN : "false",
+  ImpalaService.TImpalaQueryOptions.BATCH_SIZE : "0",
+  ImpalaService.TImpalaQueryOptions.NUM_NODES : "0",
+  ImpalaService.TImpalaQueryOptions.MAX_SCAN_RANGE_LENGTH : "0",
+  ImpalaService.TImpalaQueryOptions.MAX_IO_BUFFERS : "0"
+  ImpalaService.TImpalaQueryOptions.NUM_SCANNER_THREADS : "0"
+  ImpalaService.TImpalaQueryOptions.ALLOW_UNSUPPORTED_FORMATS : "false"
+  ImpalaService.TImpalaQueryOptions.DEFAULT_ORDER_BY_LIMIT : "-1"
+}

data/thrift/Opcodes.thrift

@@ -0,0 +1,317 @@
+// Copyright 2012 Cloudera Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This is a generated file, DO NOT EDIT.
+// To add new functions, see impala/common/function-registry/gen_opcodes.py
+
+namespace cpp impala
+namespace java com.cloudera.impala.thrift
+namespace rb Impala.Protocol
+
+enum TExprOpcode {
+  INVALID_OPCODE,
+  ADD_DOUBLE_DOUBLE,
+  ADD_LONG_LONG,
+  BITAND_CHAR_CHAR,
+  BITAND_INT_INT,
+  BITAND_LONG_LONG,
+  BITAND_SHORT_SHORT,
+  BITNOT_CHAR,
+  BITNOT_INT,
+  BITNOT_LONG,
+  BITNOT_SHORT,
+  BITOR_CHAR_CHAR,
+  BITOR_INT_INT,
+  BITOR_LONG_LONG,
+  BITOR_SHORT_SHORT,
+  BITXOR_CHAR_CHAR,
+  BITXOR_INT_INT,
+  BITXOR_LONG_LONG,
+  BITXOR_SHORT_SHORT,
+  CASE_BOOL,
+  CASE_CHAR,
+  CASE_DOUBLE,
+  CASE_FLOAT,
+  CASE_INT,
+  CASE_LONG,
+  CASE_SHORT,
+  CASE_STRINGVALUE,
+  CASE_TIMESTAMPVALUE,
+  CAST_BOOL_BOOL,
+  CAST_BOOL_CHAR,
+  CAST_BOOL_DOUBLE,
+  CAST_BOOL_FLOAT,
+  CAST_BOOL_INT,
+  CAST_BOOL_LONG,
+  CAST_BOOL_SHORT,
+  CAST_BOOL_STRINGVALUE,
+  CAST_BOOL_TIMESTAMPVALUE,
+  CAST_CHAR_BOOL,
+  CAST_CHAR_CHAR,
+  CAST_CHAR_DOUBLE,
+  CAST_CHAR_FLOAT,
+  CAST_CHAR_INT,
+  CAST_CHAR_LONG,
+  CAST_CHAR_SHORT,
+  CAST_CHAR_STRINGVALUE,
+  CAST_CHAR_TIMESTAMPVALUE,
+  CAST_DOUBLE_BOOL,
+  CAST_DOUBLE_CHAR,
+  CAST_DOUBLE_DOUBLE,
+  CAST_DOUBLE_FLOAT,
+  CAST_DOUBLE_INT,
+  CAST_DOUBLE_LONG,
+  CAST_DOUBLE_SHORT,
+  CAST_DOUBLE_STRINGVALUE,
+  CAST_DOUBLE_TIMESTAMPVALUE,
+  CAST_FLOAT_BOOL,
+  CAST_FLOAT_CHAR,
+  CAST_FLOAT_DOUBLE,
+  CAST_FLOAT_FLOAT,
+  CAST_FLOAT_INT,
+  CAST_FLOAT_LONG,
+  CAST_FLOAT_SHORT,
+  CAST_FLOAT_STRINGVALUE,
+  CAST_FLOAT_TIMESTAMPVALUE,
+  CAST_INT_BOOL,
+  CAST_INT_CHAR,
+  CAST_INT_DOUBLE,
+  CAST_INT_FLOAT,
+  CAST_INT_INT,
+  CAST_INT_LONG,
+  CAST_INT_SHORT,
+  CAST_INT_STRINGVALUE,
+  CAST_INT_TIMESTAMPVALUE,
+  CAST_LONG_BOOL,
+  CAST_LONG_CHAR,
+  CAST_LONG_DOUBLE,
+  CAST_LONG_FLOAT,
+  CAST_LONG_INT,
+  CAST_LONG_LONG,
+  CAST_LONG_SHORT,
+  CAST_LONG_STRINGVALUE,
+  CAST_LONG_TIMESTAMPVALUE,
+  CAST_SHORT_BOOL,
+  CAST_SHORT_CHAR,
+  CAST_SHORT_DOUBLE,
+  CAST_SHORT_FLOAT,
+  CAST_SHORT_INT,
+  CAST_SHORT_LONG,
+  CAST_SHORT_SHORT,
+  CAST_SHORT_STRINGVALUE,
+  CAST_SHORT_TIMESTAMPVALUE,
+  CAST_STRINGVALUE_CHAR,
+  CAST_STRINGVALUE_DOUBLE,
+  CAST_STRINGVALUE_FLOAT,
+  CAST_STRINGVALUE_INT,
+  CAST_STRINGVALUE_LONG,
+  CAST_STRINGVALUE_SHORT,
+  CAST_STRINGVALUE_TIMESTAMPVALUE,
+  CAST_TIMESTAMPVALUE_BOOL,
+  CAST_TIMESTAMPVALUE_CHAR,
+  CAST_TIMESTAMPVALUE_DOUBLE,
+  CAST_TIMESTAMPVALUE_FLOAT,
+  CAST_TIMESTAMPVALUE_INT,
+  CAST_TIMESTAMPVALUE_LONG,
+  CAST_TIMESTAMPVALUE_SHORT,
+  CAST_TIMESTAMPVALUE_STRINGVALUE,
+  COMPOUND_AND,
+  COMPOUND_NOT,
+  COMPOUND_OR,
+  CONDITIONAL_COALESCE_BOOL_VARARGS,
+  CONDITIONAL_COALESCE_DOUBLE_VARARGS,
+  CONDITIONAL_COALESCE_LONG_VARARGS,
+  CONDITIONAL_COALESCE_STRINGVALUE_VARARGS,
+  CONDITIONAL_COALESCE_TIMESTAMPVALUE_VARARGS,
+  CONDITIONAL_IF_BOOL_BOOL_BOOL,
+  CONDITIONAL_IF_BOOL_DOUBLE_DOUBLE,
+  CONDITIONAL_IF_BOOL_LONG_LONG,
+  CONDITIONAL_IF_BOOL_STRINGVALUE_STRINGVALUE,
+  CONDITIONAL_IF_BOOL_TIMESTAMPVALUE_TIMESTAMPVALUE,
+  CONSTANT_REGEX,
+  CONSTANT_SUBSTRING,
+  DIVIDE,
+  EQ_BOOL_BOOL,
+  EQ_CHAR_CHAR,
+  EQ_DOUBLE_DOUBLE,
+  EQ_FLOAT_FLOAT,
+  EQ_INT_INT,
+  EQ_LONG_LONG,
+  EQ_SHORT_SHORT,
+  EQ_STRINGVALUE_STRINGVALUE,
+  EQ_TIMESTAMPVALUE_TIMESTAMPVALUE,
+  FROM_UNIXTIME_INT,
+  FROM_UNIXTIME_INT_STRINGVALUE,
+  FROM_UTC_TIMESTAMP,
+  GE_BOOL_BOOL,
+  GE_CHAR_CHAR,
+  GE_DOUBLE_DOUBLE,
+  GE_FLOAT_FLOAT,
+  GE_INT_INT,
+  GE_LONG_LONG,
+  GE_SHORT_SHORT,
+  GE_STRINGVALUE_STRINGVALUE,
+  GE_TIMESTAMPVALUE_TIMESTAMPVALUE,
+  GT_BOOL_BOOL,
+  GT_CHAR_CHAR,
+  GT_DOUBLE_DOUBLE,
+  GT_FLOAT_FLOAT,
+  GT_INT_INT,
+  GT_LONG_LONG,
+  GT_SHORT_SHORT,
+  GT_STRINGVALUE_STRINGVALUE,
+  GT_TIMESTAMPVALUE_TIMESTAMPVALUE,
+  INT_DIVIDE_CHAR_CHAR,
+  INT_DIVIDE_INT_INT,
+  INT_DIVIDE_LONG_LONG,
+  INT_DIVIDE_SHORT_SHORT,
+  LE_BOOL_BOOL,
+  LE_CHAR_CHAR,
+  LE_DOUBLE_DOUBLE,
+  LE_FLOAT_FLOAT,
+  LE_INT_INT,
+  LE_LONG_LONG,
+  LE_SHORT_SHORT,
+  LE_STRINGVALUE_STRINGVALUE,
+  LE_TIMESTAMPVALUE_TIMESTAMPVALUE,
+  LIKE,
+  LT_BOOL_BOOL,
+  LT_CHAR_CHAR,
+  LT_DOUBLE_DOUBLE,
+  LT_FLOAT_FLOAT,
+  LT_INT_INT,
+  LT_LONG_LONG,
+  LT_SHORT_SHORT,
+  LT_STRINGVALUE_STRINGVALUE,
+  LT_TIMESTAMPVALUE_TIMESTAMPVALUE,
+  MATH_ABS,
+  MATH_ACOS,
+  MATH_ASIN,
+  MATH_ATAN,
+  MATH_BIN,
+  MATH_CEIL,
+  MATH_CONV_LONG_CHAR_CHAR,
+  MATH_CONV_STRINGVALUE_CHAR_CHAR,
+  MATH_COS,
+  MATH_DEGREES,
+  MATH_E,
+  MATH_EXP,
+  MATH_FLOOR,
+  MATH_HEX_LONG,
+  MATH_HEX_STRINGVALUE,
+  MATH_LN,
+  MATH_LOG,
+  MATH_LOG10,
+  MATH_LOG2,
+  MATH_NEGATIVE_DOUBLE,
+  MATH_NEGATIVE_LONG,
+  MATH_PI,
+  MATH_PMOD_DOUBLE_DOUBLE,
+  MATH_PMOD_LONG_LONG,
+  MATH_POSITIVE_DOUBLE,
+  MATH_POSITIVE_LONG,
+  MATH_POW,
+  MATH_RADIANS,
+  MATH_RAND,
+  MATH_RAND_INT,
+  MATH_ROUND_DOUBLE,
+  MATH_ROUND_DOUBLE_INT,
+  MATH_SIGN,
+  MATH_SIN,
+  MATH_SQRT,
+  MATH_TAN,
+  MATH_UNHEX,
+  MOD_CHAR_CHAR,
+  MOD_INT_INT,
+  MOD_LONG_LONG,
+  MOD_SHORT_SHORT,
+  MULTIPLY_DOUBLE_DOUBLE,
+  MULTIPLY_LONG_LONG,
+  NE_BOOL_BOOL,
+  NE_CHAR_CHAR,
+  NE_DOUBLE_DOUBLE,
+  NE_FLOAT_FLOAT,
+  NE_INT_INT,
+  NE_LONG_LONG,
+  NE_SHORT_SHORT,
+  NE_STRINGVALUE_STRINGVALUE,
+  NE_TIMESTAMPVALUE_TIMESTAMPVALUE,
+  REGEX,
+  STRING_ASCII,
+  STRING_CONCAT,
+  STRING_CONCAT_WS,
+  STRING_FIND_IN_SET,
+  STRING_INSTR,
+  STRING_LEFT,
+  STRING_LENGTH,
+  STRING_LOCATE_STRINGVALUE_STRINGVALUE,
+  STRING_LOCATE_STRINGVALUE_STRINGVALUE_INT,
+  STRING_LOWER,
+  STRING_LPAD,
+  STRING_LTRIM,
+  STRING_PARSE_URL_STRINGVALUE_STRINGVALUE,
+  STRING_PARSE_URL_STRINGVALUE_STRINGVALUE_STRINGVALUE,
+  STRING_REGEXP_EXTRACT,
+  STRING_REGEXP_REPLACE,
+  STRING_REPEAT,
+  STRING_REVERSE,
+  STRING_RIGHT,
+  STRING_RPAD,
+  STRING_RTRIM,
+  STRING_SPACE,
+  STRING_SUBSTRING_STRINGVALUE_INT,
+  STRING_SUBSTRING_STRINGVALUE_INT_INT,
+  STRING_TRIM,
+  STRING_UPPER,
+  SUBTRACT_DOUBLE_DOUBLE,
+  SUBTRACT_LONG_LONG,
+  TIMESTAMP_DAY,
+  TIMESTAMP_DAYOFMONTH,
+  TIMESTAMP_DAYS_ADD,
+  TIMESTAMP_DAYS_SUB,
+  TIMESTAMP_DIFF,
+  TIMESTAMP_HOUR,
+  TIMESTAMP_HOURS_ADD,
+  TIMESTAMP_HOURS_SUB,
+  TIMESTAMP_MICROSECONDS_ADD,
+  TIMESTAMP_MICROSECONDS_SUB,
+  TIMESTAMP_MILLISECONDS_ADD,
+  TIMESTAMP_MILLISECONDS_SUB,
+  TIMESTAMP_MINUTE,
+  TIMESTAMP_MINUTES_ADD,
+  TIMESTAMP_MINUTES_SUB,
+  TIMESTAMP_MONTH,
+  TIMESTAMP_MONTHS_ADD,
+  TIMESTAMP_MONTHS_SUB,
+  TIMESTAMP_NANOSECONDS_ADD,
+  TIMESTAMP_NANOSECONDS_SUB,
+  TIMESTAMP_NOW,
+  TIMESTAMP_SECOND,
+  TIMESTAMP_SECONDS_ADD,
+  TIMESTAMP_SECONDS_SUB,
+  TIMESTAMP_TO_DATE,
+  TIMESTAMP_WEEKOFYEAR,
+  TIMESTAMP_WEEKS_ADD,
+  TIMESTAMP_WEEKS_SUB,
+  TIMESTAMP_YEAR,
+  TIMESTAMP_YEARS_ADD,
+  TIMESTAMP_YEARS_SUB,
+  TO_UTC_TIMESTAMP,
+  UNIX_TIMESTAMP,
+  UNIX_TIMESTAMP_STRINGVALUE_STRINGVALUE,
+  UNIX_TIMESTAMP_TIMESTAMPVALUE,
+  UTILITY_VERSION,
+  LAST_OPCODE,
+}
+