impala 0.1.0

data/lib/impala/protocol/types_constants.rb

@@ -0,0 +1,12 @@
+#
+# Autogenerated by Thrift Compiler (0.8.0)
+#
+# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+#
+
+require 'types_types'
+
+module Impala
+  module Protocol
+  end
+end

data/lib/impala/protocol/types_types.rb

@@ -0,0 +1,86 @@
+#
+# Autogenerated by Thrift Compiler (0.8.0)
+#
+# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+#
+
+
+module Impala
+  module Protocol
+    module TPrimitiveType
+      INVALID_TYPE = 0
+      BOOLEAN = 1
+      TINYINT = 2
+      SMALLINT = 3
+      INT = 4
+      BIGINT = 5
+      FLOAT = 6
+      DOUBLE = 7
+      DATE = 8
+      DATETIME = 9
+      TIMESTAMP = 10
+      STRING = 11
+      VALUE_MAP = { 0 => "INVALID_TYPE", 1 => "BOOLEAN", 2 => "TINYINT", 3 => "SMALLINT", 4 => "INT", 5 => "BIGINT", 6 => "FLOAT", 7 => "DOUBLE", 8 => "DATE", 9 => "DATETIME", 10 => "TIMESTAMP", 11 => "STRING" }
+      VALID_VALUES = Set.new([INVALID_TYPE, BOOLEAN, TINYINT, SMALLINT, INT, BIGINT, FLOAT, DOUBLE, DATE, DATETIME, TIMESTAMP, STRING]).freeze
+    end
+
+    module TStmtType
+      QUERY = 0
+      DDL = 1
+      DML = 2
+      VALUE_MAP = { 0 => "QUERY", 1 => "DDL", 2 => "DML" }
+      VALID_VALUES = Set.new([QUERY, DDL, DML]).freeze
+    end
+
+    module TExplainLevel
+      NORMAL = 0
+      VERBOSE = 1
+      VALUE_MAP = { 0 => "NORMAL", 1 => "VERBOSE" }
+      VALID_VALUES = Set.new([NORMAL, VERBOSE]).freeze
+    end
+
+    class THostPort
+      include ::Thrift::Struct, ::Thrift::Struct_Union
+      HOSTNAME = 1
+      IPADDRESS = 2
+      PORT = 3
+
+      FIELDS = {
+      HOSTNAME => { :type => ::Thrift::Types::STRING, :name => 'hostname' },
+      IPADDRESS => { :type => ::Thrift::Types::STRING, :name => 'ipaddress' },
+      PORT => { :type => ::Thrift::Types::I32, :name => 'port' }
+      }
+
+      def struct_fields; FIELDS; end
+
+      def validate
+        raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field hostname is unset!') unless @hostname
+        raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field ipaddress is unset!') unless @ipaddress
+        raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field port is unset!') unless @port
+      end
+
+      ::Thrift::Struct.generate_accessors self
+    end
+
+    class TUniqueId
+      include ::Thrift::Struct, ::Thrift::Struct_Union
+      HI = 1
+      LO = 2
+
+      FIELDS = {
+      HI => { :type => ::Thrift::Types::I64, :name => 'hi' },
+      LO => { :type => ::Thrift::Types::I64, :name => 'lo' }
+      }
+
+      def struct_fields; FIELDS; end
+
+      def validate
+        raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field hi is unset!') unless @hi
+        raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field lo is unset!') unless @lo
+      end
+
+      ::Thrift::Struct.generate_accessors self
+    end
+
+  end
+end

data/lib/impala/version.rb

@@ -0,0 +1,3 @@
+module Impala
+  VERSION = "0.1.0"
+end

data/thrift/Data.thrift

@@ -0,0 +1,52 @@
+// Copyright 2012 Cloudera Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+namespace cpp impala
+namespace java com.cloudera.impala.thrift
+namespace rb Impala.Protocol
+namespace rb Impala.Protocol
+
+include "Types.thrift"
+
+// Serialized, self-contained version of a RowBatch (in be/src/runtime/row-batch.h).
+struct TRowBatch {
+  // total number of rows contained in this batch
+  1: required i32 num_rows
+
+  // row composition
+  2: required list<Types.TTupleId> row_tuples
+
+  // There are a total of num_rows * num_tuples_per_row offsets
+  // pointing into tuple_data.
+  // An offset of -1 records a NULL.
+  3: list<i32> tuple_offsets
+
+  // binary tuple data, broken up into chunks
+  // TODO: figure out how we can avoid copying the data during TRowBatch construction
+  4: list<string> tuple_data
+}
+
+// this is a union over all possible return types
+struct TColumnValue {
+  // TODO: use <type>_val instead of camelcase
+  1: optional bool boolVal
+  2: optional i32 intVal
+  3: optional i64 longVal
+  4: optional double doubleVal
+  5: optional string stringVal
+}
+
+struct TResultRow {
+  1: list<TColumnValue> colVals
+}

data/thrift/DataSinks.thrift

@@ -0,0 +1,61 @@
+// Copyright 2012 Cloudera Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+namespace cpp impala
+namespace java com.cloudera.impala.thrift
+namespace rb Impala.Protocol
+
+include "Exprs.thrift"
+include "Types.thrift"
+include "Descriptors.thrift"
+include "Partitions.thrift"
+
+enum TDataSinkType {
+  DATA_STREAM_SINK,
+  TABLE_SINK
+}
+
+// Sink which forwards data to a remote plan fragment,
+// according to the given output partition specification
+// (ie, the m:1 part of an m:n data stream)
+struct TDataStreamSink {
+  // destination node id
+  1: required Types.TPlanNodeId dest_node_id
+
+  // Specification of how the output of a fragment is partitioned.
+  // If the partitioning type is UNPARTITIONED, the output is broadcast
+  // to each destination host.
+  2: required Partitions.TDataPartition output_partition
+}
+
+// Creates a new Hdfs files according to the evaluation of the partitionKeyExprs,
+// and materializes all its input RowBatches as a Hdfs file.
+struct THdfsTableSink {
+  1: required list<Exprs.TExpr> partition_key_exprs
+  2: required bool overwrite
+}
+
+// Union type of all table sinks.
+// Currently, only THdfsTableSink is supported, so we don't have a separate
+// TTableSinkType yet.
+struct TTableSink {
+  1: required Types.TTableId target_table_id
+  2: required THdfsTableSink hdfs_table_sink
+}
+
+struct TDataSink {
+  1: required TDataSinkType type
+  2: optional TDataStreamSink stream_sink
+  3: optional TTableSink table_sink
+}

data/thrift/Descriptors.thrift

@@ -0,0 +1,115 @@
+// Copyright 2012 Cloudera Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+namespace cpp impala
+namespace java com.cloudera.impala.thrift
+namespace rb Impala.Protocol
+
+include "Types.thrift"
+include "Exprs.thrift"
+
+struct TSlotDescriptor {
+  1: required Types.TSlotId id
+  2: required Types.TTupleId parent
+  3: required Types.TPrimitiveType slotType
+  4: required i32 columnPos   // in originating table
+  5: required i32 byteOffset  // into tuple
+  6: required i32 nullIndicatorByte
+  7: required i32 nullIndicatorBit
+  9: required i32 slotIdx
+  10: required bool isMaterialized
+}
+
+enum TTableType {
+  HDFS_TABLE,
+  HBASE_TABLE
+}
+
+enum THdfsFileFormat {
+  TEXT,
+  LZO_TEXT,
+  RC_FILE,
+  SEQUENCE_FILE,
+  TREVNI
+}
+
+enum THdfsCompression {
+  NONE,
+  DEFAULT,
+  GZIP,
+  BZIP2,
+  SNAPPY,
+  SNAPPY_BLOCKED // Used by sequence and rc files but not stored in the metadata.
+}
+
+struct THdfsPartition {
+  1: required byte lineDelim
+  2: required byte fieldDelim
+  3: required byte collectionDelim
+  4: required byte mapKeyDelim
+  5: required byte escapeChar
+  6: required THdfsFileFormat fileFormat
+  7: list<Exprs.TExpr> partitionKeyExprs
+  8: required i32 blockSize
+  9: required THdfsCompression compression
+}
+
+struct THdfsTable {
+  1: required string hdfsBaseDir
+
+  // Partition keys are the same as clustering columns in
+  // TTableDescriptor, so there should be an equal number of each.
+  2: required list<string> partitionKeyNames
+  3: required string nullPartitionKeyValue
+
+  // map from partition id to partition metadata
+  4: required map<i64, THdfsPartition> partitions
+}
+
+struct THBaseTable {
+  1: required string tableName
+  2: required list<string> families
+  3: required list<string> qualifiers
+}
+
+// "Union" of all table types.
+struct TTableDescriptor {
+  1: required Types.TTableId id
+  2: required TTableType tableType
+  3: required i32 numCols
+  4: required i32 numClusteringCols
+  5: optional THdfsTable hdfsTable
+  6: optional THBaseTable hbaseTable
+
+  // Unqualified name of table
+  7: required string tableName;
+
+  // Name of the database that the table belongs to
+  8: required string dbName;
+}
+
+struct TTupleDescriptor {
+  1: required Types.TTupleId id
+  2: required i32 byteSize
+  3: required i32 numNullBytes
+  4: optional Types.TTableId tableId
+}
+
+struct TDescriptorTable {
+  1: optional list<TSlotDescriptor> slotDescriptors;
+  2: required list<TTupleDescriptor> tupleDescriptors;
+
+  // all table descriptors referenced by tupleDescriptors
+  3: optional list<TTableDescriptor> tableDescriptors;
+}

data/thrift/Exprs.thrift

@@ -0,0 +1,134 @@
+// Copyright 2012 Cloudera Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+namespace cpp impala
+namespace java com.cloudera.impala.thrift
+namespace rb Impala.Protocol
+
+include "Types.thrift"
+include "Opcodes.thrift"
+
+enum TExprNodeType {
+  AGG_EXPR,
+  ARITHMETIC_EXPR,
+  BINARY_PRED,
+  BOOL_LITERAL,
+  CASE_EXPR,
+  CAST_EXPR,
+  COMPOUND_PRED,
+  DATE_LITERAL,
+  FLOAT_LITERAL,
+  FUNCTION_CALL,
+  INT_LITERAL,
+  IN_PRED,
+  IS_NULL_PRED,
+  LIKE_PRED,
+  LITERAL_PRED,
+  NULL_LITERAL,
+  SLOT_REF,
+  STRING_LITERAL,
+}
+
+enum TAggregationOp {
+  INVALID,
+  COUNT,
+  MAX,
+  DISTINCT_PC,
+  MERGE_PC,
+  DISTINCT_PCSA,
+  MERGE_PCSA,
+  MIN,
+  SUM,
+}
+
+struct TAggregateExpr {
+  1: required bool is_star
+  2: required bool is_distinct
+  3: required TAggregationOp op
+}
+
+struct TBoolLiteral {
+  1: required bool value
+}
+
+struct TCaseExpr {
+  1: required bool has_case_expr
+  2: required bool has_else_expr
+}
+
+struct TDateLiteral {
+  1: required Types.TTimestamp value
+}
+
+struct TFloatLiteral {
+  1: required double value
+}
+
+struct TIntLiteral {
+  1: required i64 value
+}
+
+struct TInPredicate {
+  1: required bool is_not_in
+}
+
+struct TIsNullPredicate {
+  1: required bool is_not_null
+}
+
+struct TLikePredicate {
+  1: required string escape_char;
+}
+
+struct TLiteralPredicate {
+  1: required bool value
+  2: required bool is_null
+}
+
+struct TSlotRef {
+  1: required Types.TSlotId slot_id
+}
+
+struct TStringLiteral {
+  1: required string value;
+}
+
+// This is essentially a union over the subclasses of Expr.
+struct TExprNode {
+  1: required TExprNodeType node_type
+  2: required Types.TPrimitiveType type
+  3: optional Opcodes.TExprOpcode opcode
+  4: required i32 num_children
+
+  5: optional TAggregateExpr agg_expr
+  6: optional TBoolLiteral bool_literal
+  7: optional TCaseExpr case_expr
+  8: optional TDateLiteral date_literal
+  9: optional TFloatLiteral float_literal
+  10: optional TIntLiteral int_literal
+  11: optional TInPredicate in_predicate
+  12: optional TIsNullPredicate is_null_pred
+  13: optional TLikePredicate like_pred
+  14: optional TLiteralPredicate literal_pred
+  15: optional TSlotRef slot_ref
+  16: optional TStringLiteral string_literal
+}
+
+// A flattened representation of a tree of Expr nodes, obtained by depth-first
+// traversal.
+struct TExpr {
+  1: required list<TExprNode> nodes
+}
+
+