impala 0.1.0

data/thrift/StateStoreSubscriberService.thrift

@@ -0,0 +1,64 @@
+// Copyright 2012 Cloudera Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+namespace cpp impala
+namespace java com.cloudera.impala.thrift
+namespace rb Impala.Protocol
+
+include "StatestoreTypes.thrift"
+include "Status.thrift"
+include "Types.thrift"
+
+enum StateStoreSubscriberServiceVersion {
+  V1
+}
+
+struct TUpdateStateRequest {
+  1: required StateStoreSubscriberServiceVersion protocolVersion =
+      StateStoreSubscriberServiceVersion.V1
+
+  // Membership information for each service that the subscriber subscribed to.
+  // Required in V1.
+  2: optional list<StatestoreTypes.TServiceMembership> service_memberships
+
+  // Object updates for each service that the subscriber subscribed to.  Required in V1.
+  3: optional list<StatestoreTypes.TVersionedObject> updated_objects
+
+  // Objects that have been deleted, for each service that the subscriber has subscribed
+  // to. Required in V1.
+  4: optional list<string> deleted_object_keys
+}
+
+struct TUpdateStateResponse {
+  // Required in V1.
+  1: optional Status.TStatus status
+
+  // For each service running on the subscriber, the object updates.  Required in V1,
+  // but not yet implemented.
+  2: optional list<StatestoreTypes.TVersionedObject> updated_objects
+
+  // Objects that have been deleted, for each service that the subscriber has subscribed
+  // to. Required in V1, but not yet implemented.
+  3: optional list<string> deleted_object_keys
+}
+
+// The StateStoreSubscriber runs on all servers that need to connect to the StateStore,
+// and provides an interface between the StateStore and services that need to either push
+// updates to or receive updates from the StateStore.
+service StateStoreSubscriberService {
+  // The UpdateState() call serves two purposes: first, it allows the StateStore to push
+  // new updates to the subscriber; and second, it allows the StateStore to collect new
+  // updates from services running local with the StateStoreSubscriber.
+  TUpdateStateResponse UpdateState(1: TUpdateStateRequest request);
+}

data/thrift/StatestoreTypes.thrift

@@ -0,0 +1,50 @@
+// Copyright 2012 Cloudera Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+namespace cpp impala
+namespace java com.cloudera.impala.thrift
+namespace rb Impala.Protocol
+
+include "Types.thrift"
+
+struct TVersionedObject {
+  // Service that the object is associated with.
+  1: string service_id
+
+  // Unique (within the given service) identifier for the object.
+  2: string key
+
+  // Type of the value.
+  3: string type
+
+  // Unique (within the given key) identifier for the value.
+  4: i64 version
+
+  5: binary value
+}
+
+// Information about a running instance of a particular service.
+struct TServiceInstance {
+  // Unique identifier for the corresponding StateStoreSubscriber.
+  1: required i32 subscriber_id
+
+  2: required Types.THostPort host_port
+}
+
+// Information about all running instances of the service identified by service_id.
+struct TServiceMembership {
+  1: required string service_id
+
+  2: required list<TServiceInstance> service_instances
+}

data/thrift/Status.thrift

@@ -0,0 +1,31 @@
+// Copyright 2012 Cloudera Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+namespace cpp impala
+namespace java com.cloudera.impala.thrift
+namespace rb Impala.Protocol
+
+enum TStatusCode {
+  OK,
+  CANCELLED,
+  ANALYSIS_ERROR,
+  NOT_IMPLEMENTED_ERROR,
+  RUNTIME_ERROR,
+  INTERNAL_ERROR
+}
+
+struct TStatus {
+  1: required TStatusCode status_code
+  2: list<string> error_msgs
+}

data/thrift/Types.thrift

@@ -0,0 +1,71 @@
+// Copyright 2012 Cloudera Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+namespace cpp impala
+namespace java com.cloudera.impala.thrift
+namespace rb Impala.Protocol
+
+typedef i64 TTimestamp
+typedef i32 TPlanNodeId
+typedef i32 TTupleId
+typedef i32 TSlotId
+typedef i32 TTableId
+
+enum TPrimitiveType {
+  INVALID_TYPE,
+  BOOLEAN,
+  TINYINT,
+  SMALLINT,
+  INT,
+  BIGINT,
+  FLOAT,
+  DOUBLE,
+  DATE,
+  DATETIME,
+  TIMESTAMP,
+  STRING
+}
+
+enum TStmtType {
+  QUERY,
+  DDL, // Data definition, e.g. CREATE TABLE (includes read-only functions e.g. SHOW)
+  DML // Data modification e.g. INSERT
+}
+
+// level of verboseness for "explain" output
+// TODO: should this go somewhere else?
+enum TExplainLevel {
+  NORMAL,
+  VERBOSE
+}
+
+// A THostPort represents a general network address. It includes both
+// hostname and IP address fields in order to support use cases that may
+// require either one or both these fields set. An example is those
+// network addresses stored by the state-store which require IP addresses
+// in order for the scheduler to correctly assign data locations to
+// Impala backends but also fully qualified hostnames to be able to
+// establish secure connections with subscribers. Which of the fields are
+// set in general is usage specific.
+struct THostPort {
+  1: required string hostname
+  2: required string ipaddress
+  3: required i32 port
+}
+
+// Wire format for UniqueId
+struct TUniqueId {
+  1: required i64 hi
+  2: required i64 lo
+}

data/thrift/beeswax.thrift

@@ -0,0 +1,175 @@
+/*
+ * Licensed to Cloudera, Inc. under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  Cloudera, Inc. licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Interface for interacting with Beeswax Server
+ */
+
+namespace java com.cloudera.beeswax.api
+namespace py beeswaxd
+namespace cpp beeswax
+namespace rb Impala.Protocol.Beeswax
+
+include "hive_metastore.thrift"
+
+// A Query
+struct Query {
+  1: string query;
+  // A list of HQL commands to execute before the query.
+  // This is typically defining UDFs, setting settings, and loading resources.
+  3: list<string> configuration;
+
+  // User and groups to "act as" for purposes of Hadoop.
+  4: string hadoop_user;
+}
+
+typedef string LogContextId
+
+enum QueryState {
+  CREATED,
+  INITIALIZED,
+  COMPILED,
+  RUNNING,
+  FINISHED,
+  EXCEPTION
+}
+
+struct QueryHandle {
+  1: string id;
+  2: LogContextId log_context;
+}
+
+struct QueryExplanation {
+  1: string textual
+}
+
+struct Results {
+  // If set, data is valid.  Otherwise, results aren't ready yet.
+  1: bool ready,
+  // Columns for the results
+  2: list<string> columns,
+  // A set of results
+  3: list<string> data,
+  // The starting row of the results
+  4: i64 start_row,
+  // Whether there are more results to fetch
+  5: bool has_more
+}
+
+/**
+ * Metadata information about the results.
+ * Applicable only for SELECT.
+ */
+struct ResultsMetadata {
+  /** The schema of the results */
+  1: hive_metastore.Schema schema,
+  /** The directory containing the results. Not applicable for partition table. */
+  2: string table_dir,
+  /** If the results are straight from an existing table, the table name. */
+  3: string in_tablename,
+  /** Field delimiter */
+  4: string delim,
+}
+
+exception BeeswaxException {
+  1: string message,
+  // Use get_log(log_context) to retrieve any log related to this exception
+  2: LogContextId log_context,
+  // (Optional) The QueryHandle that caused this exception
+  3: QueryHandle handle,
+  4: optional i32 errorCode = 0,
+  5: optional string SQLState = "     "
+}
+
+exception QueryNotFoundException {
+}
+
+/** Represents a Hadoop-style configuration variable. */
+struct ConfigVariable {
+  1: string key,
+  2: string value,
+  3: string description
+}
+
+service BeeswaxService {
+  /**
+   * Submit a query and return a handle (QueryHandle). The query runs asynchronously.
+   */
+  QueryHandle query(1:Query query) throws(1:BeeswaxException error),
+
+  /**
+   * run a query synchronously and return a handle (QueryHandle).
+   */
+  QueryHandle executeAndWait(1:Query query, 2:LogContextId clientCtx)
+                        throws(1:BeeswaxException error),
+
+  /**
+   * Get the query plan for a query.
+   */
+  QueryExplanation explain(1:Query query)
+                        throws(1:BeeswaxException error),
+
+  /**
+   * Get the results of a query. This is non-blocking. Caller should check
+   * Results.ready to determine if the results are in yet. The call requests
+   * the batch size of fetch.
+   */
+  Results fetch(1:QueryHandle query_id, 2:bool start_over, 3:i32 fetch_size=-1)
+              throws(1:QueryNotFoundException error, 2:BeeswaxException error2),
+
+  /**
+   * Get the state of the query
+   */
+  QueryState get_state(1:QueryHandle handle) throws(1:QueryNotFoundException error),
+
+  /**
+   * Get the result metadata
+   */
+  ResultsMetadata get_results_metadata(1:QueryHandle handle)
+                                    throws(1:QueryNotFoundException error),
+
+  /**
+   * Used to test connection to server.  A "noop" command.
+   */
+  string echo(1:string s)
+
+  /**
+   * Returns a string representation of the configuration object being used.
+   * Handy for debugging.
+   */
+  string dump_config()
+
+  /**
+   * Get the log messages related to the given context.
+   */
+  string get_log(1:LogContextId context) throws(1:QueryNotFoundException error)
+
+  /*
+   * Returns "default" configuration.
+   */
+  list<ConfigVariable> get_default_configuration(1:bool include_hadoop)
+
+  /*
+   * closes the query with given handle
+   */
+  void close(1:QueryHandle handle) throws(1:QueryNotFoundException error,
+                            2:BeeswaxException error2)
+
+  /*
+   * clean the log context for given id
+   */
+  void clean(1:LogContextId log_context)
+}

data/thrift/fb303.thrift

@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * fb303.thrift
+ */
+
+namespace java com.facebook.fb303
+namespace cpp facebook.fb303
+namespace rb Impala.Protocol.fb303
+
+/**
+ * Common status reporting mechanism across all services
+ */
+enum fb_status {
+  DEAD = 0,
+  STARTING = 1,
+  ALIVE = 2,
+  STOPPING = 3,
+  STOPPED = 4,
+  WARNING = 5,
+}
+
+/**
+ * Standard base service
+ */
+service FacebookService {
+
+  /**
+   * Returns a descriptive name of the service
+   */
+  string getName(),
+
+  /**
+   * Returns the version of the service
+   */
+  string getVersion(),
+
+  /**
+   * Gets the status of this service
+   */
+  fb_status getStatus(),
+
+  /**
+   * User friendly description of status, such as why the service is in
+   * the dead or warning state, or what is being started or stopped.
+   */
+  string getStatusDetails(),
+
+  /**
+   * Gets the counters for this service
+   */
+  map<string, i64> getCounters(),
+
+  /**
+   * Gets the value of a single counter
+   */
+  i64 getCounter(1: string key),
+
+  /**
+   * Sets an option
+   */
+  void setOption(1: string key, 2: string value),
+
+  /**
+   * Gets an option
+   */
+  string getOption(1: string key),
+
+  /**
+   * Gets all options
+   */
+  map<string, string> getOptions(),
+
+  /**
+   * Returns a CPU profile over the given time interval (client and server
+   * must agree on the profile format).
+   */
+  string getCpuProfile(1: i32 profileDurationInSec),
+
+  /**
+   * Returns the unix time that the server has been running since
+   */
+  i64 aliveSince(),
+
+  /**
+   * Tell the server to reload its configuration, reopen log files, etc
+   */
+  oneway void reinitialize(),
+
+  /**
+   * Suggest a shutdown to the server
+   */
+  oneway void shutdown(),
+
+}