auser-poolparty 1.3.0 → 1.3.1

data/vendor/gems/thrift/lib/cpp/src/transport/TFileTransport.h

@@ -0,0 +1,442 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef _THRIFT_TRANSPORT_TFILETRANSPORT_H_
+#define _THRIFT_TRANSPORT_TFILETRANSPORT_H_ 1
+
+#include "TTransport.h"
+#include "Thrift.h"
+#include "TProcessor.h"
+
+#include <string>
+#include <stdio.h>
+
+#include <pthread.h>
+
+#include <boost/shared_ptr.hpp>
+
+namespace apache { namespace thrift { namespace transport {
+
+using apache::thrift::TProcessor;
+using apache::thrift::protocol::TProtocolFactory;
+
+// Data pertaining to a single event
+typedef struct eventInfo {
+  uint8_t* eventBuff_;
+  uint32_t eventSize_;
+  uint32_t eventBuffPos_;
+
+  eventInfo():eventBuff_(NULL), eventSize_(0), eventBuffPos_(0){};
+  ~eventInfo() {
+    if (eventBuff_) {
+      delete[] eventBuff_;
+    }
+  }
+} eventInfo;
+
+// information about current read state
+typedef struct readState {
+  eventInfo* event_;
+
+  // keep track of event size
+  uint8_t   eventSizeBuff_[4];
+  uint8_t   eventSizeBuffPos_;
+  bool      readingSize_;
+
+  // read buffer variables
+  int32_t  bufferPtr_;
+  int32_t  bufferLen_;
+
+  // last successful dispatch point
+  int32_t lastDispatchPtr_;
+
+  void resetState(uint32_t lastDispatchPtr) {
+    readingSize_ = true;
+    eventSizeBuffPos_ = 0;
+    lastDispatchPtr_ = lastDispatchPtr;
+  }
+
+  void resetAllValues() {
+    resetState(0);
+    bufferPtr_ = 0;
+    bufferLen_ = 0;
+    if (event_) {
+      delete(event_);
+    }
+    event_ = 0;
+  }
+
+  readState() {
+    event_ = 0;
+   resetAllValues();
+  }
+
+  ~readState() {
+    if (event_) {
+      delete(event_);
+    }
+  }
+
+} readState;
+
+/**
+ * TFileTransportBuffer - buffer class used by TFileTransport for queueing up events
+ * to be written to disk.  Should be used in the following way:
+ *  1) Buffer created
+ *  2) Buffer written to (addEvent)
+ *  3) Buffer read from (getNext)
+ *  4) Buffer reset (reset)
+ *  5) Go back to 2, or destroy buffer
+ *
+ * The buffer should never be written to after it is read from, unless it is reset first.
+ * Note: The above rules are enforced mainly for debugging its sole client TFileTransport
+ *       which uses the buffer in this way.
+ *
+ */
+class TFileTransportBuffer {
+  public:
+    TFileTransportBuffer(uint32_t size);
+    ~TFileTransportBuffer();
+
+    bool addEvent(eventInfo *event);
+    eventInfo* getNext();
+    void reset();
+    bool isFull();
+    bool isEmpty();
+
+  private:
+    TFileTransportBuffer(); // should not be used
+
+    enum mode {
+      WRITE,
+      READ
+    };
+    mode bufferMode_;
+
+    uint32_t writePoint_;
+    uint32_t readPoint_;
+    uint32_t size_;
+    eventInfo** buffer_;
+};
+
+/**
+ * Abstract interface for transports used to read files
+ */
+class TFileReaderTransport : virtual public TTransport {
+ public:
+  virtual int32_t getReadTimeout() = 0;
+  virtual void setReadTimeout(int32_t readTimeout) = 0;
+
+  virtual uint32_t getNumChunks() = 0;
+  virtual uint32_t getCurChunk() = 0;
+  virtual void seekToChunk(int32_t chunk) = 0;
+  virtual void seekToEnd() = 0;
+};
+
+/**
+ * Abstract interface for transports used to write files
+ */
+class TFileWriterTransport : virtual public TTransport {
+ public:
+  virtual uint32_t getChunkSize() = 0;
+  virtual void setChunkSize(uint32_t chunkSize) = 0;
+};
+
+/**
+ * File implementation of a transport. Reads and writes are done to a
+ * file on disk.
+ *
+ */
+class TFileTransport : public TFileReaderTransport,
+                       public TFileWriterTransport {
+ public:
+  TFileTransport(std::string path, bool readOnly=false);
+  ~TFileTransport();
+
+  // TODO: what is the correct behaviour for this?
+  // the log file is generally always open
+  bool isOpen() {
+    return true;
+  }
+
+  void write(const uint8_t* buf, uint32_t len);
+  void flush();
+
+  uint32_t readAll(uint8_t* buf, uint32_t len);
+  uint32_t read(uint8_t* buf, uint32_t len);
+
+  // log-file specific functions
+  void seekToChunk(int32_t chunk);
+  void seekToEnd();
+  uint32_t getNumChunks();
+  uint32_t getCurChunk();
+
+  // for changing the output file
+  void resetOutputFile(int fd, std::string filename, int64_t offset);
+
+  // Setter/Getter functions for user-controllable options
+  void setReadBuffSize(uint32_t readBuffSize) {
+    if (readBuffSize) {
+      readBuffSize_ = readBuffSize;
+    }
+  }
+  uint32_t getReadBuffSize() {
+    return readBuffSize_;
+  }
+
+  static const int32_t TAIL_READ_TIMEOUT = -1;
+  static const int32_t NO_TAIL_READ_TIMEOUT = 0;
+  void setReadTimeout(int32_t readTimeout) {
+    readTimeout_ = readTimeout;
+  }
+  int32_t getReadTimeout() {
+    return readTimeout_;
+  }
+
+  void setChunkSize(uint32_t chunkSize) {
+    if (chunkSize) {
+      chunkSize_ = chunkSize;
+    }
+  }
+  uint32_t getChunkSize() {
+    return chunkSize_;
+  }
+
+  void setEventBufferSize(uint32_t bufferSize) {
+    if (bufferAndThreadInitialized_) {
+      GlobalOutput("Cannot change the buffer size after writer thread started");
+      return;
+    }
+    eventBufferSize_ = bufferSize;
+  }
+
+  uint32_t getEventBufferSize() {
+    return eventBufferSize_;
+  }
+
+  void setFlushMaxUs(uint32_t flushMaxUs) {
+    if (flushMaxUs) {
+      flushMaxUs_ = flushMaxUs;
+    }
+  }
+  uint32_t getFlushMaxUs() {
+    return flushMaxUs_;
+  }
+
+  void setFlushMaxBytes(uint32_t flushMaxBytes) {
+    if (flushMaxBytes) {
+      flushMaxBytes_ = flushMaxBytes;
+    }
+  }
+  uint32_t getFlushMaxBytes() {
+    return flushMaxBytes_;
+  }
+
+  void setMaxEventSize(uint32_t maxEventSize) {
+    maxEventSize_ = maxEventSize;
+  }
+  uint32_t getMaxEventSize() {
+    return maxEventSize_;
+  }
+
+  void setMaxCorruptedEvents(uint32_t maxCorruptedEvents) {
+    maxCorruptedEvents_ = maxCorruptedEvents;
+  }
+  uint32_t getMaxCorruptedEvents() {
+    return maxCorruptedEvents_;
+  }
+
+  void setEofSleepTimeUs(uint32_t eofSleepTime) {
+    if (eofSleepTime) {
+      eofSleepTime_ = eofSleepTime;
+    }
+  }
+  uint32_t getEofSleepTimeUs() {
+    return eofSleepTime_;
+  }
+
+ private:
+  // helper functions for writing to a file
+  void enqueueEvent(const uint8_t* buf, uint32_t eventLen, bool blockUntilFlush);
+  bool swapEventBuffers(struct timespec* deadline);
+  bool initBufferAndWriteThread();
+
+  // control for writer thread
+  static void* startWriterThread(void* ptr) {
+    (((TFileTransport*)ptr)->writerThread());
+    return 0;
+  }
+  void writerThread();
+
+  // helper functions for reading from a file
+  eventInfo* readEvent();
+
+  // event corruption-related functions
+  bool isEventCorrupted();
+  void performRecovery();
+
+  // Utility functions
+  void openLogFile();
+  void getNextFlushTime(struct timespec* ts_next_flush);
+
+  // Class variables
+  readState readState_;
+  uint8_t* readBuff_;
+  eventInfo* currentEvent_;
+
+  uint32_t readBuffSize_;
+  static const uint32_t DEFAULT_READ_BUFF_SIZE = 1 * 1024 * 1024;
+
+  int32_t readTimeout_;
+  static const int32_t DEFAULT_READ_TIMEOUT_MS = 200;
+
+  // size of chunks that file will be split up into
+  uint32_t chunkSize_;
+  static const uint32_t DEFAULT_CHUNK_SIZE = 16 * 1024 * 1024;
+
+  // size of event buffers
+  uint32_t eventBufferSize_;
+  static const uint32_t DEFAULT_EVENT_BUFFER_SIZE = 10000;
+
+  // max number of microseconds that can pass without flushing
+  uint32_t flushMaxUs_;
+  static const uint32_t DEFAULT_FLUSH_MAX_US = 3000000;
+
+  // max number of bytes that can be written without flushing
+  uint32_t flushMaxBytes_;
+  static const uint32_t DEFAULT_FLUSH_MAX_BYTES = 1000 * 1024;
+
+  // max event size
+  uint32_t maxEventSize_;
+  static const uint32_t DEFAULT_MAX_EVENT_SIZE = 0;
+
+  // max number of corrupted events per chunk
+  uint32_t maxCorruptedEvents_;
+  static const uint32_t DEFAULT_MAX_CORRUPTED_EVENTS = 0;
+
+  // sleep duration when EOF is hit
+  uint32_t eofSleepTime_;
+  static const uint32_t DEFAULT_EOF_SLEEP_TIME_US = 500 * 1000;
+
+  // sleep duration when a corrupted event is encountered
+  uint32_t corruptedEventSleepTime_;
+  static const uint32_t DEFAULT_CORRUPTED_SLEEP_TIME_US = 1 * 1000 * 1000;
+
+  // writer thread id
+  pthread_t writerThreadId_;
+
+  // buffers to hold data before it is flushed. Each element of the buffer stores a msg that
+  // needs to be written to the file.  The buffers are swapped by the writer thread.
+  TFileTransportBuffer *dequeueBuffer_;
+  TFileTransportBuffer *enqueueBuffer_;
+
+  // conditions used to block when the buffer is full or empty
+  pthread_cond_t notFull_, notEmpty_;
+  volatile bool closing_;
+
+  // To keep track of whether the buffer has been flushed
+  pthread_cond_t flushed_;
+  volatile bool forceFlush_;
+
+  // Mutex that is grabbed when enqueueing and swapping the read/write buffers
+  pthread_mutex_t mutex_;
+
+  // File information
+  std::string filename_;
+  int fd_;
+
+  // Whether the writer thread and buffers have been initialized
+  bool bufferAndThreadInitialized_;
+
+  // Offset within the file
+  off_t offset_;
+
+  // event corruption information
+  uint32_t lastBadChunk_;
+  uint32_t numCorruptedEventsInChunk_;
+
+  bool readOnly_;
+};
+
+// Exception thrown when EOF is hit
+class TEOFException : public TTransportException {
+ public:
+  TEOFException():
+    TTransportException(TTransportException::END_OF_FILE) {};
+};
+
+
+// wrapper class to process events from a file containing thrift events
+class TFileProcessor {
+ public:
+  /**
+   * Constructor that defaults output transport to null transport
+   *
+   * @param processor processes log-file events
+   * @param protocolFactory protocol factory
+   * @param inputTransport file transport
+   */
+  TFileProcessor(boost::shared_ptr<TProcessor> processor,
+                 boost::shared_ptr<TProtocolFactory> protocolFactory,
+                 boost::shared_ptr<TFileReaderTransport> inputTransport);
+
+  TFileProcessor(boost::shared_ptr<TProcessor> processor,
+                 boost::shared_ptr<TProtocolFactory> inputProtocolFactory,
+                 boost::shared_ptr<TProtocolFactory> outputProtocolFactory,
+                 boost::shared_ptr<TFileReaderTransport> inputTransport);
+
+  /**
+   * Constructor
+   *
+   * @param processor processes log-file events
+   * @param protocolFactory protocol factory
+   * @param inputTransport input file transport
+   * @param output output transport
+   */
+  TFileProcessor(boost::shared_ptr<TProcessor> processor,
+                 boost::shared_ptr<TProtocolFactory> protocolFactory,
+                 boost::shared_ptr<TFileReaderTransport> inputTransport,
+                 boost::shared_ptr<TTransport> outputTransport);
+
+  /**
+   * processes events from the file
+   *
+   * @param numEvents number of events to process (0 for unlimited)
+   * @param tail tails the file if true
+   */
+  void process(uint32_t numEvents, bool tail);
+
+  /**
+   * process events until the end of the chunk
+   *
+   */
+  void processChunk();
+
+ private:
+  boost::shared_ptr<TProcessor> processor_;
+  boost::shared_ptr<TProtocolFactory> inputProtocolFactory_;
+  boost::shared_ptr<TProtocolFactory> outputProtocolFactory_;
+  boost::shared_ptr<TFileReaderTransport> inputTransport_;
+  boost::shared_ptr<TTransport> outputTransport_;
+};
+
+
+}}} // apache::thrift::transport
+
+#endif // _THRIFT_TRANSPORT_TFILETRANSPORT_H_

data/vendor/gems/thrift/lib/cpp/src/transport/THttpClient.cpp

@@ -0,0 +1,348 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <cstdlib>
+#include <sstream>
+
+#include "THttpClient.h"
+#include "TSocket.h"
+
+namespace apache { namespace thrift { namespace transport {
+
+using namespace std;
+
+/**
+ * Http client implementation.
+ *
+ */
+
+// Yeah, yeah, hacky to put these here, I know.
+static const char* CRLF = "\r\n";
+static const int CRLF_LEN = 2;
+
+THttpClient::THttpClient(boost::shared_ptr<TTransport> transport, string host, string path) :
+  transport_(transport),
+  host_(host),
+  path_(path),
+  readHeaders_(true),
+  chunked_(false),
+  chunkedDone_(false),
+  chunkSize_(0),
+  contentLength_(0),
+  httpBuf_(NULL),
+  httpPos_(0),
+  httpBufLen_(0),
+  httpBufSize_(1024) {
+  init();
+}
+
+THttpClient::THttpClient(string host, int port, string path) :
+  host_(host),
+  path_(path),
+  readHeaders_(true),
+  chunked_(false),
+  chunkedDone_(false),
+  chunkSize_(0),
+  contentLength_(0),
+  httpBuf_(NULL),
+  httpPos_(0),
+  httpBufLen_(0),
+  httpBufSize_(1024) {
+  transport_ = boost::shared_ptr<TTransport>(new TSocket(host, port));
+  init();
+}
+
+void THttpClient::init() {
+  httpBuf_ = (char*)std::malloc(httpBufSize_+1);
+  if (httpBuf_ == NULL) {
+    throw TTransportException("Out of memory.");
+  }
+  httpBuf_[httpBufLen_] = '\0';
+}
+
+THttpClient::~THttpClient() {
+  if (httpBuf_ != NULL) {
+    std::free(httpBuf_);
+  }
+}
+
+uint32_t THttpClient::read(uint8_t* buf, uint32_t len) {
+  if (readBuffer_.available_read() == 0) {
+    readBuffer_.resetBuffer();
+    uint32_t got = readMoreData();
+    if (got == 0) {
+      return 0;
+    }
+  }
+  return readBuffer_.read(buf, len);
+}
+
+void THttpClient::readEnd() {
+  // Read any pending chunked data (footers etc.)
+  if (chunked_) {
+    while (!chunkedDone_) {
+      readChunked();
+    }
+  }
+}
+
+uint32_t THttpClient::readMoreData() {
+  // Get more data!
+  refill();
+
+  if (readHeaders_) {
+    readHeaders();
+  }
+
+  if (chunked_) {
+    return readChunked();
+  } else {
+    return readContent(contentLength_);
+  }
+}
+
+uint32_t THttpClient::readChunked() {
+  uint32_t length = 0;
+
+  char* line = readLine();
+  uint32_t chunkSize = parseChunkSize(line);
+  if (chunkSize == 0) {
+    readChunkedFooters();
+  } else {
+    // Read data content
+    length += readContent(chunkSize);
+    // Read trailing CRLF after content
+    readLine();
+  }
+  return length;
+}
+
+void THttpClient::readChunkedFooters() {
+  // End of data, read footer lines until a blank one appears
+  while (true) {
+    char* line = readLine();
+    if (strlen(line) == 0) {
+      chunkedDone_ = true;
+      break;
+    }
+  }
+}
+
+uint32_t THttpClient::parseChunkSize(char* line) {
+  char* semi = strchr(line, ';');
+  if (semi != NULL) {
+    *semi = '\0';
+  }
+  int size = 0;
+  sscanf(line, "%x", &size);
+  return (uint32_t)size;
+}
+
+uint32_t THttpClient::readContent(uint32_t size) {
+  uint32_t need = size;
+  while (need > 0) {
+    uint32_t avail = httpBufLen_ - httpPos_;
+    if (avail == 0) {
+      // We have given all the data, reset position to head of the buffer
+      httpPos_ = 0;
+      httpBufLen_ = 0;
+      refill();
+
+      // Now have available however much we read
+      avail = httpBufLen_;
+    }
+    uint32_t give = avail;
+    if (need < give) {
+      give = need;
+    }
+    readBuffer_.write((uint8_t*)(httpBuf_+httpPos_), give);
+    httpPos_ += give;
+    need -= give;
+  }
+  return size;
+}
+
+char* THttpClient::readLine() {
+  while (true) {
+    char* eol = NULL;
+
+    eol = strstr(httpBuf_+httpPos_, CRLF);
+
+    // No CRLF yet?
+    if (eol == NULL) {
+      // Shift whatever we have now to front and refill
+      shift();
+      refill();
+    } else {
+      // Return pointer to next line
+      *eol = '\0';
+      char* line = httpBuf_+httpPos_;
+      httpPos_ = (eol-httpBuf_) + CRLF_LEN;
+      return line;
+    }
+  }
+
+}
+
+void THttpClient::shift() {
+  if (httpBufLen_ > httpPos_) {
+    // Shift down remaining data and read more
+    uint32_t length = httpBufLen_ - httpPos_;
+    memmove(httpBuf_, httpBuf_+httpPos_, length);
+    httpBufLen_ = length;
+  } else {
+    httpBufLen_ = 0;
+  }
+  httpPos_ = 0;
+  httpBuf_[httpBufLen_] = '\0';
+}
+
+void THttpClient::refill() {
+  uint32_t avail = httpBufSize_ - httpBufLen_;
+  if (avail <= (httpBufSize_ / 4)) {
+    httpBufSize_ *= 2;
+    httpBuf_ = (char*)std::realloc(httpBuf_, httpBufSize_+1);
+    if (httpBuf_ == NULL) {
+      throw TTransportException("Out of memory.");
+    }
+  }
+
+  // Read more data
+  uint32_t got = transport_->read((uint8_t*)(httpBuf_+httpBufLen_), httpBufSize_-httpBufLen_);
+  httpBufLen_ += got;
+  httpBuf_[httpBufLen_] = '\0';
+
+  if (got == 0) {
+    throw TTransportException("Could not refill buffer");
+  }
+}
+
+void THttpClient::readHeaders() {
+  // Initialize headers state variables
+  contentLength_ = 0;
+  chunked_ = false;
+  chunkedDone_ = false;
+  chunkSize_ = 0;
+
+  // Control state flow
+  bool statusLine = true;
+  bool finished = false;
+
+  // Loop until headers are finished
+  while (true) {
+    char* line = readLine();
+
+    if (strlen(line) == 0) {
+      if (finished) {
+        readHeaders_ = false;
+        return;
+      } else {
+        // Must have been an HTTP 100, keep going for another status line
+        statusLine = true;
+      }
+    } else {
+      if (statusLine) {
+        statusLine = false;
+        finished = parseStatusLine(line);
+      } else {
+        parseHeader(line);
+      }
+    }
+  }
+}
+
+bool THttpClient::parseStatusLine(char* status) {
+  char* http = status;
+
+  char* code = strchr(http, ' ');
+  if (code == NULL) {
+    throw TTransportException(string("Bad Status: ") + status);
+  }
+
+  *code = '\0';
+  while (*(code++) == ' ');
+
+  char* msg = strchr(code, ' ');
+  if (msg == NULL) {
+    throw TTransportException(string("Bad Status: ") + status);
+  }
+  *msg = '\0';
+
+  if (strcmp(code, "200") == 0) {
+    // HTTP 200 = OK, we got the response
+    return true;
+  } else if (strcmp(code, "100") == 0) {
+    // HTTP 100 = continue, just keep reading
+    return false;
+  } else {
+    throw TTransportException(string("Bad Status: ") + status);
+  }
+}
+
+void THttpClient::parseHeader(char* header) {
+  char* colon = strchr(header, ':');
+  if (colon == NULL) {
+    return;
+  }
+  uint32_t sz = colon - header;
+  char* value = colon+1;
+
+  if (strncmp(header, "Transfer-Encoding", sz) == 0) {
+    if (strstr(value, "chunked") != NULL) {
+      chunked_ = true;
+    }
+  } else if (strncmp(header, "Content-Length", sz) == 0) {
+    chunked_ = false;
+    contentLength_ = atoi(value);
+  }
+}
+
+void THttpClient::write(const uint8_t* buf, uint32_t len) {
+  writeBuffer_.write(buf, len);
+}
+
+void THttpClient::flush() {
+  // Fetch the contents of the write buffer
+  uint8_t* buf;
+  uint32_t len;
+  writeBuffer_.getBuffer(&buf, &len);
+
+  // Construct the HTTP header
+  std::ostringstream h;
+  h <<
+    "POST " << path_ << " HTTP/1.1" << CRLF <<
+    "Host: " << host_ << CRLF <<
+    "Content-Type: application/x-thrift" << CRLF <<
+    "Content-Length: " << len << CRLF <<
+    "Accept: application/x-thrift" << CRLF <<
+    "User-Agent: C++/THttpClient" << CRLF <<
+    CRLF;
+  string header = h.str();
+
+  // Write the header, then the data, then flush
+  transport_->write((const uint8_t*)header.c_str(), header.size());
+  transport_->write(buf, len);
+  transport_->flush();
+
+  // Reset the buffer and header variables
+  writeBuffer_.resetBuffer();
+  readHeaders_ = true;
+}
+
+}}} // apache::thrift::transport