extbrotli 0.0.1.PROTOTYPE

data/contrib/brotli/enc/streams.cc

@@ -0,0 +1,127 @@
+// Copyright 2009 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Convience routines to make Brotli I/O classes from some memory containers and
+// files.
+
+#include "./streams.h"
+
+#include <assert.h>
+#include <stdlib.h>
+#include <string.h>
+
+namespace brotli {
+
+BrotliMemOut::BrotliMemOut(void* buf, int len)
+    : buf_(buf),
+      len_(len),
+      pos_(0) {}
+
+void BrotliMemOut::Reset(void* buf, int len) {
+  buf_ = buf;
+  len_ = len;
+  pos_ = 0;
+}
+
+// Brotli output routine: copy n bytes to the output buffer.
+bool BrotliMemOut::Write(const void *buf, size_t n) {
+  if (n + pos_ > len_)
+    return false;
+  char* p = reinterpret_cast<char*>(buf_) + pos_;
+  memcpy(p, buf, n);
+  pos_ += n;
+  return true;
+}
+
+BrotliStringOut::BrotliStringOut(std::string* buf, int max_size)
+    : buf_(buf),
+      max_size_(max_size) {
+  assert(buf->empty());
+}
+
+void BrotliStringOut::Reset(std::string* buf, int max_size) {
+  buf_ = buf;
+  max_size_ = max_size;
+}
+
+// Brotli output routine: add n bytes to a string.
+bool BrotliStringOut::Write(const void *buf, size_t n) {
+  if (buf_->size() + n > max_size_)
+    return false;
+  buf_->append(static_cast<const char*>(buf), n);
+  return true;
+}
+
+BrotliMemIn::BrotliMemIn(const void* buf, int len)
+    : buf_(buf),
+      len_(len),
+      pos_(0) {}
+
+void BrotliMemIn::Reset(const void* buf, int len) {
+  buf_ = buf;
+  len_ = len;
+  pos_ = 0;
+}
+
+// Brotli input routine: read the next chunk of memory.
+const void* BrotliMemIn::Read(size_t n, size_t* output) {
+  if (pos_ == len_) {
+    return NULL;
+  }
+  if (n > len_ - pos_)
+    n = len_ - pos_;
+  const char* p = reinterpret_cast<const char*>(buf_) + pos_;
+  pos_ += n;
+  *output = n;
+  return p;
+}
+
+BrotliFileIn::BrotliFileIn(FILE* f, size_t max_read_size)
+    : f_(f),
+      buf_(malloc(max_read_size)),
+      buf_size_(max_read_size) {}
+
+BrotliFileIn::~BrotliFileIn() {
+  if (buf_) free(buf_);
+}
+
+const void* BrotliFileIn::Read(size_t n, size_t* bytes_read) {
+  if (buf_ == NULL) {
+    *bytes_read = 0;
+    return NULL;
+  }
+  if (n > buf_size_) {
+    n = buf_size_;
+  } else if (n == 0) {
+    return feof(f_) ? NULL : buf_;
+  }
+  *bytes_read = fread(buf_, 1, n, f_);
+  if (*bytes_read == 0) {
+    return NULL;
+  } else {
+    return buf_;
+  }
+}
+
+BrotliFileOut::BrotliFileOut(FILE* f) : f_(f) {}
+
+bool BrotliFileOut::Write(const void* buf, size_t n) {
+  if (fwrite(buf, n, 1, f_) != 1) {
+    return false;
+  }
+  return true;
+}
+
+
+}  // namespace brotli

data/contrib/brotli/enc/streams.h

@@ -0,0 +1,129 @@
+// Copyright 2009 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Input and output classes for streaming brotli compression.
+
+#ifndef BROTLI_ENC_STREAMS_H_
+#define BROTLI_ENC_STREAMS_H_
+
+#include <stddef.h>
+#include <stdio.h>
+#include <string>
+
+namespace brotli {
+
+// Input interface for the compression routines.
+class BrotliIn {
+ public:
+  virtual ~BrotliIn() {}
+
+  // Return a pointer to the next block of input of at most n bytes.
+  // Return the actual length in *nread.
+  // At end of data, return NULL. Don't return NULL if there is more data
+  // to read, even if called with n == 0.
+  // Read will only be called if some of its bytes are needed.
+  virtual const void* Read(size_t n, size_t* nread) = 0;
+};
+
+// Output interface for the compression routines.
+class BrotliOut {
+ public:
+  virtual ~BrotliOut() {}
+
+  // Write n bytes of data from buf.
+  // Return true if all written, false otherwise.
+  virtual bool Write(const void *buf, size_t n) = 0;
+};
+
+// Adapter class to make BrotliIn objects from raw memory.
+class BrotliMemIn : public BrotliIn {
+ public:
+  BrotliMemIn(const void* buf, int len);
+
+  void Reset(const void* buf, int len);
+
+  // returns the amount of data consumed
+  int position() const { return pos_; }
+
+  const void* Read(size_t n, size_t* OUTPUT) override;
+
+ private:
+  const void* buf_;  // start of input buffer
+  int len_;  // length of input
+  int pos_;  // current read position within input
+};
+
+// Adapter class to make BrotliOut objects from raw memory.
+class BrotliMemOut : public BrotliOut {
+ public:
+  BrotliMemOut(void* buf, int len);
+
+  void Reset(void* buf, int len);
+
+  // returns the amount of data written
+  int position() const { return pos_; }
+
+  bool Write(const void* buf, size_t n) override;
+
+ private:
+  void* buf_;  // start of output buffer
+  int len_;  // length of output
+  int pos_;  // current write position within output
+};
+
+// Adapter class to make BrotliOut objects from a string.
+class BrotliStringOut : public BrotliOut {
+ public:
+  // Create a writer that appends its data to buf.
+  // buf->size() will grow to at most max_size
+  // buf is expected to be empty when constructing BrotliStringOut.
+  BrotliStringOut(std::string* buf, int max_size);
+
+  void Reset(std::string* buf, int max_len);
+
+  bool Write(const void* buf, size_t n) override;
+
+ private:
+  std::string* buf_;  // start of output buffer
+  int max_size_;  // max length of output
+};
+
+// Adapter class to make BrotliIn object from a file.
+class BrotliFileIn : public BrotliIn {
+ public:
+  BrotliFileIn(FILE* f, size_t max_read_size);
+  ~BrotliFileIn();
+
+  const void* Read(size_t n, size_t* bytes_read) override;
+
+ private:
+  FILE* f_;
+  void* buf_;
+  size_t buf_size_;
+};
+
+// Adapter class to make BrotliOut object from a file.
+class BrotliFileOut : public BrotliOut {
+ public:
+  explicit BrotliFileOut(FILE* f);
+
+  bool Write(const void* buf, size_t n) override;
+ private:
+  FILE* f_;
+};
+
+
+}  // namespace brotli
+
+#endif  // BROTLI_ENC_STREAMS_H_

data/contrib/brotli/enc/transform.h

@@ -0,0 +1,250 @@
+// Copyright 2010 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Transformations on dictionary words.
+
+#ifndef BROTLI_ENC_TRANSFORM_H_
+#define BROTLI_ENC_TRANSFORM_H_
+
+#include <string>
+
+#include "./dictionary.h"
+
+namespace brotli {
+
+enum WordTransformType {
+  kIdentity       = 0,
+  kOmitLast1      = 1,
+  kOmitLast2      = 2,
+  kOmitLast3      = 3,
+  kOmitLast4      = 4,
+  kOmitLast5      = 5,
+  kOmitLast6      = 6,
+  kOmitLast7      = 7,
+  kOmitLast8      = 8,
+  kOmitLast9      = 9,
+  kUppercaseFirst = 10,
+  kUppercaseAll   = 11,
+  kOmitFirst1     = 12,
+  kOmitFirst2     = 13,
+  kOmitFirst3     = 14,
+  kOmitFirst4     = 15,
+  kOmitFirst5     = 16,
+  kOmitFirst6     = 17,
+  kOmitFirst7     = 18,
+  kOmitFirst8     = 19,
+  kOmitFirst9     = 20,
+};
+
+struct Transform {
+  const char* prefix;
+  WordTransformType word_transform;
+  const char* suffix;
+};
+
+static const Transform kTransforms[] = {
+     {         "", kIdentity,       ""           },
+     {         "", kIdentity,       " "          },
+     {        " ", kIdentity,       " "          },
+     {         "", kOmitFirst1,     ""           },
+     {         "", kUppercaseFirst, " "          },
+     {         "", kIdentity,       " the "      },
+     {        " ", kIdentity,       ""           },
+     {       "s ", kIdentity,       " "          },
+     {         "", kIdentity,       " of "       },
+     {         "", kUppercaseFirst, ""           },
+     {         "", kIdentity,       " and "      },
+     {         "", kOmitFirst2,     ""           },
+     {         "", kOmitLast1,      ""           },
+     {       ", ", kIdentity,       " "          },
+     {         "", kIdentity,       ", "         },
+     {        " ", kUppercaseFirst, " "          },
+     {         "", kIdentity,       " in "       },
+     {         "", kIdentity,       " to "       },
+     {       "e ", kIdentity,       " "          },
+     {         "", kIdentity,       "\""         },
+     {         "", kIdentity,       "."          },
+     {         "", kIdentity,       "\">"        },
+     {         "", kIdentity,       "\n"         },
+     {         "", kOmitLast3,      ""           },
+     {         "", kIdentity,       "]"          },
+     {         "", kIdentity,       " for "      },
+     {         "", kOmitFirst3,     ""           },
+     {         "", kOmitLast2,      ""           },
+     {         "", kIdentity,       " a "        },
+     {         "", kIdentity,       " that "     },
+     {        " ", kUppercaseFirst, ""           },
+     {         "", kIdentity,       ". "         },
+     {        ".", kIdentity,       ""           },
+     {        " ", kIdentity,       ", "         },
+     {         "", kOmitFirst4,     ""           },
+     {         "", kIdentity,       " with "     },
+     {         "", kIdentity,       "'"          },
+     {         "", kIdentity,       " from "     },
+     {         "", kIdentity,       " by "       },
+     {         "", kOmitFirst5,     ""           },
+     {         "", kOmitFirst6,     ""           },
+     {    " the ", kIdentity,       ""           },
+     {         "", kOmitLast4,      ""           },
+     {         "", kIdentity,       ". The "     },
+     {         "", kUppercaseAll,   ""           },
+     {         "", kIdentity,       " on "       },
+     {         "", kIdentity,       " as "       },
+     {         "", kIdentity,       " is "       },
+     {         "", kOmitLast7,      ""           },
+     {         "", kOmitLast1,      "ing "       },
+     {         "", kIdentity,       "\n\t"       },
+     {         "", kIdentity,       ":"          },
+     {        " ", kIdentity,       ". "         },
+     {         "", kIdentity,       "ed "        },
+     {         "", kOmitFirst9,     ""           },
+     {         "", kOmitFirst7,     ""           },
+     {         "", kOmitLast6,      ""           },
+     {         "", kIdentity,       "("          },
+     {         "", kUppercaseFirst, ", "         },
+     {         "", kOmitLast8,      ""           },
+     {         "", kIdentity,       " at "       },
+     {         "", kIdentity,       "ly "        },
+     {    " the ", kIdentity,       " of "       },
+     {         "", kOmitLast5,      ""           },
+     {         "", kOmitLast9,      ""           },
+     {        " ", kUppercaseFirst, ", "         },
+     {         "", kUppercaseFirst, "\""         },
+     {        ".", kIdentity,       "("          },
+     {         "", kUppercaseAll,   " "          },
+     {         "", kUppercaseFirst, "\">"        },
+     {         "", kIdentity,       "=\""        },
+     {        " ", kIdentity,       "."          },
+     {    ".com/", kIdentity,       ""           },
+     {    " the ", kIdentity,       " of the "   },
+     {         "", kUppercaseFirst, "'"          },
+     {         "", kIdentity,       ". This "    },
+     {         "", kIdentity,       ","          },
+     {        ".", kIdentity,       " "          },
+     {         "", kUppercaseFirst, "("          },
+     {         "", kUppercaseFirst, "."          },
+     {         "", kIdentity,       " not "      },
+     {        " ", kIdentity,       "=\""        },
+     {         "", kIdentity,       "er "        },
+     {        " ", kUppercaseAll,   " "          },
+     {         "", kIdentity,       "al "        },
+     {        " ", kUppercaseAll,   ""           },
+     {         "", kIdentity,       "='"         },
+     {         "", kUppercaseAll,   "\""         },
+     {         "", kUppercaseFirst, ". "         },
+     {        " ", kIdentity,       "("          },
+     {         "", kIdentity,       "ful "       },
+     {        " ", kUppercaseFirst, ". "         },
+     {         "", kIdentity,       "ive "       },
+     {         "", kIdentity,       "less "      },
+     {         "", kUppercaseAll,   "'"          },
+     {         "", kIdentity,       "est "       },
+     {        " ", kUppercaseFirst, "."          },
+     {         "", kUppercaseAll,   "\">"        },
+     {        " ", kIdentity,       "='"         },
+     {         "", kUppercaseFirst, ","          },
+     {         "", kIdentity,       "ize "       },
+     {         "", kUppercaseAll,   "."          },
+     { "\xc2\xa0", kIdentity,       ""           },
+     {        " ", kIdentity,       ","          },
+     {         "", kUppercaseFirst, "=\""        },
+     {         "", kUppercaseAll,   "=\""        },
+     {         "", kIdentity,       "ous "       },
+     {         "", kUppercaseAll,   ", "         },
+     {         "", kUppercaseFirst, "='"         },
+     {        " ", kUppercaseFirst, ","          },
+     {        " ", kUppercaseAll,   "=\""        },
+     {        " ", kUppercaseAll,   ", "         },
+     {         "", kUppercaseAll,   ","          },
+     {         "", kUppercaseAll,   "("          },
+     {         "", kUppercaseAll,   ". "         },
+     {        " ", kUppercaseAll,   "."          },
+     {         "", kUppercaseAll,   "='"         },
+     {        " ", kUppercaseAll,   ". "         },
+     {        " ", kUppercaseFirst, "=\""        },
+     {        " ", kUppercaseAll,   "='"         },
+     {        " ", kUppercaseFirst, "='"         },
+};
+
+static const int kNumTransforms = sizeof(kTransforms) / sizeof(kTransforms[0]);
+
+static const int kOmitFirstNTransforms[10] = {
+  0, 3, 11, 26, 34, 39, 40, 55, 0, 54
+};
+
+static const int kOmitLastNTransforms[10] = {
+  0, 12, 27, 23, 42, 63, 56, 48, 59, 64,
+};
+
+static int ToUpperCase(uint8_t *p, int len) {
+  if (len == 1 || p[0] < 0xc0) {
+    if (p[0] >= 'a' && p[0] <= 'z') {
+      p[0] ^= 32;
+    }
+    return 1;
+  }
+  if (p[0] < 0xe0) {
+    p[1] ^= 32;
+    return 2;
+  }
+  if (len == 2) {
+    return 2;
+  }
+  p[2] ^= 5;
+  return 3;
+}
+
+inline std::string ApplyTransform(
+    const Transform& t, const uint8_t* word, int len) {
+  std::string ret(t.prefix);
+  if (t.word_transform <= kOmitLast9) {
+    len -= t.word_transform;
+  }
+  if (len > 0) {
+    if (t.word_transform >= kOmitFirst1) {
+      const int skip = t.word_transform - (kOmitFirst1 - 1);
+      if (len > skip) {
+        ret += std::string(word + skip, word + len);
+      }
+    } else {
+      ret += std::string(word, word + len);
+      uint8_t *uppercase = reinterpret_cast<uint8_t*>(&ret[ret.size() - len]);
+      if (t.word_transform == kUppercaseFirst) {
+        ToUpperCase(uppercase, len);
+      } else if (t.word_transform == kUppercaseAll) {
+        while (len > 0) {
+          int step = ToUpperCase(uppercase, len);
+          uppercase += step;
+          len -= step;
+        }
+      }
+    }
+  }
+  ret += std::string(t.suffix);
+  return ret;
+}
+
+inline std::string GetTransformedDictionaryWord(int len_code, int word_id) {
+  int num_words = 1 << kBrotliDictionarySizeBitsByLength[len_code];
+  int offset = kBrotliDictionaryOffsetsByLength[len_code];
+  int t = word_id / num_words;
+  int word_idx = word_id % num_words;
+  offset += len_code * word_idx;
+  const uint8_t* word = &kBrotliDictionary[offset];
+  return ApplyTransform(kTransforms[t], word, len_code);
+}
+
+}  // namespace brotli
+
+#endif  // BROTLI_ENC_TRANSFORM_H_