lsqpack 0.1.0

data/ls-qpack/.cirrus.yml

@@ -0,0 +1,6 @@
+freebsd_instance:
+    image: freebsd-12-1-release-amd64
+
+task:
+    install_script: pkg install -y cmake bash perl5
+    script: cmake -DLSQPACK_TESTS=ON . && make && make test

data/ls-qpack/.travis.yml

@@ -0,0 +1,32 @@
+language: c
+sudo: false
+addons:
+  apt:
+matrix:
+    include:
+        - name: Linux (gcc-8)
+          os: linux
+          dist: xenial
+          compiler: gcc-8
+          addons:
+             apt:
+               sources:
+                 - ubuntu-toolchain-r-test
+               packages: g++-8
+        - name: Linux (clang)
+          os: linux
+          dist: xenial
+          compiler: clang
+        - name: macOS (xcode)
+          os: osx
+        - name: macOS (xcode 10.1)
+          os: osx
+          image: xcode10.1
+before_install:
+  - $CC --version
+  - cmake --version
+before_script:
+  - cmake -DLSQPACK_TESTS=ON .
+script:
+  - make
+  - make test

data/ls-qpack/CMakeLists.txt

@@ -0,0 +1,66 @@
+# The following variable can be defined on the command line:
+#
+#   BUILD_SHARED_LIBS
+#
+# The following environment variables will be taken into account when running
+# cmake for the first time:
+#
+#   CFLAGS
+#   LDFLAGS
+
+cmake_minimum_required(VERSION 3.1)
+project(ls-qpack LANGUAGES C)
+
+option(LSQPACK_TESTS "Build tests")
+option(LSQPACK_BIN "Build binaries" ON)
+option(LSQPACK_XXH "Include XXH" ON)
+
+# Use `cmake -DBUILD_SHARED_LIBS=OFF` to build a static library.
+add_library(ls-qpack "")
+target_include_directories(ls-qpack PUBLIC .)
+target_sources(ls-qpack PRIVATE lsqpack.c)
+
+target_include_directories(ls-qpack PRIVATE deps/xxhash/)
+if(LSQPACK_XXH)
+    target_sources(ls-qpack PRIVATE deps/xxhash/xxhash.c)
+endif()
+
+if(MSVC)
+    target_include_directories(ls-qpack PUBLIC wincompat)
+endif()
+
+if(MSVC)
+    target_compile_options(ls-qpack PRIVATE
+        /Wall
+        /wd4100 # unreffed parameter
+        /wd4200 # zero-sized array
+        # Apparently this C99 construct is not supported properly by VS:
+        #   https://stackoverflow.com/questions/1064930/struct-initializer-typedef-with-visual-studio
+        /wd4204 # non-constant aggregate initializer
+        /wd4255 # no function prototype (getopt)
+        /wd4820 # padding
+        /wd4668 # undefined macro
+        /wd4710 # not inlined by default
+        /wd4996 # unsafe function
+    )
+else()
+    target_compile_options(ls-qpack PRIVATE
+        -Wall
+        -Wextra
+        -Wno-unused-parameter
+        -fno-omit-frame-pointer
+    )
+endif()
+
+IF (CMAKE_BUILD_TYPE STREQUAL MinSizeRel)
+    SET(CMAKE_C_FLAGS "${CMAKE_C_CFLAGS} -DLS_QPACK_USE_LARGE_TABLES=0")
+ENDIF()
+
+if(LSQPACK_TESTS)
+    enable_testing()
+    add_subdirectory(test)
+endif()
+
+if(LSQPACK_BIN)
+    add_subdirectory(bin)
+endif()

data/ls-qpack/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2018 - 2020 LiteSpeed Tech
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/ls-qpack/README.md

@@ -0,0 +1,65 @@
+[![Build Status](https://travis-ci.org/litespeedtech/ls-qpack.svg?branch=master)](https://travis-ci.org/litespeedtech/ls-qpack)
+[![Build status](https://ci.appveyor.com/api/projects/status/kat31lt42ds0rmom?svg=true)](https://ci.appveyor.com/project/litespeedtech/ls-qpack)
+[![Build Status](https://api.cirrus-ci.com/github/litespeedtech/ls-qpack.svg)](https://cirrus-ci.com/github/litespeedtech/ls-qpack)
+
+# ls-qpack
+QPACK compression library for use with HTTP/3
+
+## Introduction
+
+QPACK is the compression mechanism used by
+[HTTP/3](https://en.wikipedia.org/wiki/HTTP/3) to compress HTTP headers.
+It is in the process of being standardazed by the QUIC Working Group.  The
+[QPACK Internet-Draft](https://tools.ietf.org/html/draft-ietf-quic-qpack-11)
+is has been stable for some time and we don't expect functional changes to
+it before the final RFC is released.
+
+## Functionality
+
+ls-qpack is a full-featured, tested, and fast QPACK library.  The QPACK encoder
+produces excellent compression results based on an innovative mnemonic technique
+(to be described in a future article).  It boasts the fastest Huffman
+[encoder](https://blog.litespeedtech.com/2019/10/03/fast-huffman-encoder/) and
+[decoder](https://blog.litespeedtech.com/2019/09/16/fast-huffman-decoder/).
+
+The library is production quality.  It is used in
+[OpenLiteSpeed](https://openlitespeed.org/),
+LiteSpeed [Web Server](https://www.litespeedtech.com/products#lsws),
+and LiteSpeed [Web ADC](https://www.litespeedtech.com/products#wadc).
+
+The library is robust:
+1. The encoder does not assume anything about usual HTTP headers such as `Server`
+   or `User-Agent`.  Instead, it uses its mnemonic compression technique to
+   achieve good compression results for any input.
+1. The decoder uses modulo arithmetic to track dynamic table insertions.  This is
+   in contrast to all other QPACK implementations, which use an integer counter,
+   meaning that at some point, the decoder will break.
+1. The decoder processes input in streaming fashion.  The caller does not have to
+   buffer the contents of HTTP/3 `HEADERS` frame.  Instead, the decoder can be
+   supplied input byte-by-byte.
+
+## Other Languages
+
+The ls-qpack library is implemented in vanilla C99.  It makes it a good candidate
+for wrapping into a library for a higher-level language.  As of this writing, we
+know of the following wrappers:
+- Go: [ls-qpack-go](https://github.com/mpiraux/ls-qpack-go)
+- Python: [pylsqpack](https://github.com/aiortc/pylsqpack)
+- TypeScript: [quicker](https://github.com/rmarx/quicker/tree/draft-20/lib/ls-qpack)
+
+## Versioning
+
+Before the QPACK RFC is released, the three parts of the version are:
+- MAJOR: set to zero;
+- MINOR: set to the number of QPACK Internet-Draft the lirbary supports; and
+- PATCH: set to the patch number
+
+Once the RFC is released, MARJO will be set to 1 and the version will follow
+the usual MAJOR.MINOR.PATCH pattern.
+
+## API
+
+The API is documented in the header file, [lsqpack.h](lsqpack.h).
+One example how it is used in real code can be seen in
+[lsquic](https://github.com/litespeedtech/lsquic), a QUIC and HTTP/3 library
+developed by LiteSpeed Technologies.

data/ls-qpack/bin/CMakeLists.txt

@@ -0,0 +1,21 @@
+function(lsqpack_add_executable TARGET)
+    add_executable(${TARGET} "")
+    target_link_libraries(${TARGET} PRIVATE ls-qpack)
+
+    target_sources(${TARGET} PRIVATE ${TARGET}.c)
+
+    if(MSVC)
+        target_include_directories(${TARGET} PRIVATE ../wincompat)
+        target_sources(${TARGET} PRIVATE 
+            ../wincompat/getopt.c 
+            ../wincompat/getopt1.c
+        )
+    else()
+        target_link_libraries(${TARGET} PRIVATE m)
+    endif()
+endfunction()
+
+lsqpack_add_executable(interop-encode)
+lsqpack_add_executable(interop-decode)
+lsqpack_add_executable(encode-int)
+lsqpack_add_executable(fuzz-decode)

data/ls-qpack/bin/encode-int.c

@@ -0,0 +1,87 @@
+/* encode-int: encode an integer into HPACK varint. */
+
+#include <assert.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdint.h>
+
+
+static unsigned char *
+enc_int (unsigned char *dst, unsigned char *const end, uint64_t value,
+                                                        unsigned prefix_bits)
+{
+    unsigned char *const dst_orig = dst;
+
+    /* This function assumes that at least one byte is available */
+    assert(dst < end);
+    if (value < (1u << prefix_bits) - 1)
+        *dst++ |= value;
+    else
+    {
+        *dst++ |= (1 << prefix_bits) - 1;
+        value -= (1 << prefix_bits) - 1;
+        while (value >= 128)
+        {
+            if (dst < end)
+            {
+                *dst++ = 0x80 | (unsigned char) value;
+                value >>= 7;
+            }
+            else
+                return dst_orig;
+        }
+        if (dst < end)
+            *dst++ = (unsigned char) value;
+        else
+            return dst_orig;
+    }
+    return dst;
+}
+
+static unsigned
+lsqpack_val2len (uint64_t value, unsigned prefix_bits)
+{
+    uint64_t mask = (1ULL << prefix_bits) - 1;
+    return 1
+         + (value >=                 mask )
+         + (value >= ((1ULL <<  7) + mask))
+         + (value >= ((1ULL << 14) + mask))
+         + (value >= ((1ULL << 21) + mask))
+         + (value >= ((1ULL << 28) + mask))
+         + (value >= ((1ULL << 35) + mask))
+         + (value >= ((1ULL << 42) + mask))
+         + (value >= ((1ULL << 49) + mask))
+         + (value >= ((1ULL << 56) + mask))
+         + (value >= ((1ULL << 63) + mask))
+         ;
+}
+
+int
+main (int argc, char **argv)
+{
+    unsigned long long val;
+    unsigned char *p;
+    unsigned prefix_bits;
+    unsigned char buf[20];
+
+    if (argc != 3)
+    {
+        fprintf(stderr, "Usage: %s prefix_bits value\n", argv[0]);
+        exit(EXIT_FAILURE);
+    }
+
+    prefix_bits = atoi(argv[1]);
+    val = strtoull(argv[2], NULL, 10);
+
+    fprintf(stderr, "expected size: %u\n", lsqpack_val2len(val, prefix_bits));
+    buf[0] = 0;
+    p = enc_int(buf, buf + sizeof(buf), val, prefix_bits);
+
+    if (p > buf)
+    {
+        fwrite(buf, 1, p - buf, stdout);
+        exit(EXIT_SUCCESS);
+    }
+    else
+        exit(EXIT_FAILURE);
+}

data/ls-qpack/bin/fuzz-decode.c

@@ -0,0 +1,247 @@
+/*
+ * fuzz-decode: special program for fuzzing.  It still reads encoded
+ * files just like interop-decode, but tries to do it faster and
+ * forgoes several advanced options.
+ */
+
+#ifdef WIN32
+
+#include <stdio.h>
+
+int
+main (int argc, char **argv)
+{
+    fprintf(stderr, "%s is not supported on Windows: need mmap(2)\n", argv[0]);
+    return 1;
+}
+
+#else
+
+#include <assert.h>
+
+#if defined(__FreeBSD__) || defined(__DragonFly__) || defined(__NetBSD__)
+#include <sys/endian.h>
+#define bswap_16 bswap16
+#define bswap_32 bswap32
+#define bswap_64 bswap64
+#elif defined(__APPLE__)
+#include <libkern/OSByteOrder.h>
+#define bswap_16 OSSwapInt16
+#define bswap_32 OSSwapInt32
+#define bswap_64 OSSwapInt64
+#elif defined(WIN32)
+#define bswap_16 _byteswap_ushort
+#define bswap_32 _byteswap_ulong
+#define bswap_64 _byteswap_uint64
+#else
+#include <byteswap.h>
+#endif
+
+#include <errno.h>
+#include <inttypes.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <sys/queue.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/mman.h>
+#include <fcntl.h>
+
+#include "lsqpack.h"
+
+#define MIN(a, b) ((a) < (b) ? (a) : (b))
+
+static void
+usage (const char *name)
+{
+    fprintf(stderr,
+"Usage: %s [options] [-i input] [-o output]\n"
+"\n"
+"Options:\n"
+"   -i FILE     Input file.\n"
+"   -f FILE     Fuzz file: this is the stuff the fuzzer will change.\n"
+"   -s NUMBER   Maximum number of risked streams.  Defaults to %u.\n"
+"   -t NUMBER   Dynamic table size.  Defaults to %u.\n"
+"\n"
+"   -h          Print this help screen and exit\n"
+    , name, LSQPACK_DEF_MAX_RISKED_STREAMS, LSQPACK_DEF_DYN_TABLE_SIZE);
+}
+
+
+static void
+hblock_unblocked (void *buf_p)
+{
+    exit(1);
+}
+
+
+int
+main (int argc, char **argv)
+{
+    int in_fd = -1, fuzz_fd = STDIN_FILENO;
+    int opt;
+    unsigned dyn_table_size     = LSQPACK_DEF_DYN_TABLE_SIZE,
+             max_risked_streams = LSQPACK_DEF_MAX_RISKED_STREAMS;
+    struct lsqpack_dec decoder;
+    const unsigned char *p, *end;
+    unsigned char *begin;
+    struct stat st;
+    uint64_t stream_id;
+    uint32_t size;
+    int r;
+
+    while (-1 != (opt = getopt(argc, argv, "i:f:s:t:h")))
+    {
+        switch (opt)
+        {
+        case 'i':
+            in_fd = open(optarg, O_RDONLY);
+            if (in_fd < 0)
+            {
+                fprintf(stderr, "cannot open `%s' for reading: %s\n",
+                                            optarg, strerror(errno));
+                exit(EXIT_FAILURE);
+            }
+            break;
+        case 'f':
+            fuzz_fd = open(optarg, O_RDONLY);
+            if (fuzz_fd < 0)
+            {
+                fprintf(stderr, "cannot open `%s' for reading: %s\n",
+                                            optarg, strerror(errno));
+                exit(EXIT_FAILURE);
+            }
+            break;
+        case 's':
+            max_risked_streams = atoi(optarg);
+            break;
+        case 't':
+            dyn_table_size = atoi(optarg);
+            break;
+        case 'h':
+            usage(argv[0]);
+            exit(EXIT_SUCCESS);
+        default:
+            exit(EXIT_FAILURE);
+        }
+    }
+
+    lsqpack_dec_init(&decoder, NULL, dyn_table_size, max_risked_streams,
+                                                        hblock_unblocked);
+
+    if (in_fd < 0)
+    {
+        fprintf(stderr, "Specify input using `-i' option\n");
+        exit(1);
+    }
+
+    if (0 != fstat(in_fd, &st))
+    {
+        perror("fstat");
+        exit(1);
+    }
+
+    begin = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, in_fd, 0);
+    if (!begin)
+    {
+        perror("mmap");
+        exit(1);
+    }
+
+    p = begin;
+    end = begin + st.st_size;
+    while (p + sizeof(stream_id) + sizeof(size) < end)
+    {
+        stream_id = * (uint64_t *) p;
+        p += sizeof(uint64_t);
+        size = * (uint32_t *) p;
+        p += sizeof(uint32_t);
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+        stream_id = bswap_64(stream_id);
+        size = bswap_32(size);
+#endif
+        if (p + size > end)
+        {
+            fprintf(stderr, "truncated input at offset %u",
+                                                (unsigned) (p - begin));
+            abort();
+        }
+        if (stream_id == 0)
+        {
+            r = lsqpack_dec_enc_in(&decoder, p, size);
+            if (r != 0)
+                abort();
+        }
+        else
+        {
+            const unsigned char *cur = p;
+            struct lsqpack_header_list *hlist;
+            enum lsqpack_read_header_status rhs;
+            rhs = lsqpack_dec_header_in(&decoder, NULL, stream_id,
+                        size, &cur, size, &hlist, NULL, NULL);
+            if (rhs != LQRHS_DONE || (uint32_t) (cur - p) != size)
+                abort();
+            lsqpack_dec_destroy_header_list(hlist);
+        }
+        p += size;
+    }
+
+    munmap(begin, st.st_size);
+    (void) close(in_fd);
+
+    /* Now let's read the fuzzed part */
+
+    if (0 != fstat(fuzz_fd, &st))
+    {
+        perror("fstat");
+        exit(1);
+    }
+
+    begin = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fuzz_fd, 0);
+    if (!begin)
+    {
+        perror("mmap");
+        exit(1);
+    }
+
+    p = begin;
+    end = begin + st.st_size;
+    while (p + sizeof(stream_id) + sizeof(size) < end)
+    {
+        stream_id = * (uint64_t *) p;
+        p += sizeof(uint64_t);
+        size = * (uint32_t *) p;
+        p += sizeof(uint32_t);
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+        stream_id = bswap_64(stream_id);
+        size = bswap_32(size);
+#endif
+        if (stream_id == 0)
+        {
+            r = lsqpack_dec_enc_in(&decoder, p, MIN(size, (uint32_t) (end - p)));
+            (void) r;
+        }
+        else
+        {
+            const unsigned char *cur = p;
+            struct lsqpack_header_list *hlist;
+            enum lsqpack_read_header_status rhs;
+            size = MIN(size, (uint32_t) (end - p));
+            rhs = lsqpack_dec_header_in(&decoder, NULL, stream_id,
+                        size, &cur, size, &hlist, NULL, NULL);
+            (void) rhs;
+        }
+        break;
+    }
+
+    munmap(begin, st.st_size);
+    (void) close(fuzz_fd);
+
+    lsqpack_dec_cleanup(&decoder);
+
+    exit(0);
+}
+
+#endif