lsqpack 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (89) hide show
  1. checksums.yaml +7 -0
  2. data/Gemfile +19 -0
  3. data/Gemfile.lock +82 -0
  4. data/LICENSE +21 -0
  5. data/README.md +36 -0
  6. data/Rakefile +19 -0
  7. data/Steepfile +6 -0
  8. data/ext/lsqpack/extconf.rb +18 -0
  9. data/ext/lsqpack/lsqpack.c +426 -0
  10. data/ext/lsqpack/lsqpack.h +6 -0
  11. data/lib/lsqpack/version.rb +5 -0
  12. data/lib/lsqpack.rb +30 -0
  13. data/ls-qpack/.appveyor.yml +14 -0
  14. data/ls-qpack/.cirrus.yml +6 -0
  15. data/ls-qpack/.travis.yml +32 -0
  16. data/ls-qpack/CMakeLists.txt +66 -0
  17. data/ls-qpack/LICENSE +21 -0
  18. data/ls-qpack/README.md +65 -0
  19. data/ls-qpack/bin/CMakeLists.txt +21 -0
  20. data/ls-qpack/bin/encode-int.c +87 -0
  21. data/ls-qpack/bin/fuzz-decode.c +247 -0
  22. data/ls-qpack/bin/interop-decode.c +433 -0
  23. data/ls-qpack/bin/interop-encode.c +554 -0
  24. data/ls-qpack/deps/xxhash/xxhash.c +941 -0
  25. data/ls-qpack/deps/xxhash/xxhash.h +160 -0
  26. data/ls-qpack/fuzz/decode/a/README +3 -0
  27. data/ls-qpack/fuzz/decode/a/preamble +0 -0
  28. data/ls-qpack/fuzz/decode/a/test-cases/id_000000,sig_06,src_000390,op_havoc,rep_4 +0 -0
  29. data/ls-qpack/fuzz/decode/a/test-cases/id_000000,sig_06,src_000579,op_flip1,pos_14 +0 -0
  30. data/ls-qpack/fuzz/decode/a/test-cases/id_000000,src_000000,op_flip2,pos_12 +0 -0
  31. data/ls-qpack/fuzz/decode/a/test-cases/id_000001,sig_11,src_000579,op_havoc,rep_4 +0 -0
  32. data/ls-qpack/fuzz/decode/a/test-cases/id_000002,sig_11,src_000481,op_int16,pos_15,val_-1 +0 -0
  33. data/ls-qpack/fuzz/decode/a/test-cases/id_000002,src_000000,op_havoc,rep_8 +0 -0
  34. data/ls-qpack/fuzz/decode/a/test-cases/id_000006,src_000285,op_flip2,pos_14 +0 -0
  35. data/ls-qpack/fuzz/decode/a/test-cases/id_000008,src_000285,op_flip2,pos_20 +0 -0
  36. data/ls-qpack/fuzz/decode/a/test-cases/id_000010,src_000306,op_flip2,pos_75 +0 -0
  37. data/ls-qpack/fuzz/decode/a/test-cases/id_000011,src_000344,op_havoc,rep_2 +0 -0
  38. data/ls-qpack/fuzz/decode/a/test-cases/id_000014,src_000366,op_flip2,pos_28 +0 -0
  39. data/ls-qpack/fuzz/decode/b/README +1 -0
  40. data/ls-qpack/fuzz/decode/b/preamble +0 -0
  41. data/ls-qpack/fuzz/decode/b/test-cases/seed +0 -0
  42. data/ls-qpack/fuzz/decode/c/setup.sh +3 -0
  43. data/ls-qpack/fuzz/decode/c/test-cases/fb-req.qif.proxygen.out.256.100.0-chopped +0 -0
  44. data/ls-qpack/fuzz/decode/d/preamble +0 -0
  45. data/ls-qpack/fuzz/decode/d/setup.sh +3 -0
  46. data/ls-qpack/fuzz/decode/d/test-cases/fb-resp.minhq.256.128.0.ack +0 -0
  47. data/ls-qpack/fuzz/input/256.100.1/fb-req.out.256.100.1 +0 -0
  48. data/ls-qpack/fuzz/input/256.100.1/fb-resp.out.256.100.1 +0 -0
  49. data/ls-qpack/fuzz/input/256.100.1/netbsd.out.256.100.1 +0 -0
  50. data/ls-qpack/huff-tables.h +136247 -0
  51. data/ls-qpack/lsqpack.c +5547 -0
  52. data/ls-qpack/lsqpack.h +768 -0
  53. data/ls-qpack/test/CMakeLists.txt +76 -0
  54. data/ls-qpack/test/lsqpack-test.h +43 -0
  55. data/ls-qpack/test/qifs/fb-req.qif +4917 -0
  56. data/ls-qpack/test/qifs/fb-resp.qif +5982 -0
  57. data/ls-qpack/test/qifs/long-codes.qif +5984 -0
  58. data/ls-qpack/test/qifs/netbsd.qif +235 -0
  59. data/ls-qpack/test/run-qif.pl +97 -0
  60. data/ls-qpack/test/run-scenario.sh +68 -0
  61. data/ls-qpack/test/scenarios/0.95-reset.sce +10 -0
  62. data/ls-qpack/test/scenarios/cancel-stream.sce +22 -0
  63. data/ls-qpack/test/scenarios/drain-2.sce +37 -0
  64. data/ls-qpack/test/scenarios/drain.sce +37 -0
  65. data/ls-qpack/test/scenarios/end-dst-2.sce +14 -0
  66. data/ls-qpack/test/scenarios/end-dst.sce +14 -0
  67. data/ls-qpack/test/scenarios/incl-name.sce +13 -0
  68. data/ls-qpack/test/scenarios/multi-byte-int-dyn-ref-1.sce +110 -0
  69. data/ls-qpack/test/scenarios/multi-byte-int-dyn-ref-2.sce +161 -0
  70. data/ls-qpack/test/scenarios/post-base-1.sce +10 -0
  71. data/ls-qpack/test/scenarios/post-base-2.sce +13 -0
  72. data/ls-qpack/test/scenarios/post-base-nr.sce +10 -0
  73. data/ls-qpack/test/scenarios/set-max-cap.sce +15 -0
  74. data/ls-qpack/test/test_enc_str.c +139 -0
  75. data/ls-qpack/test/test_get_stx_id.c +144 -0
  76. data/ls-qpack/test/test_huff_dec.c +399 -0
  77. data/ls-qpack/test/test_int.c +220 -0
  78. data/ls-qpack/test/test_qpack.c +856 -0
  79. data/ls-qpack/test/test_read_enc_stream.c +256 -0
  80. data/ls-qpack/tools/har2qif.pl +139 -0
  81. data/ls-qpack/tools/randomize-cookies.pl +41 -0
  82. data/ls-qpack/tools/sort-qif.pl +31 -0
  83. data/ls-qpack/wincompat/getopt.c +758 -0
  84. data/ls-qpack/wincompat/getopt.h +131 -0
  85. data/ls-qpack/wincompat/getopt1.c +188 -0
  86. data/ls-qpack/wincompat/sys/queue.h +859 -0
  87. data/lsqpack.gemspec +39 -0
  88. data/sig/lsqpack.rbs +29 -0
  89. metadata +135 -0
@@ -0,0 +1,433 @@
1
+ /*
2
+ * QPACK Interop -- decode from intermediate format and output QIF
3
+ *
4
+ * https://github.com/quicwg/base-drafts/wiki/QPACK-Offline-Interop
5
+ */
6
+
7
+ #include <assert.h>
8
+
9
+ #if defined(__FreeBSD__) || defined(__DragonFly__) || defined(__NetBSD__)
10
+ #include <sys/endian.h>
11
+ #define bswap_16 bswap16
12
+ #define bswap_32 bswap32
13
+ #define bswap_64 bswap64
14
+ #elif defined(__APPLE__)
15
+ #include <libkern/OSByteOrder.h>
16
+ #define bswap_16 OSSwapInt16
17
+ #define bswap_32 OSSwapInt32
18
+ #define bswap_64 OSSwapInt64
19
+ #elif defined(WIN32)
20
+ #define bswap_16 _byteswap_ushort
21
+ #define bswap_32 _byteswap_ulong
22
+ #define bswap_64 _byteswap_uint64
23
+ #else
24
+ #include <byteswap.h>
25
+ #endif
26
+
27
+ #include <errno.h>
28
+ #include <inttypes.h>
29
+ #include <stdio.h>
30
+ #include <stdlib.h>
31
+ #include <string.h>
32
+ #ifdef WIN32
33
+ #include "getopt.h"
34
+ #else
35
+ #include <unistd.h>
36
+ #endif
37
+ #include <sys/queue.h>
38
+ #include <sys/types.h>
39
+ #include <sys/stat.h>
40
+ #include <fcntl.h>
41
+
42
+ #include "lsqpack.h"
43
+
44
+ static size_t s_max_read_size = SIZE_MAX;
45
+
46
+ static int s_verbose;
47
+
48
+ static FILE *s_out;
49
+
50
+ #define MIN(a, b) ((a) < (b) ? (a) : (b))
51
+
52
+ static void
53
+ usage (const char *name)
54
+ {
55
+ fprintf(stderr,
56
+ "Usage: %s [options] [-i input] [-o output]\n"
57
+ "\n"
58
+ "Options:\n"
59
+ " -i FILE Input file. If not specified or set to `-', the input is\n"
60
+ " read from stdin.\n"
61
+ " -o FILE Output file. If not spepcified or set to `-', the output\n"
62
+ " is written to stdout.\n"
63
+ " -r FILE Recipe file. Without a recipe, buffers are processed in\n"
64
+ " order.\n"
65
+ " -s NUMBER Maximum number of risked streams. Defaults to %u.\n"
66
+ " -t NUMBER Dynamic table size. Defaults to %u.\n"
67
+ " -m NUMBER Maximum read size. Defaults to %zu.\n"
68
+ " -v Verbose: print headers and table state to stderr.\n"
69
+ "\n"
70
+ " -h Print this help screen and exit\n"
71
+ , name, LSQPACK_DEF_MAX_RISKED_STREAMS, LSQPACK_DEF_DYN_TABLE_SIZE, SIZE_MAX);
72
+ }
73
+
74
+
75
+ struct buf
76
+ {
77
+ TAILQ_ENTRY(buf) next_buf;
78
+ struct lsqpack_dec *dec;
79
+ uint64_t stream_id; /* Zero means encoder stream */
80
+ size_t size;
81
+ size_t off;
82
+ size_t file_off;
83
+ unsigned char buf[0];
84
+ };
85
+
86
+
87
+ TAILQ_HEAD(, buf) bufs = TAILQ_HEAD_INITIALIZER(bufs);
88
+
89
+
90
+ static void
91
+ hblock_unblocked (void *buf_p)
92
+ {
93
+ struct buf *buf = buf_p;
94
+ TAILQ_INSERT_HEAD(&bufs, buf, next_buf);
95
+ }
96
+
97
+
98
+ static void
99
+ header_block_done (const struct buf *buf, struct lsqpack_header_list *set)
100
+ {
101
+ unsigned n;
102
+
103
+ if (!set)
104
+ {
105
+ fprintf(stderr, "Stream %"PRIu64" has empty header list\n", buf->stream_id);
106
+ return;
107
+ }
108
+
109
+ if (s_verbose)
110
+ {
111
+ fprintf(stderr, "Have headers for stream %"PRIu64":\n", buf->stream_id);
112
+ for (n = 0; n < set->qhl_count; ++n)
113
+ fprintf(stderr, " %.*s: %.*s\n",
114
+ set->qhl_headers[n]->qh_name_len, set->qhl_headers[n]->qh_name,
115
+ set->qhl_headers[n]->qh_value_len, set->qhl_headers[n]->qh_value);
116
+ fprintf(stderr, "\n");
117
+ }
118
+
119
+ fprintf(s_out, "# stream %"PRIu64"\n", buf->stream_id);
120
+ fprintf(s_out, "# (stream ID above is used for sorting)\n");
121
+ for (n = 0; n < set->qhl_count; ++n)
122
+ fprintf(s_out, "%.*s\t%.*s\n",
123
+ set->qhl_headers[n]->qh_name_len, set->qhl_headers[n]->qh_name,
124
+ set->qhl_headers[n]->qh_value_len, set->qhl_headers[n]->qh_value);
125
+ fprintf(s_out, "\n");
126
+
127
+ lsqpack_dec_destroy_header_list(set);
128
+ }
129
+
130
+
131
+ int
132
+ main (int argc, char **argv)
133
+ {
134
+ FILE *in = stdin;
135
+ FILE *recipe = NULL;
136
+ int opt;
137
+ unsigned dyn_table_size = LSQPACK_DEF_DYN_TABLE_SIZE,
138
+ max_risked_streams = LSQPACK_DEF_MAX_RISKED_STREAMS;
139
+ struct lsqpack_dec decoder;
140
+ const struct lsqpack_dec_err *err;
141
+ const unsigned char *p;
142
+ ssize_t nr;
143
+ int r;
144
+ uint64_t stream_id;
145
+ uint32_t size;
146
+ size_t off; /* For debugging */
147
+ size_t file_off;
148
+ struct buf *buf;
149
+ unsigned lineno;
150
+ char *line, *end;
151
+ enum lsqpack_read_header_status rhs;
152
+ struct lsqpack_header_list *hlist;
153
+ char command[0x100];
154
+ char line_buf[0x100];
155
+
156
+ while (-1 != (opt = getopt(argc, argv, "i:o:r:s:t:m:hv")))
157
+ {
158
+ switch (opt)
159
+ {
160
+ case 'i':
161
+ if (0 != strcmp(optarg, "-"))
162
+ {
163
+ in = fopen(optarg, "rb");
164
+ if (!in)
165
+ {
166
+ fprintf(stderr, "cannot open `%s' for reading: %s\n",
167
+ optarg, strerror(errno));
168
+ exit(EXIT_FAILURE);
169
+ }
170
+ }
171
+ break;
172
+ case 'o':
173
+ if (0 != strcmp(optarg, "-"))
174
+ {
175
+ s_out = fopen(optarg, "w");
176
+ if (!s_out)
177
+ {
178
+ fprintf(stderr, "cannot open `%s' for writing: %s\n",
179
+ optarg, strerror(errno));
180
+ exit(EXIT_FAILURE);
181
+ }
182
+ }
183
+ break;
184
+ case 'r':
185
+ if (0 == strcmp(optarg, "-"))
186
+ recipe = stdin;
187
+ else
188
+ {
189
+ recipe = fopen(optarg, "r");
190
+ if (!recipe)
191
+ {
192
+ fprintf(stderr, "cannot open `%s' for reading: %s\n",
193
+ optarg, strerror(errno));
194
+ exit(EXIT_FAILURE);
195
+ }
196
+ }
197
+ break;
198
+ case 's':
199
+ max_risked_streams = atoi(optarg);
200
+ break;
201
+ case 't':
202
+ dyn_table_size = atoi(optarg);
203
+ break;
204
+ case 'm':
205
+ s_max_read_size = atoi(optarg);
206
+ break;
207
+ case 'h':
208
+ usage(argv[0]);
209
+ exit(EXIT_SUCCESS);
210
+ case 'v':
211
+ ++s_verbose;
212
+ break;
213
+ default:
214
+ exit(EXIT_FAILURE);
215
+ }
216
+ }
217
+
218
+ if (!s_out)
219
+ s_out = stdout;
220
+
221
+ lsqpack_dec_init(&decoder, s_verbose ? stderr : NULL, dyn_table_size,
222
+ max_risked_streams, hblock_unblocked);
223
+
224
+ off = 0;
225
+ while (1)
226
+ {
227
+ file_off = off;
228
+ nr = fread(&stream_id, 1, sizeof(stream_id), in);
229
+ if (nr == 0)
230
+ break;
231
+ if (nr != sizeof(stream_id))
232
+ {
233
+ fprintf(stderr, "could not read %zu bytes (stream id) at "
234
+ "offset %zu: %s\n", sizeof(stream_id), off, strerror(errno));
235
+ goto read_err;
236
+ }
237
+ off += nr;
238
+ nr = fread(&size, 1, sizeof(size), in);
239
+ if (nr != sizeof(size))
240
+ {
241
+ fprintf(stderr, "could not read %zu bytes (size) at "
242
+ "offset %zu: %s\n", sizeof(size), off, strerror(errno));
243
+ goto read_err;
244
+ }
245
+ off += nr;
246
+ #if __BYTE_ORDER == __LITTLE_ENDIAN
247
+ stream_id = bswap_64(stream_id);
248
+ size = bswap_32(size);
249
+ #endif
250
+ if (stream_id == 0 && size == 0)
251
+ continue;
252
+ buf = malloc(sizeof(*buf) + size);
253
+ if (!buf)
254
+ {
255
+ perror("malloc");
256
+ exit(EXIT_FAILURE);
257
+ }
258
+ memset(buf, 0, sizeof(*buf));
259
+ nr = fread(buf->buf, 1, size, in);
260
+ if (nr != (ssize_t) size)
261
+ {
262
+ fprintf(stderr, "could not read %"PRIu32" bytes (buffer) at "
263
+ "offset %zu: %s\n", size, off, strerror(errno));
264
+ goto read_err;
265
+ }
266
+ off += nr;
267
+ buf->dec = &decoder;
268
+ buf->stream_id = stream_id;
269
+ buf->size = size;
270
+ buf->file_off = file_off;
271
+ if (buf->size == 0)
272
+ exit(EXIT_FAILURE);
273
+ TAILQ_INSERT_TAIL(&bufs, buf, next_buf);
274
+ }
275
+
276
+ if (recipe)
277
+ {
278
+ lineno = 0;
279
+ while (line = fgets(line_buf, sizeof(line_buf), recipe), line != NULL)
280
+ {
281
+ ++lineno;
282
+ end = strchr(line, '\n');
283
+ if (!end)
284
+ {
285
+ fprintf(stderr, "no newline on line %u\n", lineno);
286
+ exit(EXIT_FAILURE);
287
+ }
288
+ *end = '\0';
289
+
290
+ if (*line == '#')
291
+ continue;
292
+
293
+ if (3 == sscanf(line, " %[s] %"PRIu64" %"PRIu32" ", command, &stream_id, &size))
294
+ {
295
+ TAILQ_FOREACH(buf, &bufs, next_buf)
296
+ if (stream_id == buf->stream_id)
297
+ break;
298
+ if (!buf)
299
+ {
300
+ fprintf(stderr, "stream %"PRIu64" not found (recipe line %u)\n",
301
+ stream_id, lineno);
302
+ exit(EXIT_FAILURE);
303
+ }
304
+ p = buf->buf;
305
+ rhs = lsqpack_dec_header_in(&decoder, buf, stream_id,
306
+ buf->size, &p,
307
+ buf->size /* FIXME: this should be `size' */,
308
+ &hlist, NULL, NULL);
309
+ switch (rhs)
310
+ {
311
+ case LQRHS_DONE:
312
+ assert(p == buf->buf + buf->size);
313
+ header_block_done(buf, hlist);
314
+ if (s_verbose)
315
+ fprintf(stderr, "compression ratio: %.3f\n",
316
+ lsqpack_dec_ratio(&decoder));
317
+ TAILQ_REMOVE(&bufs, buf, next_buf);
318
+ free(buf);
319
+ break;
320
+ case LQRHS_BLOCKED:
321
+ buf->off += (unsigned) (p - buf->buf);
322
+ TAILQ_REMOVE(&bufs, buf, next_buf);
323
+ break;
324
+ case LQRHS_NEED:
325
+ buf->off += (unsigned) (p - buf->buf);
326
+ break;
327
+ default:
328
+ assert(rhs == LQRHS_ERROR);
329
+ fprintf(stderr, "recipe line %u: stream %"PRIu64": "
330
+ "header_in error\n", lineno, stream_id);
331
+ exit(EXIT_FAILURE);
332
+ }
333
+ }
334
+ else if (2 == sscanf(line, " %[z] %u ", command, &size))
335
+ {
336
+ s_max_read_size = size;
337
+ }
338
+ else
339
+ {
340
+ perror("sscanf");
341
+ exit(EXIT_FAILURE);
342
+ }
343
+ }
344
+ fclose(recipe);
345
+ }
346
+
347
+ while (buf = TAILQ_FIRST(&bufs), buf != NULL)
348
+ {
349
+ TAILQ_REMOVE(&bufs, buf, next_buf);
350
+ if (buf->stream_id == 0)
351
+ {
352
+ r = lsqpack_dec_enc_in(&decoder, buf->buf, buf->size - buf->off);
353
+ if (r != 0)
354
+ {
355
+ err = lsqpack_dec_get_err_info(buf->dec);
356
+ fprintf(stderr, "encoder_in error; off %"PRIu64", line %d\n",
357
+ err->off, err->line);
358
+ exit(EXIT_FAILURE);
359
+ }
360
+ if (s_verbose)
361
+ lsqpack_dec_print_table(&decoder, stderr);
362
+ free(buf);
363
+ }
364
+ else
365
+ {
366
+ dec_header:
367
+ p = buf->buf + buf->off;
368
+ if (buf->off == 0)
369
+ rhs = lsqpack_dec_header_in(&decoder, buf, buf->stream_id,
370
+ buf->size, &p, MIN(s_max_read_size, buf->size),
371
+ &hlist, NULL, NULL);
372
+ else
373
+ rhs = lsqpack_dec_header_read(buf->dec, buf, &p,
374
+ MIN(s_max_read_size, (buf->size - buf->off)),
375
+ &hlist, NULL, NULL);
376
+ switch (rhs)
377
+ {
378
+ case LQRHS_DONE:
379
+ assert(p == buf->buf + buf->size);
380
+ header_block_done(buf, hlist);
381
+ if (s_verbose)
382
+ fprintf(stderr, "compression ratio: %.3f\n",
383
+ lsqpack_dec_ratio(&decoder));
384
+ free(buf);
385
+ break;
386
+ case LQRHS_BLOCKED:
387
+ buf->off = (unsigned) (p - buf->buf);
388
+ break;
389
+ case LQRHS_NEED:
390
+ buf->off = (unsigned) (p - buf->buf);
391
+ goto dec_header;
392
+ default:
393
+ assert(rhs == LQRHS_ERROR);
394
+ fprintf(stderr, "stream %"PRIu64": header block error "
395
+ "starting at off %zu\n", buf->stream_id, buf->off);
396
+ err = lsqpack_dec_get_err_info(&decoder);
397
+ fprintf(stderr, "encoder_in error; off %"PRIu64", line %d\n",
398
+ err->off, err->line);
399
+ exit(EXIT_FAILURE);
400
+ }
401
+ }
402
+ }
403
+
404
+ if (!TAILQ_EMPTY(&bufs))
405
+ {
406
+ fprintf(stderr, "some streams reamain\n");
407
+ exit(EXIT_FAILURE);
408
+ }
409
+ /* TODO: check if decoder has any stream references. That would be
410
+ * an error.
411
+ */
412
+
413
+ if (s_verbose)
414
+ lsqpack_dec_print_table(&decoder, stderr);
415
+
416
+ lsqpack_dec_cleanup(&decoder);
417
+
418
+ assert(TAILQ_EMPTY(&bufs));
419
+
420
+ if (s_out)
421
+ (void) fclose(s_out);
422
+
423
+ exit(EXIT_SUCCESS);
424
+
425
+ read_err:
426
+ if (nr < 0)
427
+ perror("read");
428
+ else if (nr == 0)
429
+ fprintf(stderr, "unexpected EOF\n");
430
+ else
431
+ fprintf(stderr, "not enough bytes read (%zu)\n", (size_t) nr);
432
+ exit(EXIT_FAILURE);
433
+ }