lsqpack 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (89) hide show
  1. checksums.yaml +7 -0
  2. data/Gemfile +19 -0
  3. data/Gemfile.lock +82 -0
  4. data/LICENSE +21 -0
  5. data/README.md +36 -0
  6. data/Rakefile +19 -0
  7. data/Steepfile +6 -0
  8. data/ext/lsqpack/extconf.rb +18 -0
  9. data/ext/lsqpack/lsqpack.c +426 -0
  10. data/ext/lsqpack/lsqpack.h +6 -0
  11. data/lib/lsqpack/version.rb +5 -0
  12. data/lib/lsqpack.rb +30 -0
  13. data/ls-qpack/.appveyor.yml +14 -0
  14. data/ls-qpack/.cirrus.yml +6 -0
  15. data/ls-qpack/.travis.yml +32 -0
  16. data/ls-qpack/CMakeLists.txt +66 -0
  17. data/ls-qpack/LICENSE +21 -0
  18. data/ls-qpack/README.md +65 -0
  19. data/ls-qpack/bin/CMakeLists.txt +21 -0
  20. data/ls-qpack/bin/encode-int.c +87 -0
  21. data/ls-qpack/bin/fuzz-decode.c +247 -0
  22. data/ls-qpack/bin/interop-decode.c +433 -0
  23. data/ls-qpack/bin/interop-encode.c +554 -0
  24. data/ls-qpack/deps/xxhash/xxhash.c +941 -0
  25. data/ls-qpack/deps/xxhash/xxhash.h +160 -0
  26. data/ls-qpack/fuzz/decode/a/README +3 -0
  27. data/ls-qpack/fuzz/decode/a/preamble +0 -0
  28. data/ls-qpack/fuzz/decode/a/test-cases/id_000000,sig_06,src_000390,op_havoc,rep_4 +0 -0
  29. data/ls-qpack/fuzz/decode/a/test-cases/id_000000,sig_06,src_000579,op_flip1,pos_14 +0 -0
  30. data/ls-qpack/fuzz/decode/a/test-cases/id_000000,src_000000,op_flip2,pos_12 +0 -0
  31. data/ls-qpack/fuzz/decode/a/test-cases/id_000001,sig_11,src_000579,op_havoc,rep_4 +0 -0
  32. data/ls-qpack/fuzz/decode/a/test-cases/id_000002,sig_11,src_000481,op_int16,pos_15,val_-1 +0 -0
  33. data/ls-qpack/fuzz/decode/a/test-cases/id_000002,src_000000,op_havoc,rep_8 +0 -0
  34. data/ls-qpack/fuzz/decode/a/test-cases/id_000006,src_000285,op_flip2,pos_14 +0 -0
  35. data/ls-qpack/fuzz/decode/a/test-cases/id_000008,src_000285,op_flip2,pos_20 +0 -0
  36. data/ls-qpack/fuzz/decode/a/test-cases/id_000010,src_000306,op_flip2,pos_75 +0 -0
  37. data/ls-qpack/fuzz/decode/a/test-cases/id_000011,src_000344,op_havoc,rep_2 +0 -0
  38. data/ls-qpack/fuzz/decode/a/test-cases/id_000014,src_000366,op_flip2,pos_28 +0 -0
  39. data/ls-qpack/fuzz/decode/b/README +1 -0
  40. data/ls-qpack/fuzz/decode/b/preamble +0 -0
  41. data/ls-qpack/fuzz/decode/b/test-cases/seed +0 -0
  42. data/ls-qpack/fuzz/decode/c/setup.sh +3 -0
  43. data/ls-qpack/fuzz/decode/c/test-cases/fb-req.qif.proxygen.out.256.100.0-chopped +0 -0
  44. data/ls-qpack/fuzz/decode/d/preamble +0 -0
  45. data/ls-qpack/fuzz/decode/d/setup.sh +3 -0
  46. data/ls-qpack/fuzz/decode/d/test-cases/fb-resp.minhq.256.128.0.ack +0 -0
  47. data/ls-qpack/fuzz/input/256.100.1/fb-req.out.256.100.1 +0 -0
  48. data/ls-qpack/fuzz/input/256.100.1/fb-resp.out.256.100.1 +0 -0
  49. data/ls-qpack/fuzz/input/256.100.1/netbsd.out.256.100.1 +0 -0
  50. data/ls-qpack/huff-tables.h +136247 -0
  51. data/ls-qpack/lsqpack.c +5547 -0
  52. data/ls-qpack/lsqpack.h +768 -0
  53. data/ls-qpack/test/CMakeLists.txt +76 -0
  54. data/ls-qpack/test/lsqpack-test.h +43 -0
  55. data/ls-qpack/test/qifs/fb-req.qif +4917 -0
  56. data/ls-qpack/test/qifs/fb-resp.qif +5982 -0
  57. data/ls-qpack/test/qifs/long-codes.qif +5984 -0
  58. data/ls-qpack/test/qifs/netbsd.qif +235 -0
  59. data/ls-qpack/test/run-qif.pl +97 -0
  60. data/ls-qpack/test/run-scenario.sh +68 -0
  61. data/ls-qpack/test/scenarios/0.95-reset.sce +10 -0
  62. data/ls-qpack/test/scenarios/cancel-stream.sce +22 -0
  63. data/ls-qpack/test/scenarios/drain-2.sce +37 -0
  64. data/ls-qpack/test/scenarios/drain.sce +37 -0
  65. data/ls-qpack/test/scenarios/end-dst-2.sce +14 -0
  66. data/ls-qpack/test/scenarios/end-dst.sce +14 -0
  67. data/ls-qpack/test/scenarios/incl-name.sce +13 -0
  68. data/ls-qpack/test/scenarios/multi-byte-int-dyn-ref-1.sce +110 -0
  69. data/ls-qpack/test/scenarios/multi-byte-int-dyn-ref-2.sce +161 -0
  70. data/ls-qpack/test/scenarios/post-base-1.sce +10 -0
  71. data/ls-qpack/test/scenarios/post-base-2.sce +13 -0
  72. data/ls-qpack/test/scenarios/post-base-nr.sce +10 -0
  73. data/ls-qpack/test/scenarios/set-max-cap.sce +15 -0
  74. data/ls-qpack/test/test_enc_str.c +139 -0
  75. data/ls-qpack/test/test_get_stx_id.c +144 -0
  76. data/ls-qpack/test/test_huff_dec.c +399 -0
  77. data/ls-qpack/test/test_int.c +220 -0
  78. data/ls-qpack/test/test_qpack.c +856 -0
  79. data/ls-qpack/test/test_read_enc_stream.c +256 -0
  80. data/ls-qpack/tools/har2qif.pl +139 -0
  81. data/ls-qpack/tools/randomize-cookies.pl +41 -0
  82. data/ls-qpack/tools/sort-qif.pl +31 -0
  83. data/ls-qpack/wincompat/getopt.c +758 -0
  84. data/ls-qpack/wincompat/getopt.h +131 -0
  85. data/ls-qpack/wincompat/getopt1.c +188 -0
  86. data/ls-qpack/wincompat/sys/queue.h +859 -0
  87. data/lsqpack.gemspec +39 -0
  88. data/sig/lsqpack.rbs +29 -0
  89. metadata +135 -0
@@ -0,0 +1,256 @@
1
+ /*
2
+ * Read encoder stream
3
+ */
4
+
5
+ #include <assert.h>
6
+ #include <stdlib.h>
7
+ #include <string.h>
8
+
9
+ #include "lsqpack.h"
10
+
11
+ #define MIN(a, b) ((a) < (b) ? (a) : (b))
12
+
13
+ /* Dynamic table entry: */
14
+ struct lsqpack_dec_table_entry
15
+ {
16
+ unsigned dte_name_len;
17
+ unsigned dte_val_len;
18
+ unsigned dte_refcnt;
19
+ char dte_buf[0]; /* Contains both name and value */
20
+ };
21
+
22
+ #define DTE_NAME(dte) ((dte)->dte_buf)
23
+ #define DTE_VALUE(dte) (&(dte)->dte_buf[(dte)->dte_name_len])
24
+
25
+
26
+ struct test_read_encoder_stream
27
+ {
28
+ int lineno;
29
+
30
+ /* Input */
31
+ unsigned char input[0x100];
32
+ size_t input_sz;
33
+
34
+ /* Output */
35
+ unsigned n_entries;
36
+ struct {
37
+ const char *name, *value;
38
+ } dyn_table[10];
39
+ };
40
+
41
+
42
+ static const struct test_read_encoder_stream tests[] =
43
+ {
44
+
45
+ { __LINE__,
46
+ "\xc0\x8b\xf1\xe3\xc2\xf5\x15\x31\xa2\x45\xcf\x64\xdf",
47
+ 13,
48
+ 1,
49
+ {
50
+ { ":authority", "www.netbsd.org", },
51
+ },
52
+ },
53
+
54
+ { __LINE__,
55
+ "\xc0\x00",
56
+ 2,
57
+ 1,
58
+ {
59
+ { ":authority", "", },
60
+ },
61
+ },
62
+
63
+ { __LINE__,
64
+ "\xc0\x8b\xf1\xe3\xc2\xf5\x15\x31\xa2\x45\xcf\x64\xdf\x00",
65
+ 14,
66
+ 2,
67
+ {
68
+ { ":authority", "www.netbsd.org", },
69
+ { ":authority", "www.netbsd.org", },
70
+ },
71
+ },
72
+
73
+ { __LINE__,
74
+ "\xc0\x8b\xf1\xe3\xc2\xf5\x15\x31\xa2\x45\xcf\x64\xdf\x00\x20",
75
+ 15,
76
+ 0,
77
+ {
78
+ { NULL, NULL, },
79
+ },
80
+ },
81
+
82
+ { __LINE__,
83
+ "\xc0\x15" "Respect my authorata!",
84
+ 2 + sizeof("Respect my authorata!") - 1,
85
+ 1,
86
+ {
87
+ { ":authority", "Respect my authorata!", },
88
+ },
89
+ },
90
+
91
+ { __LINE__,
92
+ "\x63\x92\xd9\x0b\x8c\xe5\x39\x6c\x2a\x86\x42\x94\xfa\x50\x83\xb3"
93
+ "\xfc",
94
+ 17,
95
+ 1,
96
+ {
97
+ { "dude", "Where is my car?", },
98
+ },
99
+ },
100
+
101
+ { __LINE__,
102
+ "\x63\x92\xd9\x0b\x03\x61\x61\x7a",
103
+ 8,
104
+ 1,
105
+ {
106
+ { "dude", "aaz", },
107
+ },
108
+ },
109
+
110
+ { __LINE__,
111
+ "\x43\x7a\x7a\x7a\x88\xcc\x6a\x0d\x48\xea\xe8\x3b\x0f",
112
+ 13,
113
+ 1,
114
+ {
115
+ { "zzz", "Kilimanjaro", },
116
+ },
117
+ },
118
+
119
+ { __LINE__,
120
+ "\x43\x7a\x7a\x7a\x03\x61\x61\x7a",
121
+ 8,
122
+ 1,
123
+ {
124
+ { "zzz", "aaz", },
125
+ },
126
+ },
127
+
128
+ { __LINE__,
129
+ "\x43\x7a\x7a\x7a\x03\x61\x61\x61\x80\x03\x61\x61\x7a",
130
+ 13,
131
+ 2,
132
+ {
133
+ { "zzz", "aaa", },
134
+ { "zzz", "aaz", },
135
+ },
136
+ },
137
+
138
+ { __LINE__,
139
+ "\x40\x88\xcc\x6a\x0d\x48\xea\xe8\x3b\x0f",
140
+ 10,
141
+ 1,
142
+ {
143
+ { "", "Kilimanjaro", },
144
+ },
145
+ },
146
+
147
+ { __LINE__,
148
+ "\x43\x7a\x7a\x7a\x00",
149
+ 5,
150
+ 1,
151
+ {
152
+ { "zzz", "", },
153
+ },
154
+ },
155
+
156
+ { __LINE__,
157
+ "\x40\x00",
158
+ 2,
159
+ 1,
160
+ {
161
+ { "", "", },
162
+ },
163
+ },
164
+
165
+ };
166
+
167
+
168
+ struct ringbuf_iter
169
+ {
170
+ const struct lsqpack_ringbuf *rbuf;
171
+ unsigned next, end;
172
+ };
173
+
174
+
175
+ #if LSQPACK_DEVEL_MODE
176
+ unsigned
177
+ ringbuf_count (const struct lsqpack_ringbuf *rbuf);
178
+
179
+ void *
180
+ ringbuf_iter_next (struct ringbuf_iter *iter);
181
+
182
+ void *
183
+ ringbuf_iter_first (struct ringbuf_iter *iter,
184
+ const struct lsqpack_ringbuf *rbuf);
185
+
186
+ static void
187
+ verify_dyn_table (const struct lsqpack_dec *dec,
188
+ const struct test_read_encoder_stream *test)
189
+ {
190
+ const struct lsqpack_dec_table_entry *entry;
191
+ unsigned n;
192
+ struct ringbuf_iter riter;
193
+
194
+ assert(ringbuf_count(&dec->qpd_dyn_table) == test->n_entries);
195
+
196
+ for (n = 0; n < test->n_entries; ++n)
197
+ {
198
+ if (n == 0)
199
+ entry = ringbuf_iter_first(&riter, &dec->qpd_dyn_table);
200
+ else
201
+ entry = ringbuf_iter_next(&riter);
202
+ assert(entry);
203
+ assert(entry->dte_name_len == strlen(test->dyn_table[n].name));
204
+ assert(entry->dte_val_len == strlen(test->dyn_table[n].value));
205
+ assert(0 == memcmp(DTE_NAME(entry), test->dyn_table[n].name, entry->dte_name_len));
206
+ assert(0 == memcmp(DTE_VALUE(entry), test->dyn_table[n].value, entry->dte_val_len));
207
+ }
208
+ }
209
+ #else
210
+ static void
211
+ verify_dyn_table (const struct lsqpack_dec *dec,
212
+ const struct test_read_encoder_stream *test)
213
+ {
214
+ fprintf(stderr, "LSQPACK_DEVEL_MODE is not compiled: cannot verify dynamic table\n");
215
+ }
216
+ #endif
217
+
218
+
219
+ static void
220
+ run_test (const struct test_read_encoder_stream *test)
221
+ {
222
+ struct lsqpack_dec dec;
223
+ size_t chunk_sz, off;
224
+ int r;
225
+
226
+ for (chunk_sz = 1; chunk_sz <= test->input_sz; ++chunk_sz)
227
+ {
228
+ lsqpack_dec_init(&dec, NULL, 0x1000, 100, NULL);
229
+
230
+ off = 0;
231
+ do
232
+ {
233
+ r = lsqpack_dec_enc_in(&dec, test->input + off,
234
+ MIN(chunk_sz, test->input_sz - off));
235
+ assert(r == 0);
236
+ off += MIN(chunk_sz, test->input_sz - off);
237
+ }
238
+ while (off < test->input_sz);
239
+
240
+ verify_dyn_table(&dec, test);
241
+
242
+ lsqpack_dec_cleanup(&dec);
243
+ }
244
+ }
245
+
246
+
247
+ int
248
+ main (void)
249
+ {
250
+ const struct test_read_encoder_stream *test;
251
+
252
+ for (test = tests; test < tests + sizeof(tests) / sizeof(tests[0]); ++test)
253
+ run_test(test);
254
+
255
+ return 0;
256
+ }
@@ -0,0 +1,139 @@
1
+ #!/usr/bin/env perl
2
+ #
3
+ # har2qif.pl - Convert HAR file to a QIF file.
4
+ #
5
+ # QIF stands for "QPACK Interop Format." It is meant to be easy to parse,
6
+ # write, and compare. The idea is that the QIF input to an encoder can
7
+ # be compared to QIF output produced by the decoder using `diff(1)':
8
+ #
9
+ # sh$ har2qif -requests my.har > in.qif
10
+ # sh$ ./qpack-encoder-A in.qif > binary-format # See wiki
11
+ # sh$ ./qpack-decoder-B binary-format > out.qif
12
+ # sh$ diff in.qif out.qif && echo "Success!"
13
+ #
14
+ # The QIF format is plain text. Each header field in on a separate line,
15
+ # with name and value separated by the TAB character. An empty line
16
+ # signifies the end of a header set. Lines beginning with '#' are ignored.
17
+ #
18
+ # HTTP/2 header sets are left untouched, while non-HTTP/2 header sets are
19
+ # transformed to resemble HTTP/2:
20
+ # - Header names are lowercased
21
+ # - `Host' header name is removed from requests
22
+ # - Requests get :method, :scheme, :authority, and :path pseudo-headers
23
+ # - Responses get :status pseudo-headers
24
+
25
+ use strict;
26
+ use warnings;
27
+
28
+ use Getopt::Long;
29
+ use JSON qw(decode_json);
30
+ use URI;
31
+
32
+ my $key = 'request';
33
+ my ($limit, $split_cookie);
34
+
35
+ GetOptions( "help" => sub {
36
+ print <<USAGE;
37
+ Usage: $0 [options] [file.har] > file.qif
38
+
39
+ Options:
40
+ -requests Print headers from requests. This is the default.
41
+ -responses Print headers from responses.
42
+ -split-cookie Split the Cookie: header.
43
+ -limit N Limit number of header sets to N. The default
44
+ is no limit.
45
+
46
+ If file.har is not specified, the HAR is read from stdin
47
+ USAGE
48
+
49
+ exit;
50
+ },
51
+ "requests" => sub { $key = "request", },
52
+ "responses" => sub { $key = "response", },
53
+ "split-cookie" => \$split_cookie,
54
+ "limit=i" => \$limit,
55
+ );
56
+
57
+ my $json = decode_json( do { undef $/; <> });
58
+ my @messages = map $$_{$key}, @{ $json->{log}{entries} };
59
+ if (defined($limit))
60
+ {
61
+ splice @messages, $limit;
62
+ }
63
+
64
+ my @header_sets = do {
65
+ if ($key eq 'request') {
66
+ map req_header_set($_), @messages
67
+ } else {
68
+ map resp_header_set($_), @messages
69
+ }
70
+ };
71
+
72
+ for (@header_sets) {
73
+ no warnings 'uninitialized';
74
+ print map "$$_[0]\t$$_[1]\n", @$_;
75
+ print "\n";
76
+ }
77
+
78
+ exit;
79
+
80
+ # Looking at capitalization of the first header is a more reliable means
81
+ # of determining HTTP version than relying on httpVersion field.
82
+ #
83
+ sub is_http2 {
84
+ my $message = shift;
85
+ if (defined($$message{headers}[0])
86
+ && defined($$message{headers}[0]{name})) {
87
+ return $$message{headers}[0]{name} =~ /^[a-z:]/;
88
+ } elsif (defined($$message{httpVersion})) {
89
+ return $$message{httpVersion} =~ m~HTTP/2~i;
90
+ } else {
91
+ return;
92
+ }
93
+ }
94
+
95
+ sub req_header_set {
96
+ my $message = shift;
97
+ my @set;
98
+ if (!is_http2($message)) {
99
+ my @headers = map [ lc($$_{name}), $$_{value}, ],
100
+ grep $$_{name} ne 'Host', @{ $$message{headers} };
101
+ my $uri = URI->new($$message{url});
102
+ @set = (
103
+ [ ':method', $$message{method}, ],
104
+ [ ':scheme', $uri->scheme, ],
105
+ [ ':authority', $uri->authority, ],
106
+ [ ':path', $uri->path_query, ],
107
+ @headers,
108
+ );
109
+ } else {
110
+ @set = map [ $$_{name}, $$_{value}, ], @{ $$message{headers} };
111
+ }
112
+ if ($split_cookie) {
113
+ return [ map {
114
+ if ('cookie' eq $$_[0]) {
115
+ map [ 'cookie', $_, ], split /;\s+/, $$_[1]
116
+ } else {
117
+ $_
118
+ }
119
+ } @set ];
120
+ } else {
121
+ return \@set;
122
+ }
123
+ }
124
+
125
+ sub resp_header_set {
126
+ my $message = shift;
127
+ no warnings 'uninitialized';
128
+ if (!is_http2($message)) {
129
+ my @headers = map [ lc($$_{name}), $$_{value}, ],
130
+ @{ $$message{headers} };
131
+ my $uri = URI->new($$message{url});
132
+ return [
133
+ [ ':status', $$message{status}, ],
134
+ @headers,
135
+ ];
136
+ } else {
137
+ return [ map [ $$_{name}, $$_{value}, ], @{ $$message{headers} } ]
138
+ }
139
+ }
@@ -0,0 +1,41 @@
1
+ #!/usr/bin/env perl
2
+ #
3
+ # Given a QIF file, randomize cookie headers
4
+ #
5
+ # Usage: randomize-cookies.pl input.qif > output.qif
6
+
7
+ @chars = ('0' .. '9', 'a' .. 'z', 'A' .. 'Z', split '', '/+*');
8
+
9
+ sub randomify {
10
+ $val = shift;
11
+
12
+ if (exists($cached_vals{$val})) {
13
+ return $cached_vals{$val};
14
+ }
15
+
16
+ @val = split '', $val;
17
+ $seen_eq = 0;
18
+ for ($i = 0; $i < @val; ++$i)
19
+ {
20
+ if ($val[$i] ne '=' or $seen_eq++)
21
+ {
22
+ $rand = $rand[$i] //= int(rand(@chars));
23
+ # Output is a function of both position and value:
24
+ $val[$i] = $chars[ ($rand + ord($val[$i])) % @chars ];
25
+ }
26
+ }
27
+
28
+ return $cached_vals{$val} = join '', @val;
29
+ }
30
+
31
+ while (<>) {
32
+ chomp;
33
+ if (m/^cookie\t(.*)/) {
34
+ print "cookie\t", randomify($1), "\n";
35
+ } elsif (m/^set-cookie\t(.*)/) {
36
+ print "set-cookie\t", join('; ', map randomify($_),
37
+ split /;\s+/, $1), "\n";
38
+ } else {
39
+ print $_, "\n";
40
+ }
41
+ }
@@ -0,0 +1,31 @@
1
+ #!/usr/bin/env perl
2
+ #
3
+ # sort-qif.pl -- sort QIF file
4
+ #
5
+ # Sort QIF file. This assumes that the header lists in the QIF file have a
6
+ # leading comment that looks like '# stream $number'.
7
+ #
8
+ # Usage: sort-qif.pl [--strip-comments] [files] [--output FILE]
9
+
10
+ use Getopt::Long;
11
+
12
+ GetOptions("strip-comments" => \$strip_comments, output => \$output);
13
+
14
+ if (defined($output)) {
15
+ open STDOUT, ">", $output or die "cannot open $output for writing: $!";
16
+ }
17
+
18
+ $/ = "\n\n";
19
+
20
+ @chunks = map $$_[1],
21
+ sort { $$a[0] <=> $$b[0] }
22
+ map { /^*#\s*stream\s+(\d+)/; [ $1, $_ ] }
23
+ <>;
24
+
25
+ if ($strip_comments) {
26
+ for (@chunks) {
27
+ s/^#.*\n//mg;
28
+ }
29
+ }
30
+
31
+ print @chunks;