protocol-quic 0.0.0

data/ext/ngtcp2/examples/tests/ngtcp2test/client.py

@@ -0,0 +1,187 @@
+import logging
+import os
+import re
+import subprocess
+from typing import List
+
+import pytest
+
+from .server import ExampleServer, ServerRun
+from .certs import Credentials
+from .tls import HandShake, HSRecord
+from .env import Env, CryptoLib
+from .log import LogFile, HexDumpScanner
+
+
+log = logging.getLogger(__name__)
+
+
+class ClientRun:
+
+    def __init__(self, env: Env, returncode, logfile: LogFile, srun: ServerRun):
+        self.env = env
+        self.returncode = returncode
+        self.logfile = logfile
+        self.log_lines = logfile.get_recent()
+        self._data_recs = None
+        self._hs_recs = None
+        self._srun = srun
+        if self.env.verbose > 1:
+            log.debug(f'read {len(self.log_lines)} lines from {logfile.path}')
+
+    @property
+    def handshake(self) -> List[HSRecord]:
+        if self._data_recs is None:
+            crypto_line =  re.compile(r'Ordered CRYPTO data in \S+ crypto level')
+            scanner = HexDumpScanner(source=self.log_lines,
+                                     leading_regex=crypto_line)
+            self._data_recs = [data for data in scanner]
+            if self.env.verbose > 1:
+                log.debug(f'detected {len(self._data_recs)} crypto hexdumps '
+                          f'in {self.logfile.path}')
+        if self._hs_recs is None:
+            self._hs_recs = [hrec for hrec in HandShake(source=self._data_recs,
+                                                        verbose=self.env.verbose)]
+            if self.env.verbose > 1:
+                log.debug(f'detected {len(self._hs_recs)} crypto '
+                          f'records in {self.logfile.path}')
+        return self._hs_recs
+
+    @property
+    def hs_stripe(self) -> str:
+        return ":".join([hrec.name for hrec in self.handshake])
+
+    @property
+    def early_data_rejected(self) -> bool:
+        for l in self.log_lines:
+            if re.match(r'^Early data was rejected by server.*', l):
+                return True
+        return False
+
+    @property
+    def server(self) -> ServerRun:
+        return self._srun
+
+    def norm_exp(self, c_hs, s_hs, allow_hello_retry=True):
+        if allow_hello_retry and self.hs_stripe.startswith('HelloRetryRequest:'):
+            c_hs = "HelloRetryRequest:" + c_hs
+            s_hs = "ClientHello:" + s_hs
+        return c_hs, s_hs
+
+    def _assert_hs(self, c_hs, s_hs):
+        if not self.hs_stripe.startswith(c_hs):
+            # what happened?
+            if self.hs_stripe == '':
+                # server send nothing
+                if self.server.hs_stripe == '':
+                    # client send nothing
+                    pytest.fail(f'client did not send a ClientHello"')
+                else:
+                    # client send sth, but server did not respond
+                    pytest.fail(f'server did not respond to ClientHello: '
+                                f'{self.server.handshake[0].to_text()}"')
+            else:
+                pytest.fail(f'Expected "{c_hs}", got "{self.hs_stripe}"')
+        assert self.server.hs_stripe == s_hs, \
+            f'Expected "{s_hs}", got "{self.server.hs_stripe}"\n'
+
+    def assert_non_resume_handshake(self, allow_hello_retry=True):
+        # for client/server where KEY_SHARE do not match, the hello is retried
+        c_hs, s_hs = self.norm_exp(
+            "ServerHello:EncryptedExtensions:Certificate:CertificateVerify:Finished",
+            "ClientHello:Finished", allow_hello_retry=allow_hello_retry)
+        self._assert_hs(c_hs, s_hs)
+
+    def assert_resume_handshake(self):
+        # for client/server where KEY_SHARE do not match, the hello is retried
+        c_hs, s_hs = self.norm_exp("ServerHello:EncryptedExtensions:Finished",
+                                   "ClientHello:Finished")
+        self._assert_hs(c_hs, s_hs)
+
+    def assert_verify_null_handshake(self):
+        c_hs, s_hs = self.norm_exp(
+            "ServerHello:EncryptedExtensions:CertificateRequest:Certificate:CertificateVerify:Finished",
+            "ClientHello:Certificate:Finished")
+        self._assert_hs(c_hs, s_hs)
+
+    def assert_verify_cert_handshake(self):
+        c_hs, s_hs = self.norm_exp(
+            "ServerHello:EncryptedExtensions:CertificateRequest:Certificate:CertificateVerify:Finished",
+            "ClientHello:Certificate:CertificateVerify:Finished")
+        self._assert_hs(c_hs, s_hs)
+
+
+class ExampleClient:
+
+    def __init__(self, env: Env, crypto_lib: str):
+        self.env = env
+        self._crypto_lib = crypto_lib
+        self._path = env.client_path(self._crypto_lib)
+        self._log_path = f'{self.env.gen_dir}/{self._crypto_lib}-client.log'
+        self._qlog_path = f'{self.env.gen_dir}/{self._crypto_lib}-client.qlog'
+        self._session_path = f'{self.env.gen_dir}/{self._crypto_lib}-client.session'
+        self._tp_path = f'{self.env.gen_dir}/{self._crypto_lib}-client.tp'
+        self._data_path = f'{self.env.gen_dir}/{self._crypto_lib}-client.data'
+
+    @property
+    def path(self):
+        return self._path
+
+    @property
+    def crypto_lib(self):
+        return self._crypto_lib
+
+    @property
+    def uses_cipher_config(self):
+        return CryptoLib.uses_cipher_config(self.crypto_lib)
+
+    def supports_cipher(self, cipher):
+        return CryptoLib.supports_cipher(self.crypto_lib, cipher)
+
+    def exists(self):
+        return os.path.isfile(self.path)
+
+    def clear_session(self):
+        if os.path.isfile(self._session_path):
+            os.remove(self._session_path)
+        if os.path.isfile(self._tp_path):
+            os.remove(self._tp_path)
+
+    def http_get(self, server: ExampleServer, url: str, extra_args: List[str] = None,
+                 use_session=False, data=None,
+                 credentials: Credentials = None,
+                 ciphers: str = None):
+        args = [
+            self.path, '--exit-on-all-streams-close',
+            f'--qlog-file={self._qlog_path}'
+        ]
+        if use_session:
+            args.append(f'--session-file={self._session_path}')
+            args.append(f'--tp-file={self._tp_path}')
+        if data is not None:
+            with open(self._data_path, 'w') as fd:
+                fd.write(data)
+            args.append(f'--data={self._data_path}')
+        if ciphers is not None:
+            ciphers = CryptoLib.adjust_ciphers(self.crypto_lib, ciphers)
+            args.append(f'--ciphers={ciphers}')
+        if credentials is not None:
+            args.append(f'--key={credentials.pkey_file}')
+            args.append(f'--cert={credentials.cert_file}')
+        if extra_args is not None:
+            args.extend(extra_args)
+        args.extend([
+            'localhost', str(self.env.examples_port),
+            url
+        ])
+        if os.path.isfile(self._qlog_path):
+            os.remove(self._qlog_path)
+        with open(self._log_path, 'w') as log_file:
+            logfile = LogFile(path=self._log_path)
+            server.log.advance()
+            process = subprocess.Popen(args=args, text=True,
+                                       stdout=log_file, stderr=log_file)
+            process.wait()
+            return ClientRun(env=self.env, returncode=process.returncode,
+                             logfile=logfile, srun=server.get_run())
+

data/ext/ngtcp2/examples/tests/ngtcp2test/env.py

@@ -0,0 +1,191 @@
+import logging
+import os
+from configparser import ConfigParser, ExtendedInterpolation
+from typing import Dict, Optional
+
+from .certs import CertificateSpec, Ngtcp2TestCA, Credentials
+
+log = logging.getLogger(__name__)
+
+
+class CryptoLib:
+
+    IGNORES_CIPHER_CONFIG = [
+        'picotls', 'boringssl'
+    ]
+    UNSUPPORTED_CIPHERS = {
+        'wolfssl': [
+            'TLS_AES_128_CCM_SHA256',  # no plans to
+        ],
+        'picotls': [
+            'TLS_AES_128_CCM_SHA256',  # no plans to
+        ],
+        'boringssl': [
+            'TLS_AES_128_CCM_SHA256',  # no plans to
+        ]
+    }
+    GNUTLS_CIPHERS = {
+        'TLS_AES_128_GCM_SHA256': 'AES-128-GCM',
+        'TLS_AES_256_GCM_SHA384': 'AES-256-GCM',
+        'TLS_CHACHA20_POLY1305_SHA256': 'CHACHA20-POLY1305',
+        'TLS_AES_128_CCM_SHA256': 'AES-128-CCM',
+    }
+
+    @classmethod
+    def uses_cipher_config(cls, crypto_lib):
+        return crypto_lib not in cls.IGNORES_CIPHER_CONFIG
+
+    @classmethod
+    def supports_cipher(cls, crypto_lib, cipher):
+        return crypto_lib not in cls.UNSUPPORTED_CIPHERS or \
+               cipher not in cls.UNSUPPORTED_CIPHERS[crypto_lib]
+
+    @classmethod
+    def adjust_ciphers(cls, crypto_lib, ciphers: str) -> str:
+        if crypto_lib == 'gnutls':
+            gciphers = "NORMAL:-VERS-ALL:+VERS-TLS1.3:-CIPHER-ALL"
+            for cipher in ciphers.split(':'):
+                gciphers += f':+{cls.GNUTLS_CIPHERS[cipher]}'
+            return gciphers
+        return ciphers
+
+
+def init_config_from(conf_path):
+    if os.path.isfile(conf_path):
+        config = ConfigParser(interpolation=ExtendedInterpolation())
+        config.read(conf_path)
+        return config
+    return None
+
+
+TESTS_PATH = os.path.dirname(os.path.dirname(__file__))
+EXAMPLES_PATH = os.path.dirname(TESTS_PATH)
+DEF_CONFIG = init_config_from(os.path.join(TESTS_PATH, 'config.ini'))
+
+
+class Env:
+
+    @classmethod
+    def get_crypto_libs(cls, configurable_ciphers=None):
+        names = [name for name in DEF_CONFIG['examples']
+                 if DEF_CONFIG['examples'][name] == 'yes']
+        if configurable_ciphers is not None:
+            names = [n for n in names if CryptoLib.uses_cipher_config(n)]
+        return names
+
+    def __init__(self, examples_dir=None, tests_dir=None, config=None,
+                 pytestconfig=None):
+        self._verbose = pytestconfig.option.verbose if pytestconfig is not None else 0
+        self._examples_dir = examples_dir if examples_dir is not None else EXAMPLES_PATH
+        self._tests_dir = examples_dir if tests_dir is not None else TESTS_PATH
+        self._gen_dir = os.path.join(self._tests_dir, 'gen')
+        self.config = config if config is not None else DEF_CONFIG
+        self._version = self.config['ngtcp2']['version']
+        self._crypto_libs = [name for name in self.config['examples']
+                             if self.config['examples'][name] == 'yes']
+        self._clients = [self.config['clients'][lib] for lib in self._crypto_libs
+                         if lib in self.config['clients']]
+        self._servers = [self.config['servers'][lib] for lib in self._crypto_libs
+                         if lib in self.config['servers']]
+        self._examples_pem = {
+            'key': 'xxx',
+            'cert': 'xxx',
+        }
+        self._htdocs_dir = os.path.join(self._gen_dir, 'htdocs')
+        self._tld = 'tests.ngtcp2.nghttp2.org'
+        self._example_domain = f"one.{self._tld}"
+        self._ca = None
+        self._cert_specs = [
+            CertificateSpec(domains=[self._example_domain], key_type='rsa2048'),
+            CertificateSpec(name="clientsX", sub_specs=[
+               CertificateSpec(name="user1", client=True),
+            ]),
+        ]
+
+    def issue_certs(self):
+        if self._ca is None:
+            self._ca = Ngtcp2TestCA.create_root(name=self._tld,
+                                               store_dir=os.path.join(self.gen_dir, 'ca'),
+                                               key_type="rsa2048")
+        self._ca.issue_certs(self._cert_specs)
+
+    def setup(self):
+        os.makedirs(self._gen_dir, exist_ok=True)
+        os.makedirs(self._htdocs_dir, exist_ok=True)
+        self.issue_certs()
+
+    def get_server_credentials(self) -> Optional[Credentials]:
+        creds = self.ca.get_credentials_for_name(self._example_domain)
+        if len(creds) > 0:
+            return creds[0]
+        return None
+
+    @property
+    def verbose(self) -> int:
+        return self._verbose
+
+    @property
+    def version(self) -> str:
+        return self._version
+
+    @property
+    def gen_dir(self) -> str:
+        return self._gen_dir
+
+    @property
+    def ca(self):
+        return self._ca
+
+    @property
+    def htdocs_dir(self) -> str:
+        return self._htdocs_dir
+
+    @property
+    def example_domain(self) -> str:
+        return self._example_domain
+
+    @property
+    def examples_dir(self) -> str:
+        return self._examples_dir
+
+    @property
+    def examples_port(self) -> int:
+        return int(self.config['examples']['port'])
+
+    @property
+    def examples_pem(self) -> Dict[str, str]:
+        return self._examples_pem
+
+    @property
+    def crypto_libs(self):
+        return self._crypto_libs
+
+    @property
+    def clients(self):
+        return self._clients
+
+    @property
+    def servers(self):
+        return self._servers
+
+    def client_name(self, crypto_lib):
+        if crypto_lib in self.config['clients']:
+            return self.config['clients'][crypto_lib]
+        return None
+
+    def client_path(self, crypto_lib):
+        cname = self.client_name(crypto_lib)
+        if cname is not None:
+            return os.path.join(self.examples_dir, cname)
+        return None
+
+    def server_name(self, crypto_lib):
+        if crypto_lib in self.config['servers']:
+            return self.config['servers'][crypto_lib]
+        return None
+
+    def server_path(self, crypto_lib):
+        sname = self.server_name(crypto_lib)
+        if sname is not None:
+            return os.path.join(self.examples_dir, sname)
+        return None

data/ext/ngtcp2/examples/tests/ngtcp2test/log.py

@@ -0,0 +1,101 @@
+import binascii
+import os
+import re
+import sys
+import time
+from datetime import timedelta, datetime
+from io import SEEK_END
+from typing import List
+
+
+class LogFile:
+
+    def __init__(self, path: str):
+        self._path = path
+        self._start_pos = 0
+        self._last_pos = self._start_pos
+
+    @property
+    def path(self) -> str:
+        return self._path
+
+    def reset(self):
+        self._start_pos = 0
+        self._last_pos = self._start_pos
+
+    def advance(self) -> None:
+        if os.path.isfile(self._path):
+            with open(self._path) as fd:
+                self._start_pos = fd.seek(0, SEEK_END)
+
+    def get_recent(self, advance=True) -> List[str]:
+        lines = []
+        if os.path.isfile(self._path):
+            with open(self._path) as fd:
+                fd.seek(self._last_pos, os.SEEK_SET)
+                for line in fd:
+                    lines.append(line)
+                if advance:
+                    self._last_pos = fd.tell()
+        return lines
+
+    def scan_recent(self, pattern: re, timeout=10) -> bool:
+        if not os.path.isfile(self.path):
+            return False
+        with open(self.path) as fd:
+            end = datetime.now() + timedelta(seconds=timeout)
+            while True:
+                fd.seek(self._last_pos, os.SEEK_SET)
+                for line in fd:
+                    if pattern.match(line):
+                        return True
+                if datetime.now() > end:
+                    raise TimeoutError(f"pattern not found in error log after {timeout} seconds")
+                time.sleep(.1)
+        return False
+
+
+class HexDumpScanner:
+
+    def __init__(self, source, leading_regex=None):
+        self._source = source
+        self._leading_regex = leading_regex
+
+    def __iter__(self):
+        data = b''
+        offset = 0 if self._leading_regex is None else -1
+        idx = 0
+        for l in self._source:
+            if offset == -1:
+                pass
+            elif offset == 0:
+                # possible start of a hex dump
+                m = re.match(r'^\s*0+(\s+-)?((\s+[0-9a-f]{2}){1,16})(\s+.*)$',
+                             l, re.IGNORECASE)
+                if m:
+                    data = binascii.unhexlify(re.sub(r'\s+', '', m.group(2)))
+                    offset = 16
+                    idx = 1
+                    continue
+            else:
+                # possible continuation of a hexdump
+                m = re.match(r'^\s*([0-9a-f]+)(\s+-)?((\s+[0-9a-f]{2}){1,16})'
+                             r'(\s+.*)$', l, re.IGNORECASE)
+                if m:
+                    loffset = int(m.group(1), 16)
+                    if loffset == offset or loffset == idx:
+                        data += binascii.unhexlify(re.sub(r'\s+', '',
+                                                          m.group(3)))
+                        offset += 16
+                        idx += 1
+                        continue
+                    else:
+                        sys.stderr.write(f'wrong offset {loffset}, expected {offset} or {idx}\n')
+            # not a hexdump line, produce any collected data
+            if len(data) > 0:
+                yield data
+                data = b''
+            offset = 0 if self._leading_regex is None \
+                or self._leading_regex.match(l) else -1
+        if len(data) > 0:
+            yield data

data/ext/ngtcp2/examples/tests/ngtcp2test/server.py

@@ -0,0 +1,137 @@
+import logging
+import os
+import re
+import subprocess
+import time
+from datetime import datetime, timedelta
+from threading import Thread
+
+from .tls import HandShake
+from .env import Env, CryptoLib
+from .log import LogFile, HexDumpScanner
+
+
+log = logging.getLogger(__name__)
+
+
+class ServerRun:
+
+    def __init__(self, env: Env, logfile: LogFile):
+        self.env = env
+        self._logfile = logfile
+        self.log_lines = self._logfile.get_recent()
+        self._data_recs = None
+        self._hs_recs = None
+        if self.env.verbose > 1:
+            log.debug(f'read {len(self.log_lines)} lines from {logfile.path}')
+
+    @property
+    def handshake(self):
+        if self._data_recs is None:
+            self._data_recs = [data for data in HexDumpScanner(source=self.log_lines)]
+            if self.env.verbose > 1:
+                log.debug(f'detected {len(self._data_recs)} hexdumps '
+                          f'in {self._logfile.path}')
+        if self._hs_recs is None:
+            self._hs_recs = [hrec for hrec in HandShake(source=self._data_recs,
+                                                        verbose=self.env.verbose)]
+            if self.env.verbose > 1:
+                log.debug(f'detected {len(self._hs_recs)} crypto records '
+                          f'in {self._logfile.path}')
+        return self._hs_recs
+
+    @property
+    def hs_stripe(self):
+        return ":".join([hrec.name for hrec in self.handshake])
+
+
+def monitor_proc(env: Env, proc):
+    _env = env
+    proc.wait()
+
+
+class ExampleServer:
+
+    def __init__(self, env: Env, crypto_lib: str, verify_client=False):
+        self.env = env
+        self._crypto_lib = crypto_lib
+        self._path = env.server_path(self._crypto_lib)
+        self._logpath = f'{self.env.gen_dir}/{self._crypto_lib}-server.log'
+        self._log = LogFile(path=self._logpath)
+        self._logfile = None
+        self._process = None
+        self._verify_client = verify_client
+
+    @property
+    def path(self):
+        return self._path
+
+    @property
+    def crypto_lib(self):
+        return self._crypto_lib
+
+    @property
+    def uses_cipher_config(self):
+        return CryptoLib.uses_cipher_config(self.crypto_lib)
+
+    def supports_cipher(self, cipher):
+        return CryptoLib.supports_cipher(self.crypto_lib, cipher)
+
+    @property
+    def log(self):
+        return self._log
+
+    def exists(self):
+        return os.path.isfile(self.path)
+
+    def start(self):
+        if self._process is not None:
+            return False
+        creds = self.env.get_server_credentials()
+        assert creds
+        args = [
+            self.path,
+            f'--htdocs={self.env.htdocs_dir}',
+        ]
+        if self._verify_client:
+            args.append('--verify-client')
+        args.extend([
+            '*', str(self.env.examples_port),
+            creds.pkey_file, creds.cert_file
+        ])
+        self._logfile = open(self._logpath, 'w')
+        self._process = subprocess.Popen(args=args, text=True,
+                                         stdout=self._logfile, stderr=self._logfile)
+        t = Thread(target=monitor_proc, daemon=True, args=(self.env, self._process))
+        t.start()
+        timeout = 5
+        end = datetime.now() + timedelta(seconds=timeout)
+        while True:
+            if self._process.poll():
+                return False
+            try:
+                if self.log.scan_recent(pattern=re.compile(r'^Using document root'), timeout=0.5):
+                    break
+            except TimeoutError:
+                pass
+            if datetime.now() > end:
+                raise TimeoutError(f"pattern not found in error log after {timeout} seconds")
+        self.log.advance()
+        return True
+
+    def stop(self):
+        if self._process:
+            self._process.terminate()
+            self._process = None
+        if self._logfile:
+            self._logfile.close()
+            self._logfile = None
+        return True
+
+    def restart(self):
+        self.stop()
+        self._log.reset()
+        return self.start()
+
+    def get_run(self) -> ServerRun:
+        return ServerRun(env=self.env, logfile=self.log)