data-structure-typed 2.4.0 → 2.4.2

package/build/_deps/benchmark-src/tools/gbench/report.py

@@ -1,1201 +0,0 @@
-"""report.py - Utilities for reporting statistics about benchmark results
-"""
-
-import unittest
-import os
-import re
-import copy
-import random
-
-from scipy.stats import mannwhitneyu, gmean
-from numpy import array
-
-
-class BenchmarkColor(object):
-    def __init__(self, name, code):
-        self.name = name
-        self.code = code
-
-    def __repr__(self):
-        return '%s%r' % (self.__class__.__name__,
-                         (self.name, self.code))
-
-    def __format__(self, format):
-        return self.code
-
-
-# Benchmark Colors Enumeration
-BC_NONE = BenchmarkColor('NONE', '')
-BC_MAGENTA = BenchmarkColor('MAGENTA', '\033[95m')
-BC_CYAN = BenchmarkColor('CYAN', '\033[96m')
-BC_OKBLUE = BenchmarkColor('OKBLUE', '\033[94m')
-BC_OKGREEN = BenchmarkColor('OKGREEN', '\033[32m')
-BC_HEADER = BenchmarkColor('HEADER', '\033[92m')
-BC_WARNING = BenchmarkColor('WARNING', '\033[93m')
-BC_WHITE = BenchmarkColor('WHITE', '\033[97m')
-BC_FAIL = BenchmarkColor('FAIL', '\033[91m')
-BC_ENDC = BenchmarkColor('ENDC', '\033[0m')
-BC_BOLD = BenchmarkColor('BOLD', '\033[1m')
-BC_UNDERLINE = BenchmarkColor('UNDERLINE', '\033[4m')
-
-UTEST_MIN_REPETITIONS = 2
-UTEST_OPTIMAL_REPETITIONS = 9  # Lowest reasonable number, More is better.
-UTEST_COL_NAME = "_pvalue"
-
-_TIME_UNIT_TO_SECONDS_MULTIPLIER = {
-    "s": 1.0,
-    "ms": 1e-3,
-    "us": 1e-6,
-    "ns": 1e-9,
-}
-
-
-def color_format(use_color, fmt_str, *args, **kwargs):
-    """
-    Return the result of 'fmt_str.format(*args, **kwargs)' after transforming
-    'args' and 'kwargs' according to the value of 'use_color'. If 'use_color'
-    is False then all color codes in 'args' and 'kwargs' are replaced with
-    the empty string.
-    """
-    assert use_color is True or use_color is False
-    if not use_color:
-        args = [arg if not isinstance(arg, BenchmarkColor) else BC_NONE
-                for arg in args]
-        kwargs = {key: arg if not isinstance(arg, BenchmarkColor) else BC_NONE
-                  for key, arg in kwargs.items()}
-    return fmt_str.format(*args, **kwargs)
-
-
-def find_longest_name(benchmark_list):
-    """
-    Return the length of the longest benchmark name in a given list of
-    benchmark JSON objects
-    """
-    longest_name = 1
-    for bc in benchmark_list:
-        if len(bc['name']) > longest_name:
-            longest_name = len(bc['name'])
-    return longest_name
-
-
-def calculate_change(old_val, new_val):
-    """
-    Return a float representing the decimal change between old_val and new_val.
-    """
-    if old_val == 0 and new_val == 0:
-        return 0.0
-    if old_val == 0:
-        return float(new_val - old_val) / (float(old_val + new_val) / 2)
-    return float(new_val - old_val) / abs(old_val)
-
-
-def filter_benchmark(json_orig, family, replacement=""):
-    """
-    Apply a filter to the json, and only leave the 'family' of benchmarks.
-    """
-    regex = re.compile(family)
-    filtered = {}
-    filtered['benchmarks'] = []
-    for be in json_orig['benchmarks']:
-        if not regex.search(be['name']):
-            continue
-        filteredbench = copy.deepcopy(be)  # Do NOT modify the old name!
-        filteredbench['name'] = regex.sub(replacement, filteredbench['name'])
-        filtered['benchmarks'].append(filteredbench)
-    return filtered
-
-
-def get_unique_benchmark_names(json):
-    """
-    While *keeping* the order, give all the unique 'names' used for benchmarks.
-    """
-    seen = set()
-    uniqued = [x['name'] for x in json['benchmarks']
-               if x['name'] not in seen and
-               (seen.add(x['name']) or True)]
-    return uniqued
-
-
-def intersect(list1, list2):
-    """
-    Given two lists, get a new list consisting of the elements only contained
-    in *both of the input lists*, while preserving the ordering.
-    """
-    return [x for x in list1 if x in list2]
-
-
-def is_potentially_comparable_benchmark(x):
-    return ('time_unit' in x and 'real_time' in x and 'cpu_time' in x)
-
-
-def partition_benchmarks(json1, json2):
-    """
-    While preserving the ordering, find benchmarks with the same names in
-    both of the inputs, and group them.
-    (i.e. partition/filter into groups with common name)
-    """
-    json1_unique_names = get_unique_benchmark_names(json1)
-    json2_unique_names = get_unique_benchmark_names(json2)
-    names = intersect(json1_unique_names, json2_unique_names)
-    partitions = []
-    for name in names:
-        time_unit = None
-        # Pick the time unit from the first entry of the lhs benchmark.
-        # We should be careful not to crash with unexpected input.
-        for x in json1['benchmarks']:
-            if (x['name'] == name and is_potentially_comparable_benchmark(x)):
-                time_unit = x['time_unit']
-                break
-        if time_unit is None:
-            continue
-        # Filter by name and time unit.
-        # All the repetitions are assumed to be comparable.
-        lhs = [x for x in json1['benchmarks'] if x['name'] == name and
-               x['time_unit'] == time_unit]
-        rhs = [x for x in json2['benchmarks'] if x['name'] == name and
-               x['time_unit'] == time_unit]
-        partitions.append([lhs, rhs])
-    return partitions
-
-
-def get_timedelta_field_as_seconds(benchmark, field_name):
-    """
-    Get value of field_name field of benchmark, which is time with time unit
-    time_unit, as time in seconds.
-    """
-    timedelta = benchmark[field_name]
-    time_unit = benchmark.get('time_unit', 's')
-    return timedelta * _TIME_UNIT_TO_SECONDS_MULTIPLIER.get(time_unit)
-
-
-def calculate_geomean(json):
-    """
-    Extract all real/cpu times from all the benchmarks as seconds,
-    and calculate their geomean.
-    """
-    times = []
-    for benchmark in json['benchmarks']:
-        if 'run_type' in benchmark and benchmark['run_type'] == 'aggregate':
-            continue
-        times.append([get_timedelta_field_as_seconds(benchmark, 'real_time'),
-                      get_timedelta_field_as_seconds(benchmark, 'cpu_time')])
-    return gmean(times) if times else array([])
-
-
-def extract_field(partition, field_name):
-    # The count of elements may be different. We want *all* of them.
-    lhs = [x[field_name] for x in partition[0]]
-    rhs = [x[field_name] for x in partition[1]]
-    return [lhs, rhs]
-
-
-def calc_utest(timings_cpu, timings_time):
-    min_rep_cnt = min(len(timings_time[0]),
-                      len(timings_time[1]),
-                      len(timings_cpu[0]),
-                      len(timings_cpu[1]))
-
-    # Does *everything* has at least UTEST_MIN_REPETITIONS repetitions?
-    if min_rep_cnt < UTEST_MIN_REPETITIONS:
-        return False, None, None
-
-    time_pvalue = mannwhitneyu(
-        timings_time[0], timings_time[1], alternative='two-sided').pvalue
-    cpu_pvalue = mannwhitneyu(
-        timings_cpu[0], timings_cpu[1], alternative='two-sided').pvalue
-
-    return (min_rep_cnt >= UTEST_OPTIMAL_REPETITIONS), cpu_pvalue, time_pvalue
-
-
-def print_utest(bc_name, utest, utest_alpha, first_col_width, use_color=True):
-    def get_utest_color(pval):
-        return BC_FAIL if pval >= utest_alpha else BC_OKGREEN
-
-    # Check if we failed miserably with minimum required repetitions for utest
-    if not utest['have_optimal_repetitions'] and utest['cpu_pvalue'] is None and utest['time_pvalue'] is None:
-        return []
-
-    dsc = "U Test, Repetitions: {} vs {}".format(
-        utest['nr_of_repetitions'], utest['nr_of_repetitions_other'])
-    dsc_color = BC_OKGREEN
-
-    # We still got some results to show but issue a warning about it.
-    if not utest['have_optimal_repetitions']:
-        dsc_color = BC_WARNING
-        dsc += ". WARNING: Results unreliable! {}+ repetitions recommended.".format(
-            UTEST_OPTIMAL_REPETITIONS)
-
-    special_str = "{}{:<{}s}{endc}{}{:16.4f}{endc}{}{:16.4f}{endc}{}      {}"
-
-    return [color_format(use_color,
-                         special_str,
-                         BC_HEADER,
-                         "{}{}".format(bc_name, UTEST_COL_NAME),
-                         first_col_width,
-                         get_utest_color(
-                             utest['time_pvalue']), utest['time_pvalue'],
-                         get_utest_color(
-                             utest['cpu_pvalue']), utest['cpu_pvalue'],
-                         dsc_color, dsc,
-                         endc=BC_ENDC)]
-
-
-def get_difference_report(
-        json1,
-        json2,
-        utest=False):
-    """
-    Calculate and report the difference between each test of two benchmarks
-    runs specified as 'json1' and 'json2'. Output is another json containing
-    relevant details for each test run.
-    """
-    assert utest is True or utest is False
-
-    diff_report = []
-    partitions = partition_benchmarks(json1, json2)
-    for partition in partitions:
-        benchmark_name = partition[0][0]['name']
-        label = partition[0][0]['label'] if 'label' in partition[0][0] else ''
-        time_unit = partition[0][0]['time_unit']
-        measurements = []
-        utest_results = {}
-        # Careful, we may have different repetition count.
-        for i in range(min(len(partition[0]), len(partition[1]))):
-            bn = partition[0][i]
-            other_bench = partition[1][i]
-            measurements.append({
-                'real_time': bn['real_time'],
-                'cpu_time': bn['cpu_time'],
-                'real_time_other': other_bench['real_time'],
-                'cpu_time_other': other_bench['cpu_time'],
-                'time': calculate_change(bn['real_time'], other_bench['real_time']),
-                'cpu': calculate_change(bn['cpu_time'], other_bench['cpu_time'])
-            })
-
-        # After processing the whole partition, if requested, do the U test.
-        if utest:
-            timings_cpu = extract_field(partition, 'cpu_time')
-            timings_time = extract_field(partition, 'real_time')
-            have_optimal_repetitions, cpu_pvalue, time_pvalue = calc_utest(
-                timings_cpu, timings_time)
-            if cpu_pvalue and time_pvalue:
-                utest_results = {
-                    'have_optimal_repetitions': have_optimal_repetitions,
-                    'cpu_pvalue': cpu_pvalue,
-                    'time_pvalue': time_pvalue,
-                    'nr_of_repetitions': len(timings_cpu[0]),
-                    'nr_of_repetitions_other': len(timings_cpu[1])
-                }
-
-        # Store only if we had any measurements for given benchmark.
-        # E.g. partition_benchmarks will filter out the benchmarks having
-        # time units which are not compatible with other time units in the
-        # benchmark suite.
-        if measurements:
-            run_type = partition[0][0]['run_type'] if 'run_type' in partition[0][0] else ''
-            aggregate_name = partition[0][0]['aggregate_name'] if run_type == 'aggregate' and 'aggregate_name' in partition[0][0] else ''
-            diff_report.append({
-                'name': benchmark_name,
-                'label': label,
-                'measurements': measurements,
-                'time_unit': time_unit,
-                'run_type': run_type,
-                'aggregate_name': aggregate_name,
-                'utest': utest_results
-            })
-
-    lhs_gmean = calculate_geomean(json1)
-    rhs_gmean = calculate_geomean(json2)
-    if lhs_gmean.any() and rhs_gmean.any():
-        diff_report.append({
-            'name': 'OVERALL_GEOMEAN',
-            'label': '',
-            'measurements': [{
-                'real_time': lhs_gmean[0],
-                'cpu_time': lhs_gmean[1],
-                'real_time_other': rhs_gmean[0],
-                'cpu_time_other': rhs_gmean[1],
-                'time': calculate_change(lhs_gmean[0], rhs_gmean[0]),
-                'cpu': calculate_change(lhs_gmean[1], rhs_gmean[1])
-            }],
-            'time_unit': 's',
-            'run_type': 'aggregate',
-            'aggregate_name': 'geomean',
-            'utest': {}
-        })
-
-    return diff_report
-
-
-def print_difference_report(
-        json_diff_report,
-        include_aggregates_only=False,
-        utest=False,
-        utest_alpha=0.05,
-        use_color=True):
-    """
-    Calculate and report the difference between each test of two benchmarks
-    runs specified as 'json1' and 'json2'.
-    """
-    assert utest is True or utest is False
-
-    def get_color(res):
-        if res > 0.05:
-            return BC_FAIL
-        elif res > -0.07:
-            return BC_WHITE
-        else:
-            return BC_CYAN
-
-    first_col_width = find_longest_name(json_diff_report)
-    first_col_width = max(
-        first_col_width,
-        len('Benchmark'))
-    first_col_width += len(UTEST_COL_NAME)
-    first_line = "{:<{}s}Time             CPU      Time Old      Time New       CPU Old       CPU New".format(
-        'Benchmark', 12 + first_col_width)
-    output_strs = [first_line, '-' * len(first_line)]
-
-    fmt_str = "{}{:<{}s}{endc}{}{:+16.4f}{endc}{}{:+16.4f}{endc}{:14.0f}{:14.0f}{endc}{:14.0f}{:14.0f}"
-    for benchmark in json_diff_report:
-        # *If* we were asked to only include aggregates,
-        # and if it is non-aggregate, then don't print it.
-        if not include_aggregates_only or not 'run_type' in benchmark or benchmark['run_type'] == 'aggregate':
-            for measurement in benchmark['measurements']:
-                output_strs += [color_format(use_color,
-                                             fmt_str,
-                                             BC_HEADER,
-                                             benchmark['name'],
-                                             first_col_width,
-                                             get_color(measurement['time']),
-                                             measurement['time'],
-                                             get_color(measurement['cpu']),
-                                             measurement['cpu'],
-                                             measurement['real_time'],
-                                             measurement['real_time_other'],
-                                             measurement['cpu_time'],
-                                             measurement['cpu_time_other'],
-                                             endc=BC_ENDC)]
-
-        # After processing the measurements, if requested and
-        # if applicable (e.g. u-test exists for given benchmark),
-        # print the U test.
-        if utest and benchmark['utest']:
-            output_strs += print_utest(benchmark['name'],
-                                       benchmark['utest'],
-                                       utest_alpha=utest_alpha,
-                                       first_col_width=first_col_width,
-                                       use_color=use_color)
-
-    return output_strs
-
-
-###############################################################################
-# Unit tests
-
-
-class TestGetUniqueBenchmarkNames(unittest.TestCase):
-    def load_results(self):
-        import json
-        testInputs = os.path.join(
-            os.path.dirname(
-                os.path.realpath(__file__)),
-            'Inputs')
-        testOutput = os.path.join(testInputs, 'test3_run0.json')
-        with open(testOutput, 'r') as f:
-            json = json.load(f)
-        return json
-
-    def test_basic(self):
-        expect_lines = [
-            'BM_One',
-            'BM_Two',
-            'short',  # These two are not sorted
-            'medium',  # These two are not sorted
-        ]
-        json = self.load_results()
-        output_lines = get_unique_benchmark_names(json)
-        print("\n")
-        print("\n".join(output_lines))
-        self.assertEqual(len(output_lines), len(expect_lines))
-        for i in range(0, len(output_lines)):
-            self.assertEqual(expect_lines[i], output_lines[i])
-
-
-class TestReportDifference(unittest.TestCase):
-    @classmethod
-    def setUpClass(cls):
-        def load_results():
-            import json
-            testInputs = os.path.join(
-                os.path.dirname(
-                    os.path.realpath(__file__)),
-                'Inputs')
-            testOutput1 = os.path.join(testInputs, 'test1_run1.json')
-            testOutput2 = os.path.join(testInputs, 'test1_run2.json')
-            with open(testOutput1, 'r') as f:
-                json1 = json.load(f)
-            with open(testOutput2, 'r') as f:
-                json2 = json.load(f)
-            return json1, json2
-
-        json1, json2 = load_results()
-        cls.json_diff_report = get_difference_report(json1, json2)
-
-    def test_json_diff_report_pretty_printing(self):
-        expect_lines = [
-            ['BM_SameTimes', '+0.0000', '+0.0000', '10', '10', '10', '10'],
-            ['BM_2xFaster', '-0.5000', '-0.5000', '50', '25', '50', '25'],
-            ['BM_2xSlower', '+1.0000', '+1.0000', '50', '100', '50', '100'],
-            ['BM_1PercentFaster', '-0.0100', '-0.0100', '100', '99', '100', '99'],
-            ['BM_1PercentSlower', '+0.0100', '+0.0100', '100', '101', '100', '101'],
-            ['BM_10PercentFaster', '-0.1000', '-0.1000', '100', '90', '100', '90'],
-            ['BM_10PercentSlower', '+0.1000', '+0.1000', '100', '110', '100', '110'],
-            ['BM_100xSlower', '+99.0000', '+99.0000',
-                '100', '10000', '100', '10000'],
-            ['BM_100xFaster', '-0.9900', '-0.9900',
-                '10000', '100', '10000', '100'],
-            ['BM_10PercentCPUToTime', '+0.1000',
-                '-0.1000', '100', '110', '100', '90'],
-            ['BM_ThirdFaster', '-0.3333', '-0.3334', '100', '67', '100', '67'],
-            ['BM_NotBadTimeUnit', '-0.9000', '+0.2000', '0', '0', '0', '1'],
-            ['BM_hasLabel', '+0.0000', '+0.0000', '1', '1', '1', '1'],
-            ['OVERALL_GEOMEAN', '-0.8113', '-0.7779', '0', '0', '0', '0']
-        ]
-        output_lines_with_header = print_difference_report(
-            self.json_diff_report, use_color=False)
-        output_lines = output_lines_with_header[2:]
-        print("\n")
-        print("\n".join(output_lines_with_header))
-        self.assertEqual(len(output_lines), len(expect_lines))
-        for i in range(0, len(output_lines)):
-            parts = [x for x in output_lines[i].split(' ') if x]
-            self.assertEqual(len(parts), 7)
-            self.assertEqual(expect_lines[i], parts)
-
-    def test_json_diff_report_output(self):
-        expected_output = [
-            {
-                'name': 'BM_SameTimes',
-                'label': '',
-                'measurements': [{'time': 0.0000, 'cpu': 0.0000,
-                                  'real_time': 10, 'real_time_other': 10,
-                                  'cpu_time': 10, 'cpu_time_other': 10}],
-                'time_unit': 'ns',
-                'utest': {}
-            },
-            {
-                'name': 'BM_2xFaster',
-                'label': '',
-                'measurements': [{'time': -0.5000, 'cpu': -0.5000,
-                                  'real_time': 50, 'real_time_other': 25,
-                                  'cpu_time': 50, 'cpu_time_other': 25}],
-                'time_unit': 'ns',
-                'utest': {}
-            },
-            {
-                'name': 'BM_2xSlower',
-                'label': '',
-                'measurements': [{'time': 1.0000, 'cpu': 1.0000,
-                                  'real_time': 50, 'real_time_other': 100,
-                                  'cpu_time': 50, 'cpu_time_other': 100}],
-                'time_unit': 'ns',
-                'utest': {}
-            },
-            {
-                'name': 'BM_1PercentFaster',
-                'label': '',
-                'measurements': [{'time': -0.0100, 'cpu': -0.0100,
-                                  'real_time': 100, 'real_time_other': 98.9999999,
-                                  'cpu_time': 100, 'cpu_time_other': 98.9999999}],
-                'time_unit': 'ns',
-                'utest': {}
-            },
-            {
-                'name': 'BM_1PercentSlower',
-                'label': '',
-                'measurements': [{'time': 0.0100, 'cpu': 0.0100,
-                                  'real_time': 100, 'real_time_other': 101,
-                                  'cpu_time': 100, 'cpu_time_other': 101}],
-                'time_unit': 'ns',
-                'utest': {}
-            },
-            {
-                'name': 'BM_10PercentFaster',
-                'label': '',
-                'measurements': [{'time': -0.1000, 'cpu': -0.1000,
-                                  'real_time': 100, 'real_time_other': 90,
-                                  'cpu_time': 100, 'cpu_time_other': 90}],
-                'time_unit': 'ns',
-                'utest': {}
-            },
-            {
-                'name': 'BM_10PercentSlower',
-                'label': '',
-                'measurements': [{'time': 0.1000, 'cpu': 0.1000,
-                                  'real_time': 100, 'real_time_other': 110,
-                                  'cpu_time': 100, 'cpu_time_other': 110}],
-                'time_unit': 'ns',
-                'utest': {}
-            },
-            {
-                'name': 'BM_100xSlower',
-                'label': '',
-                'measurements': [{'time': 99.0000, 'cpu': 99.0000,
-                                  'real_time': 100, 'real_time_other': 10000,
-                                  'cpu_time': 100, 'cpu_time_other': 10000}],
-                'time_unit': 'ns',
-                'utest': {}
-            },
-            {
-                'name': 'BM_100xFaster',
-                'label': '',
-                'measurements': [{'time': -0.9900, 'cpu': -0.9900,
-                                  'real_time': 10000, 'real_time_other': 100,
-                                  'cpu_time': 10000, 'cpu_time_other': 100}],
-                'time_unit': 'ns',
-                'utest': {}
-            },
-            {
-                'name': 'BM_10PercentCPUToTime',
-                'label': '',
-                'measurements': [{'time': 0.1000, 'cpu': -0.1000,
-                                  'real_time': 100, 'real_time_other': 110,
-                                  'cpu_time': 100, 'cpu_time_other': 90}],
-                'time_unit': 'ns',
-                'utest': {}
-            },
-            {
-                'name': 'BM_ThirdFaster',
-                'label': '',
-                'measurements': [{'time': -0.3333, 'cpu': -0.3334,
-                                  'real_time': 100, 'real_time_other': 67,
-                                  'cpu_time': 100, 'cpu_time_other': 67}],
-                'time_unit': 'ns',
-                'utest': {}
-            },
-            {
-                'name': 'BM_NotBadTimeUnit',
-                'label': '',
-                'measurements': [{'time': -0.9000, 'cpu': 0.2000,
-                                  'real_time': 0.4, 'real_time_other': 0.04,
-                                  'cpu_time': 0.5, 'cpu_time_other': 0.6}],
-                'time_unit': 's',
-                'utest': {}
-            },
-            {
-                'name': 'BM_hasLabel',
-                'label': 'a label',
-                'measurements': [{'time': 0.0000, 'cpu': 0.0000,
-                                  'real_time': 1, 'real_time_other': 1,
-                                  'cpu_time': 1, 'cpu_time_other': 1}],
-                'time_unit': 's',
-                'utest': {}
-            },
-            {
-                'name': 'OVERALL_GEOMEAN',
-                'label': '',
-                'measurements': [{'real_time': 3.1622776601683826e-06, 'cpu_time': 3.2130844755623912e-06,
-                                  'real_time_other': 1.9768988699420897e-07, 'cpu_time_other': 2.397447755209533e-07,
-                                  'time': -0.8112976497120911, 'cpu': -0.7778551721181174}],
-                'time_unit': 's',
-                'run_type': 'aggregate',
-                'aggregate_name': 'geomean', 'utest': {}
-            },
-        ]
-        self.assertEqual(len(self.json_diff_report), len(expected_output))
-        for out, expected in zip(
-                self.json_diff_report, expected_output):
-            self.assertEqual(out['name'], expected['name'])
-            self.assertEqual(out['label'], expected['label'])
-            self.assertEqual(out['time_unit'], expected['time_unit'])
-            assert_utest(self, out, expected)
-            assert_measurements(self, out, expected)
-
-
-class TestReportDifferenceBetweenFamilies(unittest.TestCase):
-    @classmethod
-    def setUpClass(cls):
-        def load_result():
-            import json
-            testInputs = os.path.join(
-                os.path.dirname(
-                    os.path.realpath(__file__)),
-                'Inputs')
-            testOutput = os.path.join(testInputs, 'test2_run.json')
-            with open(testOutput, 'r') as f:
-                json = json.load(f)
-            return json
-
-        json = load_result()
-        json1 = filter_benchmark(json, "BM_Z.ro", ".")
-        json2 = filter_benchmark(json, "BM_O.e", ".")
-        cls.json_diff_report = get_difference_report(json1, json2)
-
-    def test_json_diff_report_pretty_printing(self):
-        expect_lines = [
-            ['.', '-0.5000', '-0.5000', '10', '5', '10', '5'],
-            ['./4', '-0.5000', '-0.5000', '40', '20', '40', '20'],
-            ['Prefix/.', '-0.5000', '-0.5000', '20', '10', '20', '10'],
-            ['Prefix/./3', '-0.5000', '-0.5000', '30', '15', '30', '15'],
-            ['OVERALL_GEOMEAN', '-0.5000', '-0.5000', '0', '0', '0', '0']
-        ]
-        output_lines_with_header = print_difference_report(
-            self.json_diff_report, use_color=False)
-        output_lines = output_lines_with_header[2:]
-        print("\n")
-        print("\n".join(output_lines_with_header))
-        self.assertEqual(len(output_lines), len(expect_lines))
-        for i in range(0, len(output_lines)):
-            parts = [x for x in output_lines[i].split(' ') if x]
-            self.assertEqual(len(parts), 7)
-            self.assertEqual(expect_lines[i], parts)
-
-    def test_json_diff_report(self):
-        expected_output = [
-            {
-                'name': u'.',
-                'measurements': [{'time': -0.5, 'cpu': -0.5, 'real_time': 10, 'real_time_other': 5, 'cpu_time': 10, 'cpu_time_other': 5}],
-                'time_unit': 'ns',
-                'utest': {}
-            },
-            {
-                'name': u'./4',
-                'measurements': [{'time': -0.5, 'cpu': -0.5, 'real_time': 40, 'real_time_other': 20, 'cpu_time': 40, 'cpu_time_other': 20}],
-                'time_unit': 'ns',
-                'utest': {},
-            },
-            {
-                'name': u'Prefix/.',
-                'measurements': [{'time': -0.5, 'cpu': -0.5, 'real_time': 20, 'real_time_other': 10, 'cpu_time': 20, 'cpu_time_other': 10}],
-                'time_unit': 'ns',
-                'utest': {}
-            },
-            {
-                'name': u'Prefix/./3',
-                'measurements': [{'time': -0.5, 'cpu': -0.5, 'real_time': 30, 'real_time_other': 15, 'cpu_time': 30, 'cpu_time_other': 15}],
-                'time_unit': 'ns',
-                'utest': {}
-            },
-            {
-                'name': 'OVERALL_GEOMEAN',
-                'measurements': [{'real_time': 2.213363839400641e-08, 'cpu_time': 2.213363839400641e-08,
-                                  'real_time_other': 1.1066819197003185e-08, 'cpu_time_other': 1.1066819197003185e-08,
-                                  'time': -0.5000000000000009, 'cpu': -0.5000000000000009}],
-                'time_unit': 's',
-                'run_type': 'aggregate',
-                'aggregate_name': 'geomean',
-                'utest': {}
-            }
-        ]
-        self.assertEqual(len(self.json_diff_report), len(expected_output))
-        for out, expected in zip(
-                self.json_diff_report, expected_output):
-            self.assertEqual(out['name'], expected['name'])
-            self.assertEqual(out['time_unit'], expected['time_unit'])
-            assert_utest(self, out, expected)
-            assert_measurements(self, out, expected)
-
-
-class TestReportDifferenceWithUTest(unittest.TestCase):
-    @classmethod
-    def setUpClass(cls):
-        def load_results():
-            import json
-            testInputs = os.path.join(
-                os.path.dirname(
-                    os.path.realpath(__file__)),
-                'Inputs')
-            testOutput1 = os.path.join(testInputs, 'test3_run0.json')
-            testOutput2 = os.path.join(testInputs, 'test3_run1.json')
-            with open(testOutput1, 'r') as f:
-                json1 = json.load(f)
-            with open(testOutput2, 'r') as f:
-                json2 = json.load(f)
-            return json1, json2
-
-        json1, json2 = load_results()
-        cls.json_diff_report = get_difference_report(
-            json1, json2, utest=True)
-
-    def test_json_diff_report_pretty_printing(self):
-        expect_lines = [
-            ['BM_One', '-0.1000', '+0.1000', '10', '9', '100', '110'],
-            ['BM_Two', '+0.1111', '-0.0111', '9', '10', '90', '89'],
-            ['BM_Two', '-0.1250', '-0.1628', '8', '7', '86', '72'],
-            ['BM_Two_pvalue',
-             '1.0000',
-             '0.6667',
-             'U',
-             'Test,',
-             'Repetitions:',
-             '2',
-             'vs',
-             '2.',
-             'WARNING:',
-             'Results',
-             'unreliable!',
-             '9+',
-             'repetitions',
-             'recommended.'],
-            ['short', '-0.1250', '-0.0625', '8', '7', '80', '75'],
-            ['short', '-0.4325', '-0.1351', '8', '5', '77', '67'],
-            ['short_pvalue',
-             '0.7671',
-             '0.2000',
-             'U',
-             'Test,',
-             'Repetitions:',
-             '2',
-             'vs',
-             '3.',
-             'WARNING:',
-             'Results',
-             'unreliable!',
-             '9+',
-             'repetitions',
-             'recommended.'],
-            ['medium', '-0.3750', '-0.3375', '8', '5', '80', '53'],
-            ['OVERALL_GEOMEAN', '+1.6405', '-0.6985', '0', '0', '0', '0']
-        ]
-        output_lines_with_header = print_difference_report(
-            self.json_diff_report, utest=True, utest_alpha=0.05, use_color=False)
-        output_lines = output_lines_with_header[2:]
-        print("\n")
-        print("\n".join(output_lines_with_header))
-        self.assertEqual(len(output_lines), len(expect_lines))
-        for i in range(0, len(output_lines)):
-            parts = [x for x in output_lines[i].split(' ') if x]
-            self.assertEqual(expect_lines[i], parts)
-
-    def test_json_diff_report_pretty_printing_aggregates_only(self):
-        expect_lines = [
-            ['BM_One', '-0.1000', '+0.1000', '10', '9', '100', '110'],
-            ['BM_Two_pvalue',
-             '1.0000',
-             '0.6667',
-             'U',
-             'Test,',
-             'Repetitions:',
-             '2',
-             'vs',
-             '2.',
-             'WARNING:',
-             'Results',
-             'unreliable!',
-             '9+',
-             'repetitions',
-             'recommended.'],
-            ['short', '-0.1250', '-0.0625', '8', '7', '80', '75'],
-            ['short', '-0.4325', '-0.1351', '8', '5', '77', '67'],
-            ['short_pvalue',
-             '0.7671',
-             '0.2000',
-             'U',
-             'Test,',
-             'Repetitions:',
-             '2',
-             'vs',
-             '3.',
-             'WARNING:',
-             'Results',
-             'unreliable!',
-             '9+',
-             'repetitions',
-             'recommended.'],
-            ['OVERALL_GEOMEAN', '+1.6405', '-0.6985', '0', '0', '0', '0']
-        ]
-        output_lines_with_header = print_difference_report(
-            self.json_diff_report, include_aggregates_only=True, utest=True, utest_alpha=0.05, use_color=False)
-        output_lines = output_lines_with_header[2:]
-        print("\n")
-        print("\n".join(output_lines_with_header))
-        self.assertEqual(len(output_lines), len(expect_lines))
-        for i in range(0, len(output_lines)):
-            parts = [x for x in output_lines[i].split(' ') if x]
-            self.assertEqual(expect_lines[i], parts)
-
-    def test_json_diff_report(self):
-        expected_output = [
-            {
-                'name': u'BM_One',
-                'measurements': [
-                    {'time': -0.1,
-                     'cpu': 0.1,
-                     'real_time': 10,
-                     'real_time_other': 9,
-                     'cpu_time': 100,
-                     'cpu_time_other': 110}
-                ],
-                'time_unit': 'ns',
-                'utest': {}
-            },
-            {
-                'name': u'BM_Two',
-                'measurements': [
-                    {'time': 0.1111111111111111,
-                     'cpu': -0.011111111111111112,
-                     'real_time': 9,
-                     'real_time_other': 10,
-                     'cpu_time': 90,
-                     'cpu_time_other': 89},
-                    {'time': -0.125, 'cpu': -0.16279069767441862, 'real_time': 8,
-                        'real_time_other': 7, 'cpu_time': 86, 'cpu_time_other': 72}
-                ],
-                'time_unit': 'ns',
-                'utest': {
-                    'have_optimal_repetitions': False, 'cpu_pvalue': 0.6666666666666666, 'time_pvalue': 1.0
-                }
-            },
-            {
-                'name': u'short',
-                'measurements': [
-                    {'time': -0.125,
-                     'cpu': -0.0625,
-                     'real_time': 8,
-                     'real_time_other': 7,
-                     'cpu_time': 80,
-                     'cpu_time_other': 75},
-                    {'time': -0.4325,
-                     'cpu': -0.13506493506493514,
-                     'real_time': 8,
-                     'real_time_other': 4.54,
-                     'cpu_time': 77,
-                     'cpu_time_other': 66.6}
-                ],
-                'time_unit': 'ns',
-                'utest': {
-                    'have_optimal_repetitions': False, 'cpu_pvalue': 0.2, 'time_pvalue': 0.7670968684102772
-                }
-            },
-            {
-                'name': u'medium',
-                'measurements': [
-                    {'time': -0.375,
-                     'cpu': -0.3375,
-                     'real_time': 8,
-                     'real_time_other': 5,
-                     'cpu_time': 80,
-                     'cpu_time_other': 53}
-                ],
-                'time_unit': 'ns',
-                'utest': {}
-            },
-            {
-                'name': 'OVERALL_GEOMEAN',
-                'measurements': [{'real_time': 8.48528137423858e-09, 'cpu_time': 8.441336246629233e-08,
-                                  'real_time_other': 2.2405267593145244e-08, 'cpu_time_other': 2.5453661413660466e-08,
-                                  'time': 1.6404861082353634, 'cpu': -0.6984640740519662}],
-                'time_unit': 's',
-                'run_type': 'aggregate',
-                'aggregate_name': 'geomean',
-                'utest': {}
-            }
-        ]
-        self.assertEqual(len(self.json_diff_report), len(expected_output))
-        for out, expected in zip(
-                self.json_diff_report, expected_output):
-            self.assertEqual(out['name'], expected['name'])
-            self.assertEqual(out['time_unit'], expected['time_unit'])
-            assert_utest(self, out, expected)
-            assert_measurements(self, out, expected)
-
-
-class TestReportDifferenceWithUTestWhileDisplayingAggregatesOnly(
-        unittest.TestCase):
-    @classmethod
-    def setUpClass(cls):
-        def load_results():
-            import json
-            testInputs = os.path.join(
-                os.path.dirname(
-                    os.path.realpath(__file__)),
-                'Inputs')
-            testOutput1 = os.path.join(testInputs, 'test3_run0.json')
-            testOutput2 = os.path.join(testInputs, 'test3_run1.json')
-            with open(testOutput1, 'r') as f:
-                json1 = json.load(f)
-            with open(testOutput2, 'r') as f:
-                json2 = json.load(f)
-            return json1, json2
-
-        json1, json2 = load_results()
-        cls.json_diff_report = get_difference_report(
-            json1, json2, utest=True)
-
-    def test_json_diff_report_pretty_printing(self):
-        expect_lines = [
-            ['BM_One', '-0.1000', '+0.1000', '10', '9', '100', '110'],
-            ['BM_Two', '+0.1111', '-0.0111', '9', '10', '90', '89'],
-            ['BM_Two', '-0.1250', '-0.1628', '8', '7', '86', '72'],
-            ['BM_Two_pvalue',
-             '1.0000',
-             '0.6667',
-             'U',
-             'Test,',
-             'Repetitions:',
-             '2',
-             'vs',
-             '2.',
-             'WARNING:',
-             'Results',
-             'unreliable!',
-             '9+',
-             'repetitions',
-             'recommended.'],
-            ['short', '-0.1250', '-0.0625', '8', '7', '80', '75'],
-            ['short', '-0.4325', '-0.1351', '8', '5', '77', '67'],
-            ['short_pvalue',
-             '0.7671',
-             '0.2000',
-             'U',
-             'Test,',
-             'Repetitions:',
-             '2',
-             'vs',
-             '3.',
-             'WARNING:',
-             'Results',
-             'unreliable!',
-             '9+',
-             'repetitions',
-             'recommended.'],
-            ['medium', '-0.3750', '-0.3375', '8', '5', '80', '53'],
-            ['OVERALL_GEOMEAN', '+1.6405', '-0.6985', '0', '0', '0', '0']
-        ]
-        output_lines_with_header = print_difference_report(
-            self.json_diff_report,
-            utest=True, utest_alpha=0.05, use_color=False)
-        output_lines = output_lines_with_header[2:]
-        print("\n")
-        print("\n".join(output_lines_with_header))
-        self.assertEqual(len(output_lines), len(expect_lines))
-        for i in range(0, len(output_lines)):
-            parts = [x for x in output_lines[i].split(' ') if x]
-            self.assertEqual(expect_lines[i], parts)
-
-    def test_json_diff_report(self):
-        expected_output = [
-            {
-                'name': u'BM_One',
-                'measurements': [
-                    {'time': -0.1,
-                     'cpu': 0.1,
-                     'real_time': 10,
-                     'real_time_other': 9,
-                     'cpu_time': 100,
-                     'cpu_time_other': 110}
-                ],
-                'time_unit': 'ns',
-                'utest': {}
-            },
-            {
-                'name': u'BM_Two',
-                'measurements': [
-                    {'time': 0.1111111111111111,
-                     'cpu': -0.011111111111111112,
-                     'real_time': 9,
-                     'real_time_other': 10,
-                     'cpu_time': 90,
-                     'cpu_time_other': 89},
-                    {'time': -0.125, 'cpu': -0.16279069767441862, 'real_time': 8,
-                        'real_time_other': 7, 'cpu_time': 86, 'cpu_time_other': 72}
-                ],
-                'time_unit': 'ns',
-                'utest': {
-                    'have_optimal_repetitions': False, 'cpu_pvalue': 0.6666666666666666, 'time_pvalue': 1.0
-                }
-            },
-            {
-                'name': u'short',
-                'measurements': [
-                    {'time': -0.125,
-                     'cpu': -0.0625,
-                     'real_time': 8,
-                     'real_time_other': 7,
-                     'cpu_time': 80,
-                     'cpu_time_other': 75},
-                    {'time': -0.4325,
-                     'cpu': -0.13506493506493514,
-                     'real_time': 8,
-                     'real_time_other': 4.54,
-                     'cpu_time': 77,
-                     'cpu_time_other': 66.6}
-                ],
-                'time_unit': 'ns',
-                'utest': {
-                    'have_optimal_repetitions': False, 'cpu_pvalue': 0.2, 'time_pvalue': 0.7670968684102772
-                }
-            },
-            {
-                'name': u'medium',
-                'measurements': [
-                    {'real_time_other': 5,
-                     'cpu_time': 80,
-                     'time': -0.375,
-                     'real_time': 8,
-                     'cpu_time_other': 53,
-                     'cpu': -0.3375
-                     }
-                ],
-                'utest': {},
-                'time_unit': u'ns',
-                'aggregate_name': ''
-            },
-            {
-                'name': 'OVERALL_GEOMEAN',
-                'measurements': [{'real_time': 8.48528137423858e-09, 'cpu_time': 8.441336246629233e-08,
-                                  'real_time_other': 2.2405267593145244e-08, 'cpu_time_other': 2.5453661413660466e-08,
-                                  'time': 1.6404861082353634, 'cpu': -0.6984640740519662}],
-                'time_unit': 's',
-                'run_type': 'aggregate',
-                'aggregate_name': 'geomean',
-                'utest': {}
-            }
-        ]
-        self.assertEqual(len(self.json_diff_report), len(expected_output))
-        for out, expected in zip(
-                self.json_diff_report, expected_output):
-            self.assertEqual(out['name'], expected['name'])
-            self.assertEqual(out['time_unit'], expected['time_unit'])
-            assert_utest(self, out, expected)
-            assert_measurements(self, out, expected)
-
-
-class TestReportDifferenceForPercentageAggregates(
-        unittest.TestCase):
-    @classmethod
-    def setUpClass(cls):
-        def load_results():
-            import json
-            testInputs = os.path.join(
-                os.path.dirname(
-                    os.path.realpath(__file__)),
-                'Inputs')
-            testOutput1 = os.path.join(testInputs, 'test4_run0.json')
-            testOutput2 = os.path.join(testInputs, 'test4_run1.json')
-            with open(testOutput1, 'r') as f:
-                json1 = json.load(f)
-            with open(testOutput2, 'r') as f:
-                json2 = json.load(f)
-            return json1, json2
-
-        json1, json2 = load_results()
-        cls.json_diff_report = get_difference_report(
-            json1, json2, utest=True)
-
-    def test_json_diff_report_pretty_printing(self):
-        expect_lines = [
-            ['whocares', '-0.5000', '+0.5000', '0', '0', '0', '0']
-        ]
-        output_lines_with_header = print_difference_report(
-            self.json_diff_report,
-            utest=True, utest_alpha=0.05, use_color=False)
-        output_lines = output_lines_with_header[2:]
-        print("\n")
-        print("\n".join(output_lines_with_header))
-        self.assertEqual(len(output_lines), len(expect_lines))
-        for i in range(0, len(output_lines)):
-            parts = [x for x in output_lines[i].split(' ') if x]
-            self.assertEqual(expect_lines[i], parts)
-
-    def test_json_diff_report(self):
-        expected_output = [
-            {
-                'name': u'whocares',
-                'measurements': [
-                    {'time': -0.5,
-                     'cpu': 0.5,
-                     'real_time': 0.01,
-                     'real_time_other': 0.005,
-                     'cpu_time': 0.10,
-                     'cpu_time_other': 0.15}
-                ],
-                'time_unit': 'ns',
-                'utest': {}
-            }
-        ]
-        self.assertEqual(len(self.json_diff_report), len(expected_output))
-        for out, expected in zip(
-                self.json_diff_report, expected_output):
-            self.assertEqual(out['name'], expected['name'])
-            self.assertEqual(out['time_unit'], expected['time_unit'])
-            assert_utest(self, out, expected)
-            assert_measurements(self, out, expected)
-
-
-class TestReportSorting(unittest.TestCase):
-    @classmethod
-    def setUpClass(cls):
-        def load_result():
-            import json
-            testInputs = os.path.join(
-                os.path.dirname(
-                    os.path.realpath(__file__)),
-                'Inputs')
-            testOutput = os.path.join(testInputs, 'test4_run.json')
-            with open(testOutput, 'r') as f:
-                json = json.load(f)
-            return json
-
-        cls.json = load_result()
-
-    def test_json_diff_report_pretty_printing(self):
-        import util
-
-        expected_names = [
-            "99 family 0 instance 0 repetition 0",
-            "98 family 0 instance 0 repetition 1",
-            "97 family 0 instance 0 aggregate",
-            "96 family 0 instance 1 repetition 0",
-            "95 family 0 instance 1 repetition 1",
-            "94 family 0 instance 1 aggregate",
-            "93 family 1 instance 0 repetition 0",
-            "92 family 1 instance 0 repetition 1",
-            "91 family 1 instance 0 aggregate",
-            "90 family 1 instance 1 repetition 0",
-            "89 family 1 instance 1 repetition 1",
-            "88 family 1 instance 1 aggregate"
-        ]
-
-        for n in range(len(self.json['benchmarks']) ** 2):
-            random.shuffle(self.json['benchmarks'])
-            sorted_benchmarks = util.sort_benchmark_results(self.json)[
-                'benchmarks']
-            self.assertEqual(len(expected_names), len(sorted_benchmarks))
-            for out, expected in zip(sorted_benchmarks, expected_names):
-                self.assertEqual(out['name'], expected)
-
-
-def assert_utest(unittest_instance, lhs, rhs):
-    if lhs['utest']:
-        unittest_instance.assertAlmostEqual(
-            lhs['utest']['cpu_pvalue'],
-            rhs['utest']['cpu_pvalue'])
-        unittest_instance.assertAlmostEqual(
-            lhs['utest']['time_pvalue'],
-            rhs['utest']['time_pvalue'])
-        unittest_instance.assertEqual(
-            lhs['utest']['have_optimal_repetitions'],
-            rhs['utest']['have_optimal_repetitions'])
-    else:
-        # lhs is empty. assert if rhs is not.
-        unittest_instance.assertEqual(lhs['utest'], rhs['utest'])
-
-
-def assert_measurements(unittest_instance, lhs, rhs):
-    for m1, m2 in zip(lhs['measurements'], rhs['measurements']):
-        unittest_instance.assertEqual(m1['real_time'], m2['real_time'])
-        unittest_instance.assertEqual(m1['cpu_time'], m2['cpu_time'])
-        # m1['time'] and m1['cpu'] hold values which are being calculated,
-        # and therefore we must use almost-equal pattern.
-        unittest_instance.assertAlmostEqual(m1['time'], m2['time'], places=4)
-        unittest_instance.assertAlmostEqual(m1['cpu'], m2['cpu'], places=4)
-
-
-if __name__ == '__main__':
-    unittest.main()
-
-# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4
-# kate: tab-width: 4; replace-tabs on; indent-width 4; tab-indents: off;
-# kate: indent-mode python; remove-trailing-spaces modified;