datasketches 0.3.2 → 0.4.0

data/vendor/datasketches-cpp/python/tests/count_min_test.py

@@ -1,86 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-  
-import unittest
-from datasketches import count_min_sketch
-
-class CountMinTest(unittest.TestCase):
-  def test_count_min_example(self):
-    # we'll define target confidence and relative error and use the built-in
-    # methods to determine how many hashes and buckets to use
-    confidence = 0.95
-    num_hashes = count_min_sketch.suggest_num_hashes(confidence)
-    relative_error = 0.01
-    num_buckets = count_min_sketch.suggest_num_buckets(relative_error)
-
-    # now we can create a few empty sketches
-    cm = count_min_sketch(num_hashes, num_buckets)
-    cm2 = count_min_sketch(num_hashes, num_buckets)
-    self.assertTrue(cm.is_empty())
-
-    # we'll use a moderate number of distinct items with
-    # increasing weights, with each item's weight being
-    # equal to its value
-    n = 1000
-    total_wt = 0
-    for i in range(1, n+1):
-      cm.update(i, i)
-      total_wt += i
-    self.assertFalse(cm.is_empty())
-    self.assertEqual(cm.get_total_weight(), total_wt)
-
-    # querying the items, each of them should
-    # have a non-zero count.  the estimate should
-    # be at least i with appropriately behaved bounds.
-    for i in range(1, n+1):
-      val = cm.get_estimate(i)
-      self.assertGreaterEqual(val, i)
-      self.assertGreaterEqual(val, cm.get_lower_bound(i))
-      self.assertGreater(cm.get_upper_bound(i), val)
-
-    # values not in the sketch should have lower estimates, but
-    # are not guaranteed to be zero and will succeed
-    self.assertIsNotNone(cm.get_estimate("not in set"))
-
-    # we can create another sketch with partial overlap
-    # and merge them
-    for i in range(int(n / 2), int(3 * n / 2)):
-      cm2.update(i, i)
-    cm.merge(cm2)
-
-    # and the estimated weight for the overlapped meerged values
-    # (n/2 to n) should now be at least 2x the value
-    self.assertGreaterEqual(cm.get_estimate(n), 2 * n)
-
-    # finally, serialize and reconstruct
-    cm_bytes = cm.serialize()
-    self.assertEqual(cm.get_serialized_size_bytes(), len(cm_bytes))
-    new_cm = count_min_sketch.deserialize(cm_bytes)
-
-    # and now interrogate the sketch
-    self.assertFalse(new_cm.is_empty())
-    self.assertEqual(new_cm.get_num_hashes(), cm.get_num_hashes())
-    self.assertEqual(new_cm.get_num_buckets(), cm.get_num_buckets())
-    self.assertEqual(new_cm.get_total_weight(), cm.get_total_weight())
-    
-    # we can also iterate through values in and out of the sketch to ensure
-    # the estimates match
-    for i in range(0, 2 * n):
-      self.assertEqual(cm.get_estimate(i), new_cm.get_estimate(i))
-
-if __name__ == '__main__':
-    unittest.main()

data/vendor/datasketches-cpp/python/tests/cpc_test.py

@@ -1,64 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import unittest
-from datasketches import cpc_sketch, cpc_union
-
-class CpcTest(unittest.TestCase):
-  def test_cpc_example(self):
-    lgk = 12    # 2^k = 4096 rows in the table
-    n = 1 << 18 # ~256k distinct values
-
-    # create a couple sketches and inject some values
-    # we'll have 1/4 of the values overlap
-    cpc  = cpc_sketch(lgk)
-    cpc2 = cpc_sketch(lgk)
-    offset = int(3 * n / 4) # it's a float w/o cast
-    # because we hash on the bits, not an abstract numeric value,
-    # cpc.update(1) and cpc.update(1.0) give different results.
-    for i in range(0, n):
-        cpc.update(i)
-        cpc2.update(i + offset)
-
-    # although we provide get_composite_estimate() and get_estimate(),
-    # the latter will always give the best available estimate.  we
-    # recommend using get_estimate().
-    # we can check that the upper and lower bounds bracket the
-    # estimate, without needing to know the exact value.
-    self.assertLessEqual(cpc.get_lower_bound(1), cpc.get_estimate())
-    self.assertGreaterEqual(cpc.get_upper_bound(1), cpc.get_estimate())
-
-    # union is a separate class, so we need to get_result()
-    # to query the unioned sketches
-    union = cpc_union(lgk)
-    union.update(cpc)
-    union.update(cpc2)
-    result = union.get_result()
-
-    # since our process here (including post-union CPC) is
-    # deterministic, we have checked and know the exact
-    # answer is within one standard deviation of the estimate
-    self.assertLessEqual(result.get_lower_bound(1), 7 * n / 4)
-    self.assertGreaterEqual(result.get_upper_bound(1), 7 * n / 4)
-
-    # serialize for storage and reconstruct
-    sk_bytes = result.serialize()
-    new_cpc = cpc_sketch.deserialize(sk_bytes)
-    self.assertFalse(new_cpc.is_empty())
-
-if __name__ == '__main__':
-    unittest.main()

data/vendor/datasketches-cpp/python/tests/density_test.py

@@ -1,93 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import unittest
-from datasketches import density_sketch, KernelFunction
-import numpy as np
-
-class UnitSphereKernel(KernelFunction):
-  def __call__(self, a: np.array, b: np.array) -> float:
-    if np.linalg.norm(a - b) < 1.0:
-      return 1.0
-    else:
-      return 0.0
-
-class densityTest(unittest.TestCase):
-  def test_density_sketch(self):
-    k = 10
-    dim = 3
-    n = 1000
-
-    sketch = density_sketch(k, dim)
-
-    self.assertEqual(sketch.get_k(), k)
-    self.assertEqual(sketch.get_dim(), dim)
-    self.assertTrue(sketch.is_empty())
-    self.assertFalse(sketch.is_estimation_mode())
-    self.assertEqual(sketch.get_n(), 0)
-    self.assertEqual(sketch.get_num_retained(), 0)
-
-    for i in range(n):
-      sketch.update([i, i, i])
-
-    self.assertFalse(sketch.is_empty())
-    self.assertTrue(sketch.is_estimation_mode())
-    self.assertEqual(sketch.get_n(), n)
-    self.assertGreater(sketch.get_num_retained(), k)
-    self.assertLess(sketch.get_num_retained(), n)
-    self.assertGreater(sketch.get_estimate([n - 1, n - 1, n - 1]), 0)
-
-    for tuple in sketch:
-      vector = tuple[0]
-      weight = tuple[1]
-      self.assertEqual(len(vector), dim)
-      self.assertGreaterEqual(weight, 1)
-
-    sk_bytes = sketch.serialize()
-    sketch2 = density_sketch.deserialize(sk_bytes)
-    self.assertEqual(sketch.get_estimate([1.5, 2.5, 3.5]), sketch2.get_estimate([1.5, 2.5, 3.5]))
-
-  def test_density_merge(self):
-    sketch1 = density_sketch(10, 2)
-    sketch1.update([0, 0])
-    sketch2 = density_sketch(10, 2)
-    sketch2.update([0, 1])
-    sketch1.merge(sketch2)
-    self.assertEqual(sketch1.get_n(), 2)
-    self.assertEqual(sketch1.get_num_retained(), 2)
-
-  def test_custom_kernel(self):
-    gaussianSketch = density_sketch(10, 2) # default kernel
-    sphericalSketch = density_sketch(10, 2, UnitSphereKernel())
-
-    p = [1, 1]
-    gaussianSketch.update(p)
-    sphericalSketch.update(p)
-
-    # Spherical kernel should return 1.0 for a nearby point, 0 farther
-    # Gaussian kernel should return something nonzero when farther away
-    self.assertEqual(sphericalSketch.get_estimate([1.001, 1]), 1.0)
-    self.assertEqual(sphericalSketch.get_estimate([2, 2]), 0.0)
-    self.assertGreater(gaussianSketch.get_estimate([2, 2]), 0.0)
-
-    # We can also use a custom kernel when deserializing
-    sk_bytes = sphericalSketch.serialize()
-    sphericalRebuilt = density_sketch.deserialize(sk_bytes, UnitSphereKernel())
-    self.assertEqual(sphericalSketch.get_estimate([1.001, 1]), sphericalRebuilt.get_estimate([1.001, 1]))
-
-if __name__ == '__main__':
-    unittest.main()

data/vendor/datasketches-cpp/python/tests/fi_test.py

@@ -1,149 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
- 
-import unittest
-from datasketches import frequent_strings_sketch, frequent_items_sketch
-from datasketches import frequent_items_error_type, PyIntsSerDe
-
-class FiTest(unittest.TestCase):
-  def test_fi_strings_example(self):
-    k = 3  # a small value so we can easily fill the sketch
-    fi = frequent_strings_sketch(k)
-
-    # we'll use a small number of distinct items so we
-    # can use exponentially increasing weights and have
-    # some frequent items, decreasing so we have some
-    # small items inserted after a purge
-    n = 8
-    for i in range(0, n):
-      fi.update(str(i), 2 ** (n - i))
-
-    # there are two ways to extract items :
-    # * NO_FALSE_POSITIVES includes all items with a lower bound
-    #   above the a posteriori error
-    # * NO_FALSE_NEGATIVES includes all items with an uper bound
-    #   above the a posteriori error
-    # a more complete discussion may be found at
-    # https://datasketches.github.io/docs/Frequency/FrequentItemsOverview.html
-    items_no_fp = fi.get_frequent_items(frequent_items_error_type.NO_FALSE_POSITIVES)
-    items_no_fn = fi.get_frequent_items(frequent_items_error_type.NO_FALSE_NEGATIVES)
-    self.assertLessEqual(len(items_no_fp), len(items_no_fn))
-
-    # the items list returns a decreasing weight-sorted list, and
-    # for each item we have (item, estimate, lower_bound, upper_bound)
-    item = items_no_fp[1]
-    self.assertLessEqual(item[2], item[1]) # lower bound vs estimate 
-    self.assertLessEqual(item[1], item[3]) # estimate vs upper bound
-
-    # we can also query directly for a specific item
-    id = items_no_fn[0][0]
-    est = fi.get_estimate(id)
-    lb = fi.get_lower_bound(id)
-    ub = fi.get_upper_bound(id)
-    self.assertLessEqual(lb, est)
-    self.assertLessEqual(est, ub)
-
-    # the values are zero if the item isn't in our list
-    self.assertEqual(fi.get_estimate("NaN"), 0)
-
-    # now create a second sketch with a lot of unique
-    # values but all with equal weight (of 1) such that
-    # the total weight is much larger than the first sketch
-    fi2 = frequent_strings_sketch(k)
-    wt = fi.get_total_weight()
-    for i in range(0, 4*wt):
-      fi2.update(str(i))
-
-    # merge the second sketch into the first
-    fi.merge(fi2)
-
-    # we can see that the weight is much larger
-    self.assertEqual(5 * wt, fi.get_total_weight())
-
-    # querying with NO_FALSE_POSITIVES means we don't find anything
-    # heavy enough to return
-    items_no_fp = fi.get_frequent_items(frequent_items_error_type.NO_FALSE_POSITIVES)
-    self.assertEqual(len(items_no_fp), 0)
-
-    # we do, however, find a few potential heavy items
-    # if querying with NO_FALSE_NEGATIVES
-    items_no_fn = fi.get_frequent_items(frequent_items_error_type.NO_FALSE_NEGATIVES)
-    self.assertGreater(len(items_no_fn), 0)
-
-    # finally, serialize and reconstruct
-    fi_bytes = fi.serialize()
-    self.assertEqual(len(fi_bytes), fi.get_serialized_size_bytes())
-    new_fi = frequent_strings_sketch.deserialize(fi_bytes)
-
-    # and now interrogate the sketch
-    self.assertFalse(new_fi.is_empty())
-    self.assertGreater(new_fi.get_num_active_items(), 0)
-    self.assertEqual(5 * wt, new_fi.get_total_weight())
-
-  # This example uses generic objects but is otherwise identical
-  def test_fi_items_example(self):
-    k = 3  # a small value so we can easily fill the sketch
-    fi = frequent_items_sketch(k)
-
-    # as above, but in this case inserting ints
-    n = 8
-    for i in range(0, n):
-      fi.update(i, 2 ** (n - i))
-
-    # everything else works identically, so let's jump straight
-    # to merging and serialization
-
-    # now create a second sketch with a lot of unique
-    # values but all with equal weight (of 1) such that
-    # the total weight is much larger than the first sketch
-    fi2 = frequent_items_sketch(k)
-    wt = fi.get_total_weight()
-    for i in range(0, 4*wt):
-      fi2.update(i)
-
-    # merge the second sketch into the first
-    fi.merge(fi2)
-
-    # we can see that the weight is much larger
-    self.assertEqual(5 * wt, fi.get_total_weight())
-
-    # finally, serialize and reconstruct -- now we need a serde to tell
-    # (de)serialization how to interpret the objects
-    fi_bytes = fi.serialize(PyIntsSerDe())
-    self.assertEqual(len(fi_bytes), fi.get_serialized_size_bytes(PyIntsSerDe()))
-    new_fi = frequent_items_sketch.deserialize(fi_bytes, PyIntsSerDe())
-
-    # and again interrogate the sketch to check that it's what we serialized
-    self.assertFalse(new_fi.is_empty())
-    self.assertGreater(new_fi.get_num_active_items(), 0)
-    self.assertEqual(5 * wt, new_fi.get_total_weight())
-
-
-  def test_fi_sketch(self):
-    # only testing a few things not used in the above example
-    k = 12
-    wt = 10000
-    fi = frequent_strings_sketch(k)
-
-    self.assertAlmostEqual(fi.get_sketch_epsilon(), 0.0008545, delta=1e-6)
-
-    sk_apriori_error = fi.get_sketch_epsilon() * wt
-    reference_apriori_error = frequent_strings_sketch.get_apriori_error(k, wt)
-    self.assertAlmostEqual(sk_apriori_error, reference_apriori_error, delta=1e-6)
-
-if __name__ == '__main__':
-  unittest.main()

data/vendor/datasketches-cpp/python/tests/hll_test.py

@@ -1,129 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import unittest
-from datasketches import hll_sketch, hll_union, tgt_hll_type
-
-class HllTest(unittest.TestCase):
-    def test_hll_example(self):
-        lgk = 12    # 2^k = 4096 rows in the table
-        n = 1 << 18 # ~256k unique values
-
-        # create a couple sketches and inject some values
-        # we'll have 1/4 of the values overlap
-        hll  = hll_sketch(lgk, tgt_hll_type.HLL_8)
-        hll2 = hll_sketch(lgk, tgt_hll_type.HLL_6)
-        offset = int(3 * n / 4) # it's a float w/o cast
-        # because we hash on the bits, not an abstract numeric value,
-        # hll.update(1) and hll.update(1.0) give different results.
-        for i in range(0, n):
-            hll.update(i)
-            hll2.update(i + offset)
-
-        # we can check that the upper and lower bounds bracket the
-        # estimate, without needing to know the exact value.
-        self.assertLessEqual(hll.get_lower_bound(1), hll.get_estimate())
-        self.assertGreaterEqual(hll.get_upper_bound(1), hll.get_estimate())
-
-        # union is a separate class, and we can either get a result
-        # sketch or query the union object directly
-        union = hll_union(lgk)
-        union.update(hll)
-        union.update(hll2)
-        result = union.get_result()
-        self.assertEqual(result.get_estimate(), union.get_estimate())
-
-        # since our process here (including post-union HLL) is
-        # deterministic, we have checked and know the exact
-        # answer is within one standard deviation of the estimate
-        self.assertLessEqual(union.get_lower_bound(1), 7 * n / 4)
-        self.assertGreaterEqual(union.get_upper_bound(1), 7 * n / 4)
-
-        # serialize for storage and reconstruct
-        sk_bytes = result.serialize_compact()
-        self.assertEqual(len(sk_bytes), result.get_compact_serialization_bytes())
-        new_hll = hll_sketch.deserialize(sk_bytes)
-
-        # the sketch can self-report its configuration and status
-        self.assertEqual(new_hll.lg_config_k, lgk)
-        self.assertEqual(new_hll.tgt_type, tgt_hll_type.HLL_4)
-        self.assertFalse(new_hll.is_empty())
-
-        # if we want to reduce some object overhead, we can also reset
-        new_hll.reset()
-        self.assertTrue(new_hll.is_empty())
-
-    def test_hll_sketch(self):
-        lgk = 8
-        n = 117
-        hll = self.generate_sketch(n, lgk, tgt_hll_type.HLL_6)
-        hll.update('string data')
-        hll.update(3.14159) # double data
-
-        self.assertLessEqual(hll.get_lower_bound(1), hll.get_estimate())
-        self.assertGreaterEqual(hll.get_upper_bound(1), hll.get_estimate())
-
-        self.assertEqual(hll.lg_config_k, lgk)
-        self.assertEqual(hll.tgt_type, tgt_hll_type.HLL_6)
-
-        bytes_compact = hll.serialize_compact()
-        bytes_update = hll.serialize_updatable()
-        self.assertEqual(len(bytes_compact), hll.get_compact_serialization_bytes())
-        self.assertEqual(len(bytes_update), hll.get_updatable_serialization_bytes())
-
-        self.assertFalse(hll.is_compact())
-        self.assertFalse(hll.is_empty())
-
-        self.assertTrue(isinstance(hll_sketch.deserialize(bytes_compact), hll_sketch))
-        self.assertTrue(isinstance(hll_sketch.deserialize(bytes_update), hll_sketch))
-
-        self.assertIsNotNone(hll_sketch.get_rel_err(True, False, 12, 1))
-        self.assertIsNotNone(hll_sketch.get_max_updatable_serialization_bytes(20, tgt_hll_type.HLL_6))
-
-        hll.reset()
-        self.assertTrue(hll.is_empty())
-
-    def test_hll_union(self):
-        lgk = 7
-        n = 53
-        union = hll_union(lgk)
-
-        sk = self.generate_sketch(n, lgk, tgt_hll_type.HLL_4, 0)
-        union.update(sk)
-        sk = self.generate_sketch(3 * n, lgk, tgt_hll_type.HLL_4, n)
-        union.update(sk)
-        union.update('string data')
-        union.update(1.4142136)
-
-        self.assertLessEqual(union.get_lower_bound(1), union.get_estimate())
-        self.assertGreaterEqual(union.get_upper_bound(1), union.get_estimate())
-
-        self.assertEqual(union.lg_config_k, lgk)
-        self.assertFalse(union.is_empty())
-
-        sk = union.get_result()
-        self.assertTrue(isinstance(sk, hll_sketch))
-        self.assertEqual(sk.tgt_type, tgt_hll_type.HLL_4)
-        
-    def generate_sketch(self, n, lgk, sk_type=tgt_hll_type.HLL_4, st_idx=0):
-        sk = hll_sketch(lgk, sk_type)
-        for i in range(st_idx, st_idx + n):
-            sk.update(i)
-        return sk
-
-if __name__ == '__main__':
-    unittest.main()

data/vendor/datasketches-cpp/python/tests/kll_test.py

@@ -1,159 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import unittest
-from datasketches import kll_ints_sketch, kll_floats_sketch, kll_doubles_sketch
-from datasketches import kll_items_sketch, ks_test, PyStringsSerDe
-import numpy as np
-
-class KllTest(unittest.TestCase):
-    def test_kll_floats_example(self):
-      k = 160
-      n = 2 ** 20
-
-      # create a sketch and inject ~1 million N(0,1) points as an array and as a single item
-      kll = kll_floats_sketch(k)
-      kll.update(np.random.normal(size=n-1))
-      kll.update(0.0)
-
-      # 0 should be near the median
-      self.assertAlmostEqual(0.5, kll.get_rank(0.0), delta=0.035)
-      
-      # the median should be near 0
-      self.assertAlmostEqual(0.0, kll.get_quantile(0.5), delta=0.035)
-
-      # we also track the min/max independently from the rest of the data
-      # which lets us know the full observed data range
-      self.assertLessEqual(kll.get_min_value(), kll.get_quantile(0.01))
-      self.assertLessEqual(0.0, kll.get_rank(kll.get_min_value()))
-      self.assertGreaterEqual(kll.get_max_value(), kll.get_quantile(0.99))
-      self.assertGreaterEqual(1.0, kll.get_rank(kll.get_max_value()))
-
-      # we can also extract a list of values at a time,
-      # here the values should give us something close to [-2, -1, 0, 1, 2].
-      # then get the CDF, which will return something close to
-      # the original values used in get_quantiles()
-      # finally, can check the normalized rank error bound
-      pts = kll.get_quantiles([0.0228, 0.1587, 0.5, 0.8413, 0.9772])
-      cdf = kll.get_cdf(pts)  # include 1.0 at end to account for all probability mass
-      self.assertEqual(len(cdf), len(pts)+1)
-      err = kll.normalized_rank_error(False)
-      self.assertEqual(err, kll_floats_sketch.get_normalized_rank_error(k, False))
-
-      # and a few basic queries about the sketch
-      self.assertFalse(kll.is_empty())
-      self.assertTrue(kll.is_estimation_mode())
-      self.assertEqual(kll.get_n(), n)
-      self.assertEqual(kll.get_k(), k)
-      self.assertLess(kll.get_num_retained(), n)
-
-      # merging itself will double the number of items the sketch has seen
-      # but need to do that with a copy
-      kll_copy = kll_floats_sketch(kll)
-      kll.merge(kll_copy)
-      self.assertEqual(kll.get_n(), 2*n)
-
-      # we can then serialize and reconstruct the sketch
-      kll_bytes = kll.serialize()
-      new_kll = kll_floats_sketch.deserialize(kll_bytes)
-      self.assertEqual(kll.get_num_retained(), new_kll.get_num_retained())
-      self.assertEqual(kll.get_min_value(), new_kll.get_min_value())
-      self.assertEqual(kll.get_max_value(), new_kll.get_max_value())
-      self.assertEqual(kll.get_quantile(0.7), new_kll.get_quantile(0.7))
-      self.assertEqual(kll.get_rank(0.0), new_kll.get_rank(0.0))
-
-      # A Kolmogorov-Smirnov Test of kll and new_kll should match, even for
-      # a fairly small p-value -- cannot reject the null hypothesis that
-      # they come from the same distribution (since they do)
-      self.assertFalse(ks_test(kll, new_kll, 0.001))
-
-      total_weight = 0
-      for tuple in kll:
-        item = tuple[0]
-        weight = tuple[1]
-        total_weight = total_weight + weight
-      self.assertEqual(total_weight, kll.get_n())
-
-    def test_kll_ints_sketch(self):
-        k = 100
-        n = 10
-        kll = kll_ints_sketch(k)
-        for i in range(0, n):
-          kll.update(i)
-
-        self.assertEqual(kll.get_min_value(), 0)
-        self.assertEqual(kll.get_max_value(), n-1)
-        self.assertEqual(kll.get_n(), n)
-        self.assertFalse(kll.is_empty())
-        self.assertFalse(kll.is_estimation_mode()) # n < k
-        self.assertEqual(kll.get_k(), k)
-
-        pmf = kll.get_pmf([round(n/2)])
-        self.assertIsNotNone(pmf)
-        self.assertEqual(len(pmf), 2)
-
-        cdf = kll.get_cdf([round(n/2)])
-        self.assertIsNotNone(cdf)
-        self.assertEqual(len(cdf), 2)
-
-        self.assertEqual(kll.get_quantile(0.5), round(n/2))
-        quants = kll.get_quantiles([0.25, 0.5, 0.75])
-        self.assertIsNotNone(quants)
-        self.assertEqual(len(quants), 3)
-
-        self.assertEqual(kll.get_rank(round(n/2)), 0.5)
-
-        # merge copy of self
-        kll_copy = kll_ints_sketch(kll)
-        kll.merge(kll_copy)
-        self.assertEqual(kll.get_n(), 2 * n)
-
-        sk_bytes = kll.serialize()
-        self.assertTrue(isinstance(kll_ints_sketch.deserialize(sk_bytes), kll_ints_sketch))
-
-    def test_kll_doubles_sketch(self):
-      # already tested float and ints and it's templatized, so just make sure it instantiates properly
-      k = 75
-      kll = kll_doubles_sketch(k)
-      self.assertTrue(kll.is_empty())
-
-    def test_kll_items_sketch(self):
-      # most functionality has been tested, but we need to ensure objects and sorting work
-      # as well as serialization
-      k = 100
-      n = 2 ** 16
-
-      # create a sketch and inject enough points to force compaction
-      kll = kll_items_sketch(k)
-      for i in range(0, n):
-        kll.update(str(i))
-      
-      kll_copy = kll_items_sketch(kll)
-      kll.merge(kll_copy)
-      self.assertEqual(kll.get_n(), 2 * n)
-      
-      kll_bytes = kll.serialize(PyStringsSerDe())
-      new_kll = kll_items_sketch.deserialize(kll_bytes, PyStringsSerDe())
-      self.assertEqual(kll.get_num_retained(), new_kll.get_num_retained())
-      self.assertEqual(kll.get_min_value(), new_kll.get_min_value())
-      self.assertEqual(kll.get_max_value(), new_kll.get_max_value())
-      self.assertEqual(kll.get_quantile(0.7), new_kll.get_quantile(0.7))
-      self.assertEqual(kll.get_rank(str(n/4)), new_kll.get_rank(str(n/4)))
-
-
-if __name__ == '__main__':
-    unittest.main()