hypothesis-specs 0.0.8 → 0.0.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 23fd4398a4ec86d30001513e3f003dcc76ebc1e5
-  data.tar.gz: 733ad33a2f5518119e6d0336131b8b46d4c6135e
+  metadata.gz: 7e8eb07704e06280a79469b76b6b7267cf356797
+  data.tar.gz: edd8bb7476a3a84dedf2751016385dbe2d6afcac
 SHA512:
-  metadata.gz: 9cc1fcf63b61384f84a92cd7289b0ac1104f2f579bd45bf31089a3cf3cfd9d7caff604063be04c3102c12e40e1e557a6fe26cd520c845201d9058b2881c1b21d
-  data.tar.gz: 84aa608a1d2076778e2f686a15c1d01ad7ac557ce057303151ea4b0e72d89129c9a17314ee26df29a312ad956151fdc35d571a06a733d81346a614c1a621da36
+  metadata.gz: 8e72ef276034266146155092d14827e75f5f20ec3986dd57245f94694e1f7da09d5cc62b74fced3829e0d8e5dbff522ad44d6fc88e2e49d274979eaae3f06e0e
+  data.tar.gz: b6917a3a24f98febffaab647ecec0393827e3f116390762d3163cd9d2fe7e1cac49cfd79bc2db1e448beb044e862b0c96c79a5e3e8d7aed986ff718eef755d40

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+## Hypothesis for Ruby 0.0.9 (2018-04-20)
+
+This improves Hypothesis for Ruby's shrinking to be much closer
+to Hypothesis for Python's. It's still far from complete, and even
+in cases where it has the same level of quality it will often be
+significantly slower, but examples should now be much more consistent,
+especially in cases where you are using e.g. `built_as`.
+
 ## Hypothesis for Ruby 0.0.8 (2018-02-20)
 
 This release fixes the dependency on Rake to be in a more sensible range.

data/lib/hypothesis/testcase.rb

@@ -30,7 +30,9 @@ module Hypothesis
       begin
         @depth += 1
         possible ||= block
+        @wrapped_data.start_draw
         result = possible.provide(&block)
+        @wrapped_data.stop_draw
         if top_level
           draws&.push(result)
           print_log&.push([name, result.inspect])

data/src/data.rs

@@ -2,6 +2,7 @@
 // needs.
 
 use rand::{ChaChaRng, Rng};
+use std::collections::HashSet;
 
 pub type DataStream = Vec<u64>;
 
@@ -14,6 +15,22 @@ enum BitGenerator {
     Recorded(DataStream),
 }
 
+// Records information corresponding to a single draw call.
+#[derive(Debug, Clone)]
+pub struct DrawInProgress {
+    depth: usize,
+    start: usize,
+    end: Option<usize>,
+}
+
+// Records information corresponding to a single draw call.
+#[derive(Debug, Clone)]
+pub struct Draw {
+    pub depth: usize,
+    pub start: usize,
+    pub end: usize,
+}
+
 // Main entry point for running a test:
 // A test function takes a DataSource, uses it to
 // produce some data, and the DataSource records the
@@ -22,9 +39,61 @@ enum BitGenerator {
 pub struct DataSource {
     bitgenerator: BitGenerator,
     record: DataStream,
+    draws: Vec<DrawInProgress>,
+    draw_stack: Vec<usize>,
+    written_indices: HashSet<usize>,
 }
 
 impl DataSource {
+    fn new(generator: BitGenerator) -> DataSource {
+        return DataSource {
+            bitgenerator: generator,
+            record: DataStream::new(),
+            draws: Vec::new(),
+            draw_stack: Vec::new(),
+            written_indices: HashSet::new(),
+        };
+    }
+
+    pub fn from_random(random: ChaChaRng) -> DataSource {
+        return DataSource::new(BitGenerator::Random(random));
+    }
+
+    pub fn from_vec(record: DataStream) -> DataSource {
+        return DataSource::new(BitGenerator::Recorded(record));
+    }
+
+    pub fn start_draw(&mut self) {
+        let i = self.draws.len();
+        let depth = self.draw_stack.len();
+        let start = self.record.len();
+
+        self.draw_stack.push(i);
+        self.draws.push(DrawInProgress {
+            start: start,
+            end: None,
+            depth: depth,
+        });
+    }
+
+    pub fn stop_draw(&mut self) {
+        assert!(self.draws.len() > 0);
+        assert!(self.draw_stack.len() > 0);
+        let i = self.draw_stack.pop().unwrap();
+        let end = self.record.len();
+        self.draws[i].end = Some(end);
+    }
+
+    pub fn write(&mut self, value: u64) -> Result<(), FailedDraw> {
+        match self.bitgenerator {
+            BitGenerator::Recorded(ref mut v) if self.record.len() >= v.len() => Err(FailedDraw),
+            _ => {
+                self.record.push(value);
+                Ok(())
+            }
+        }
+    }
+
     pub fn bits(&mut self, n_bits: u64) -> Result<u64, FailedDraw> {
         let mut result = match self.bitgenerator {
             BitGenerator::Random(ref mut random) => random.next_u64(),
@@ -45,32 +114,40 @@ impl DataSource {
         return Ok(result);
     }
 
-    fn new(generator: BitGenerator) -> DataSource {
-        return DataSource {
-            bitgenerator: generator,
-            record: DataStream::new(),
-        };
-    }
-
-    pub fn from_random(random: ChaChaRng) -> DataSource {
-        return DataSource::new(BitGenerator::Random(random));
-    }
-
-    pub fn from_vec(record: DataStream) -> DataSource {
-        return DataSource::new(BitGenerator::Recorded(record));
-    }
-
-    pub fn to_result(self, status: Status) -> TestResult {
+    pub fn to_result(mut self, status: Status) -> TestResult {
         TestResult {
             record: self.record,
             status: status,
+            written_indices: self.written_indices,
+            draws: self.draws
+                .drain(..)
+                .filter_map(|d| match d {
+                    DrawInProgress {
+                        depth,
+                        start,
+                        end: Some(end),
+                    } if start < end =>
+                    {
+                        Some(Draw {
+                            start: start,
+                            end: end,
+                            depth: depth,
+                        })
+                    }
+                    DrawInProgress { end: None, .. } => {
+                        assert!(status == Status::Invalid || status == Status::Overflow);
+                        None
+                    }
+                    _ => None,
+                })
+                .collect(),
         }
     }
 }
 
 // Status indicates the result that we got from completing
 // a single test execution.
-#[derive(Debug, Clone, Eq, PartialEq)]
+#[derive(Debug, Clone, Eq, PartialEq, Copy)]
 pub enum Status {
     // The test tried to read more data than we had for it.
     Overflow,
@@ -96,4 +173,6 @@ pub enum Status {
 pub struct TestResult {
     pub record: DataStream,
     pub status: Status,
+    pub draws: Vec<Draw>,
+    pub written_indices: HashSet<usize>,
 }

data/src/distributions.rs

@@ -4,6 +4,8 @@ use std::collections::BinaryHeap;
 use std::mem;
 use std::cmp::{Ord, Ordering, PartialOrd, Reverse};
 
+use std::u64::MAX as MAX64;
+
 type Draw<T> = Result<T, FailedDraw>;
 
 pub fn weighted(source: &mut DataSource, probability: f64) -> Result<bool, FailedDraw> {
@@ -11,12 +13,19 @@ pub fn weighted(source: &mut DataSource, probability: f64) -> Result<bool, Faile
 
     let truthy = (probability * (u64::max_value() as f64 + 1.0)).floor() as u64;
     let probe = source.bits(64)?;
-    return Ok(probe >= u64::max_value() - truthy + 1);
+    Ok(match (truthy, probe) {
+        (0, _) => false,
+        (MAX64, _) => true,
+        (_, 0) => false,
+        (_, 1) => true,
+        _ => probe >= MAX64 - truthy,
+    })
 }
 
 pub fn bounded_int(source: &mut DataSource, max: u64) -> Draw<u64> {
     let bitlength = 64 - max.leading_zeros() as u64;
     if bitlength == 0 {
+        source.write(0)?;
         return Ok(0);
     }
     loop {
@@ -46,30 +55,25 @@ impl Repeat {
         }
     }
 
-    fn draw_until(&self, source: &mut DataSource, value: bool) -> Result<(), FailedDraw> {
-        // Force a draw until we get the desired outcome. By having this we get much better
-        // shrinking when min_size or max_size are set because all decisions are represented
-        // somewhere in the bit stream.
-        loop {
-            let d = weighted(source, self.p_continue)?;
-            if d == value {
-                return Ok(());
-            }
-        }
-    }
-
     pub fn reject(&mut self) {
         assert!(self.current_count > 0);
         self.current_count -= 1;
     }
 
     pub fn should_continue(&mut self, source: &mut DataSource) -> Result<bool, FailedDraw> {
-        if self.current_count < self.min_count {
-            self.draw_until(source, true)?;
+        if self.min_count == self.max_count {
+            if self.current_count < self.max_count {
+                self.current_count += 1;
+                return Ok(true);
+            } else {
+                return Ok(false);
+            }
+        } else if self.current_count < self.min_count {
+            source.write(1)?;
             self.current_count += 1;
             return Ok(true);
         } else if self.current_count >= self.max_count {
-            self.draw_until(source, false)?;
+            source.write(0)?;
             return Ok(false);
         }
 

data/src/engine.rs

@@ -13,9 +13,7 @@ use data::{DataSource, DataStream, Status, TestResult};
 enum LoopExitReason {
     Complete,
     MaxExamples,
-    //MaxShrinks,
     Shutdown,
-    //Error(String),
 }
 
 #[derive(Debug)]
@@ -108,6 +106,7 @@ struct Shrinker<'owner, Predicate> {
     _predicate: Predicate,
     shrink_target: TestResult,
     changes: u64,
+    expensive_passes_enabled: bool,
     main_loop: &'owner mut MainGenerationLoop,
 }
 
@@ -126,12 +125,22 @@ where
             _predicate: predicate,
             shrink_target: shrink_target,
             changes: 0,
+            expensive_passes_enabled: false,
         }
     }
 
     fn predicate(&mut self, result: &TestResult) -> bool {
         let succeeded = (self._predicate)(result);
-        if succeeded {
+        if succeeded
+            && (
+          // In the presence of writes it may be the case that we thought
+          // we were going to shrink this but didn't actually succeed because
+          // the written value was used.
+          result.record.len() < self.shrink_target.record.len() || (
+            result.record.len() == self.shrink_target.record.len()  &&
+            result.record < self.shrink_target.record
+          )
+        ) {
             self.changes += 1;
             self.shrink_target = result.clone();
         }
@@ -143,15 +152,167 @@ where
 
         while prev != self.changes {
             prev = self.changes;
+            self.adaptive_delete()?;
             self.binary_search_blocks()?;
-            self.remove_intervals()?;
+            if prev == self.changes {
+                self.expensive_passes_enabled = true;
+            }
+            if !self.expensive_passes_enabled {
+                continue;
+            }
+
+            self.reorder_blocks()?;
+            self.lower_and_delete()?;
+            self.delete_all_ranges()?;
         }
         Ok(())
     }
 
-    fn remove_intervals(&mut self) -> StepResult {
-        // TODO: Actually track the data we need to make this
-        // not quadratic.
+    fn lower_and_delete(&mut self) -> StepResult {
+        let mut i = 0;
+        while i < self.shrink_target.record.len() {
+            if self.shrink_target.record[i] > 0 {
+                let mut attempt = self.shrink_target.record.clone();
+                attempt[i] -= 1;
+                let (succeeded, result) = self.execute(&attempt)?;
+                if !succeeded && result.record.len() < self.shrink_target.record.len() {
+                    let mut j = 0;
+                    while j < self.shrink_target.draws.len() {
+                        // Having to copy this is an annoying consequence of lexical lifetimes -
+                        // if we borrowed it immutably then we'd not be allowed to call self.incorporate
+                        // down below. Fortunately these things are tiny structs of integers so it doesn't
+                        // really matter.
+                        let d = self.shrink_target.draws[j].clone();
+                        if d.start > i {
+                            let mut attempt2 = attempt.clone();
+                            attempt2.drain(d.start..d.end);
+                            if self.incorporate(&attempt2)? {
+                                break;
+                            }
+                        }
+                        j += 1;
+                    }
+                }
+            }
+            i += 1;
+        }
+        Ok(())
+    }
+
+    fn reorder_blocks(&mut self) -> StepResult {
+        let mut i = 0;
+        while i < self.shrink_target.record.len() {
+            let mut j = i + 1;
+            while j < self.shrink_target.record.len() {
+                assert!(i < self.shrink_target.record.len());
+                if self.shrink_target.record[i] == 0 {
+                    break;
+                }
+                if self.shrink_target.record[j] < self.shrink_target.record[i] {
+                    let mut attempt = self.shrink_target.record.clone();
+                    attempt.swap(i, j);
+                    self.incorporate(&attempt)?;
+                }
+                j += 1;
+            }
+            i += 1;
+        }
+        Ok(())
+    }
+
+    fn try_delete_range(
+        &mut self,
+        target: &TestResult,
+        i: usize,
+        k: usize,
+    ) -> Result<bool, LoopExitReason> {
+        // Attempts to delete k non-overlapping draws starting from the draw at index i.
+
+        let mut stack: Vec<(usize, usize)> = Vec::new();
+        let mut j = i;
+        while j < target.draws.len() && stack.len() < k {
+            let m = target.draws[j].start;
+            let n = target.draws[j].end;
+            assert!(m < n);
+            if m < n && (stack.len() == 0 || stack[stack.len() - 1].1 <= m) {
+                stack.push((m, n))
+            }
+            j += 1;
+        }
+
+        let mut attempt = target.record.clone();
+        while stack.len() > 0 {
+            let (m, n) = stack.pop().unwrap();
+            attempt.drain(m..n);
+        }
+
+        if attempt.len() >= self.shrink_target.record.len() {
+            Ok(false)
+        } else {
+            self.incorporate(&attempt)
+        }
+    }
+
+    fn adaptive_delete(&mut self) -> StepResult {
+        let mut i = 0;
+        let target = self.shrink_target.clone();
+
+        while i < target.draws.len() {
+            // This is an adaptive pass loosely modelled after timsort. If
+            // little or nothing is deletable here then we don't try any more
+            // deletions than the naive greedy algorithm would, but if it looks
+            // like we have an opportunity to delete a lot then we try to do so.
+
+            // What we're trying to do is to find a large k such that we can
+            // delete k but not k + 1 draws starting from this point, and we
+            // want to do that in O(log(k)) rather than O(k) test executions.
+
+            // We try a quite careful sequence of small shrinks here before we
+            // move on to anything big. This is because if we try to be
+            // aggressive too early on we'll tend to find that we lose out when
+            // the example is "nearly minimal".
+            if self.try_delete_range(&target, i, 2)? {
+                if self.try_delete_range(&target, i, 3)? && self.try_delete_range(&target, i, 4)? {
+                    let mut hi = 5;
+                    // At this point it looks like we've got a pretty good
+                    // opportunity for a long run here. We do an exponential
+                    // probe upwards to try and find some k where we can't
+                    // delete many intervals. We do this rather than choosing
+                    // that upper bound to immediately be large because we
+                    // don't really expect k to be huge. If it turns out that
+                    // it is, the subsequent example is going to be so tiny that
+                    // it doesn't really matter if we waste a bit of extra time
+                    // here.
+                    while self.try_delete_range(&target, i, hi)? {
+                        assert!(hi <= target.draws.len());
+                        hi *= 2;
+                    }
+                    // We now know that we can delete the first lo intervals but
+                    // not the first hi. We preserve that property while doing
+                    // a binary search to find the point at which we stop being
+                    // able to delete intervals.
+                    let mut lo = 4;
+                    while lo + 1 < hi {
+                        let mid = lo + (hi - lo) / 2;
+                        if self.try_delete_range(&target, i, mid)? {
+                            lo = mid;
+                        } else {
+                            hi = mid;
+                        }
+                    }
+                }
+            } else {
+                self.try_delete_range(&target, i, 1)?;
+            }
+            // We unconditionally bump i because we have always tried deleting
+            // one more example than we succeeded at deleting, so we expect the
+            // next example to be undeletable.
+            i += 1;
+        }
+        return Ok(());
+    }
+
+    fn delete_all_ranges(&mut self) -> StepResult {
         let mut i = 0;
         while i < self.shrink_target.record.len() {
             let start_length = self.shrink_target.record.len();
@@ -174,38 +335,45 @@ where
         Ok(())
     }
 
-    fn binary_search_blocks(&mut self) -> StepResult {
-        let mut i = 0;
+    fn try_lowering_value(&mut self, i: usize, v: u64) -> Result<bool, LoopExitReason> {
+        if v >= self.shrink_target.record[i] {
+            return Ok(false);
+        }
 
         let mut attempt = self.shrink_target.record.clone();
+        attempt[i] = v;
+        let (succeeded, result) = self.execute(&attempt)?;
+        assert!(result.record.len() <= self.shrink_target.record.len());
+        let lost_bytes = self.shrink_target.record.len() - result.record.len();
+        if !succeeded && result.status == Status::Valid && lost_bytes > 0 {
+            attempt.drain(i + 1..i + lost_bytes + 1);
+            assert!(attempt.len() + lost_bytes == self.shrink_target.record.len());
+            self.incorporate(&attempt)
+        } else {
+            Ok(succeeded)
+        }
+    }
 
-        while i < self.shrink_target.record.len() {
-            assert!(attempt.len() >= self.shrink_target.record.len());
+    fn binary_search_blocks(&mut self) -> StepResult {
+        let mut i = 0;
 
+        while i < self.shrink_target.record.len() {
             let mut hi = self.shrink_target.record[i];
 
-            if hi > 0 {
-                attempt[i] = 0;
-                let zeroed = self.incorporate(&attempt)?;
+            if hi > 0 && !self.shrink_target.written_indices.contains(&i) {
+                let zeroed = self.try_lowering_value(i, 0)?;
                 if !zeroed {
                     let mut lo = 0;
                     // Binary search to find the smallest value we can
                     // replace this with.
                     while lo + 1 < hi {
                         let mid = lo + (hi - lo) / 2;
-                        attempt[i] = mid;
-                        let succeeded = self.incorporate(&attempt)?;
-                        if succeeded {
-                            attempt = self.shrink_target.record.clone();
+                        if self.try_lowering_value(i, mid)? {
                             hi = mid;
                         } else {
-                            attempt[i] = self.shrink_target.record[i];
                             lo = mid;
                         }
                     }
-                    attempt[i] = hi;
-                } else {
-                    attempt = self.shrink_target.record.clone();
                 }
             }
 
@@ -215,6 +383,12 @@ where
         Ok(())
     }
 
+    fn execute(&mut self, buf: &DataStream) -> Result<(bool, TestResult), LoopExitReason> {
+        // TODO: Later there will be caching here
+        let result = self.main_loop.execute(DataSource::from_vec(buf.clone()))?;
+        Ok((self.predicate(&result), result))
+    }
+
     fn incorporate(&mut self, buf: &DataStream) -> Result<bool, LoopExitReason> {
         assert!(
             buf.len() <= self.shrink_target.record.len(),
@@ -229,8 +403,8 @@ where
         if self.shrink_target.record.starts_with(buf) {
             return Ok(false);
         }
-        let result = self.main_loop.execute(DataSource::from_vec(buf.clone()))?;
-        return Ok(self.predicate(&result));
+        let (succeeded, _) = self.execute(buf)?;
+        Ok(succeeded)
     }
 }
 

data/src/lib.rs

@@ -33,6 +33,18 @@ ruby! {
       mem::swap(&mut result.source, &mut engine.pending);
       return result;
     }
+
+    def start_draw(&mut self){
+      if let &mut Some(ref mut source) = &mut self.source {
+        source.start_draw();
+      }
+    }
+
+    def stop_draw(&mut self){
+      if let &mut Some(ref mut source) = &mut self.source {
+        source.stop_draw();
+      }
+    }
   }
 
   class HypothesisCoreEngine {

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: hypothesis-specs
 version: !ruby/object:Gem::Version
-  version: 0.0.8
+  version: 0.0.9
 platform: ruby
 authors:
 - David R. Maciver
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-02-20 00:00:00.000000000 Z
+date: 2018-04-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: helix_runtime