passenger 6.0.19 → 6.0.20

data/src/cxx_supportlib/vendor-modified/boost/unordered/detail/{foa.hpp → foa/core.hpp}

@@ -1,6 +1,7 @@
-/* Fast open-addressing hash table.
+/* Common base for Boost.Unordered open-addressing tables.
  *
- * Copyright 2022 Joaquin M Lopez Munoz.
+ * Copyright 2022-2023 Joaquin M Lopez Munoz.
+ * Copyright 2023 Christian Mazakas.
  * Distributed under the Boost Software License, Version 1.0.
  * (See accompanying file LICENSE_1_0.txt or copy at
  * http://www.boost.org/LICENSE_1_0.txt)
@@ -8,8 +9,8 @@
  * See https://www.boost.org/libs/unordered for library home page.
  */
 
-#ifndef BOOST_UNORDERED_DETAIL_FOA_HPP
-#define BOOST_UNORDERED_DETAIL_FOA_HPP
+#ifndef BOOST_UNORDERED_DETAIL_FOA_CORE_HPP
+#define BOOST_UNORDERED_DETAIL_FOA_CORE_HPP
 
 #include <boost/assert.hpp>
 #include <boost/config.hpp>
@@ -21,25 +22,43 @@
 #include <boost/core/pointer_traits.hpp>
 #include <boost/cstdint.hpp>
 #include <boost/predef.h>
+#include <boost/static_assert.hpp>
+#include <boost/type_traits/has_trivial_constructor.hpp>
+#include <boost/type_traits/has_trivial_copy.hpp>
+#include <boost/type_traits/has_trivial_assign.hpp>
 #include <boost/type_traits/is_nothrow_swappable.hpp>
-#include <boost/unordered/detail/xmx.hpp>
+#include <boost/unordered/detail/narrow_cast.hpp>
+#include <boost/unordered/detail/mulx.hpp>
 #include <boost/unordered/hash_traits.hpp>
 #include <climits>
 #include <cmath>
 #include <cstddef>
 #include <cstring>
-#include <iterator>
 #include <limits>
+#include <memory>
+#include <new>
 #include <tuple>
 #include <type_traits>
 #include <utility>
 
-#if defined(__SSE2__)||\
+#if !defined(BOOST_UNORDERED_DISABLE_SSE2)
+#if defined(BOOST_UNORDERED_ENABLE_SSE2)|| \
+    defined(__SSE2__)|| \
     defined(_M_X64)||(defined(_M_IX86_FP)&&_M_IX86_FP>=2)
 #define BOOST_UNORDERED_SSE2
-#include <emmintrin.h>
-#elif defined(__ARM_NEON)&&!defined(__ARM_BIG_ENDIAN)
+#endif
+#endif
+
+#if !defined(BOOST_UNORDERED_DISABLE_NEON)
+#if defined(BOOST_UNORDERED_ENABLE_NEON)||\
+    (defined(__ARM_NEON)&&!defined(__ARM_BIG_ENDIAN))
 #define BOOST_UNORDERED_LITTLE_ENDIAN_NEON
+#endif
+#endif
+
+#if defined(BOOST_UNORDERED_SSE2)
+#include <emmintrin.h>
+#elif defined(BOOST_UNORDERED_LITTLE_ENDIAN_NEON)
 #include <arm_neon.h>
 #endif
 
@@ -67,15 +86,70 @@
   }while(0)
 #endif
 
+/* We use BOOST_UNORDERED_PREFETCH[_ELEMENTS] macros rather than proper
+ * functions because of https://gcc.gnu.org/bugzilla/show_bug.cgi?id=109985
+ */
+
+#if defined(BOOST_GCC)||defined(BOOST_CLANG)
+#define BOOST_UNORDERED_PREFETCH(p) __builtin_prefetch((const char*)(p))
+#elif defined(BOOST_UNORDERED_SSE2)
+#define BOOST_UNORDERED_PREFETCH(p) _mm_prefetch((const char*)(p),_MM_HINT_T0)
+#else
+#define BOOST_UNORDERED_PREFETCH(p) ((void)0)
+#endif
+
+/* We have experimentally confirmed that ARM architectures get a higher
+ * speedup when around the first half of the element slots in a group are
+ * prefetched, whereas for Intel just the first cache line is best.
+ * Please report back if you find better tunings for some particular
+ * architectures.
+ */
+
+#if BOOST_ARCH_ARM
+/* Cache line size can't be known at compile time, so we settle on
+ * the very frequent value of 64B.
+ */
+
+#define BOOST_UNORDERED_PREFETCH_ELEMENTS(p,N)                          \
+  do{                                                                   \
+    auto           BOOST_UNORDERED_P=(p);                               \
+    constexpr int  cache_line=64;                                       \
+    const char    *p0=reinterpret_cast<const char*>(BOOST_UNORDERED_P), \
+                  *p1=p0+sizeof(*BOOST_UNORDERED_P)*(N)/2;              \
+    for(;p0<p1;p0+=cache_line)BOOST_UNORDERED_PREFETCH(p0);             \
+  }while(0)
+#else
+#define BOOST_UNORDERED_PREFETCH_ELEMENTS(p,N) BOOST_UNORDERED_PREFETCH(p)
+#endif
+
+#ifdef __has_feature
+#define BOOST_UNORDERED_HAS_FEATURE(x) __has_feature(x)
+#else
+#define BOOST_UNORDERED_HAS_FEATURE(x) 0
+#endif
+
+#if BOOST_UNORDERED_HAS_FEATURE(thread_sanitizer)|| \
+    defined(__SANITIZE_THREAD__)
+#define BOOST_UNORDERED_THREAD_SANITIZER
+#endif
+
+#define BOOST_UNORDERED_STATIC_ASSERT_HASH_PRED(Hash, Pred)                    \
+  static_assert(boost::is_nothrow_swappable<Hash>::value,                      \
+    "Template parameter Hash is required to be nothrow Swappable.");           \
+  static_assert(boost::is_nothrow_swappable<Pred>::value,                      \
+    "Template parameter Pred is required to be nothrow Swappable");
+
 namespace boost{
 namespace unordered{
 namespace detail{
 namespace foa{
 
-static const std::size_t default_bucket_count = 0;
+static constexpr std::size_t default_bucket_count=0;
 
-/* foa::table is an open-addressing hash table serving as the foundational core
- * of boost::unordered_flat_[map|set]. Its main internal design aspects are:
+/* foa::table_core is the common base of foa::table and foa::concurrent_table,
+ * which in their turn serve as the foundational core of
+ * boost::unordered_(flat|node)_(map|set) and boost::concurrent_flat_map,
+ * respectively. Its main internal design aspects are:
  * 
  *   - Element slots are logically split into groups of size N=15. The number
  *     of groups is always a power of two, so the number of allocated slots
@@ -128,23 +202,32 @@ static const std::size_t default_bucket_count = 0;
  * "logical" 128-bit word, and so forth. With this layout, match can be
  * implemented with 4 ANDs, 3 shifts, 2 XORs, 1 OR and 1 NOT.
  * 
- * group15 has no user-defined ctor so that it's a trivial type and can be
- * initialized via memset etc. Where needed, group15::initialize sets the
- * metadata to all zeros.
+ * IntegralWrapper<Integral> is used to implement group15's underlying
+ * metadata: it behaves as a plain integral for foa::table or introduces
+ * atomic ops for foa::concurrent_table. If IntegralWrapper<...> is trivially
+ * constructible, so is group15, in which case it can be initialized via memset
+ * etc. Where needed, group15::initialize resets the metadata to the all
+ * zeros (default state).
  */
 
 #if defined(BOOST_UNORDERED_SSE2)
 
+template<template<typename> class IntegralWrapper>
 struct group15
 {
-  static constexpr int N=15;
+  static constexpr std::size_t N=15;
+  static constexpr bool        regular_layout=true;
 
   struct dummy_group_type
   {
     alignas(16) unsigned char storage[N+1]={0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0};
   };
 
-  inline void initialize(){m=_mm_setzero_si128();}
+  inline void initialize()
+  {
+    _mm_store_si128(
+      reinterpret_cast<__m128i*>(m),_mm_setzero_si128());
+  }
 
   inline void set(std::size_t pos,std::size_t hash)
   {
@@ -163,6 +246,11 @@ struct group15
     return at(pos)==sentinel_;
   }
 
+  static inline bool is_sentinel(unsigned char* pc)noexcept
+  {
+    return *pc==sentinel_;
+  }
+
   inline void reset(std::size_t pos)
   {
     BOOST_ASSERT(pos<N);
@@ -171,13 +259,13 @@ struct group15
 
   static inline void reset(unsigned char* pc)
   {
-    *pc=available_;
+    *reinterpret_cast<slot_type*>(pc)=available_;
   }
 
   inline int match(std::size_t hash)const
   {
     return _mm_movemask_epi8(
-      _mm_cmpeq_epi8(m,_mm_set1_epi32(match_word(hash))))&0x7FFF;
+      _mm_cmpeq_epi8(load_metadata(),_mm_set1_epi32(match_word(hash))))&0x7FFF;
   }
 
   inline bool is_not_overflowed(std::size_t hash)const
@@ -189,11 +277,7 @@ struct group15
 
   inline void mark_overflow(std::size_t hash)
   {
-#if BOOST_WORKAROUND(BOOST_GCC, >= 50000 && BOOST_GCC < 60000)
-    overflow() = static_cast<unsigned char>( overflow() | static_cast<unsigned char>(1<<(hash%8)) );
-#else
     overflow()|=static_cast<unsigned char>(1<<(hash%8));
-#endif
   }
 
   static inline bool maybe_caused_overflow(unsigned char* pc)
@@ -201,28 +285,54 @@ struct group15
     std::size_t pos=reinterpret_cast<uintptr_t>(pc)%sizeof(group15);
     group15    *pg=reinterpret_cast<group15*>(pc-pos);
     return !pg->is_not_overflowed(*pc);
-  };
+  }
 
   inline int match_available()const
   {
     return _mm_movemask_epi8(
-      _mm_cmpeq_epi8(m,_mm_setzero_si128()))&0x7FFF;
+      _mm_cmpeq_epi8(load_metadata(),_mm_setzero_si128()))&0x7FFF;
   }
 
-  inline int match_occupied()const
+  inline bool is_occupied(std::size_t pos)const
   {
-    return (~match_available())&0x7FFF;
+    BOOST_ASSERT(pos<N);
+    return at(pos)!=available_;
   }
 
-  inline int match_really_occupied()const /* excluding sentinel */
+  static inline bool is_occupied(unsigned char* pc)noexcept
+  {
+    return *reinterpret_cast<slot_type*>(pc)!=available_;
+  }
+
+  inline int match_occupied()const
   {
-    return at(N-1)==sentinel_?match_occupied()&0x3FFF:match_occupied();
+    return (~match_available())&0x7FFF;
   }
 
 private:
+  using slot_type=IntegralWrapper<unsigned char>;
+  BOOST_STATIC_ASSERT(sizeof(slot_type)==1);
+
   static constexpr unsigned char available_=0,
                                  sentinel_=1;
 
+  inline __m128i load_metadata()const
+  {
+#if defined(BOOST_UNORDERED_THREAD_SANITIZER)
+    /* ThreadSanitizer complains on 1-byte atomic writes combined with
+     * 16-byte atomic reads.
+     */
+
+    return _mm_set_epi8(
+      (char)m[15],(char)m[14],(char)m[13],(char)m[12],
+      (char)m[11],(char)m[10],(char)m[ 9],(char)m[ 8],
+      (char)m[ 7],(char)m[ 6],(char)m[ 5],(char)m[ 4],
+      (char)m[ 3],(char)m[ 2],(char)m[ 1],(char)m[ 0]);
+#else
+    return _mm_load_si128(reinterpret_cast<const __m128i*>(m));
+#endif
+  }
+
   inline static int match_word(std::size_t hash)
   {
     static constexpr boost::uint32_t word[]=
@@ -261,57 +371,54 @@ private:
       0xF8F8F8F8u,0xF9F9F9F9u,0xFAFAFAFAu,0xFBFBFBFBu,0xFCFCFCFCu,0xFDFDFDFDu,0xFEFEFEFEu,0xFFFFFFFFu,
     };
 
-#if defined(__MSVC_RUNTIME_CHECKS)
-    return (int)word[hash&0xffu];
-#else
-    return (int)word[(unsigned char)hash];
-#endif
+    return (int)word[narrow_cast<unsigned char>(hash)];
   }
 
   inline static unsigned char reduced_hash(std::size_t hash)
   {
-#if defined(__MSVC_RUNTIME_CHECKS)
-    return match_word(hash)&0xffu;
-#else
-    return (unsigned char)match_word(hash);
-#endif
+    return narrow_cast<unsigned char>(match_word(hash));
   }
 
-  inline unsigned char& at(std::size_t pos)
+  inline slot_type& at(std::size_t pos)
   {
-    return reinterpret_cast<unsigned char*>(&m)[pos];
+    return m[pos];
   }
 
-  inline unsigned char at(std::size_t pos)const
+  inline const slot_type& at(std::size_t pos)const
   {
-    return reinterpret_cast<const unsigned char*>(&m)[pos];
+    return m[pos];
   }
 
-  inline unsigned char& overflow()
+  inline slot_type& overflow()
   {
     return at(N);
   }
 
-  inline unsigned char overflow()const
+  inline const slot_type& overflow()const
   {
     return at(N);
   }
 
-  alignas(16) __m128i m;
+  alignas(16) slot_type m[16];
 };
 
 #elif defined(BOOST_UNORDERED_LITTLE_ENDIAN_NEON)
 
+template<template<typename> class IntegralWrapper>
 struct group15
 {
-  static constexpr int N=15;
+  static constexpr std::size_t N=15;
+  static constexpr bool        regular_layout=true;
 
   struct dummy_group_type
   {
     alignas(16) unsigned char storage[N+1]={0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0};
   };
 
-  inline void initialize(){m=vdupq_n_s8(0);}
+  inline void initialize()
+  {
+    vst1q_u8(reinterpret_cast<uint8_t*>(m),vdupq_n_u8(0));
+  }
 
   inline void set(std::size_t pos,std::size_t hash)
   {
@@ -330,6 +437,11 @@ struct group15
     return pos==N-1&&at(N-1)==sentinel_;
   }
 
+  static inline bool is_sentinel(unsigned char* pc)noexcept
+  {
+    return *reinterpret_cast<slot_type*>(pc)==sentinel_;
+  }
+
   inline void reset(std::size_t pos)
   {
     BOOST_ASSERT(pos<N);
@@ -338,13 +450,13 @@ struct group15
 
   static inline void reset(unsigned char* pc)
   {
-    *pc=available_;
+    *reinterpret_cast<slot_type*>(pc)=available_;
   }
 
   inline int match(std::size_t hash)const
   {
-    return simde_mm_movemask_epi8(
-      vceqq_s8(m,vdupq_n_s8(reduced_hash(hash))))&0x7FFF;
+    return simde_mm_movemask_epi8(vceqq_u8(
+      load_metadata(),vdupq_n_u8(reduced_hash(hash))))&0x7FFF;
   }
 
   inline bool is_not_overflowed(std::size_t hash)const
@@ -368,24 +480,50 @@ struct group15
 
   inline int match_available()const
   {
-    return simde_mm_movemask_epi8(vceqq_s8(m,vdupq_n_s8(0)))&0x7FFF;
+    return simde_mm_movemask_epi8(vceqq_u8(
+      load_metadata(),vdupq_n_u8(0)))&0x7FFF;
   }
 
-  inline int match_occupied()const
+  inline bool is_occupied(std::size_t pos)const
+  {
+    BOOST_ASSERT(pos<N);
+    return at(pos)!=available_;
+  }
+
+  static inline bool is_occupied(unsigned char* pc)noexcept
   {
-    return simde_mm_movemask_epi8(
-      vcgtq_u8(vreinterpretq_u8_s8(m),vdupq_n_u8(0)))&0x7FFF;
+    return *reinterpret_cast<slot_type*>(pc)!=available_;
   }
 
-  inline int match_really_occupied()const /* excluding sentinel */
+  inline int match_occupied()const
   {
-    return at(N-1)==sentinel_?match_occupied()&0x3FFF:match_occupied();
+    return simde_mm_movemask_epi8(vcgtq_u8(
+      load_metadata(),vdupq_n_u8(0)))&0x7FFF;
   }
 
 private:
+  using slot_type=IntegralWrapper<unsigned char>;
+  BOOST_STATIC_ASSERT(sizeof(slot_type)==1);
+
   static constexpr unsigned char available_=0,
                                  sentinel_=1;
 
+  inline uint8x16_t load_metadata()const
+  {
+#if defined(BOOST_UNORDERED_THREAD_SANITIZER)
+    /* ThreadSanitizer complains on 1-byte atomic writes combined with
+     * 16-byte atomic reads.
+     */
+
+    alignas(16) uint8_t data[16]={
+      m[ 0],m[ 1],m[ 2],m[ 3],m[ 4],m[ 5],m[ 6],m[ 7],
+      m[ 8],m[ 9],m[10],m[11],m[12],m[13],m[14],m[15]};
+    return vld1q_u8(data);
+#else
+    return vld1q_u8(reinterpret_cast<const uint8_t*>(m));
+#endif
+  }
+
   inline static unsigned char reduced_hash(std::size_t hash)
   {
     static constexpr unsigned char table[]={
@@ -435,34 +573,36 @@ private:
 #endif
   }
 
-  inline unsigned char& at(std::size_t pos)
+  inline slot_type& at(std::size_t pos)
   {
-    return reinterpret_cast<unsigned char*>(&m)[pos];
+    return m[pos];
   }
 
-  inline unsigned char at(std::size_t pos)const
+  inline const slot_type& at(std::size_t pos)const
   {
-    return reinterpret_cast<const unsigned char*>(&m)[pos];
+    return m[pos];
   }
 
-  inline unsigned char& overflow()
+  inline slot_type& overflow()
   {
     return at(N);
   }
 
-  inline unsigned char overflow()const
+  inline const slot_type& overflow()const
   {
     return at(N);
   }
 
-  alignas(16) int8x16_t m;
+  alignas(16) slot_type m[16];
 };
 
 #else /* non-SIMD */
 
+template<template<typename> class IntegralWrapper>
 struct group15
 {
-  static constexpr int N=15;
+  static constexpr std::size_t N=15;
+  static constexpr bool        regular_layout=false;
 
   struct dummy_group_type
   {
@@ -525,11 +665,7 @@ struct group15
     std::size_t     pos=reinterpret_cast<uintptr_t>(pc)%sizeof(group15);
     group15        *pg=reinterpret_cast<group15*>(pc-pos);
     boost::uint64_t x=((pg->m[0])>>pos)&0x000100010001ull;
-#if defined(__MSVC_RUNTIME_CHECKS)
-    boost::uint32_t y=(x|(x>>15)|(x>>30))&0xffffffffu;
-#else
-    boost::uint32_t y=static_cast<boost::uint32_t>(x|(x>>15)|(x>>30));
-#endif
+    boost::uint32_t y=narrow_cast<boost::uint32_t>(x|(x>>15)|(x>>30));
     return !pg->is_not_overflowed(y);
   };
 
@@ -541,24 +677,25 @@ struct group15
     return y&0x7FFF;
   }
 
-  inline int match_occupied()const
+  inline bool is_occupied(std::size_t pos)const
   {
+    BOOST_ASSERT(pos<N);
     boost::uint64_t x=m[0]|m[1];
-#if defined(__MSVC_RUNTIME_CHECKS)
-    boost::uint32_t y=(x|(x>>32))&0xffffffffu;
-#else
-    boost::uint32_t y=static_cast<boost::uint32_t>(x|(x>>32));
-#endif
-    y|=y>>16;
-    return y&0x7FFF;
+    return (x&(0x0001000100010001ull<<pos))!=0;
   }
 
-  inline int match_really_occupied()const /* excluding sentinel */
+  inline int match_occupied()const
   {
-    return ~(match_impl(0)|match_impl(1))&0x7FFF;
+    boost::uint64_t x=m[0]|m[1];
+    boost::uint32_t y=narrow_cast<boost::uint32_t>(x|(x>>32));
+    y|=y>>16;
+    return y&0x7FFF;
   }
 
 private:
+  using word_type=IntegralWrapper<uint64_t>;
+  BOOST_STATIC_ASSERT(sizeof(word_type)==8);
+
   static constexpr unsigned char available_=0,
                                  sentinel_=1;
 
@@ -583,11 +720,7 @@ private:
       240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,
     };
     
-#if defined(__MSVC_RUNTIME_CHECKS)
-  return table[hash&0xffu];
-#else
-    return table[(unsigned char)hash];
-#endif
+    return table[narrow_cast<unsigned char>(hash)];
   }
 
   inline void set_impl(std::size_t pos,std::size_t n)
@@ -597,7 +730,7 @@ private:
     set_impl(m[1],pos,n>>4);
   }
 
-  static inline void set_impl(boost::uint64_t& x,std::size_t pos,std::size_t n)
+  static inline void set_impl(word_type& x,std::size_t pos,std::size_t n)
   {
     static constexpr boost::uint64_t mask[]=
     {
@@ -643,14 +776,14 @@ private:
     return          y&0x7FFF;
   }
 
-  alignas(16) boost::uint64_t m[2];
+  alignas(16) word_type m[2];
 };
 
 #endif
 
-/* foa::table uses a size policy to obtain the permissible sizes of the group
- * array (and, by implication, the element array) and to do the hash->group
- * mapping.
+/* foa::table_core uses a size policy to obtain the permissible sizes of the
+ * group array (and, by implication, the element array) and to do the
+ * hash->group mapping.
  * 
  *   - size_index(n) returns an unspecified "index" number used in other policy
  *     operations.
@@ -736,10 +869,12 @@ private:
   std::size_t pos,step=0;
 };
 
-/* Mixing policies: no_mix is the identity function and xmx_mix uses the
- * xmx function defined in <boost/unordered/detail/xmx.hpp>.
- * foa::table mixes hash results with xmx_mix unless the hash is marked as
- * avalanching, i.e. of good quality (see <boost/unordered/hash_traits.hpp>).
+/* Mixing policies: no_mix is the identity function, and mulx_mix
+ * uses the mulx function from <boost/unordered/detail/mulx.hpp>.
+ *
+ * foa::table_core mixes hash results with mulx_mix unless the hash is marked
+ * as avalanching, i.e. of good quality
+ * (see <boost/unordered/hash_traits.hpp>).
  */
 
 struct no_mix
@@ -751,12 +886,12 @@ struct no_mix
   }
 };
 
-struct xmx_mix
+struct mulx_mix
 {
   template<typename Hash,typename T>
   static inline std::size_t mix(const Hash& h,const T& x)
   {
-    return xmx(h(x));
+    return mulx(h(x));
   }
 };
 
@@ -776,112 +911,6 @@ inline unsigned int unchecked_countr_zero(int x)
 #endif
 }
 
-template<typename,typename,typename,typename>
-class table;
-
-/* table_iterator keeps two pointers:
- * 
- *   - A pointer p to the element slot.
- *   - A pointer pc to the n-th byte of the associated group metadata, where n
- *     is the position of the element in the group.
- *
- * A simpler solution would have been to keep a pointer p to the element, a
- * pointer pg to the group, and the position n, but that would increase
- * sizeof(table_iterator) by 4/8 bytes. In order to make this compact
- * representation feasible, it is required that group objects are aligned
- * to their size, so that we can recover pg and n as
- * 
- *   - n = pc%sizeof(group)
- *   - pg = pc-n
- * 
- * (for explanatory purposes pg and pc are treated above as if they were memory
- * addresses rather than pointers).The main drawback of this two-pointer
- * representation is that iterator increment is relatively slow.
- * 
- * p = nullptr is conventionally used to mark end() iterators.
- */
-
-/* internal conversion from const_iterator to iterator */
-class const_iterator_cast_tag {}; 
-
-template<typename Value,typename Group,bool Const>
-class table_iterator
-{
-public:
-  using difference_type=std::ptrdiff_t;
-  using value_type=Value;
-  using pointer=
-    typename std::conditional<Const,value_type const*,value_type*>::type;
-  using reference=
-    typename std::conditional<Const,value_type const&,value_type&>::type;
-  using iterator_category=std::forward_iterator_tag;
-  using element_type=
-    typename std::conditional<Const,value_type const,value_type>::type;
-
-  table_iterator()=default;
-  template<bool Const2,typename std::enable_if<!Const2>::type* =nullptr>
-  table_iterator(const table_iterator<Value,Group,Const2>& x):
-    pc{x.pc},p{x.p}{}
-  table_iterator(
-    const_iterator_cast_tag, const table_iterator<Value,Group,true>& x):
-    pc{x.pc},p{x.p}{}
-
-  inline reference operator*()const noexcept{return *p;}
-  inline pointer operator->()const noexcept{return p;}
-  inline table_iterator& operator++()noexcept{increment();return *this;}
-  inline table_iterator operator++(int)noexcept
-    {auto x=*this;increment();return x;}
-  friend inline bool operator==(
-    const table_iterator& x,const table_iterator& y)
-    {return x.p==y.p;}
-  friend inline bool operator!=(
-    const table_iterator& x,const table_iterator& y)
-    {return !(x==y);}
-
-private:
-  template<typename,typename,bool> friend class table_iterator;
-  template<typename,typename,typename,typename> friend class table;
-
-  table_iterator(Group* pg,std::size_t n,const Value* p_):
-    pc{reinterpret_cast<unsigned char*>(const_cast<Group*>(pg))+n},
-    p{const_cast<Value*>(p_)}
-    {}
-
-  inline std::size_t rebase() noexcept
-  {
-    std::size_t off=reinterpret_cast<uintptr_t>(pc)%sizeof(Group);
-    pc-=off;
-    return off;
-  }
-
-  inline void increment()noexcept
-  {
-    std::size_t n0=rebase();
-
-    int mask=(reinterpret_cast<Group*>(pc)->match_occupied()>>(n0+1))<<(n0+1);
-    if(!mask){
-      do{
-        pc+=sizeof(Group);
-        p+=Group::N;
-      }
-      while((mask=reinterpret_cast<Group*>(pc)->match_occupied())==0);
-    }
-
-    auto n=unchecked_countr_zero(mask);
-    if(BOOST_UNLIKELY(reinterpret_cast<Group*>(pc)->is_sentinel(n))){
-      p=nullptr;
-    }
-    else{
-      pc+=n;
-      p-=n0;
-      p+=n;
-    }
-  }
-
-  unsigned char *pc=nullptr;
-  Value         *p=nullptr;
-};
-
 /* table_arrays controls allocation, initialization and deallocation of
  * paired arrays of groups and element slots. Only one chunk of memory is
  * allocated to place both arrays: this is not done for efficiency reasons,
@@ -917,10 +946,15 @@ struct table_arrays
   static constexpr auto N=group_type::N;
   using size_policy=SizePolicy;
 
+  table_arrays(std::size_t gsi,std::size_t gsm,group_type *pg,value_type *pe):
+    groups_size_index{gsi},groups_size_mask{gsm},groups{pg},elements{pe}{}
+
   template<typename Allocator>
   static table_arrays new_(Allocator& al,std::size_t n)
   {
-    using alloc_traits=boost::allocator_traits<Allocator>;
+    using storage_allocator=
+      typename boost::allocator_rebind<Allocator, Value>::type;
+    using storage_traits=boost::allocator_traits<storage_allocator>;
 
     auto         groups_size_index=size_index_for<group_type,size_policy>(n);
     auto         groups_size=size_policy::size(groups_size_index);
@@ -930,8 +964,9 @@ struct table_arrays
       arrays.groups=dummy_groups<group_type,size_policy::min_size()>();
     }
     else{
-      arrays.elements=
-        boost::to_address(alloc_traits::allocate(al,buffer_size(groups_size)));
+      auto sal=storage_allocator(al);
+      arrays.elements=boost::to_address(
+        storage_traits::allocate(sal,buffer_size(groups_size)));
       
       /* Align arrays.groups to sizeof(group_type). table_iterator critically
        * depends on such alignment for its increment operation.
@@ -942,11 +977,17 @@ struct table_arrays
           reinterpret_cast<uintptr_t>(p))%sizeof(group_type);
       arrays.groups=reinterpret_cast<group_type*>(p);
 
-      /* memset is faster/not slower than initializing groups individually.
-       * This assumes all zeros is group_type's default layout. 
-       */
-
-      std::memset(arrays.groups,0,sizeof(group_type)*groups_size);
+      initialize_groups(
+        arrays.groups,groups_size,
+        std::integral_constant<
+          bool,
+#if BOOST_WORKAROUND(BOOST_LIBSTDCXX_VERSION,<50000)
+        /* std::is_trivially_constructible not provided */
+        boost::has_trivial_constructor<group_type>::value
+#else
+        std::is_trivially_constructible<group_type>::value
+#endif  
+        >{});
       arrays.groups[groups_size-1].set_sentinel();
     }
     return arrays;
@@ -955,13 +996,15 @@ struct table_arrays
   template<typename Allocator>
   static void delete_(Allocator& al,table_arrays& arrays)noexcept
   {
-    using alloc_traits=boost::allocator_traits<Allocator>;
-    using pointer=typename alloc_traits::pointer;
+    using storage_alloc=typename boost::allocator_rebind<Allocator,Value>::type;
+    using storage_traits=boost::allocator_traits<storage_alloc>;
+    using pointer=typename storage_traits::pointer;
     using pointer_traits=boost::pointer_traits<pointer>;
 
+    auto sal=storage_alloc(al);
     if(arrays.elements){
-      alloc_traits::deallocate(
-        al,pointer_traits::pointer_to(*arrays.elements),
+      storage_traits::deallocate(
+        sal,pointer_traits::pointer_to(*arrays.elements),
         buffer_size(arrays.groups_size_mask+1));
     }
   }
@@ -980,6 +1023,25 @@ struct table_arrays
     return (buffer_bytes+sizeof(value_type)-1)/sizeof(value_type);
   }
 
+  static void initialize_groups(
+    group_type* groups_,std::size_t size,std::true_type /* memset */)
+  {
+    /* memset faster/not slower than manual, assumes all zeros is group_type's
+     * default layout.
+     * reinterpret_cast: GCC may complain about group_type not being trivially
+     * copy-assignable when we're relying on trivial copy constructibility.
+     */
+
+    std::memset(
+      reinterpret_cast<unsigned char*>(groups_),0,sizeof(group_type)*size);
+  }
+
+  static void initialize_groups(
+    group_type* groups_,std::size_t size,std::false_type /* manual */)
+  {
+    while(size--!=0)::new (groups_++) group_type();
+  }
+
   std::size_t  groups_size_index;
   std::size_t  groups_size_mask;
   group_type  *groups;
@@ -1012,130 +1074,195 @@ void swap_if(T& x,T& y){using std::swap; swap(x,y);}
 template<bool B,typename T,typename std::enable_if<!B>::type* =nullptr>
 void swap_if(T&,T&){}
 
-inline void prefetch(const void* p)
-{
-  (void) p;
-#if defined(BOOST_GCC)||defined(BOOST_CLANG)
-  __builtin_prefetch((const char*)p);
-#elif defined(BOOST_UNORDERED_SSE2)
-  _mm_prefetch((const char*)p,_MM_HINT_T0);
-#endif    
-}
+template<typename Allocator>
+struct is_std_allocator:std::false_type{};
 
-#if defined(BOOST_GCC)
-/* GCC's -Wshadow triggers at scenarios like this: 
- *
- *   struct foo{};
- *   template<typename Base>
- *   struct derived:Base
- *   {
- *     void f(){int foo;}
- *   };
- * 
- *   derived<foo>x;
- *   x.f(); // declaration of "foo" in derived::f shadows base type "foo"
- *
- * This makes shadowing warnings unavoidable in general when a class template
- * derives from user-provided classes, as is the case with table and
- * empty_value's below.
- */
+template<typename T>
+struct is_std_allocator<std::allocator<T>>:std::true_type{};
 
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wshadow"
+/* std::allocator::construct marked as deprecated */
+#if defined(_LIBCPP_SUPPRESS_DEPRECATED_PUSH)
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
+#elif defined(_STL_DISABLE_DEPRECATED_WARNING)
+_STL_DISABLE_DEPRECATED_WARNING
+#elif defined(_MSC_VER)
+#pragma warning(push)
+#pragma warning(disable:4996)
 #endif
 
-#if defined(BOOST_MSVC)
-#pragma warning(push)
-#pragma warning(disable:4714) /* marked as __forceinline not inlined */
+template<typename Allocator,typename Ptr,typename... Args>
+struct alloc_has_construct
+{
+private:
+  template<typename Allocator2>
+  static decltype(
+    std::declval<Allocator2&>().construct(
+      std::declval<Ptr>(),std::declval<Args&&>()...),
+    std::true_type{}
+  ) check(int);
+
+  template<typename> static std::false_type check(...);
+
+public:
+  static constexpr bool value=decltype(check<Allocator>(0))::value;
+};
+
+#if defined(_LIBCPP_SUPPRESS_DEPRECATED_POP)
+_LIBCPP_SUPPRESS_DEPRECATED_POP
+#elif defined(_STL_RESTORE_DEPRECATED_WARNING)
+_STL_RESTORE_DEPRECATED_WARNING
+#elif defined(_MSC_VER)
+#pragma warning(pop)
 #endif
 
-#if BOOST_WORKAROUND(BOOST_MSVC,<=1900)
-/* VS2015 marks as unreachable generic catch clauses around non-throwing
- * code.
+/* We expose the hard-coded max load factor so that tests can use it without
+ * needing to pull it from an instantiated class template such as the table
+ * class.
  */
-#pragma warning(push)
-#pragma warning(disable:4702)
-#endif
+static constexpr float mlf=0.875f;
 
-/* foa::table interface departs in a number of ways from that of C++ unordered
- * associative containers because it's not for end-user consumption
- * (boost::unordered_flat_[map|set] wrappers complete it as appropriate) and,
- * more importantly, because of fundamental restrictions imposed by open
- * addressing:
- * 
- *   - value_type must be moveable.
- *   - Pointer stability is not kept under rehashing.
- *   - begin() is not O(1).
- *   - No bucket API.
- *   - Load factor is fixed and can't be set by the user.
- *   - No extract API.
- * 
- * The TypePolicy template parameter is used to generate instantiations
- * suitable for either maps or sets, and introduces non-standard init_type:
- * 
+template<typename Group,typename Element>
+struct table_locator
+{
+  table_locator()=default;
+  table_locator(Group* pg_,unsigned int n_,Element* p_):pg{pg_},n{n_},p{p_}{}
+
+  explicit operator bool()const noexcept{return p!=nullptr;}
+
+  Group        *pg=nullptr;
+  unsigned int  n=0;
+  Element      *p=nullptr;
+};
+
+struct try_emplace_args_t{};
+
+template<typename TypePolicy,typename Allocator,typename... Args>
+class alloc_cted_insert_type
+{
+  using emplace_type=typename std::conditional<
+    std::is_constructible<typename TypePolicy::init_type,Args...>::value,
+    typename TypePolicy::init_type,
+    typename TypePolicy::value_type
+  >::type;
+
+  using insert_type=typename std::conditional<
+    std::is_constructible<typename TypePolicy::value_type,emplace_type>::value,
+    emplace_type,typename TypePolicy::element_type
+  >::type;
+
+  alignas(insert_type) unsigned char storage[sizeof(insert_type)];
+  Allocator                          al;
+
+public:
+  alloc_cted_insert_type(const Allocator& al_,Args&&... args):al{al_}
+  {
+    TypePolicy::construct(al,data(),std::forward<Args>(args)...);
+  }
+
+  ~alloc_cted_insert_type()
+  {
+    TypePolicy::destroy(al,data());
+  }
+
+  insert_type* data(){return reinterpret_cast<insert_type*>(&storage);}
+  insert_type& value(){return *data();}
+};
+
+template<typename TypePolicy,typename Allocator,typename... Args>
+alloc_cted_insert_type<TypePolicy,Allocator,Args...>
+alloc_make_insert_type(const Allocator& al,Args&&... args)
+{
+  return {al,std::forward<Args>(args)...};
+}
+
+/* table_core. The TypePolicy template parameter is used to generate
+ * instantiations suitable for either maps or sets, and introduces non-standard
+ * init_type and element_type:
+ *
  *   - TypePolicy::key_type and TypePolicy::value_type have the obvious
  *     meaning.
+ *
  *   - TypePolicy::init_type is the type implicitly converted to when
  *     writing x.insert({...}). For maps, this is std::pair<Key,T> rather
  *     than std::pair<const Key,T> so that, for instance, x.insert({"hello",0})
  *     produces a cheaply moveable std::string&& ("hello") rather than
  *     a copyable const std::string&&. foa::table::insert is extended to accept
  *     both init_type and value_type references.
- *   - TypePolicy::move(value_type&) returns a temporary object for value
- *     transfer on rehashing, move copy/assignment, and merge. For maps, this
- *     object is a std::pair<Key&&,T&&>, which is generally cheaper to move
- *     than std::pair<const Key,T>&& because of the constness in Key.
+ *
+ *   - TypePolicy::construct and TypePolicy::destroy are used for the
+ *     construction and destruction of the internal types: value_type, init_type
+ *     and element_type.
+ * 
+ *   - TypePolicy::move is used to provide move semantics for the internal
+ *     types used by the container during rehashing and emplace. These types
+ *     are init_type, value_type and emplace_type. During insertion, a
+ *     stack-local type will be created based on the constructibility of the
+ *     value_type and the supplied arguments. TypePolicy::move is used here
+ *     for transfer of ownership. Similarly, TypePolicy::move is also used
+ *     during rehashing when elements are moved to the new table.
+ *
  *   - TypePolicy::extract returns a const reference to the key part of
- *     a value of type value_type, init_type or
+ *     a value of type value_type, init_type, element_type or
  *     decltype(TypePolicy::move(...)).
- * 
- *  try_emplace, erase and find support heterogenous lookup by default, that is,
- *  without checking for any ::is_transparent typedefs --the checking is done by
- *  boost::unordered_flat_[map|set].
- * 
- *  At the moment, we're not supporting allocators with fancy pointers.
- *  Allocator::pointer must be convertible to/from regular pointers.
+ *
+ *   - TypePolicy::element_type is the type that table_arrays uses when
+ *     allocating buckets, which allows us to have flat and node container.
+ *     For flat containers, element_type is value_type. For node
+ *     containers, it is a strong typedef to value_type*.
+ *
+ *   - TypePolicy::value_from returns a mutable reference to value_type from
+ *     a given element_type. This is used when elements of the table themselves
+ *     need to be moved, such as during move construction/assignment when
+ *     allocators are unequal and there is no propagation. For all other cases,
+ *     the element_type itself is moved.
  */
 
-/* We pull this out so the tests don't have to rely on a magic constant or
- * instantiate the table class template as it can be quite gory.
+#include <boost/unordered/detail/foa/ignore_wshadow.hpp>
+
+#if defined(BOOST_MSVC)
+#pragma warning(push)
+#pragma warning(disable:4714) /* marked as __forceinline not inlined */
+#endif
+
+#if BOOST_WORKAROUND(BOOST_MSVC,<=1900)
+/* VS2015 marks as unreachable generic catch clauses around non-throwing
+ * code.
  */
-constexpr static float const mlf = 0.875f;
+#pragma warning(push)
+#pragma warning(disable:4702)
+#endif
 
-template<typename TypePolicy,typename Hash,typename Pred,typename Allocator>
+template<
+  typename TypePolicy,typename Group,template<typename...> class Arrays,
+  typename SizeControl,typename Hash,typename Pred,typename Allocator
+>
 class 
 
 #if defined(_MSC_VER)&&_MSC_FULL_VER>=190023918
 __declspec(empty_bases) /* activate EBO with multiple inheritance */
 #endif
 
-table:empty_value<Hash,0>,empty_value<Pred,1>,empty_value<Allocator,2>
+table_core:empty_value<Hash,0>,empty_value<Pred,1>,empty_value<Allocator,2>
 {
-  using hash_base=empty_value<Hash,0>;
-  using pred_base=empty_value<Pred,1>;
-  using allocator_base=empty_value<Allocator,2>;
+public:
   using type_policy=TypePolicy;
-  using group_type=group15;
+  using group_type=Group;
   static constexpr auto N=group_type::N;
   using size_policy=pow2_size_policy;
   using prober=pow2_quadratic_prober;
   using mix_policy=typename std::conditional<
     hash_is_avalanching<Hash>::value,
     no_mix,
-    xmx_mix
+    mulx_mix
   >::type;
   using alloc_traits=boost::allocator_traits<Allocator>;
+  using element_type=typename type_policy::element_type;
+  using arrays_type=Arrays<element_type,group_type,size_policy>;
+  using size_ctrl_type=SizeControl;
 
-public:
   using key_type=typename type_policy::key_type;
   using init_type=typename type_policy::init_type;
   using value_type=typename type_policy::value_type;
-
-private:
-  static constexpr bool has_mutable_iterator=
-    !std::is_same<key_type,value_type>::value;
-
-public:
   using hasher=Hash;
   using key_equal=Pred;
   using allocator_type=Allocator;
@@ -1145,24 +1272,20 @@ public:
   using const_reference=const value_type&;
   using size_type=std::size_t;
   using difference_type=std::ptrdiff_t;
-  using const_iterator=table_iterator<value_type,group_type,true>;
-  using iterator=typename std::conditional<
-    has_mutable_iterator,
-    table_iterator<value_type,group_type,false>,
-    const_iterator>::type;
-
-  table(
-    std::size_t n=0,const Hash& h_=Hash(),const Pred& pred_=Pred(),
-    const Allocator& al_=Allocator()):
+  using locator=table_locator<group_type,element_type>;
+
+  table_core(
+    std::size_t n=default_bucket_count,const Hash& h_=Hash(),
+    const Pred& pred_=Pred(),const Allocator& al_=Allocator()):
     hash_base{empty_init,h_},pred_base{empty_init,pred_},
-    allocator_base{empty_init,al_},size_{0},arrays(new_arrays(n)),
-    ml{initial_max_load()}
+    allocator_base{empty_init,al_},arrays(new_arrays(n)),
+    size_ctrl{initial_max_load(),0}
     {}
 
-  table(const table& x):
-    table{x,alloc_traits::select_on_container_copy_construction(x.al())}{}
+  table_core(const table_core& x):
+    table_core{x,alloc_traits::select_on_container_copy_construction(x.al())}{}
 
-  table(table&& x)
+  table_core(table_core&& x)
     noexcept(
       std::is_nothrow_move_constructible<Hash>::value&&
       std::is_nothrow_move_constructible<Pred>::value&&
@@ -1170,28 +1293,26 @@ public:
     hash_base{empty_init,std::move(x.h())},
     pred_base{empty_init,std::move(x.pred())},
     allocator_base{empty_init,std::move(x.al())},
-    size_{x.size_},arrays(x.arrays),ml{x.ml}
+    arrays(x.arrays),size_ctrl(x.size_ctrl)
   {
-    x.size_=0;
     x.arrays=x.new_arrays(0);
-    x.ml=x.initial_max_load();
+    x.size_ctrl.ml=x.initial_max_load();
+    x.size_ctrl.size=0;
   }
 
-  table(const table& x,const Allocator& al_):
-    table{std::size_t(std::ceil(float(x.size())/mlf)),x.h(),x.pred(),al_}
+  table_core(const table_core& x,const Allocator& al_):
+    table_core{std::size_t(std::ceil(float(x.size())/mlf)),x.h(),x.pred(),al_}
   {
-    x.for_all_elements([this](value_type* p){
-      unchecked_insert(*p);
-    });
+    copy_elements_from(x);
   }
 
-  table(table&& x,const Allocator& al_):
-    table{0,std::move(x.h()),std::move(x.pred()),al_}
+  table_core(table_core&& x,const Allocator& al_):
+    table_core{std::move(x.h()),std::move(x.pred()),al_}
   {
     if(al()==x.al()){
-      std::swap(size_,x.size_);
-      std::swap(arrays,x.arrays);
-      std::swap(ml,x.ml);
+      using std::swap;
+      swap(arrays,x.arrays);
+      swap(size_ctrl,x.size_ctrl);
     }
     else{
       reserve(x.size());
@@ -1201,38 +1322,54 @@ public:
       /* This works because subsequent x.clear() does not depend on the
        * elements' values.
        */
-      x.for_all_elements([this](value_type* p){
-        unchecked_insert(type_policy::move(*p));
+      x.for_all_elements([this](element_type* p){
+        unchecked_insert(type_policy::move(type_policy::value_from(*p)));
       });
     }
   }
 
-  ~table()noexcept
+  ~table_core()noexcept
   {
-    for_all_elements([this](value_type* p){
+    for_all_elements([this](element_type* p){
       destroy_element(p);
     });
     delete_arrays(arrays);
   }
 
-  table& operator=(const table& x)
+  table_core& operator=(const table_core& x)
   {
+    BOOST_UNORDERED_STATIC_ASSERT_HASH_PRED(Hash, Pred)
+
     static constexpr auto pocca=
       alloc_traits::propagate_on_container_copy_assignment::value;
 
     if(this!=std::addressof(x)){
+      /* If copy construction here winds up throwing, the container is still
+       * left intact so we perform these operations first.
+       */
+      hasher    tmp_h=x.h();
+      key_equal tmp_p=x.pred();
+
+      /* already noexcept, clear() before we swap the Hash, Pred just in case
+       * the clear() impl relies on them at some point in the future.
+       */
       clear();
-      h()=x.h();
-      pred()=x.pred();
+
+      /* Because we've asserted at compile-time that Hash and Pred are nothrow
+       * swappable, we can safely mutate our source container and maintain
+       * consistency between the Hash, Pred compatibility.
+       */
+      using std::swap;
+      swap(h(),tmp_h);
+      swap(pred(),tmp_p);
+
       if_constexpr<pocca>([&,this]{
         if(al()!=x.al())reserve(0);
         copy_assign_if<pocca>(al(),x.al());
       });
       /* noshrink: favor memory reuse over tightness */
-      noshrink_reserve(x.size()); 
-      x.for_all_elements([this](value_type* p){
-        unchecked_insert(*p);
-      });
+      noshrink_reserve(x.size());
+      copy_elements_from(x);
     }
     return *this;
   }
@@ -1242,26 +1379,38 @@ public:
 #pragma warning(disable:4127) /* conditional expression is constant */
 #endif
 
-  table& operator=(table&& x)
+  table_core& operator=(table_core&& x)
     noexcept(
-      alloc_traits::is_always_equal::value&&
-      std::is_nothrow_move_assignable<Hash>::value&&
-      std::is_nothrow_move_assignable<Pred>::value)
+      alloc_traits::propagate_on_container_move_assignment::value||
+      alloc_traits::is_always_equal::value)
   {
+    BOOST_UNORDERED_STATIC_ASSERT_HASH_PRED(Hash, Pred)
+
     static constexpr auto pocma=
       alloc_traits::propagate_on_container_move_assignment::value;
 
     if(this!=std::addressof(x)){
+      /* Given ambiguity in implementation strategies briefly discussed here:
+       * https://www.open-std.org/jtc1/sc22/wg21/docs/lwg-active.html#2227
+       *
+       * we opt into requiring nothrow swappability and eschew the move
+       * operations associated with Hash, Pred.
+       *
+       * To this end, we ensure that the user never has to consider the
+       * moved-from state of their Hash, Pred objects
+       */
+
+      using std::swap;
+
       clear();
-      h()=std::move(x.h());
-      pred()=std::move(x.pred());
+      swap(h(),x.h());
+      swap(pred(),x.pred());
+
       if(pocma||al()==x.al()){
-        using std::swap;
         reserve(0);
         move_assign_if<pocma>(al(),x.al());
-        swap(size_,x.size_);
         swap(arrays,x.arrays);
-        swap(ml,x.ml);
+        swap(size_ctrl,x.size_ctrl);
       }
       else{
         /* noshrink: favor memory reuse over tightness */
@@ -1272,8 +1421,8 @@ public:
         /* This works because subsequent x.clear() does not depend on the
          * elements' values.
          */
-        x.for_all_elements([this](value_type* p){
-          unchecked_insert(type_policy::move(*p));
+        x.for_all_elements([this](element_type* p){
+          unchecked_insert(type_policy::move(type_policy::value_from(*p)));
         });
       }
     }
@@ -1286,103 +1435,81 @@ public:
 
   allocator_type get_allocator()const noexcept{return al();}
 
-  iterator begin()noexcept
-  {
-    iterator it{arrays.groups,0,arrays.elements};
-    if(!(arrays.groups[0].match_occupied()&0x1))++it;
-    return it;
-  }
-
-  const_iterator begin()const noexcept
-                   {return const_cast<table*>(this)->begin();}
-  iterator       end()noexcept{return {};}
-  const_iterator end()const noexcept{return const_cast<table*>(this)->end();}
-  const_iterator cbegin()const noexcept{return begin();}
-  const_iterator cend()const noexcept{return end();}
-
   bool        empty()const noexcept{return size()==0;}
-  std::size_t size()const noexcept{return size_;}
+  std::size_t size()const noexcept{return size_ctrl.size;}
   std::size_t max_size()const noexcept{return SIZE_MAX;}
 
-  template<typename... Args>
-  BOOST_FORCEINLINE std::pair<iterator,bool> emplace(Args&&... args)
+  BOOST_FORCEINLINE
+  void erase(group_type* pg,unsigned int pos,element_type* p)noexcept
   {
-    using emplace_type = typename std::conditional<
-      std::is_constructible<
-        init_type, Args...
-      >::value,
-      init_type,
-      value_type
-    >::type;
-    return emplace_impl(emplace_type(std::forward<Args>(args)...));
+    destroy_element(p);
+    recover_slot(pg,pos);
   }
 
-  template<typename Key,typename... Args>
-  BOOST_FORCEINLINE std::pair<iterator,bool> try_emplace(
-    Key&& k,Args&&... args)
+  BOOST_FORCEINLINE
+  void erase(unsigned char* pc,element_type* p)noexcept
   {
-    return emplace_impl(
-      std::piecewise_construct,
-      std::forward_as_tuple(std::forward<Key>(k)),
-      std::forward_as_tuple(std::forward<Args>(args)...));
+    destroy_element(p);
+    recover_slot(pc);
   }
 
-  BOOST_FORCEINLINE std::pair<iterator,bool>
-  insert(const init_type& x){return emplace_impl(x);}
-
-  BOOST_FORCEINLINE std::pair<iterator,bool>
-  insert(init_type&& x){return emplace_impl(std::move(x));}
-
-  /* template<typename=void> tilts call ambiguities in favor of init_type */
-
-  template<typename=void>
-  BOOST_FORCEINLINE std::pair<iterator,bool>
-  insert(const value_type& x){return emplace_impl(x);}
-
-  template<typename=void>
-  BOOST_FORCEINLINE std::pair<iterator,bool>
-  insert(value_type&& x){return emplace_impl(std::move(x));}
-
-  template<
-    bool dependent_value=false,
-    typename std::enable_if<
-      has_mutable_iterator||dependent_value>::type* =nullptr
-  >
-  void erase(iterator pos)noexcept{return erase(const_iterator(pos));}
-
-  BOOST_FORCEINLINE
-  void erase(const_iterator pos)noexcept
+  template<typename Key>
+  BOOST_FORCEINLINE locator find(const Key& x)const
   {
-    destroy_element(pos.p);
-    recover_slot(pos.pc);
+    auto hash=hash_for(x);
+    return find(x,position_for(hash),hash);
   }
 
+#if defined(BOOST_MSVC)
+/* warning: forcing value to bool 'true' or 'false' in bool(pred()...) */
+#pragma warning(push)
+#pragma warning(disable:4800)
+#endif
+
   template<typename Key>
-  BOOST_FORCEINLINE
-  auto erase(Key&& x) -> typename std::enable_if<
-    !std::is_convertible<Key,iterator>::value&&
-    !std::is_convertible<Key,const_iterator>::value, std::size_t>::type
-  {
-    auto it=find(x);
-    if(it!=end()){
-      erase(it);
-      return 1;
+  BOOST_FORCEINLINE locator find(
+    const Key& x,std::size_t pos0,std::size_t hash)const
+  {    
+    prober pb(pos0);
+    do{
+      auto pos=pb.get();
+      auto pg=arrays.groups+pos;
+      auto mask=pg->match(hash);
+      if(mask){
+        BOOST_UNORDERED_ASSUME(arrays.elements!=nullptr);
+        auto p=arrays.elements+pos*N;
+        BOOST_UNORDERED_PREFETCH_ELEMENTS(p,N);
+        do{
+          auto n=unchecked_countr_zero(mask);
+          if(BOOST_LIKELY(bool(pred()(x,key_from(p[n]))))){
+            return {pg,n,p+n};
+          }
+          mask&=mask-1;
+        }while(mask);
+      }
+      if(BOOST_LIKELY(pg->is_not_overflowed(hash))){
+        return {};
+      }
     }
-    else return 0;
+    while(BOOST_LIKELY(pb.next(arrays.groups_size_mask)));
+    return {};
   }
 
-  void swap(table& x)
+#if defined(BOOST_MSVC)
+#pragma warning(pop) /* C4800 */
+#endif
+
+  void swap(table_core& x)
     noexcept(
-      alloc_traits::is_always_equal::value&&
-      boost::is_nothrow_swappable<Hash>::value&&
-      boost::is_nothrow_swappable<Pred>::value)
+      alloc_traits::propagate_on_container_swap::value||
+      alloc_traits::is_always_equal::value)
   {
+    BOOST_UNORDERED_STATIC_ASSERT_HASH_PRED(Hash, Pred)
+
     static constexpr auto pocs=
       alloc_traits::propagate_on_container_swap::value;
 
     using std::swap;
-    swap(h(),x.h());
-    swap(pred(),x.pred());
     if_constexpr<pocs>([&,this]{
       swap_if<pocs>(al(),x.al());
     },
@@ -1390,9 +1517,11 @@ public:
       BOOST_ASSERT(al()==x.al());
       (void)this; /* makes sure captured this is used */
     });
-    swap(size_,x.size_);
+
+    swap(h(),x.h());
+    swap(pred(),x.pred());
     swap(arrays,x.arrays);
-    swap(ml,x.ml);
+    swap(size_ctrl,x.size_ctrl);
   }
 
   void clear()noexcept
@@ -1401,7 +1530,7 @@ public:
     if(p){
       for(auto pg=arrays.groups,last=pg+arrays.groups_size_mask+1;
           pg!=last;++pg,p+=N){
-        auto mask=pg->match_really_occupied();
+        auto mask=match_really_occupied(pg,last);
         while(mask){
           destroy_element(p+unchecked_countr_zero(mask));
           mask&=mask-1;
@@ -1410,41 +1539,14 @@ public:
         pg->initialize();
       }
       arrays.groups[arrays.groups_size_mask].set_sentinel();
-      size_=0;
-      ml=initial_max_load();
+      size_ctrl.ml=initial_max_load();
+      size_ctrl.size=0;
     }
   }
 
-  // TODO: should we accept different allocator too?
-  template<typename Hash2,typename Pred2>
-  void merge(table<TypePolicy,Hash2,Pred2,Allocator>& x)
-  {
-    x.for_all_elements([&,this](group_type* pg,unsigned int n,value_type* p){
-      if(emplace_impl(type_policy::move(*p)).second){
-        x.erase(iterator{pg,n,p});
-      }
-    });
-  }
-
-  template<typename Hash2,typename Pred2>
-  void merge(table<TypePolicy,Hash2,Pred2,Allocator>&& x){merge(x);}
-
   hasher hash_function()const{return h();}
   key_equal key_eq()const{return pred();}
 
-  template<typename Key>
-  BOOST_FORCEINLINE iterator find(const Key& x)
-  {
-    auto hash=hash_for(x);
-    return find_impl(x,position_for(hash),hash);
-  }
-
-  template<typename Key>
-  BOOST_FORCEINLINE const_iterator find(const Key& x)const
-  {
-    return const_cast<table*>(this)->find(x);
-  }
-
   std::size_t capacity()const noexcept
   {
     return arrays.elements?(arrays.groups_size_mask+1)*N-1:0;
@@ -1452,13 +1554,13 @@ public:
   
   float load_factor()const noexcept
   {
-    if (capacity() == 0) { return 0; }
-    return float(size())/float(capacity());
+    if(capacity()==0)return 0;
+    else             return float(size())/float(capacity());
   }
 
   float max_load_factor()const noexcept{return mlf;}
 
-  std::size_t max_load()const noexcept{return ml;}
+  std::size_t max_load()const noexcept{return size_ctrl.ml;}
 
   void rehash(std::size_t n)
   {
@@ -1474,20 +1576,27 @@ public:
     rehash(std::size_t(std::ceil(float(n)/mlf)));
   }
 
-  template<typename Predicate>
-  friend std::size_t erase_if(table& x,Predicate pr)
+  friend bool operator==(const table_core& x,const table_core& y)
   {
-    return x.erase_if_impl(pr);
+    return
+      x.size()==y.size()&&
+      x.for_all_elements_while([&](element_type* p){
+        auto loc=y.find(key_from(*p));
+        return loc&&
+          const_cast<const value_type&>(type_policy::value_from(*p))==
+          const_cast<const value_type&>(type_policy::value_from(*loc.p));
+      });
   }
 
-private:
-  template<typename,typename,typename,typename> friend class table;
-  using arrays_type=table_arrays<value_type,group_type,size_policy>;
+  friend bool operator!=(const table_core& x,const table_core& y)
+  {
+    return !(x==y);
+  }
 
   struct clear_on_exit
   {
     ~clear_on_exit(){x.clear();}
-    table& x;
+    table_core& x;
   };
 
   Hash&            h(){return hash_base::get();}
@@ -1497,63 +1606,33 @@ private:
   Allocator&       al(){return allocator_base::get();}
   const Allocator& al()const{return allocator_base::get();}
 
-  arrays_type new_arrays(std::size_t n)
-  {
-    return arrays_type::new_(al(),n);
-  }
-
-  void delete_arrays(arrays_type& arrays_)noexcept
+  template<typename... Args>
+  void construct_element(element_type* p,Args&&... args)
   {
-    arrays_type::delete_(al(),arrays_);
+    type_policy::construct(al(),p,std::forward<Args>(args)...);
   }
 
   template<typename... Args>
-  void construct_element(value_type* p,Args&&... args)
+  void construct_element(element_type* p,try_emplace_args_t,Args&&... args)
   {
-    alloc_traits::construct(al(),p,std::forward<Args>(args)...);
+    construct_element_from_try_emplace_args(
+      p,
+      std::integral_constant<bool,std::is_same<key_type,value_type>::value>{},
+      std::forward<Args>(args)...);
   }
 
-  void destroy_element(value_type* p)noexcept
+  void destroy_element(element_type* p)noexcept
   {
-    alloc_traits::destroy(al(),p);
+    type_policy::destroy(al(),p);
   }
 
   struct destroy_element_on_exit
   {
     ~destroy_element_on_exit(){this_->destroy_element(p);}
-    table      *this_;
-    value_type *p;
+    table_core   *this_;
+    element_type *p;
   };
 
-  void recover_slot(unsigned char* pc)
-  {
-    /* If this slot potentially caused overflow, we decrease the maximum load so
-     * that average probe length won't increase unboundedly in repeated
-     * insert/erase cycles (drift).
-     */
-    ml-=group_type::maybe_caused_overflow(pc);
-    group_type::reset(pc);
-    --size_;
-  }
-
-  void recover_slot(group_type* pg,std::size_t pos)
-  {
-    recover_slot(reinterpret_cast<unsigned char*>(pg)+pos);
-  }
-
-  std::size_t initial_max_load()const
-  {
-    static constexpr std::size_t small_capacity=2*N-1;
-
-    auto capacity_=capacity();
-    if(capacity_<=small_capacity){
-      return capacity_; /* we allow 100% usage */
-    }
-    else{
-      return (std::size_t)(mlf*(float)(capacity_));
-    }
-  }
-
   template<typename T>
   static inline auto key_from(const T& x)
     ->decltype(type_policy::extract(x))
@@ -1561,12 +1640,11 @@ private:
     return type_policy::extract(x);
   }
 
-  template<typename Arg1,typename Arg2>
-  static inline auto key_from(
-    std::piecewise_construct_t,const Arg1& k,const Arg2&)
-    ->decltype(std::get<0>(k))
+  template<typename Key,typename... Args>
+  static inline const Key& key_from(
+    try_emplace_args_t,const Key& x,const Args&...)
   {
-    return std::get<0>(k);
+    return x;
   }
 
   template<typename Key>
@@ -1586,129 +1664,332 @@ private:
     return size_policy::position(hash,arrays_.groups_size_index);
   }
 
-  static inline void prefetch_elements(const value_type* p)
+  static inline int match_really_occupied(group_type* pg,group_type* last)
   {
-    /* We have experimentally confirmed that ARM architectures get a higher
-     * speedup when around the first half of the element slots in a group are
-     * prefetched, whereas for Intel just the first cache line is best.
-     * Please report back if you find better tunings for some particular
-     * architectures.
-     */
+    /* excluding the sentinel */
+    return pg->match_occupied()&~(int(pg==last-1)<<(N-1));
+  }
 
-#if BOOST_ARCH_ARM
-    /* Cache line size can't be known at compile time, so we settle on
-     * the very frequent value of 64B.
-     */
-    constexpr int  cache_line=64;
-    const char    *p0=reinterpret_cast<const char*>(p),
-                  *p1=p0+sizeof(value_type)*N/2;
-    for(;p0<p1;p0+=cache_line)prefetch(p0);
-#else
-    prefetch(p);
-#endif
+  template<typename... Args>
+  locator unchecked_emplace_at(
+    std::size_t pos0,std::size_t hash,Args&&... args)
+  {
+    auto res=nosize_unchecked_emplace_at(
+      arrays,pos0,hash,std::forward<Args>(args)...);
+    ++size_ctrl.size;
+    return res;
   }
 
-#if defined(BOOST_MSVC)
-/* warning: forcing value to bool 'true' or 'false' in bool(pred()...) */
-#pragma warning(push)
-#pragma warning(disable:4800)
-#endif
+  BOOST_NOINLINE void unchecked_rehash_for_growth()
+  {
+    auto new_arrays_=new_arrays_for_growth();
+    unchecked_rehash(new_arrays_);
+  }
 
-  template<typename Key>
-  BOOST_FORCEINLINE iterator find_impl(
-    const Key& x,std::size_t pos0,std::size_t hash)const
-  {    
-    prober pb(pos0);
-    do{
-      auto pos=pb.get();
-      auto pg=arrays.groups+pos;
-      auto mask=pg->match(hash);
-      if(mask){
-        auto p=arrays.elements+pos*N;
-        prefetch_elements(p);
-        do{
+  template<typename... Args>
+  BOOST_NOINLINE locator
+  unchecked_emplace_with_rehash(std::size_t hash,Args&&... args)
+  {
+    auto    new_arrays_=new_arrays_for_growth();
+    locator it;
+    BOOST_TRY{
+      /* strong exception guarantee -> try insertion before rehash */
+      it=nosize_unchecked_emplace_at(
+        new_arrays_,position_for(hash,new_arrays_),
+        hash,std::forward<Args>(args)...);
+    }
+    BOOST_CATCH(...){
+      delete_arrays(new_arrays_);
+      BOOST_RETHROW
+    }
+    BOOST_CATCH_END
+
+    /* new_arrays_ lifetime taken care of by unchecked_rehash */
+    unchecked_rehash(new_arrays_);
+    ++size_ctrl.size;
+    return it;
+  }
+
+  void noshrink_reserve(std::size_t n)
+  {
+    /* used only on assignment after element clearance */
+    BOOST_ASSERT(empty());
+
+    if(n){
+      n=std::size_t(std::ceil(float(n)/mlf)); /* elements -> slots */
+      n=capacity_for(n); /* exact resulting capacity */
+
+      if(n>capacity()){
+        auto new_arrays_=new_arrays(n);
+        delete_arrays(arrays);
+        arrays=new_arrays_;
+        size_ctrl.ml=initial_max_load();
+      }
+    }
+  }
+
+  template<typename F>
+  void for_all_elements(F f)const
+  {
+    for_all_elements(arrays,f);
+  }
+
+  template<typename F>
+  static auto for_all_elements(const arrays_type& arrays_,F f)
+    ->decltype(f(nullptr),void())
+  {
+    for_all_elements_while(arrays_,[&](element_type* p){f(p);return true;});
+  }
+
+  template<typename F>
+  static auto for_all_elements(const arrays_type& arrays_,F f)
+    ->decltype(f(nullptr,0,nullptr),void())
+  {
+    for_all_elements_while(
+      arrays_,[&](group_type* pg,unsigned int n,element_type* p)
+        {f(pg,n,p);return true;});
+  }
+
+  template<typename F>
+  bool for_all_elements_while(F f)const
+  {
+    return for_all_elements_while(arrays,f);
+  }
+
+  template<typename F>
+  static auto for_all_elements_while(const arrays_type& arrays_,F f)
+    ->decltype(f(nullptr),bool())
+  {
+    return for_all_elements_while(
+      arrays_,[&](group_type*,unsigned int,element_type* p){return f(p);});
+  }
+
+  template<typename F>
+  static auto for_all_elements_while(const arrays_type& arrays_,F f)
+    ->decltype(f(nullptr,0,nullptr),bool())
+  {
+    auto p=arrays_.elements;
+    if(p){
+      for(auto pg=arrays_.groups,last=pg+arrays_.groups_size_mask+1;
+          pg!=last;++pg,p+=N){
+        auto mask=match_really_occupied(pg,last);
+        while(mask){
           auto n=unchecked_countr_zero(mask);
-          if(BOOST_LIKELY(bool(pred()(x,key_from(p[n]))))){
-            return {pg,n,p+n};
-          }
+          if(!f(pg,n,p+n))return false;
           mask&=mask-1;
-        }while(mask);
-      }
-      if(BOOST_LIKELY(pg->is_not_overflowed(hash))){
-        return {}; /* end() */
+        }
       }
     }
-    while(BOOST_LIKELY(pb.next(arrays.groups_size_mask)));
-    return {}; /* end() */
+    return true;
   }
 
-#if defined(BOOST_MSVC)
-#pragma warning(pop) /* C4800 */
-#endif
+  arrays_type    arrays;
+  size_ctrl_type size_ctrl;
 
-  template<typename... Args>
-  BOOST_FORCEINLINE std::pair<iterator,bool> emplace_impl(Args&&... args)
+private:
+  template<
+    typename,typename,template<typename...> class,
+    typename,typename,typename,typename
+  >
+  friend class table_core;
+
+  using hash_base=empty_value<Hash,0>;
+  using pred_base=empty_value<Pred,1>;
+  using allocator_base=empty_value<Allocator,2>;
+
+  /* used by allocator-extended move ctor */
+
+  table_core(Hash&& h_,Pred&& pred_,const Allocator& al_):
+    hash_base{empty_init,std::move(h_)},
+    pred_base{empty_init,std::move(pred_)},
+    allocator_base{empty_init,al_},arrays(new_arrays(0)),
+    size_ctrl{initial_max_load(),0}
+    {}
+
+  arrays_type new_arrays(std::size_t n)
   {
-    const auto &k=key_from(std::forward<Args>(args)...);
-    auto        hash=hash_for(k);
-    auto        pos0=position_for(hash);
-    auto        it=find_impl(k,pos0,hash);
+    return arrays_type::new_(al(),n);
+  }
 
-    if(it!=end()){
-      return {it,false};
-    }
-    if(BOOST_LIKELY(size_<ml)){
-      return {
-        unchecked_emplace_at(pos0,hash,std::forward<Args>(args)...),
-        true
-      };  
+  arrays_type new_arrays_for_growth()
+  {
+    /* Due to the anti-drift mechanism (see recover_slot), the new arrays may
+     * be of the same size as the old arrays; in the limit, erasing one
+     * element at full load and then inserting could bring us back to the same
+     * capacity after a costly rehash. To avoid this, we jump to the next
+     * capacity level when the number of erased elements is <= 10% of total
+     * elements at full load, which is implemented by requesting additional
+     * F*size elements, with F = P * 10% / (1 - P * 10%), where P is the
+     * probability of an element having caused overflow; P has been measured as
+     * ~0.162 under ideal conditions, yielding F ~ 0.0165 ~ 1/61.
+     */
+    return new_arrays(std::size_t(
+      std::ceil(static_cast<float>(size()+size()/61+1)/mlf)));
+  }
+
+  void delete_arrays(arrays_type& arrays_)noexcept
+  {
+    arrays_type::delete_(al(),arrays_);
+  }
+
+  template<typename Key,typename... Args>
+  void construct_element_from_try_emplace_args(
+    element_type* p,std::false_type,Key&& x,Args&&... args)
+  {
+    type_policy::construct(
+      this->al(),p,
+      std::piecewise_construct,
+      std::forward_as_tuple(std::forward<Key>(x)),
+      std::forward_as_tuple(std::forward<Args>(args)...));
+  }
+
+  /* This overload allows boost::unordered_flat_set to internally use
+   * try_emplace to implement heterogeneous insert (P2363).
+   */
+
+  template<typename Key>
+  void construct_element_from_try_emplace_args(
+    element_type* p,std::true_type,Key&& x)
+  {
+    type_policy::construct(this->al(),p,std::forward<Key>(x));
+  }
+
+  void copy_elements_from(const table_core& x)
+  {
+    BOOST_ASSERT(empty());
+    BOOST_ASSERT(this!=std::addressof(x));
+    if(arrays.groups_size_mask==x.arrays.groups_size_mask){
+      fast_copy_elements_from(x);
     }
     else{
-      return {
-        unchecked_emplace_with_rehash(hash,std::forward<Args>(args)...),
-        true
-      };  
+      x.for_all_elements([this](const element_type* p){
+        unchecked_insert(*p);
+      });
     }
   }
 
-  static std::size_t capacity_for(std::size_t n)
+  void fast_copy_elements_from(const table_core& x)
   {
-    return size_policy::size(size_index_for<group_type,size_policy>(n))*N-1;
+    if(arrays.elements){
+      copy_elements_array_from(x);
+      copy_groups_array_from(x);
+      size_ctrl.size=std::size_t(x.size_ctrl.size);
+    }
   }
 
-  template<typename... Args>
-  BOOST_NOINLINE iterator
-  unchecked_emplace_with_rehash(std::size_t hash,Args&&... args)
+  void copy_elements_array_from(const table_core& x)
   {
-    /* Due to the anti-drift mechanism (see recover_slot), new_arrays_ may be
-     * of the same size as the old arrays; in the limit, erasing one element at
-     * full load and then inserting could bring us back to the same capacity
-     * after a costly rehash. To avoid this, we jump to the next capacity level
-     * when the number of erased elements is <= 10% of total elements at full
-     * load, which is implemented by requesting additional F*size elements,
-     * with F = P * 10% / (1 - P * 10%), where P is the probability of an
-     * element having caused overflow; P has been measured as ~0.162 under
-     * ideal conditions, yielding F ~ 0.0165 ~ 1/61.
+    copy_elements_array_from(
+      x,
+      std::integral_constant<
+        bool,
+#if BOOST_WORKAROUND(BOOST_LIBSTDCXX_VERSION,<50000)
+        /* std::is_trivially_copy_constructible not provided */
+        boost::has_trivial_copy<element_type>::value
+#else
+        std::is_trivially_copy_constructible<element_type>::value
+#endif
+        &&(
+          is_std_allocator<Allocator>::value||
+          !alloc_has_construct<Allocator,value_type*,const value_type&>::value)
+      >{}
+    );
+  }
+
+  void copy_elements_array_from(
+    const table_core& x,std::true_type /* -> memcpy */)
+  {
+    /* reinterpret_cast: GCC may complain about value_type not being trivially
+     * copy-assignable when we're relying on trivial copy constructibility.
      */
-    auto     new_arrays_=new_arrays(std::size_t(
-               std::ceil(static_cast<float>(size_+size_/61+1)/mlf)));
-    iterator it;
+    std::memcpy(
+      reinterpret_cast<unsigned char*>(arrays.elements),
+      reinterpret_cast<unsigned char*>(x.arrays.elements),
+      x.capacity()*sizeof(value_type));
+  }
+
+  void copy_elements_array_from(
+    const table_core& x,std::false_type /* -> manual */)
+  {
+    std::size_t num_constructed=0;
     BOOST_TRY{
-      /* strong exception guarantee -> try insertion before rehash */
-      it=nosize_unchecked_emplace_at(
-        new_arrays_,position_for(hash,new_arrays_),
-        hash,std::forward<Args>(args)...);
+      x.for_all_elements([&,this](const element_type* p){
+        construct_element(arrays.elements+(p-x.arrays.elements),*p);
+        ++num_constructed;
+      });
     }
     BOOST_CATCH(...){
-      delete_arrays(new_arrays_);
+      if(num_constructed){
+        x.for_all_elements_while([&,this](const element_type* p){
+          destroy_element(arrays.elements+(p-x.arrays.elements));
+          return --num_constructed!=0;
+        });
+      }
       BOOST_RETHROW
     }
     BOOST_CATCH_END
+  }
 
-    /* new_arrays_ lifetime taken care of by unchecked_rehash */
-    unchecked_rehash(new_arrays_);
-    ++size_;
-    return it;
+  void copy_groups_array_from(const table_core& x) {
+    copy_groups_array_from(x, std::integral_constant<bool,
+#if BOOST_WORKAROUND(BOOST_LIBSTDCXX_VERSION,<50000)
+      /* std::is_trivially_copy_assignable not provided */
+      boost::has_trivial_assign<group_type>::value
+#else
+      std::is_trivially_copy_assignable<group_type>::value
+#endif
+      >{}
+    );
+  }
+
+  void copy_groups_array_from(
+    const table_core& x, std::true_type /* -> memcpy */)
+  {
+    std::memcpy(
+      arrays.groups,x.arrays.groups,
+      (arrays.groups_size_mask+1)*sizeof(group_type));
+  }
+
+  void copy_groups_array_from(
+    const table_core& x, std::false_type /* -> manual */) 
+  {
+    for(std::size_t i=0;i<arrays.groups_size_mask+1;++i){
+      arrays.groups[i]=x.arrays.groups[i];
+    }
+  }
+
+  void recover_slot(unsigned char* pc)
+  {
+    /* If this slot potentially caused overflow, we decrease the maximum load so
+     * that average probe length won't increase unboundedly in repeated
+     * insert/erase cycles (drift).
+     */
+    size_ctrl.ml-=group_type::maybe_caused_overflow(pc);
+    group_type::reset(pc);
+    --size_ctrl.size;
+  }
+
+  void recover_slot(group_type* pg,std::size_t pos)
+  {
+    recover_slot(reinterpret_cast<unsigned char*>(pg)+pos);
+  }
+
+  std::size_t initial_max_load()const
+  {
+    static constexpr std::size_t small_capacity=2*N-1;
+
+    auto capacity_=capacity();
+    if(capacity_<=small_capacity){
+      return capacity_; /* we allow 100% usage */
+    }
+    else{
+      return (std::size_t)(mlf*(float)(capacity_));
+    }
+  }  
+
+  static std::size_t capacity_for(std::size_t n)
+  {
+    return size_policy::size(size_index_for<group_type,size_policy>(n))*N-1;
   }
 
   BOOST_NOINLINE void unchecked_rehash(std::size_t n)
@@ -1721,24 +2002,20 @@ private:
   {
     std::size_t num_destroyed=0;
     BOOST_TRY{
-      for_all_elements([&,this](value_type* p){
+      for_all_elements([&,this](element_type* p){
         nosize_transfer_element(p,new_arrays_,num_destroyed);
       });
     }
     BOOST_CATCH(...){
       if(num_destroyed){
-        for(auto pg=arrays.groups;;++pg){
-          auto mask=pg->match_occupied();
-          while(mask){
-            auto nz=unchecked_countr_zero(mask);
-            recover_slot(pg,nz);
-            if(!(--num_destroyed))goto continue_;
-            mask&=mask-1;
+        for_all_elements_while(
+          [&,this](group_type* pg,unsigned int n,element_type*){
+            recover_slot(pg,n);
+            return --num_destroyed!=0;
           }
-        }
+        );
       }
-      continue_:
-      for_all_elements(new_arrays_,[this](value_type* p){
+      for_all_elements(new_arrays_,[this](element_type* p){
         destroy_element(p);
       });
       delete_arrays(new_arrays_);
@@ -1749,31 +2026,13 @@ private:
     /* either all moved and destroyed or all copied */
     BOOST_ASSERT(num_destroyed==size()||num_destroyed==0);
     if(num_destroyed!=size()){
-      for_all_elements([this](value_type* p){
+      for_all_elements([this](element_type* p){
         destroy_element(p);
       });
     }
     delete_arrays(arrays);
     arrays=new_arrays_;
-    ml=initial_max_load();
-  }
-
-  void noshrink_reserve(std::size_t n)
-  {
-    /* used only on assignment after element clearance */
-    BOOST_ASSERT(empty());
-
-    if(n){
-      n=std::size_t(std::ceil(float(n)/mlf)); /* elements -> slots */
-      n=capacity_for(n); /* exact resulting capacity */
-
-      if(n>capacity()){
-        auto new_arrays_=new_arrays(n);
-        delete_arrays(arrays);
-        arrays=new_arrays_;
-        ml=initial_max_load();
-      }
-    }
+    size_ctrl.ml=initial_max_load();
   }
 
   template<typename Value>
@@ -1784,18 +2043,19 @@ private:
   }
 
   void nosize_transfer_element(
-    value_type* p,const arrays_type& arrays_,std::size_t& num_destroyed)
+    element_type* p,const arrays_type& arrays_,std::size_t& num_destroyed)
   {
     nosize_transfer_element(
       p,hash_for(key_from(*p)),arrays_,num_destroyed,
       std::integral_constant< /* std::move_if_noexcept semantics */
         bool,
         std::is_nothrow_move_constructible<init_type>::value||
-        !std::is_copy_constructible<init_type>::value>{});
+        !std::is_same<element_type,value_type>::value||
+        !std::is_copy_constructible<element_type>::value>{});
   }
 
   void nosize_transfer_element(
-    value_type* p,std::size_t hash,const arrays_type& arrays_,
+    element_type* p,std::size_t hash,const arrays_type& arrays_,
     std::size_t& num_destroyed,std::true_type /* ->move */)
   {
     /* Destroy p even if an an exception is thrown in the middle of move
@@ -1809,26 +2069,16 @@ private:
   }
 
   void nosize_transfer_element(
-    value_type* p,std::size_t hash,const arrays_type& arrays_,
+    element_type* p,std::size_t hash,const arrays_type& arrays_,
     std::size_t& /*num_destroyed*/,std::false_type /* ->copy */)
   {
     nosize_unchecked_emplace_at(
       arrays_,position_for(hash,arrays_),hash,
-      const_cast<const value_type&>(*p));
-  }
-
-  template<typename... Args>
-  iterator unchecked_emplace_at(
-    std::size_t pos0,std::size_t hash,Args&&... args)
-  {
-    auto res=nosize_unchecked_emplace_at(
-      arrays,pos0,hash,std::forward<Args>(args)...);
-    ++size_;
-    return res;
+      const_cast<const element_type&>(*p));
   }
 
   template<typename... Args>
-  iterator nosize_unchecked_emplace_at(
+  locator nosize_unchecked_emplace_at(
     const arrays_type& arrays_,std::size_t pos0,std::size_t hash,
     Args&&... args)
   {
@@ -1846,51 +2096,6 @@ private:
       else pg->mark_overflow(hash);
     }
   }
-
-  template<typename Predicate>
-  std::size_t erase_if_impl(Predicate pr)
-  {
-    std::size_t s=size();
-    for_all_elements([&,this](group_type* pg,unsigned int n,value_type* p){
-      if(pr(*p)) erase(iterator{pg,n,p});
-    });
-    return std::size_t(s-size());
-  }
-
-  template<typename F>
-  void for_all_elements(F f)const
-  {
-    for_all_elements(arrays,f);
-  }
-
-  template<typename F>
-  static auto for_all_elements(const arrays_type& arrays_,F f)
-    ->decltype(f(nullptr),void())
-  {
-    for_all_elements(
-      arrays_,[&](group_type*,unsigned int,value_type* p){return f(p);});
-  }
-
-  template<typename F>
-  static auto for_all_elements(const arrays_type& arrays_,F f)
-    ->decltype(f(nullptr,0,nullptr),void())
-  {
-    auto p=arrays_.elements;
-    if(!p){return;}
-    for(auto pg=arrays_.groups,last=pg+arrays_.groups_size_mask+1;
-        pg!=last;++pg,p+=N){
-      auto mask=pg->match_really_occupied();
-      while(mask){
-        auto n=unchecked_countr_zero(mask);
-        f(pg,n,p+n);
-        mask&=mask-1;
-      }
-    }
-  }
-
-  std::size_t size_;
-  arrays_type arrays;
-  std::size_t ml;
 };
 
 #if BOOST_WORKAROUND(BOOST_MSVC,<=1900)
@@ -1901,21 +2106,14 @@ private:
 #pragma warning(pop) /* C4714 */
 #endif
 
-#if defined(BOOST_GCC)
-#pragma GCC diagnostic pop /* ignored "-Wshadow" */
-#endif
+#include <boost/unordered/detail/foa/restore_wshadow.hpp>
 
 } /* namespace foa */
 } /* namespace detail */
 } /* namespace unordered */
 } /* namespace boost */
 
-#undef BOOST_UNORDERED_ASSUME
+#undef BOOST_UNORDERED_STATIC_ASSERT_HASH_PRED
+#undef BOOST_UNORDERED_HAS_FEATURE
 #undef BOOST_UNORDERED_HAS_BUILTIN
-#ifdef BOOST_UNORDERED_LITTLE_ENDIAN_NEON
-#undef BOOST_UNORDERED_LITTLE_ENDIAN_NEON
-#endif
-#ifdef BOOST_UNORDERED_SSE2
-#undef BOOST_UNORDERED_SSE2
-#endif
 #endif