@delali/narsil-react-native 0.0.0-stage → 0.3.1-next.151

package/cpp/core/src/kernels_float.c

@@ -0,0 +1,197 @@
+#include "internal.h"
+
+#include <math.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#define LANE_WIDTH 4U
+#define UNROLLED_LANES 4U
+#define UNROLLED_WIDTH (LANE_WIDTH * UNROLLED_LANES)
+
+#ifdef NARSIL_NEON
+#include <arm_neon.h>
+typedef float32x4_t lanes;
+static inline lanes lanes_zero(void) { return vdupq_n_f32(0); }
+static inline lanes lanes_load(const float *source) { return vld1q_f32(source); }
+static inline lanes lanes_add(lanes lhs, lanes rhs) { return vaddq_f32(lhs, rhs); }
+static inline lanes lanes_sub(lanes lhs, lanes rhs) { return vsubq_f32(lhs, rhs); }
+static inline lanes lanes_mul(lanes lhs, lanes rhs) { return vmulq_f32(lhs, rhs); }
+static inline float lanes_sum_in_pairs(lanes values) {
+  return (vgetq_lane_f32(values, 0) + vgetq_lane_f32(values, 1)) +
+         (vgetq_lane_f32(values, 2) + vgetq_lane_f32(values, 3));
+}
+#elif defined(NARSIL_SSE2)
+#include <emmintrin.h>
+#if (defined(__GNUC__) || defined(__clang__)) && !defined(NARSIL_SSE2_ONLY)
+#define NARSIL_AVX2_DOT 1
+#include <cpuid.h>
+#include <immintrin.h>
+#include <stdatomic.h>
+#endif
+typedef __m128 lanes;
+static inline lanes lanes_zero(void) { return _mm_setzero_ps(); }
+static inline lanes lanes_load(const float *source) { return _mm_loadu_ps(source); }
+static inline lanes lanes_add(lanes lhs, lanes rhs) { return _mm_add_ps(lhs, rhs); }
+static inline lanes lanes_sub(lanes lhs, lanes rhs) { return _mm_sub_ps(lhs, rhs); }
+static inline lanes lanes_mul(lanes lhs, lanes rhs) { return _mm_mul_ps(lhs, rhs); }
+static inline float lanes_sum_in_pairs(lanes values) {
+  float lane[LANE_WIDTH];
+  _mm_storeu_ps(lane, values);
+  return (lane[0] + lane[1]) + (lane[2] + lane[3]);
+}
+#else
+typedef struct {
+  float lane[LANE_WIDTH];
+} lanes;
+static inline lanes lanes_zero(void) { return (lanes){{0, 0, 0, 0}}; }
+static inline lanes lanes_load(const float *source) { return (lanes){{source[0], source[1], source[2], source[3]}}; }
+static inline lanes lanes_add(lanes lhs, lanes rhs) {
+  return (lanes){
+      {lhs.lane[0] + rhs.lane[0], lhs.lane[1] + rhs.lane[1], lhs.lane[2] + rhs.lane[2], lhs.lane[3] + rhs.lane[3]}};
+}
+static inline lanes lanes_sub(lanes lhs, lanes rhs) {
+  return (lanes){
+      {lhs.lane[0] - rhs.lane[0], lhs.lane[1] - rhs.lane[1], lhs.lane[2] - rhs.lane[2], lhs.lane[3] - rhs.lane[3]}};
+}
+static inline lanes lanes_mul(lanes lhs, lanes rhs) {
+  return (lanes){
+      {lhs.lane[0] * rhs.lane[0], lhs.lane[1] * rhs.lane[1], lhs.lane[2] * rhs.lane[2], lhs.lane[3] * rhs.lane[3]}};
+}
+static inline float lanes_sum_in_pairs(lanes values) {
+  return (values.lane[0] + values.lane[1]) + (values.lane[2] + values.lane[3]);
+}
+#endif
+
+static inline uint32_t whole_steps(uint32_t count, uint32_t width) { return count - (count % width); }
+
+static inline lanes lane_product(const float *lhs, const float *rhs, size_t lane) {
+  return lanes_mul(lanes_load(lhs + (lane * LANE_WIDTH)), lanes_load(rhs + (lane * LANE_WIDTH)));
+}
+
+typedef struct {
+  lanes sums[UNROLLED_LANES];
+  uint32_t offset;
+} unrolled_dot;
+
+static inline float finish_dot(const float *lhs, const float *rhs, uint32_t dimension, unrolled_dot dot) {
+  uint32_t lanes_end = whole_steps(dimension, LANE_WIDTH);
+  uint32_t offset = dot.offset;
+  lanes first = dot.sums[0];
+  for (; offset < lanes_end; offset += LANE_WIDTH) {
+    first = lanes_add(first, lane_product(lhs + offset, rhs + offset, 0));
+  }
+  float sum = lanes_sum_in_pairs(lanes_add(lanes_add(first, dot.sums[1]), lanes_add(dot.sums[2], dot.sums[3])));
+  for (; offset < dimension; offset++) { sum = sum + (lhs[offset] * rhs[offset]); }
+  return sum;
+}
+
+#ifdef NARSIL_AVX2_DOT
+#define CPUID_FEATURES_LEAF 1U
+#define CPUID_EXTENDED_FEATURES_LEAF 7U
+#define XCR0_SSE_AND_AVX_STATE 6U
+#define WIDE_LANE_WIDTH 8U
+#define AVX2_UNKNOWN 0
+#define AVX2_ABSENT 1
+#define AVX2_PRESENT 2
+
+static atomic_int avx2_state = AVX2_UNKNOWN;
+
+static uint32_t enabled_register_state(void) {
+  uint32_t low = 0;
+  uint32_t high = 0;
+  __asm__ volatile("xgetbv" : "=a"(low), "=d"(high) : "c"(0));
+  (void)high;
+  return low;
+}
+
+static int processor_runs_avx2(void) {
+  unsigned int eax = 0;
+  unsigned int ebx = 0;
+  unsigned int ecx = 0;
+  unsigned int edx = 0;
+  if (__get_cpuid_max(0, NULL) < CPUID_EXTENDED_FEATURES_LEAF) { return 0; }
+  if (!__get_cpuid(CPUID_FEATURES_LEAF, &eax, &ebx, &ecx, &edx)) { return 0; }
+  if ((ecx & bit_OSXSAVE) == 0 || (ecx & bit_AVX) == 0) { return 0; }
+  if ((enabled_register_state() & XCR0_SSE_AND_AVX_STATE) != XCR0_SSE_AND_AVX_STATE) { return 0; }
+  __cpuid_count(CPUID_EXTENDED_FEATURES_LEAF, 0, eax, ebx, ecx, edx);
+  return (ebx & bit_AVX2) != 0;
+}
+
+static int uses_avx2(void) {
+  int state = atomic_load_explicit(&avx2_state, memory_order_relaxed);
+  if (state == AVX2_UNKNOWN) {
+    state = processor_runs_avx2() ? AVX2_PRESENT : AVX2_ABSENT;
+    atomic_store_explicit(&avx2_state, state, memory_order_relaxed);
+  }
+  return state == AVX2_PRESENT;
+}
+
+__attribute__((target("avx2"))) static float dot_in_wide_lanes(const float *lhs, const float *rhs, uint32_t dimension) {
+  uint32_t unrolled_end = whole_steps(dimension, UNROLLED_WIDTH);
+  __m256 first_half = _mm256_setzero_ps();
+  __m256 second_half = _mm256_setzero_ps();
+  uint32_t offset = 0;
+  for (; offset < unrolled_end; offset += UNROLLED_WIDTH) {
+    first_half = _mm256_add_ps(first_half, _mm256_mul_ps(_mm256_loadu_ps(lhs + offset), _mm256_loadu_ps(rhs + offset)));
+    second_half = _mm256_add_ps(second_half, _mm256_mul_ps(_mm256_loadu_ps(lhs + offset + WIDE_LANE_WIDTH),
+                                                           _mm256_loadu_ps(rhs + offset + WIDE_LANE_WIDTH)));
+  }
+  unrolled_dot dot = {{_mm256_castps256_ps128(first_half), _mm256_extractf128_ps(first_half, 1),
+                       _mm256_castps256_ps128(second_half), _mm256_extractf128_ps(second_half, 1)},
+                      offset};
+  return finish_dot(lhs, rhs, dimension, dot);
+}
+#endif
+
+static float dot_in_narrow_lanes(const float *lhs, const float *rhs, uint32_t dimension) {
+  uint32_t unrolled_end = whole_steps(dimension, UNROLLED_WIDTH);
+  lanes sum0 = lanes_zero();
+  lanes sum1 = lanes_zero();
+  lanes sum2 = lanes_zero();
+  lanes sum3 = lanes_zero();
+  uint32_t offset = 0;
+  for (; offset < unrolled_end; offset += UNROLLED_WIDTH) {
+    sum0 = lanes_add(sum0, lane_product(lhs + offset, rhs + offset, 0));
+    sum1 = lanes_add(sum1, lane_product(lhs + offset, rhs + offset, 1));
+    sum2 = lanes_add(sum2, lane_product(lhs + offset, rhs + offset, 2));
+    sum3 = lanes_add(sum3, lane_product(lhs + offset, rhs + offset, 3));
+  }
+  unrolled_dot dot = {{sum0, sum1, sum2, sum3}, offset};
+  return finish_dot(lhs, rhs, dimension, dot);
+}
+
+float kernel_dot(const float *lhs, const float *rhs, uint32_t dimension) {
+#ifdef NARSIL_AVX2_DOT
+  if (uses_avx2()) { return dot_in_wide_lanes(lhs, rhs, dimension); }
+#endif
+  return dot_in_narrow_lanes(lhs, rhs, dimension);
+}
+
+float kernel_squared_distance(const float *lhs, const float *rhs, uint32_t dimension) {
+  uint32_t lanes_end = whole_steps(dimension, LANE_WIDTH);
+  lanes running = lanes_zero();
+  uint32_t offset = 0;
+  for (; offset < lanes_end; offset += LANE_WIDTH) {
+    lanes difference = lanes_sub(lanes_load(lhs + offset), lanes_load(rhs + offset));
+    running = lanes_add(running, lanes_mul(difference, difference));
+  }
+  float sum = lanes_sum_in_pairs(running);
+  for (; offset < dimension; offset++) {
+    float difference = lhs[offset] - rhs[offset];
+    sum = sum + (difference * difference);
+  }
+  return sum;
+}
+
+float kernel_magnitude(const float *values, uint32_t dimension) {
+  uint32_t lanes_end = whole_steps(dimension, LANE_WIDTH);
+  lanes running = lanes_zero();
+  uint32_t offset = 0;
+  for (; offset < lanes_end; offset += LANE_WIDTH) {
+    lanes loaded = lanes_load(values + offset);
+    running = lanes_add(running, lanes_mul(loaded, loaded));
+  }
+  float sum = lanes_sum_in_pairs(running);
+  for (; offset < dimension; offset++) { sum = sum + (values[offset] * values[offset]); }
+  return sqrtf(sum);
+}

package/cpp/core/src/kernels_pair.c

@@ -0,0 +1,133 @@
+#include "internal.h"
+
+#include <stddef.h>
+#include <stdint.h>
+#include <string.h>
+
+#define VECTOR_BYTES 16U
+#define WORD_BYTES 8U
+#define LOW_NIBBLE_MASK 15U
+#define NIBBLE_BITS 4
+#define TWO_BIT_LOW_BITS UINT64_C(0x5555555555555555)
+
+#ifdef NARSIL_NEON
+#include <arm_neon.h>
+#endif
+
+#ifdef NARSIL_SSE2
+#include <emmintrin.h>
+#endif
+
+#ifdef NARSIL_SSE2
+static inline __m128i load_bytes(const uint8_t *source) {
+  __m128i loaded;
+  memcpy(&loaded, source, sizeof loaded);
+  return loaded;
+}
+
+static inline uint32_t sum_of_lanes(__m128i running) {
+  uint32_t lane[4];
+  memcpy(lane, &running, sizeof lane);
+  return lane[0] + lane[1] + lane[2] + lane[3];
+}
+
+static inline __m128i widened_products(__m128i lhs, __m128i rhs) {
+  __m128i zero = _mm_setzero_si128();
+  return _mm_add_epi32(_mm_madd_epi16(_mm_unpacklo_epi8(lhs, zero), _mm_unpacklo_epi8(rhs, zero)),
+                       _mm_madd_epi16(_mm_unpackhi_epi8(lhs, zero), _mm_unpackhi_epi8(rhs, zero)));
+}
+#endif
+
+static uint32_t nibble_pair_products(const uint8_t *first, const uint8_t *second, uint32_t bytes) {
+  uint32_t total = 0;
+  uint32_t offset = 0;
+#ifdef NARSIL_NEON
+  uint8x16_t mask = vdupq_n_u8(LOW_NIBBLE_MASK);
+  uint32x4_t running = vdupq_n_u32(0);
+  for (; offset + VECTOR_BYTES <= bytes; offset += VECTOR_BYTES) {
+    uint8x16_t first_bytes = vld1q_u8(first + offset);
+    uint8x16_t second_bytes = vld1q_u8(second + offset);
+    uint8x16_t first_low = vandq_u8(first_bytes, mask);
+    uint8x16_t first_high = vshrq_n_u8(first_bytes, NIBBLE_BITS);
+    uint8x16_t second_low = vandq_u8(second_bytes, mask);
+    uint8x16_t second_high = vshrq_n_u8(second_bytes, NIBBLE_BITS);
+    uint16x8_t lower = vmlal_u8(vmull_u8(vget_low_u8(first_low), vget_low_u8(second_low)), vget_low_u8(first_high),
+                                vget_low_u8(second_high));
+    uint16x8_t upper = vmlal_u8(vmull_u8(vget_high_u8(first_low), vget_high_u8(second_low)), vget_high_u8(first_high),
+                                vget_high_u8(second_high));
+    running = vpadalq_u16(vpadalq_u16(running, lower), upper);
+  }
+  total = vaddvq_u32(running);
+#elif defined(NARSIL_SSE2)
+  __m128i mask = _mm_set1_epi8((char)LOW_NIBBLE_MASK);
+  __m128i running = _mm_setzero_si128();
+  for (; offset + VECTOR_BYTES <= bytes; offset += VECTOR_BYTES) {
+    __m128i first_bytes = load_bytes(first + offset);
+    __m128i second_bytes = load_bytes(second + offset);
+    __m128i low = widened_products(_mm_and_si128(first_bytes, mask), _mm_and_si128(second_bytes, mask));
+    __m128i high = widened_products(_mm_and_si128(_mm_srli_epi16(first_bytes, NIBBLE_BITS), mask),
+                                    _mm_and_si128(_mm_srli_epi16(second_bytes, NIBBLE_BITS), mask));
+    running = _mm_add_epi32(running, _mm_add_epi32(low, high));
+  }
+  total = sum_of_lanes(running);
+#endif
+  for (; offset < bytes; offset++) {
+    total += ((uint32_t)(first[offset] & LOW_NIBBLE_MASK) * (uint32_t)(second[offset] & LOW_NIBBLE_MASK)) +
+             ((uint32_t)(first[offset] >> NIBBLE_BITS) * (uint32_t)(second[offset] >> NIBBLE_BITS));
+  }
+  return total;
+}
+
+INLINED_IN_EACH_KERNEL uint32_t shared_bits(uint64_t first, uint64_t second) {
+  return (uint32_t)__builtin_popcountll(first & second);
+}
+
+typedef struct {
+  uint64_t first;
+  uint64_t second;
+} word_pair;
+
+INLINED_IN_EACH_KERNEL uint32_t two_bit_products(word_pair words) {
+  uint64_t first_low = words.first & TWO_BIT_LOW_BITS;
+  uint64_t first_high = (words.first >> 1) & TWO_BIT_LOW_BITS;
+  uint64_t second_low = words.second & TWO_BIT_LOW_BITS;
+  uint64_t second_high = (words.second >> 1) & TWO_BIT_LOW_BITS;
+  return shared_bits(first_low, second_low) + (2 * shared_bits(first_low, second_high)) +
+         (2 * shared_bits(first_high, second_low)) + (4 * shared_bits(first_high, second_high));
+}
+
+INLINED_IN_EACH_KERNEL uint32_t narrow_word_products(word_pair words, uint32_t bits) {
+  return bits == 1 ? shared_bits(words.first, words.second) : two_bit_products(words);
+}
+
+INLINED_IN_EACH_KERNEL uint32_t narrow_pair_products(packed_code first, const uint8_t *second) {
+  uint32_t total = 0;
+  uint32_t offset = 0;
+  for (; offset + WORD_BYTES <= first.byte_count; offset += WORD_BYTES) {
+    word_pair words = {0, 0};
+    memcpy(&words.first, first.bytes + offset, sizeof words.first);
+    memcpy(&words.second, second + offset, sizeof words.second);
+    total += narrow_word_products(words, first.bits);
+  }
+  for (; offset < first.byte_count; offset++) {
+    word_pair words = {first.bytes[offset], second[offset]};
+    total += narrow_word_products(words, first.bits);
+  }
+  return total;
+}
+
+#ifdef NARSIL_HARDWARE_POPCOUNT
+__attribute__((target("popcnt"))) static uint32_t narrow_pair_products_with_popcnt(packed_code first,
+                                                                                   const uint8_t *second) {
+  return narrow_pair_products(first, second);
+}
+#endif
+
+uint32_t kernel_pair_products(packed_code first, const uint8_t *second, uint32_t dimension) {
+  if (first.bits == BITS_PER_BYTE) { return kernel_products_8x8(first.bytes, second, dimension); }
+  if (first.bits == NIBBLE_BITS) { return nibble_pair_products(first.bytes, second, first.byte_count); }
+#ifdef NARSIL_HARDWARE_POPCOUNT
+  if (uses_hardware_popcount()) { return narrow_pair_products_with_popcnt(first, second); }
+#endif
+  return narrow_pair_products(first, second);
+}

package/cpp/core/src/last_links.c

@@ -0,0 +1,145 @@
+#include "internal.h"
+
+#include "../include/narsil_core.h"
+
+#include <stdatomic.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <string.h>
+
+static _Atomic(int32_t) *incoming_word(const walk_context *context, int32_t ordinal) {
+  if (ordinal < 0 || context->graph->incoming == NULL) { return NULL; }
+  if (ordinal >= load_word(context->graph->header, NARSIL_GRAPH_WORD_SLOT_CAPACITY)) { return NULL; }
+  return atomic_word(context->graph->incoming, (uint32_t)ordinal);
+}
+
+int32_t incoming_count(const walk_context *context, int32_t ordinal) {
+  _Atomic(int32_t) *word = incoming_word(context, ordinal);
+  return word == NULL ? 0 : atomic_load(word);
+}
+
+void note_linked(const walk_context *context, int32_t ordinal) {
+  _Atomic(int32_t) *word = incoming_word(context, ordinal);
+  if (word != NULL) { atomic_fetch_add(word, 1); }
+}
+
+void note_nearest_link_due(const walk_context *context, int32_t ordinal) {
+  if (incoming_word(context, ordinal) == NULL) { return; }
+  atomic_store(atomic_byte(context->graph->nearest_link_due, (uint32_t)ordinal), 1);
+}
+
+void note_unlinked(const walk_context *context, int32_t ordinal) {
+  _Atomic(int32_t) *word = incoming_word(context, ordinal);
+  if (word == NULL) { return; }
+  note_nearest_link_due(context, ordinal);
+  int32_t seen = atomic_load(word);
+  while (seen > 0 && !atomic_compare_exchange_weak(word, &seen, seen - 1)) {}
+}
+
+void note_node_left_unlinked(const walk_context *context) {
+  atomic_store(atomic_word(context->graph->header, NARSIL_GRAPH_WORD_UNLINKED_NODES), 1);
+}
+
+int release_unless_last(const walk_context *context, int32_t ordinal) {
+  _Atomic(int32_t) *word = incoming_word(context, ordinal);
+  if (word == NULL) { return 1; }
+  int32_t seen = atomic_load(word);
+  while (seen > 1) {
+    if (!atomic_compare_exchange_weak(word, &seen, seen - 1)) { continue; }
+    note_nearest_link_due(context, ordinal);
+    return 1;
+  }
+  return 0;
+}
+
+int is_live(const walk_context *context, int32_t ordinal) {
+  return node_top_layer(context, ordinal) >= 0 && !is_tombstoned(context, ordinal);
+}
+
+narsil_status reserve_last_links(narsil_workspace *workspace) {
+  return list_reserve(&workspace->dropped_links, MAX_REPAIR_CANDIDATES);
+}
+
+int list_holds(held_links list, int32_t ordinal) {
+  for (uint32_t i = 0; i < list.count; i++) {
+    if (list.ordinals[i] == ordinal) { return 1; }
+  }
+  return 0;
+}
+
+static int was_held(held_links held, int32_t ordinal) { return held.every_candidate || list_holds(held, ordinal); }
+
+static double distance_in(const scored_list *sorted, int32_t ordinal) {
+  for (uint32_t i = 0; i < sorted->count; i++) {
+    if (sorted->ordinals[i] == ordinal) { return sorted->distances[i]; }
+  }
+  return 0;
+}
+
+static void collect_dropped(const scored_list *sorted, const int32_t *kept, uint32_t kept_count, scored_list *dropped) {
+  dropped->count = 0;
+  held_links chosen = {kept, kept_count, 0};
+  for (uint32_t i = 0; i < sorted->count && dropped->count < dropped->capacity; i++) {
+    if (list_holds(chosen, sorted->ordinals[i])) { continue; }
+    dropped->ordinals[dropped->count] = sorted->ordinals[i];
+    dropped->distances[dropped->count] = sorted->distances[i];
+    dropped->count += 1;
+  }
+}
+
+static int replace_farthest_shared(const walk_context *context, int32_t *kept, uint32_t kept_count, held_links held,
+                                   scored_node replacement) {
+  narsil_workspace *workspace = context->workspace;
+  double *distances = workspace->kept_distances;
+  uint8_t *tried = workspace->replacement_tried;
+  memset(tried, 0, kept_count);
+  for (;;) {
+    int64_t farthest = -1;
+    for (uint32_t k = 0; k < kept_count; k++) {
+      if (tried[k] || incoming_count(context, kept[k]) < 2) { continue; }
+      if (farthest == -1 || distances[k] > distances[farthest]) { farthest = k; }
+    }
+    if (farthest == -1) { return 0; }
+    tried[farthest] = 1;
+    int32_t replaced = kept[farthest];
+    if (was_held(held, replaced) && !release_unless_last(context, replaced)) { continue; }
+    kept[farthest] = replacement.ordinal;
+    distances[farthest] = replacement.distance;
+    return 1;
+  }
+}
+
+uint32_t write_keeping_last_links(const walk_context *context, int32_t ordinal, const scored_list *sorted,
+                                  int32_t *kept, uint32_t kept_count, held_links held) {
+  narsil_workspace *workspace = context->workspace;
+  uint32_t limit = list_limit(context->graph, 0);
+  scored_list *dropped = &workspace->dropped_links;
+  collect_dropped(sorted, kept, kept_count, dropped);
+  for (uint32_t k = 0; k < kept_count; k++) { workspace->kept_distances[k] = distance_in(sorted, kept[k]); }
+  uint32_t count = kept_count;
+  for (uint32_t i = 0; i < dropped->count; i++) {
+    int32_t candidate = dropped->ordinals[i];
+    if (!was_held(held, candidate)) { continue; }
+    if (!is_live(context, candidate)) {
+      note_unlinked(context, candidate);
+      continue;
+    }
+    if (release_unless_last(context, candidate)) { continue; }
+    if (count < limit && count < MAX_NEIGHBOURS_PER_LIST) {
+      kept[count] = candidate;
+      workspace->kept_distances[count] = dropped->distances[i];
+      count += 1;
+      continue;
+    }
+    scored_node replacement = {candidate, dropped->distances[i]};
+    if (replace_farthest_shared(context, kept, count, held, replacement)) { continue; }
+    note_unlinked(context, candidate);
+    note_node_left_unlinked(context);
+  }
+  for (uint32_t k = 0; k < count; k++) {
+    if (!was_held(held, kept[k])) { note_linked(context, kept[k]); }
+  }
+  node_layer position = {ordinal, 0};
+  list_replace(context, position, kept, count);
+  return count;
+}

package/cpp/core/src/links.c

@@ -0,0 +1,195 @@
+#include "internal.h"
+
+#include "../include/narsil_core.h"
+
+#include <math.h>
+#include <stdatomic.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+#define INITIAL_LIST_CAPACITY 256
+#define FARTHEST_REJECTED_LINKS_KEPT 1U
+
+narsil_status list_reserve(scored_list *list, uint32_t capacity) {
+  if (capacity <= list->capacity) { return NARSIL_OK; }
+  uint32_t held = list->capacity == 0 ? INITIAL_LIST_CAPACITY : list->capacity;
+  uint32_t next = grown_capacity(held, capacity);
+  int32_t *ordinals = realloc(list->ordinals, (size_t)next * sizeof *ordinals);
+  if (ordinals == NULL) { return NARSIL_OUT_OF_MEMORY; }
+  list->ordinals = ordinals;
+  double *distances = realloc(list->distances, (size_t)next * sizeof *distances);
+  if (distances == NULL) { return NARSIL_OUT_OF_MEMORY; }
+  list->distances = distances;
+  list->capacity = next;
+  return NARSIL_OK;
+}
+
+void list_release(scored_list *list) {
+  free(list->ordinals);
+  free(list->distances);
+  list->ordinals = NULL;
+  list->distances = NULL;
+  list->capacity = 0;
+  list->count = 0;
+}
+
+void list_sort_nearest_first(scored_list *list) {
+  for (uint32_t i = 1; i < list->count; i++) {
+    int32_t ordinal = list->ordinals[i];
+    double distance = list->distances[i];
+    uint32_t slot = i;
+    while (slot > 0 && list->distances[slot - 1] > distance) {
+      list->distances[slot] = list->distances[slot - 1];
+      list->ordinals[slot] = list->ordinals[slot - 1];
+      slot -= 1;
+    }
+    list->distances[slot] = distance;
+    list->ordinals[slot] = ordinal;
+  }
+}
+
+uint32_t stored_count(const int32_t *list, uint32_t capacity) {
+  if (list[0] < 0) { return 0; }
+  return (uint32_t)list[0] > capacity ? capacity : (uint32_t)list[0];
+}
+
+void list_add(const walk_context *context, node_layer position, int32_t neighbour) {
+  uint32_t capacity = 0;
+  int32_t *list = list_at(context, position, &capacity);
+  if (list == NULL) { return; }
+  uint32_t count = stored_count(list, capacity);
+  for (uint32_t i = 1; i <= count; i++) {
+    if (list[i] == neighbour) { return; }
+  }
+  if (count + 1 > capacity) { return; }
+  list[count + 1] = neighbour;
+  list[0] = (int32_t)(count + 1);
+  if (position.layer == 0) { note_linked(context, neighbour); }
+}
+
+void list_take_out(const walk_context *context, node_layer position, int32_t neighbour) {
+  uint32_t capacity = 0;
+  int32_t *list = list_at(context, position, &capacity);
+  if (list == NULL) { return; }
+  uint32_t count = stored_count(list, capacity);
+  for (uint32_t i = 1; i <= count; i++) {
+    if (list[i] != neighbour) { continue; }
+    for (uint32_t moved = i; moved < count; moved++) { list[moved] = list[moved + 1]; }
+    list[0] = (int32_t)(count - 1);
+    if (position.layer == 0) { note_unlinked(context, neighbour); }
+    return;
+  }
+}
+
+void list_write_counted(const walk_context *context, node_layer position, const int32_t *neighbours, uint32_t count) {
+  uint32_t capacity = 0;
+  int32_t *list = list_at(context, position, &capacity);
+  if (list == NULL) { return; }
+  if (position.layer > 0) {
+    list_replace(context, position, neighbours, count);
+    return;
+  }
+  held_links held = {list + 1, stored_count(list, capacity), 0};
+  held_links written = {neighbours, count > capacity ? capacity : count, 0};
+  for (uint32_t i = 0; i < held.count; i++) {
+    if (!list_holds(written, held.ordinals[i])) { note_unlinked(context, held.ordinals[i]); }
+  }
+  for (uint32_t i = 0; i < written.count; i++) {
+    if (!list_holds(held, written.ordinals[i])) { note_linked(context, written.ordinals[i]); }
+  }
+  list_replace(context, position, neighbours, count);
+}
+
+void list_replace(const walk_context *context, node_layer position, const int32_t *neighbours, uint32_t count) {
+  uint32_t capacity = 0;
+  int32_t *list = list_at(context, position, &capacity);
+  if (list == NULL) { return; }
+  uint32_t kept = count > capacity ? capacity : count;
+  for (uint32_t i = 0; i < kept; i++) { list[i + 1] = neighbours[i]; }
+  list[0] = (int32_t)kept;
+}
+
+static uint32_t keep_farthest_rejected(const scored_list *sorted, uint32_t limit, int32_t *selected, uint32_t count) {
+  if (count * 2 >= limit) { return count; }
+  uint32_t kept = 0;
+  for (uint32_t i = sorted->count; i > 0 && kept < FARTHEST_REJECTED_LINKS_KEPT; i--) {
+    if (count >= limit) { return count; }
+    int32_t candidate = sorted->ordinals[i - 1];
+    held_links chosen = {selected, count, 0};
+    if (list_holds(chosen, candidate)) { continue; }
+    selected[count] = candidate;
+    count += 1;
+    kept += 1;
+  }
+  return count;
+}
+
+uint32_t select_neighbours(const walk_context *context, scored_list *candidates, uint32_t limit, int32_t *selected) {
+  list_sort_nearest_first(candidates);
+  uint32_t count = 0;
+  for (uint32_t i = 0; i < candidates->count && count < limit; i++) {
+    int32_t candidate = candidates->ordinals[i];
+    double distance = candidates->distances[i];
+    int accepted = 1;
+    for (uint32_t chosen = 0; chosen < count && accepted; chosen++) {
+      if (distance >= pair_distance(context, candidate, selected[chosen])) { accepted = 0; }
+    }
+    if (!accepted) { continue; }
+    selected[count] = candidate;
+    count += 1;
+  }
+  return keep_farthest_rejected(candidates, limit, selected, count);
+}
+
+void list_prune(const walk_context *context, node_layer position) {
+  uint32_t capacity = 0;
+  int32_t *list = list_at(context, position, &capacity);
+  if (list == NULL) { return; }
+  uint32_t limit = list_limit(context->graph, position.layer);
+  uint32_t count = stored_count(list, capacity);
+  if (count <= limit) { return; }
+  scored_list *candidates = &context->workspace->link_candidates;
+  if (list_reserve(candidates, count) != NARSIL_OK) { return; }
+  candidates->count = 0;
+  for (uint32_t i = 1; i <= count; i++) {
+    double distance = pair_distance(context, position.ordinal, list[i]);
+    if (distance == HUGE_VAL) {
+      if (position.layer == 0) { note_unlinked(context, list[i]); }
+      continue;
+    }
+    candidates->ordinals[candidates->count] = list[i];
+    candidates->distances[candidates->count] = distance;
+    candidates->count += 1;
+  }
+  uint32_t kept = select_neighbours(context, candidates, limit, context->workspace->kept);
+  if (position.layer == 0) {
+    held_links every_candidate = {NULL, 0, 1};
+    (void)write_keeping_last_links(context, position.ordinal, candidates, context->workspace->kept, kept,
+                                   every_candidate);
+    return;
+  }
+  list_replace(context, position, context->workspace->kept, kept);
+}
+
+int32_t highest_node(const walk_context *context, int include_tombstoned) {
+  int32_t best = NO_NODE;
+  int32_t best_layer = -1;
+  for (uint32_t ordinal = 0; ordinal < context->slots; ordinal++) {
+    int32_t level = node_top_layer(context, (int32_t)ordinal);
+    if (level < 0) { continue; }
+    if (!include_tombstoned && atomic_load(atomic_byte(context->graph->tombstones, ordinal)) == 1) { continue; }
+    if (level > best_layer) {
+      best_layer = level;
+      best = (int32_t)ordinal;
+    }
+  }
+  return best;
+}
+
+void store_entry_point(const walk_context *context, int32_t ordinal) {
+  int32_t *header = context->graph->header;
+  atomic_store(atomic_word(header, NARSIL_GRAPH_WORD_ENTRY_POINT), ordinal);
+  atomic_store(atomic_word(header, NARSIL_GRAPH_WORD_TOP_LAYER),
+               ordinal == NO_NODE ? -1 : node_top_layer(context, ordinal));
+}