@delali/narsil-react-native 0.0.0-stage → 0.3.1-next.146

package/cpp/core/src/files.c

@@ -0,0 +1,99 @@
+#include "../include/narsil_core.h"
+
+#include <stddef.h>
+#include <stdint.h>
+
+static void forget_mapping(narsil_vector_file *file) {
+  file->bytes = NULL;
+  file->length = 0;
+  file->mapping = NULL;
+}
+
+#ifdef _WIN32
+
+#include <stdlib.h>
+#include <windows.h>
+
+static WCHAR *wide_path_of(const char *path) {
+  int length = MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, path, -1, NULL, 0);
+  if (length <= 0) { return NULL; }
+  WCHAR *wide = calloc((size_t)length, sizeof *wide);
+  if (wide == NULL) { return NULL; }
+  if (MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, path, -1, wide, length) != length) {
+    free(wide);
+    return NULL;
+  }
+  return wide;
+}
+
+static narsil_status map_open_file(HANDLE opened, narsil_vector_file *file) {
+  LARGE_INTEGER size;
+  if (!GetFileSizeEx(opened, &size) || size.QuadPart <= 0) { return NARSIL_FILE_UNREADABLE; }
+  HANDLE mapping = CreateFileMappingW(opened, NULL, PAGE_READONLY, 0, 0, NULL);
+  if (mapping == NULL) { return NARSIL_FILE_UNREADABLE; }
+  void *view = MapViewOfFile(mapping, FILE_MAP_READ, 0, 0, 0);
+  CloseHandle(mapping);
+  if (view == NULL) { return NARSIL_FILE_UNREADABLE; }
+  file->mapping = view;
+  file->bytes = view;
+  file->length = (uint64_t)size.QuadPart;
+  return NARSIL_OK;
+}
+
+narsil_status narsil_vector_file_map(const char *path, narsil_vector_file *file) {
+  if (path == NULL || file == NULL) { return NARSIL_INVALID_ARGUMENT; }
+  forget_mapping(file);
+  WCHAR *wide = wide_path_of(path);
+  if (wide == NULL) { return NARSIL_FILE_UNREADABLE; }
+  HANDLE opened = CreateFileW(wide, GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, NULL,
+                              OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
+  free(wide);
+  if (opened == INVALID_HANDLE_VALUE) { return NARSIL_FILE_UNREADABLE; }
+  narsil_status status = map_open_file(opened, file);
+  CloseHandle(opened);
+  return status;
+}
+
+void narsil_vector_file_unmap(narsil_vector_file *file) {
+  if (file == NULL || file->mapping == NULL) { return; }
+  UnmapViewOfFile(file->mapping);
+  forget_mapping(file);
+}
+
+#else
+
+#include <fcntl.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+static narsil_status map_open_file(int descriptor, narsil_vector_file *file) {
+  struct stat status;
+  if (fstat(descriptor, &status) != 0 || status.st_size <= 0) { return NARSIL_FILE_UNREADABLE; }
+  size_t length = (size_t)status.st_size;
+  void *view = mmap(NULL, length, PROT_READ, MAP_SHARED, descriptor, 0);
+  if (view == MAP_FAILED) { return NARSIL_FILE_UNREADABLE; }
+  (void)posix_madvise(view, length, POSIX_MADV_RANDOM);
+  file->mapping = view;
+  file->bytes = view;
+  file->length = (uint64_t)length;
+  return NARSIL_OK;
+}
+
+narsil_status narsil_vector_file_map(const char *path, narsil_vector_file *file) {
+  if (path == NULL || file == NULL) { return NARSIL_INVALID_ARGUMENT; }
+  forget_mapping(file);
+  int descriptor = open(path, O_RDONLY | O_CLOEXEC);
+  if (descriptor < 0) { return NARSIL_FILE_UNREADABLE; }
+  narsil_status status = map_open_file(descriptor, file);
+  (void)close(descriptor);
+  return status;
+}
+
+void narsil_vector_file_unmap(narsil_vector_file *file) {
+  if (file == NULL || file->mapping == NULL) { return; }
+  (void)munmap(file->mapping, (size_t)file->length);
+  forget_mapping(file);
+}
+
+#endif

package/cpp/core/src/heap.c

@@ -0,0 +1,84 @@
+#include "internal.h"
+
+#include "../include/narsil_core.h"
+
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+#define INITIAL_HEAP_CAPACITY 256
+
+narsil_status heap_reserve(distance_heap *heap, uint32_t capacity) {
+  if (capacity <= heap->capacity) { return NARSIL_OK; }
+  uint32_t held = heap->capacity == 0 ? INITIAL_HEAP_CAPACITY : heap->capacity;
+  uint32_t next = grown_capacity(held, capacity);
+  int32_t *ordinals = realloc(heap->ordinals, (size_t)next * sizeof *ordinals);
+  if (ordinals == NULL) { return NARSIL_OUT_OF_MEMORY; }
+  heap->ordinals = ordinals;
+  double *distances = realloc(heap->distances, (size_t)next * sizeof *distances);
+  if (distances == NULL) { return NARSIL_OUT_OF_MEMORY; }
+  heap->distances = distances;
+  heap->capacity = next;
+  return NARSIL_OK;
+}
+
+void heap_release(distance_heap *heap) {
+  free(heap->ordinals);
+  free(heap->distances);
+  heap->ordinals = NULL;
+  heap->distances = NULL;
+  heap->capacity = 0;
+  heap->size = 0;
+}
+
+static inline int leaves_first(const distance_heap *heap, double candidate, double other) {
+  return heap->greatest_first ? candidate > other : candidate < other;
+}
+
+narsil_status heap_push(distance_heap *heap, scored_node node) {
+  narsil_status status = heap_reserve(heap, heap->size + 1);
+  if (status != NARSIL_OK) { return status; }
+  uint32_t index = heap->size;
+  heap->size += 1;
+  while (index > 0) {
+    uint32_t parent = (index - 1) >> 1;
+    if (!leaves_first(heap, node.distance, heap->distances[parent])) { break; }
+    heap->ordinals[index] = heap->ordinals[parent];
+    heap->distances[index] = heap->distances[parent];
+    index = parent;
+  }
+  heap->ordinals[index] = node.ordinal;
+  heap->distances[index] = node.distance;
+  return NARSIL_OK;
+}
+
+int heap_pop(distance_heap *heap) {
+  if (heap->size == 0) { return 0; }
+  heap->top.ordinal = heap->ordinals[0];
+  heap->top.distance = heap->distances[0];
+  heap->size -= 1;
+  if (heap->size == 0) { return 1; }
+
+  uint32_t size = heap->size;
+  int32_t ordinal = heap->ordinals[size];
+  double distance = heap->distances[size];
+  uint32_t index = 0;
+  for (;;) {
+    uint32_t left = (2 * index) + 1;
+    uint32_t right = left + 1;
+    uint32_t first = index;
+    double first_distance = distance;
+    if (left < size && leaves_first(heap, heap->distances[left], first_distance)) {
+      first = left;
+      first_distance = heap->distances[left];
+    }
+    if (right < size && leaves_first(heap, heap->distances[right], first_distance)) { first = right; }
+    if (first == index) { break; }
+    heap->ordinals[index] = heap->ordinals[first];
+    heap->distances[index] = heap->distances[first];
+    index = first;
+  }
+  heap->ordinals[index] = ordinal;
+  heap->distances[index] = distance;
+  return 1;
+}

package/cpp/core/src/internal.h

@@ -0,0 +1,284 @@
+#ifndef NARSIL_CORE_INTERNAL_H
+#define NARSIL_CORE_INTERNAL_H
+
+#include "../include/narsil_core.h"
+
+#include <stdatomic.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#define NODE_WRITE_HELD 1
+#define NODE_WRITER_WAITING 2
+#define NODE_VERSION_STEP 4U
+
+#define QUERY_PLANES 4
+#define MAX_M 512
+#define MAX_NEIGHBOURS_PER_LIST ((2 * MAX_M) + 1)
+#define MAX_REPAIR_CANDIDATES (2 * MAX_NEIGHBOURS_PER_LIST)
+#define BITS_PER_BYTE 8
+#define TRAILER_LOWER 0
+#define TRAILER_UPPER 4
+#define TRAILER_CORRECTION 8
+#define TRAILER_SUM 12
+#define NO_ENTRY_POINT (-1)
+#define NO_NODE (-1)
+
+#ifdef NARSIL_PORTABLE_KERNELS
+#elif defined(__aarch64__) || defined(_M_ARM64)
+#define NARSIL_NEON 1
+#elif defined(__SSE2__) || defined(_M_X64)
+#define NARSIL_SSE2 1
+#endif
+
+#if defined(NARSIL_SSE2) && (defined(__GNUC__) || defined(__clang__)) && !defined(NARSIL_SSE2_ONLY)
+#define NARSIL_HARDWARE_POPCOUNT 1
+#endif
+
+#define INLINED_IN_EACH_KERNEL static inline __attribute__((always_inline))
+
+typedef struct {
+  int32_t ordinal;
+  double distance;
+} scored_node;
+
+typedef struct {
+  int32_t *ordinals;
+  double *distances;
+  uint32_t size;
+  uint32_t capacity;
+  int greatest_first;
+  scored_node top;
+} distance_heap;
+
+typedef struct {
+  int32_t *ordinals;
+  double *distances;
+  uint32_t count;
+  uint32_t capacity;
+} scored_list;
+
+typedef struct {
+  double lower;
+  double upper;
+} level_range;
+
+typedef struct {
+  level_range range;
+  double correction;
+  const double *centred;
+} quantised_vector;
+
+typedef struct {
+  uint32_t bits;
+  level_range range;
+  double correction;
+  double sum;
+  uint8_t *levels;
+  uint8_t *low_levels;
+  uint8_t *high_levels;
+  uint8_t *planes;
+} prepared_code;
+
+typedef struct {
+  const uint8_t *bytes;
+  uint32_t byte_count;
+  uint32_t bits;
+} packed_code;
+
+typedef struct {
+  const float *values;
+  double magnitude;
+} float_query;
+
+typedef struct {
+  int32_t ordinal;
+  int32_t layer;
+} node_layer;
+
+typedef struct {
+  uint32_t candidate_limit;
+  int32_t layer;
+  int excludes_tombstones;
+  narsil_ordinal_set accepted;
+} layer_search;
+
+typedef struct {
+  int32_t graph_top;
+  int32_t link_top;
+} link_span;
+
+typedef struct {
+  int32_t ordinal;
+  uint32_t former_count;
+} removed_node;
+
+typedef struct {
+  const narsil_graph *graph;
+  const narsil_store *store;
+  narsil_workspace *workspace;
+  narsil_metric metric;
+  uint32_t thread_slot;
+  int uses_codes;
+  int pairs_use_codes;
+  uint32_t slots;
+  uint32_t store_slots;
+  uint32_t code_bytes;
+  uint32_t record_bytes;
+  double centroid_dot;
+  float_query query;
+  int32_t *fence;
+  prepared_code code;
+} walk_context;
+
+struct narsil_workspace {
+  distance_heap frontier;
+  distance_heap found;
+  uint32_t *visited;
+  uint32_t visited_capacity;
+  uint32_t stamp;
+  int32_t *entry_points;
+  uint32_t entry_point_count;
+  uint32_t entry_point_capacity;
+  uint32_t dimension_capacity;
+  size_t dimension_scratch_bytes;
+  double *normalised;
+  double *centred;
+  float *placed_vector;
+  float *stored_vector;
+  float *other_vector;
+  prepared_code code;
+  scored_list layer_candidates;
+  scored_list link_candidates;
+  scored_list repair_candidates;
+  scored_list dropped_links;
+  int32_t *selections;
+  uint32_t *selection_counts;
+  uint32_t selection_layers;
+  uint32_t selection_stride;
+  int32_t fence;
+  int32_t neighbours[MAX_NEIGHBOURS_PER_LIST];
+  int32_t former[MAX_NEIGHBOURS_PER_LIST];
+  int32_t kept[MAX_NEIGHBOURS_PER_LIST];
+  int32_t repaired[MAX_NEIGHBOURS_PER_LIST];
+  double kept_distances[MAX_NEIGHBOURS_PER_LIST];
+  uint8_t replacement_tried[MAX_NEIGHBOURS_PER_LIST];
+};
+
+typedef struct {
+  const int32_t *ordinals;
+  uint32_t count;
+  int every_candidate;
+} held_links;
+
+static inline int32_t load_word(const int32_t *words, uint32_t index) {
+  return atomic_load((const _Atomic(int32_t) *)(words + index));
+}
+
+static inline _Atomic(int32_t) *atomic_word(int32_t *words, uint32_t index) {
+  return (_Atomic(int32_t) *)(words + index);
+}
+
+static inline _Atomic(uint8_t) *atomic_byte(uint8_t *bytes, uint32_t index) {
+  return (_Atomic(uint8_t) *)(bytes + index);
+}
+
+static inline int is_tombstoned(const walk_context *context, int32_t ordinal) {
+  return atomic_load(atomic_byte(context->graph->tombstones, (uint32_t)ordinal)) == 1;
+}
+
+uint32_t grown_capacity(uint32_t first, uint32_t needed);
+
+narsil_status heap_reserve(distance_heap *heap, uint32_t capacity);
+void heap_release(distance_heap *heap);
+narsil_status heap_push(distance_heap *heap, scored_node node);
+int heap_pop(distance_heap *heap);
+
+narsil_status list_reserve(scored_list *list, uint32_t capacity);
+void list_release(scored_list *list);
+void list_sort_nearest_first(scored_list *list);
+
+void graph_lock_shared(const narsil_graph *graph, uint32_t thread_slot);
+void graph_unlock_shared(const narsil_graph *graph, uint32_t thread_slot);
+void node_lock(const walk_context *context, int32_t ordinal);
+void node_unlock(const walk_context *context, int32_t ordinal);
+void entry_lock(const walk_context *context);
+void entry_unlock(const walk_context *context);
+uint32_t read_neighbours(const walk_context *context, node_layer position, int32_t *out);
+
+int32_t node_top_layer(const walk_context *context, int32_t ordinal);
+int32_t *list_at(const walk_context *context, node_layer position, uint32_t *capacity);
+uint32_t list_limit(const narsil_graph *graph, int32_t layer);
+void list_add(const walk_context *context, node_layer position, int32_t neighbour);
+void list_take_out(const walk_context *context, node_layer position, int32_t neighbour);
+void list_replace(const walk_context *context, node_layer position, const int32_t *neighbours, uint32_t count);
+uint32_t select_neighbours(const walk_context *context, scored_list *candidates, uint32_t limit, int32_t *selected);
+void list_prune(const walk_context *context, node_layer position);
+uint32_t stored_count(const int32_t *list, uint32_t capacity);
+void list_write_counted(const walk_context *context, node_layer position, const int32_t *neighbours, uint32_t count);
+int32_t highest_node(const walk_context *context, int include_tombstoned);
+
+int32_t incoming_count(const walk_context *context, int32_t ordinal);
+void note_linked(const walk_context *context, int32_t ordinal);
+void note_unlinked(const walk_context *context, int32_t ordinal);
+void note_node_left_unlinked(const walk_context *context);
+void note_nearest_link_due(const walk_context *context, int32_t ordinal);
+narsil_status link_from_nearest(const walk_context *context);
+int release_unless_last(const walk_context *context, int32_t ordinal);
+int is_live(const walk_context *context, int32_t ordinal);
+narsil_status reserve_last_links(narsil_workspace *workspace);
+int list_holds(held_links list, int32_t ordinal);
+uint32_t write_keeping_last_links(const walk_context *context, int32_t ordinal, const scored_list *sorted,
+                                  int32_t *kept, uint32_t kept_count, held_links held);
+void store_entry_point(const walk_context *context, int32_t ordinal);
+
+narsil_status workspace_reserve_dimension(narsil_workspace *workspace, uint32_t dimension);
+narsil_status workspace_reserve_visited(narsil_workspace *workspace, uint32_t slots);
+narsil_status workspace_reserve_entry_points(narsil_workspace *workspace, uint32_t count);
+narsil_status workspace_reserve_links(narsil_workspace *workspace, const narsil_graph *graph, uint32_t layers);
+
+narsil_status begin_walk(walk_context *context, narsil_workspace *workspace, const narsil_graph *graph,
+                         const narsil_store *store, uint32_t metric);
+void walk_with_query(walk_context *context, const float *vector);
+void walk_with_placement_query(walk_context *context, const float *vector);
+narsil_status read_graph_extent(walk_context *context);
+void use_single_entry(narsil_workspace *workspace, int32_t ordinal);
+narsil_status search_layer(const walk_context *context, layer_search search, narsil_candidates *out);
+narsil_status descend_layers(const walk_context *context, int32_t first, int32_t last, narsil_candidates *out);
+
+void normalise_into(double *normalised, const float *vector, uint32_t dimension);
+void quantise_vector(narsil_workspace *workspace, const narsil_store *store, narsil_metric metric, const float *vector,
+                     uint32_t bits, quantised_vector *quantised);
+double quantised_level(double value, level_range range, uint32_t bits);
+void prepare_code(narsil_workspace *workspace, const narsil_store *store, narsil_metric metric, const float *vector,
+                  prepared_code *code);
+void prepare_code_with_bits(narsil_workspace *workspace, const narsil_store *store, narsil_metric metric,
+                            const float *vector, uint32_t query_bits, prepared_code *code);
+uint32_t code_bytes_of(uint32_t dimension, uint32_t bits);
+uint32_t query_bits_of(uint32_t bits);
+uint32_t placement_query_bits_of(uint32_t bits);
+double estimate_distance(const walk_context *context, const uint8_t *record);
+double estimate_pair_distance(const walk_context *context, const uint8_t *first, const uint8_t *second);
+double pair_distance(const walk_context *context, int32_t first, int32_t second);
+
+float kernel_dot(const float *lhs, const float *rhs, uint32_t dimension);
+float kernel_squared_distance(const float *lhs, const float *rhs, uint32_t dimension);
+float kernel_magnitude(const float *values, uint32_t dimension);
+uint32_t kernel_products_8x8(const uint8_t *document, const uint8_t *query, uint32_t dimension);
+uint32_t kernel_products_4x4(const uint8_t *document, const uint8_t *low, const uint8_t *high, uint32_t bytes);
+uint32_t kernel_products_bits(packed_code document, const uint8_t *planes);
+uint32_t kernel_pair_products(packed_code first, const uint8_t *second, uint32_t dimension);
+#ifdef NARSIL_HARDWARE_POPCOUNT
+int uses_hardware_popcount(void);
+#endif
+
+int is_metric(uint32_t metric);
+int is_code_width(uint32_t bits);
+int store_is_complete(const narsil_store *store);
+int graph_is_complete(const narsil_graph *graph);
+const float *stored_vector(const narsil_store *store, int32_t ordinal, float *scratch);
+uint8_t *stored_record(const narsil_store *store, int32_t ordinal);
+double float_distance(const walk_context *context, float_query query, int32_t ordinal);
+double walk_distance(const walk_context *context, int32_t ordinal);
+void prefetch_for_walk(const walk_context *context, int32_t ordinal);
+
+#endif

package/cpp/core/src/kernels_code.c

@@ -0,0 +1,209 @@
+#include "internal.h"
+
+#include <stddef.h>
+#include <stdint.h>
+#include <string.h>
+
+#define VECTOR_BYTES 16U
+#define WORD_BYTES 8U
+#define LOW_NIBBLE_MASK 15U
+#define NIBBLE_BITS 4
+#define TWO_BIT_LOW_BITS UINT64_C(0x5555555555555555)
+
+#ifdef NARSIL_NEON
+#include <arm_neon.h>
+#endif
+
+#ifdef NARSIL_SSE2
+#include <emmintrin.h>
+#endif
+
+#ifdef NARSIL_HARDWARE_POPCOUNT
+#include <cpuid.h>
+#include <stdatomic.h>
+#endif
+
+#ifdef NARSIL_SSE2
+static inline __m128i load_bytes(const uint8_t *source) {
+  __m128i loaded;
+  memcpy(&loaded, source, sizeof loaded);
+  return loaded;
+}
+
+static inline uint32_t sum_of_lanes(__m128i running) {
+  uint32_t lane[4];
+  memcpy(lane, &running, sizeof lane);
+  return lane[0] + lane[1] + lane[2] + lane[3];
+}
+#endif
+
+uint32_t kernel_products_8x8(const uint8_t *document, const uint8_t *query, uint32_t dimension) {
+  uint32_t total = 0;
+  uint32_t offset = 0;
+#ifdef NARSIL_NEON
+  uint32x4_t running = vdupq_n_u32(0);
+  for (; offset + VECTOR_BYTES <= dimension; offset += VECTOR_BYTES) {
+    uint8x16_t document_bytes = vld1q_u8(document + offset);
+    uint8x16_t query_bytes = vld1q_u8(query + offset);
+    running = vpadalq_u16(running, vmull_u8(vget_low_u8(document_bytes), vget_low_u8(query_bytes)));
+    running = vpadalq_u16(running, vmull_u8(vget_high_u8(document_bytes), vget_high_u8(query_bytes)));
+  }
+  total = vaddvq_u32(running);
+#elif defined(NARSIL_SSE2)
+  __m128i zero = _mm_setzero_si128();
+  __m128i running = zero;
+  for (; offset + VECTOR_BYTES <= dimension; offset += VECTOR_BYTES) {
+    __m128i document_bytes = load_bytes(document + offset);
+    __m128i query_bytes = load_bytes(query + offset);
+    __m128i low = _mm_mullo_epi16(_mm_unpacklo_epi8(document_bytes, zero), _mm_unpacklo_epi8(query_bytes, zero));
+    __m128i high = _mm_mullo_epi16(_mm_unpackhi_epi8(document_bytes, zero), _mm_unpackhi_epi8(query_bytes, zero));
+    running = _mm_add_epi32(running, _mm_add_epi32(_mm_unpacklo_epi16(low, zero), _mm_unpackhi_epi16(low, zero)));
+    running = _mm_add_epi32(running, _mm_add_epi32(_mm_unpacklo_epi16(high, zero), _mm_unpackhi_epi16(high, zero)));
+  }
+  total = sum_of_lanes(running);
+#endif
+  for (; offset < dimension; offset++) { total += (uint32_t)document[offset] * (uint32_t)query[offset]; }
+  return total;
+}
+
+uint32_t kernel_products_4x4(const uint8_t *document, const uint8_t *low, const uint8_t *high, uint32_t bytes) {
+  uint32_t total = 0;
+  uint32_t offset = 0;
+#ifdef NARSIL_NEON
+  uint8x16_t mask = vdupq_n_u8(LOW_NIBBLE_MASK);
+  uint32x4_t running = vdupq_n_u32(0);
+  for (; offset + VECTOR_BYTES <= bytes; offset += VECTOR_BYTES) {
+    uint8x16_t document_bytes = vld1q_u8(document + offset);
+    uint8x16_t document_low = vandq_u8(document_bytes, mask);
+    uint8x16_t document_high = vshrq_n_u8(document_bytes, NIBBLE_BITS);
+    uint8x16_t low_bytes = vld1q_u8(low + offset);
+    uint8x16_t high_bytes = vld1q_u8(high + offset);
+    uint16x8_t first = vmlal_u8(vmull_u8(vget_low_u8(document_low), vget_low_u8(low_bytes)), vget_low_u8(document_high),
+                                vget_low_u8(high_bytes));
+    uint16x8_t second = vmlal_u8(vmull_u8(vget_high_u8(document_low), vget_high_u8(low_bytes)),
+                                 vget_high_u8(document_high), vget_high_u8(high_bytes));
+    running = vpadalq_u16(vpadalq_u16(running, first), second);
+  }
+  total = vaddvq_u32(running);
+#elif defined(NARSIL_SSE2)
+  __m128i zero = _mm_setzero_si128();
+  __m128i mask = _mm_set1_epi8((char)LOW_NIBBLE_MASK);
+  __m128i running = zero;
+  for (; offset + VECTOR_BYTES <= bytes; offset += VECTOR_BYTES) {
+    __m128i document_bytes = load_bytes(document + offset);
+    __m128i low_bytes = load_bytes(low + offset);
+    __m128i high_bytes = load_bytes(high + offset);
+    __m128i document_low = _mm_and_si128(document_bytes, mask);
+    __m128i document_high = _mm_and_si128(_mm_srli_epi16(document_bytes, NIBBLE_BITS), mask);
+    __m128i even =
+        _mm_add_epi32(_mm_madd_epi16(_mm_unpacklo_epi8(document_low, zero), _mm_unpacklo_epi8(low_bytes, zero)),
+                      _mm_madd_epi16(_mm_unpackhi_epi8(document_low, zero), _mm_unpackhi_epi8(low_bytes, zero)));
+    __m128i odd =
+        _mm_add_epi32(_mm_madd_epi16(_mm_unpacklo_epi8(document_high, zero), _mm_unpacklo_epi8(high_bytes, zero)),
+                      _mm_madd_epi16(_mm_unpackhi_epi8(document_high, zero), _mm_unpackhi_epi8(high_bytes, zero)));
+    running = _mm_add_epi32(running, _mm_add_epi32(even, odd));
+  }
+  total = sum_of_lanes(running);
+#endif
+  for (; offset < bytes; offset++) {
+    total += ((uint32_t)(document[offset] & LOW_NIBBLE_MASK) * low[offset]) +
+             ((uint32_t)(document[offset] >> NIBBLE_BITS) * high[offset]);
+  }
+  return total;
+}
+
+INLINED_IN_EACH_KERNEL uint32_t one_bit_products(packed_code document, const uint8_t *planes) {
+  uint32_t total = 0;
+  uint32_t offset = 0;
+  for (; offset + WORD_BYTES <= document.byte_count; offset += WORD_BYTES) {
+    uint64_t word = 0;
+    memcpy(&word, document.bytes + offset, sizeof word);
+    for (uint32_t plane = 0; plane < QUERY_PLANES; plane++) {
+      uint64_t query = 0;
+      memcpy(&query, planes + ((size_t)plane * document.byte_count) + offset, sizeof query);
+      total += (uint32_t)__builtin_popcountll(word & query) << plane;
+    }
+  }
+  for (; offset < document.byte_count; offset++) {
+    for (uint32_t plane = 0; plane < QUERY_PLANES; plane++) {
+      uint32_t query = planes[((size_t)plane * document.byte_count) + offset];
+      total += (uint32_t)__builtin_popcount(document.bytes[offset] & query) << plane;
+    }
+  }
+  return total;
+}
+
+INLINED_IN_EACH_KERNEL uint32_t weighted_bits(uint64_t low, uint64_t high, uint64_t plane) {
+  return (uint32_t)__builtin_popcountll(low & plane) + (2 * (uint32_t)__builtin_popcountll(high & plane));
+}
+
+INLINED_IN_EACH_KERNEL uint32_t two_bit_products(packed_code document, const uint8_t *planes) {
+  uint32_t total = 0;
+  uint32_t offset = 0;
+  for (; offset + WORD_BYTES <= document.byte_count; offset += WORD_BYTES) {
+    uint64_t word = 0;
+    memcpy(&word, document.bytes + offset, sizeof word);
+    uint64_t low = word & TWO_BIT_LOW_BITS;
+    uint64_t high = (word >> 1) & TWO_BIT_LOW_BITS;
+    for (uint32_t plane = 0; plane < QUERY_PLANES; plane++) {
+      uint64_t query = 0;
+      memcpy(&query, planes + ((size_t)plane * document.byte_count) + offset, sizeof query);
+      total += weighted_bits(low, high, query) << plane;
+    }
+  }
+  for (; offset < document.byte_count; offset++) {
+    uint64_t low = document.bytes[offset] & TWO_BIT_LOW_BITS;
+    uint64_t high = ((uint64_t)document.bytes[offset] >> 1) & TWO_BIT_LOW_BITS;
+    for (uint32_t plane = 0; plane < QUERY_PLANES; plane++) {
+      total += weighted_bits(low, high, planes[((size_t)plane * document.byte_count) + offset]) << plane;
+    }
+  }
+  return total;
+}
+
+#ifdef NARSIL_HARDWARE_POPCOUNT
+#define POPCOUNT_UNKNOWN 0
+#define POPCOUNT_ABSENT 1
+#define POPCOUNT_PRESENT 2
+#define CPUID_FEATURES_LEAF 1U
+
+static atomic_int popcount_state = POPCOUNT_UNKNOWN;
+
+static int processor_counts_bits(void) {
+  unsigned int eax = 0;
+  unsigned int ebx = 0;
+  unsigned int ecx = 0;
+  unsigned int edx = 0;
+  if (!__get_cpuid(CPUID_FEATURES_LEAF, &eax, &ebx, &ecx, &edx)) { return 0; }
+  return (ecx & bit_POPCNT) != 0;
+}
+
+int uses_hardware_popcount(void) {
+  int state = atomic_load_explicit(&popcount_state, memory_order_relaxed);
+  if (state == POPCOUNT_UNKNOWN) {
+    state = processor_counts_bits() ? POPCOUNT_PRESENT : POPCOUNT_ABSENT;
+    atomic_store_explicit(&popcount_state, state, memory_order_relaxed);
+  }
+  return state == POPCOUNT_PRESENT;
+}
+
+__attribute__((target("popcnt"))) static uint32_t one_bit_products_with_popcnt(packed_code document,
+                                                                               const uint8_t *planes) {
+  return one_bit_products(document, planes);
+}
+
+__attribute__((target("popcnt"))) static uint32_t two_bit_products_with_popcnt(packed_code document,
+                                                                               const uint8_t *planes) {
+  return two_bit_products(document, planes);
+}
+#endif
+
+uint32_t kernel_products_bits(packed_code document, const uint8_t *planes) {
+#ifdef NARSIL_HARDWARE_POPCOUNT
+  if (uses_hardware_popcount()) {
+    return document.bits == 2 ? two_bit_products_with_popcnt(document, planes)
+                              : one_bit_products_with_popcnt(document, planes);
+  }
+#endif
+  return document.bits == 2 ? two_bit_products(document, planes) : one_bit_products(document, planes);
+}