@delali/narsil-react-native 0.3.1-next.154 → 0.3.1-next.156

package/cpp/core/src/analysis/normalise.c

@@ -0,0 +1,242 @@
+#include "normalise.h"
+
+#include "../../include/narsil_core.h"
+#include "text.h"
+#include "unicode.h"
+
+#include <stddef.h>
+#include <stdint.h>
+
+#define CAPITAL_SIGMA 0x3A3U
+#define SMALL_FINAL_SIGMA 0x3C2U
+#define ASCII_CAPITAL_A 0x41U
+#define ASCII_CAPITAL_Z 0x5AU
+#define ASCII_CASE_OFFSET 0x20U
+#define APOSTROPHE 0x27U
+#define LATIN_SMALL_I 0x69U
+#define COMBINING_DOT_ABOVE 0x307U
+#define DOTTED_CAPITAL_I 0x130U
+#define ARMENIAN_MARKS_FIRST 0x55BU
+#define ARMENIAN_MARKS_LAST 0x55FU
+#define LEFT_SINGLE_QUOTE 0x2018U
+#define RIGHT_SINGLE_QUOTE 0x2019U
+#define MODIFIER_APOSTROPHE 0x2BCU
+#define MODIFIER_TURNED_COMMA 0x2BBU
+#define FULLWIDTH_ASCII_FIRST 0xFF01U
+#define FULLWIDTH_ASCII_LAST 0xFF5EU
+#define FULLWIDTH_ASCII_OFFSET 0xFEE0U
+#define HALFWIDTH_KANA_FIRST 0xFF61U
+#define HALFWIDTH_KANA_LAST 0xFF9DU
+#define HALFWIDTH_VOICED_MARK 0xFF9EU
+#define HALFWIDTH_SEMI_VOICED_MARK 0xFF9FU
+#define COMBINING_VOICED_MARK 0x3099U
+#define COMBINING_SEMI_VOICED_MARK 0x309AU
+#define COMBINING_DIACRITICS_FIRST 0x300U
+#define COMBINING_DIACRITICS_LAST 0x36FU
+#define NON_STARTER_BEFORE_ANY_STARTER 256U
+
+static const uint16_t halfwidth_kana_targets[HALFWIDTH_KANA_LAST - HALFWIDTH_KANA_FIRST + 1] = {
+    0x3002, 0x300C, 0x300D, 0x3001, 0x30FB, 0x30F2, 0x30A1, 0x30A3, 0x30A5, 0x30A7, 0x30A9, 0x30E3, 0x30E5,
+    0x30E7, 0x30C3, 0x30FC, 0x30A2, 0x30A4, 0x30A6, 0x30A8, 0x30AA, 0x30AB, 0x30AD, 0x30AF, 0x30B1, 0x30B3,
+    0x30B5, 0x30B7, 0x30B9, 0x30BB, 0x30BD, 0x30BF, 0x30C1, 0x30C4, 0x30C6, 0x30C8, 0x30CA, 0x30CB, 0x30CC,
+    0x30CD, 0x30CE, 0x30CF, 0x30D2, 0x30D5, 0x30D8, 0x30DB, 0x30DE, 0x30DF, 0x30E0, 0x30E1, 0x30E2, 0x30E4,
+    0x30E6, 0x30E8, 0x30E9, 0x30EA, 0x30EB, 0x30EC, 0x30ED, 0x30EF, 0x30F3,
+};
+
+void narsil_normalise_scratch_release(narsil_normalise_scratch *scratch) {
+  narsil_text_release(&scratch->first);
+  narsil_text_release(&scratch->second);
+  narsil_code_points_release(&scratch->points);
+}
+
+static void reorder_canonically(uint32_t *values, size_t length) {
+  for (size_t i = 1; i < length; i++) {
+    uint32_t value = values[i];
+    uint32_t value_class = narsil_unicode_combining_class(value);
+    if (value_class == 0) { continue; }
+    size_t position = i;
+    while (position > 0) {
+      uint32_t before_class = narsil_unicode_combining_class(values[position - 1]);
+      if (before_class <= value_class) { break; }
+      values[position] = values[position - 1];
+      position--;
+    }
+    values[position] = value;
+  }
+}
+
+static int decompose_text(const uint16_t *units, size_t length, narsil_code_points *points) {
+  points->length = 0;
+  int status = narsil_code_points_reserve(points, length);
+  if (status != NARSIL_OK) { return status; }
+  uint32_t parts[NARSIL_UNICODE_MAX_DECOMPOSITION];
+  size_t index = 0;
+  while (index < length) {
+    uint32_t count = narsil_unicode_decompose(narsil_decode_next(units, length, &index), parts);
+    for (uint32_t part = 0; part < count; part++) {
+      status = narsil_code_points_push(points, parts[part]);
+      if (status != NARSIL_OK) { return status; }
+    }
+  }
+  reorder_canonically(points->values, points->length);
+  return NARSIL_OK;
+}
+
+static size_t compose_in_place(uint32_t *values, size_t length) {
+  if (length == 0) { return 0; }
+  size_t starter_position = 0;
+  uint32_t starter = values[0];
+  uint32_t last_class = narsil_unicode_combining_class(starter);
+  if (last_class != 0) { last_class = NON_STARTER_BEFORE_ANY_STARTER; }
+  size_t kept = 1;
+  for (size_t i = 1; i < length; i++) {
+    uint32_t value = values[i];
+    uint32_t value_class = narsil_unicode_combining_class(value);
+    uint32_t composite = narsil_unicode_compose(starter, value);
+    if (composite != NARSIL_UNICODE_NO_COMPOSITE && (last_class < value_class || last_class == 0)) {
+      values[starter_position] = composite;
+      starter = composite;
+      continue;
+    }
+    if (value_class == 0) {
+      starter_position = kept;
+      starter = value;
+    }
+    last_class = value_class;
+    values[kept++] = value;
+  }
+  return kept;
+}
+
+static int encode_points(const uint32_t *values, size_t length, narsil_text *out) {
+  out->length = 0;
+  int status = narsil_text_reserve(out, length);
+  for (size_t i = 0; i < length && status == NARSIL_OK; i++) { status = narsil_text_push_code_point(out, values[i]); }
+  return status;
+}
+
+int narsil_nfd(const uint16_t *units, size_t length, narsil_text *out, narsil_code_points *points) {
+  int status = decompose_text(units, length, points);
+  if (status != NARSIL_OK) { return status; }
+  return encode_points(points->values, points->length, out);
+}
+
+int narsil_nfc(const uint16_t *units, size_t length, narsil_text *out, narsil_code_points *points) {
+  int status = decompose_text(units, length, points);
+  if (status != NARSIL_OK) { return status; }
+  points->length = compose_in_place(points->values, points->length);
+  return encode_points(points->values, points->length, out);
+}
+
+static int cased_letter_beside(const uint16_t *units, size_t length, size_t index, int forward) {
+  while (forward ? index < length : index > 0) {
+    uint32_t code_point = forward ? narsil_decode_next(units, length, &index) : narsil_decode_previous(units, &index);
+    uint32_t properties = narsil_unicode_properties(code_point);
+    if ((properties & NARSIL_UNICODE_CASE_IGNORABLE) != 0) { continue; }
+    return (properties & NARSIL_UNICODE_CASED) != 0;
+  }
+  return 0;
+}
+
+int narsil_lowercase(const uint16_t *units, size_t length, narsil_text *out) {
+  out->length = 0;
+  int status = narsil_text_reserve(out, length);
+  size_t index = 0;
+  while (index < length && status == NARSIL_OK) {
+    size_t start = index;
+    uint32_t code_point = narsil_decode_next(units, length, &index);
+    if (code_point == CAPITAL_SIGMA && cased_letter_beside(units, length, start, 0) &&
+        !cased_letter_beside(units, length, index, 1)) {
+      status = narsil_text_push(out, SMALL_FINAL_SIGMA);
+      continue;
+    }
+    uint32_t second = 0;
+    uint32_t lower = narsil_unicode_lowercase_of(code_point, &second);
+    status = narsil_text_push_code_point(out, lower);
+    if (status == NARSIL_OK && second != 0) { status = narsil_text_push_code_point(out, second); }
+  }
+  return status;
+}
+
+static uint16_t fold_width(uint16_t unit) {
+  if (unit >= FULLWIDTH_ASCII_FIRST && unit <= FULLWIDTH_ASCII_LAST) {
+    return (uint16_t)(unit - FULLWIDTH_ASCII_OFFSET);
+  }
+  if (unit >= HALFWIDTH_KANA_FIRST && unit <= HALFWIDTH_KANA_LAST) {
+    return halfwidth_kana_targets[unit - HALFWIDTH_KANA_FIRST];
+  }
+  if (unit == HALFWIDTH_VOICED_MARK) { return COMBINING_VOICED_MARK; }
+  if (unit == HALFWIDTH_SEMI_VOICED_MARK) { return COMBINING_SEMI_VOICED_MARK; }
+  return unit;
+}
+
+static int fold_forms(const narsil_text *source, narsil_text *out) {
+  out->length = 0;
+  int status = narsil_text_reserve(out, source->length);
+  for (size_t i = 0; i < source->length && status == NARSIL_OK; i++) {
+    uint16_t unit = source->units[i];
+    if (unit == COMBINING_DOT_ABOVE || (unit >= ARMENIAN_MARKS_FIRST && unit <= ARMENIAN_MARKS_LAST)) { continue; }
+    if (unit == DOTTED_CAPITAL_I) {
+      status = narsil_text_push(out, LATIN_SMALL_I);
+    } else if (unit == LEFT_SINGLE_QUOTE || unit == RIGHT_SINGLE_QUOTE || unit == MODIFIER_APOSTROPHE ||
+               unit == MODIFIER_TURNED_COMMA) {
+      status = narsil_text_push(out, APOSTROPHE);
+    } else {
+      status = narsil_text_push(out, unit);
+    }
+  }
+  return status;
+}
+
+int narsil_fold_for_splitting(const uint16_t *units, size_t length, narsil_text *out,
+                              narsil_normalise_scratch *scratch) {
+  if (narsil_units_are_ascii(units, length)) {
+    out->length = 0;
+    int status = narsil_text_reserve(out, length);
+    if (status != NARSIL_OK) { return status; }
+    for (size_t i = 0; i < length; i++) {
+      uint16_t unit = units[i];
+      out->units[i] = unit >= ASCII_CAPITAL_A && unit <= ASCII_CAPITAL_Z ? (uint16_t)(unit + ASCII_CASE_OFFSET) : unit;
+    }
+    out->length = length;
+    return NARSIL_OK;
+  }
+  scratch->first.length = 0;
+  int status = narsil_text_reserve(&scratch->first, length);
+  if (status != NARSIL_OK) { return status; }
+  for (size_t i = 0; i < length; i++) { scratch->first.units[i] = fold_width(units[i]); }
+  scratch->first.length = length;
+  status = narsil_nfc(scratch->first.units, scratch->first.length, &scratch->second, &scratch->points);
+  if (status == NARSIL_OK) { status = fold_forms(&scratch->second, &scratch->first); }
+  if (status == NARSIL_OK) { status = narsil_lowercase(scratch->first.units, scratch->first.length, out); }
+  return status;
+}
+
+static int is_combining_diacritic(uint32_t code_point) {
+  return code_point >= COMBINING_DIACRITICS_FIRST && code_point <= COMBINING_DIACRITICS_LAST;
+}
+
+int narsil_strip_diacritics(const uint16_t *units, size_t length, narsil_text *out, narsil_code_points *points) {
+  int status = decompose_text(units, length, points);
+  if (status != NARSIL_OK) { return status; }
+  size_t kept = 0;
+  for (size_t i = 0; i < points->length; i++) {
+    if (!is_combining_diacritic(points->values[i])) { points->values[kept++] = points->values[i]; }
+  }
+  return encode_points(points->values, kept, out);
+}
+
+int narsil_remove_marks(const uint16_t *units, size_t length, narsil_mark_test is_removed, narsil_text *out,
+                        narsil_normalise_scratch *scratch) {
+  if (narsil_units_are_ascii(units, length)) { return narsil_text_assign(out, units, length); }
+  int status = decompose_text(units, length, &scratch->points);
+  if (status != NARSIL_OK) { return status; }
+  size_t kept = 0;
+  for (size_t i = 0; i < scratch->points.length; i++) {
+    if (!is_removed(scratch->points.values[i])) { scratch->points.values[kept++] = scratch->points.values[i]; }
+  }
+  if (kept == scratch->points.length) { return narsil_text_assign(out, units, length); }
+  status = encode_points(scratch->points.values, kept, &scratch->first);
+  if (status != NARSIL_OK) { return status; }
+  return narsil_nfc(scratch->first.units, scratch->first.length, out, &scratch->points);
+}

package/cpp/core/src/analysis/normalise.h

@@ -0,0 +1,28 @@
+#ifndef NARSIL_ANALYSIS_NORMALISE_H
+#define NARSIL_ANALYSIS_NORMALISE_H
+
+#include "text.h"
+
+#include <stddef.h>
+#include <stdint.h>
+
+typedef struct {
+  narsil_text first;
+  narsil_text second;
+  narsil_code_points points;
+} narsil_normalise_scratch;
+
+typedef int (*narsil_mark_test)(uint32_t code_point);
+
+void narsil_normalise_scratch_release(narsil_normalise_scratch *scratch);
+
+int narsil_nfd(const uint16_t *units, size_t length, narsil_text *out, narsil_code_points *points);
+int narsil_nfc(const uint16_t *units, size_t length, narsil_text *out, narsil_code_points *points);
+int narsil_lowercase(const uint16_t *units, size_t length, narsil_text *out);
+int narsil_fold_for_splitting(const uint16_t *units, size_t length, narsil_text *out,
+                              narsil_normalise_scratch *scratch);
+int narsil_strip_diacritics(const uint16_t *units, size_t length, narsil_text *out, narsil_code_points *points);
+int narsil_remove_marks(const uint16_t *units, size_t length, narsil_mark_test is_removed, narsil_text *out,
+                        narsil_normalise_scratch *scratch);
+
+#endif

package/cpp/core/src/analysis/split.c

@@ -0,0 +1,183 @@
+#include "split.h"
+
+#include "../../include/narsil_core.h"
+#include "languages.h"
+#include "text.h"
+#include "unicode.h"
+
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+#define MIN_SPANS 16U
+
+#define PROLONGED_SOUND_MARK 0x30FCU
+#define IDEOGRAPHIC_ITERATION_MARK 0x3005U
+#define LOWEST_NGRAM_CODE_POINT 0x0E00U
+#define LOWEST_MARK_CODE_POINT 0x0300U
+#define THAI_BLOCK_END 0x0E7FU
+#define LAO_BLOCK_END 0x0EFFU
+#define MYANMAR_BLOCK_START 0x1000U
+#define MYANMAR_BLOCK_END 0x109FU
+#define KHMER_BLOCK_START 0x1780U
+#define KHMER_BLOCK_END 0x19FFU
+#define SOUTHEAST_ASIAN_BLOCKS_END 0x1A00U
+#define HIRAGANA_BLOCK_START 0x3040U
+#define HIRAGANA_BLOCK_END 0x309FU
+#define COMBINING_VOICED_MARK 0x3099U
+#define COMBINING_SEMI_VOICED_MARK 0x309AU
+#define KATAKANA_BLOCK_START 0x30A0U
+#define KATAKANA_BLOCK_END 0x30FFU
+#define HAN_EXTENSION_A_START 0x3400U
+#define HAN_EXTENSION_A_END 0x4DBFU
+#define HAN_UNIFIED_START 0x4E00U
+#define HAN_UNIFIED_END 0x9FFFU
+#define HANGUL_SYLLABLE_START 0xAC00U
+#define HANGUL_SYLLABLE_END 0xD7A3U
+#define HAN_COMPATIBILITY_START 0xF900U
+#define HAN_COMPATIBILITY_END 0xFAFFU
+
+int narsil_spans_push(narsil_spans *spans, narsil_span span) {
+  if (spans->length >= spans->capacity) {
+    size_t capacity = spans->capacity < MIN_SPANS ? MIN_SPANS : spans->capacity;
+    while (capacity <= spans->length) {
+      if (capacity > (SIZE_MAX / sizeof(narsil_span)) / 2) { return NARSIL_OUT_OF_MEMORY; }
+      capacity *= 2;
+    }
+    narsil_span *grown = realloc(spans->spans, capacity * sizeof(narsil_span));
+    if (grown == NULL) { return NARSIL_OUT_OF_MEMORY; }
+    spans->spans = grown;
+    spans->capacity = capacity;
+  }
+  spans->spans[spans->length] = span;
+  spans->length++;
+  return NARSIL_OK;
+}
+
+static narsil_span span_of(size_t begin, size_t end) {
+  narsil_span span = {(uint32_t)begin, (uint32_t)(end - begin)};
+  return span;
+}
+
+void narsil_spans_release(narsil_spans *spans) {
+  free(spans->spans);
+  spans->spans = NULL;
+  spans->length = 0;
+  spans->capacity = 0;
+}
+
+static uint32_t next_split_unit(const narsil_language *language, const uint16_t *units, uint32_t length,
+                                size_t *index) {
+  if (language->splits_code_points) { return narsil_decode_next(units, length, index); }
+  return units[(*index)++];
+}
+
+int narsil_split(const narsil_language *language, const uint16_t *units, uint32_t length, narsil_spans *parts) {
+  parts->length = 0;
+  size_t start = 0;
+  size_t index = 0;
+  while (index < length) {
+    size_t separator = index;
+    if (narsil_language_keeps(language, next_split_unit(language, units, length, &index))) { continue; }
+    int status = narsil_spans_push(parts, span_of(start, separator));
+    if (status != NARSIL_OK) { return status; }
+    for (;;) {
+      size_t before = index;
+      if (index >= length || narsil_language_keeps(language, next_split_unit(language, units, length, &index))) {
+        index = before;
+        break;
+      }
+    }
+    start = index;
+  }
+  return narsil_spans_push(parts, span_of(start, length));
+}
+
+static int is_mark(uint32_t code_point) {
+  if (code_point < LOWEST_MARK_CODE_POINT) { return 0; }
+  if (code_point >= HAN_UNIFIED_START && code_point <= HAN_UNIFIED_END) { return 0; }
+  if (code_point >= HIRAGANA_BLOCK_START && code_point <= KATAKANA_BLOCK_END) {
+    return code_point == COMBINING_VOICED_MARK || code_point == COMBINING_SEMI_VOICED_MARK;
+  }
+  if (code_point >= HANGUL_SYLLABLE_START && code_point <= HANGUL_SYLLABLE_END) { return 0; }
+  if (code_point >= HAN_EXTENSION_A_START && code_point <= HAN_EXTENSION_A_END) { return 0; }
+  return (narsil_unicode_properties(code_point) & NARSIL_UNICODE_MARK) != 0;
+}
+
+static uint32_t script_in_block(uint32_t code_point, uint32_t block_script) {
+  return narsil_unicode_script(code_point) == block_script ? block_script : NARSIL_UNICODE_SCRIPT_OTHER;
+}
+
+static uint32_t classify_southeast_asian(uint32_t code_point) {
+  if (code_point <= THAI_BLOCK_END) { return script_in_block(code_point, NARSIL_UNICODE_SCRIPT_THAI); }
+  if (code_point <= LAO_BLOCK_END) { return script_in_block(code_point, NARSIL_UNICODE_SCRIPT_LAO); }
+  if (code_point >= MYANMAR_BLOCK_START && code_point <= MYANMAR_BLOCK_END) {
+    return script_in_block(code_point, NARSIL_UNICODE_SCRIPT_MYANMAR);
+  }
+  if (code_point >= KHMER_BLOCK_START && code_point <= KHMER_BLOCK_END) {
+    return script_in_block(code_point, NARSIL_UNICODE_SCRIPT_KHMER);
+  }
+  return narsil_unicode_script(code_point);
+}
+
+static uint32_t prolonged_sound_kind(uint32_t current) {
+  return current == NARSIL_UNICODE_SCRIPT_HIRAGANA ? NARSIL_UNICODE_SCRIPT_HIRAGANA : NARSIL_UNICODE_SCRIPT_KATAKANA;
+}
+
+static uint32_t classify(uint32_t code_point) {
+  if (code_point < LOWEST_NGRAM_CODE_POINT) { return NARSIL_UNICODE_SCRIPT_OTHER; }
+  if (code_point < SOUTHEAST_ASIAN_BLOCKS_END) { return classify_southeast_asian(code_point); }
+  if (code_point >= HIRAGANA_BLOCK_START && code_point <= HIRAGANA_BLOCK_END) { return NARSIL_UNICODE_SCRIPT_HIRAGANA; }
+  if (code_point >= KATAKANA_BLOCK_START && code_point <= KATAKANA_BLOCK_END) { return NARSIL_UNICODE_SCRIPT_KATAKANA; }
+  if (code_point >= HAN_UNIFIED_START && code_point <= HAN_UNIFIED_END) { return NARSIL_UNICODE_SCRIPT_HAN; }
+  if (code_point >= HAN_EXTENSION_A_START && code_point <= HAN_EXTENSION_A_END) { return NARSIL_UNICODE_SCRIPT_HAN; }
+  if (code_point >= HANGUL_SYLLABLE_START && code_point <= HANGUL_SYLLABLE_END) { return NARSIL_UNICODE_SCRIPT_HANGUL; }
+  if (code_point >= HAN_COMPATIBILITY_START && code_point <= HAN_COMPATIBILITY_END) {
+    return NARSIL_UNICODE_SCRIPT_HAN;
+  }
+  if (code_point == IDEOGRAPHIC_ITERATION_MARK) { return NARSIL_UNICODE_SCRIPT_HAN; }
+  return narsil_unicode_script(code_point);
+}
+
+static int emit_run(narsil_span part, const narsil_code_points *offsets, uint32_t kind, uint32_t size,
+                    narsil_spans *out) {
+  if (offsets->length < 2) { return NARSIL_OK; }
+  size_t count = offsets->length - 1;
+  const uint32_t *boundaries = offsets->values;
+  if (kind == NARSIL_UNICODE_SCRIPT_OTHER || count <= size) {
+    return narsil_spans_push(out, span_of((size_t)part.start + boundaries[0], (size_t)part.start + boundaries[count]));
+  }
+  for (size_t first = 0; first + size <= count; first++) {
+    int status = narsil_spans_push(
+        out, span_of((size_t)part.start + boundaries[first], (size_t)part.start + boundaries[first + size]));
+    if (status != NARSIL_OK) { return status; }
+  }
+  return NARSIL_OK;
+}
+
+int narsil_expand_ngrams(const uint16_t *units, narsil_span part, uint32_t size, narsil_spans *out,
+                         narsil_code_points *offsets) {
+  const uint16_t *text = units + part.start;
+  size_t length = part.length;
+  offsets->length = 0;
+  if (length == 0) { return NARSIL_OK; }
+  uint32_t kind = NARSIL_UNICODE_SCRIPT_OTHER;
+  size_t index = 0;
+  int status = NARSIL_OK;
+  while (index < length && status == NARSIL_OK) {
+    size_t start = index;
+    uint32_t code_point = narsil_decode_next(text, length, &index);
+    if (offsets->length > 0 && is_mark(code_point)) { continue; }
+    uint32_t next = code_point == PROLONGED_SOUND_MARK ? prolonged_sound_kind(kind) : classify(code_point);
+    if (offsets->length > 0 && next != kind) {
+      status = narsil_code_points_push(offsets, (uint32_t)start);
+      if (status == NARSIL_OK) { status = emit_run(part, offsets, kind, size, out); }
+      offsets->length = 0;
+    }
+    kind = next;
+    if (status == NARSIL_OK) { status = narsil_code_points_push(offsets, (uint32_t)start); }
+  }
+  if (status == NARSIL_OK) { status = narsil_code_points_push(offsets, (uint32_t)length); }
+  if (status == NARSIL_OK) { status = emit_run(part, offsets, kind, size, out); }
+  return status;
+}

package/cpp/core/src/analysis/split.h

@@ -0,0 +1,27 @@
+#ifndef NARSIL_ANALYSIS_SPLIT_H
+#define NARSIL_ANALYSIS_SPLIT_H
+
+#include "languages.h"
+
+#include <stddef.h>
+#include <stdint.h>
+
+typedef struct {
+  uint32_t start;
+  uint32_t length;
+} narsil_span;
+
+typedef struct {
+  narsil_span *spans;
+  size_t length;
+  size_t capacity;
+} narsil_spans;
+
+int narsil_spans_push(narsil_spans *spans, narsil_span span);
+void narsil_spans_release(narsil_spans *spans);
+
+int narsil_split(const narsil_language *language, const uint16_t *units, uint32_t length, narsil_spans *parts);
+int narsil_expand_ngrams(const uint16_t *units, narsil_span part, uint32_t size, narsil_spans *out,
+                         narsil_code_points *offsets);
+
+#endif

package/cpp/core/src/analysis/stop_words.c

@@ -0,0 +1,82 @@
+#include "stop_words.h"
+
+#include "../../include/narsil_core.h"
+#include "text.h"
+
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+#define FNV_OFFSET_BASIS 2166136261U
+#define FNV_PRIME 16777619U
+#define MIN_SLOTS 8U
+#define MAX_STOP_WORDS (1U << 28)
+#define EMPTY_SLOT 0U
+
+static uint32_t hash_units(const uint16_t *units, size_t length) {
+  uint32_t hash = FNV_OFFSET_BASIS;
+  for (size_t i = 0; i < length; i++) {
+    hash ^= units[i];
+    hash *= FNV_PRIME;
+  }
+  return hash;
+}
+
+static uint32_t word_length(const narsil_stop_words *set, uint32_t word) {
+  return set->offsets[word + 1] - set->offsets[word];
+}
+
+static int slot_holds(const narsil_stop_words *set, uint32_t slot, const uint16_t *units, size_t length) {
+  uint32_t word = set->slots[slot] - 1;
+  return narsil_units_equal(set->units + set->offsets[word], word_length(set, word), units, length);
+}
+
+void narsil_stop_words_release(narsil_stop_words *set) {
+  free(set->units);
+  free(set->offsets);
+  free(set->slots);
+  memset(set, 0, sizeof(*set));
+}
+
+int narsil_stop_words_build(narsil_stop_words *set, const uint16_t *units, const uint32_t *offsets, uint32_t count) {
+  memset(set, 0, sizeof(*set));
+  if (count > MAX_STOP_WORDS) { return NARSIL_INVALID_ARGUMENT; }
+  for (uint32_t word = 0; word < count; word++) {
+    if (offsets[word + 1] < offsets[word]) { return NARSIL_INVALID_ARGUMENT; }
+  }
+  size_t total = offsets[count];
+  uint32_t slots = MIN_SLOTS;
+  while (slots < count * 2U) { slots *= 2U; }
+  set->units = malloc((total == 0 ? 1 : total) * sizeof(uint16_t));
+  set->offsets = malloc(((size_t)count + 1) * sizeof(uint32_t));
+  set->slots = calloc(slots, sizeof(uint32_t));
+  if (set->units == NULL || set->offsets == NULL || set->slots == NULL) {
+    narsil_stop_words_release(set);
+    return NARSIL_OUT_OF_MEMORY;
+  }
+  if (total > 0) { memcpy(set->units, units, total * sizeof(uint16_t)); }
+  memcpy(set->offsets, offsets, ((size_t)count + 1) * sizeof(uint32_t));
+  set->count = count;
+  set->slot_mask = slots - 1U;
+  for (uint32_t word = 0; word < count; word++) {
+    const uint16_t *word_units = set->units + offsets[word];
+    uint32_t length = word_length(set, word);
+    uint32_t slot = hash_units(word_units, length) & set->slot_mask;
+    while (set->slots[slot] != EMPTY_SLOT && !slot_holds(set, slot, word_units, length)) {
+      slot = (slot + 1U) & set->slot_mask;
+    }
+    set->slots[slot] = word + 1U;
+  }
+  return NARSIL_OK;
+}
+
+int narsil_stop_words_contains(const narsil_stop_words *set, const uint16_t *units, size_t length) {
+  if (set->slots == NULL) { return 0; }
+  uint32_t slot = hash_units(units, length) & set->slot_mask;
+  while (set->slots[slot] != EMPTY_SLOT) {
+    if (slot_holds(set, slot, units, length)) { return 1; }
+    slot = (slot + 1U) & set->slot_mask;
+  }
+  return 0;
+}

package/cpp/core/src/analysis/stop_words.h

@@ -0,0 +1,19 @@
+#ifndef NARSIL_ANALYSIS_STOP_WORDS_H
+#define NARSIL_ANALYSIS_STOP_WORDS_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+typedef struct {
+  uint16_t *units;
+  uint32_t *offsets;
+  uint32_t count;
+  uint32_t *slots;
+  uint32_t slot_mask;
+} narsil_stop_words;
+
+int narsil_stop_words_build(narsil_stop_words *set, const uint16_t *units, const uint32_t *offsets, uint32_t count);
+int narsil_stop_words_contains(const narsil_stop_words *set, const uint16_t *units, size_t length);
+void narsil_stop_words_release(narsil_stop_words *set);
+
+#endif

package/cpp/core/src/analysis/term_cache.c

@@ -0,0 +1,88 @@
+#include "term_cache.h"
+
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+#define TERM_CACHE_SLOTS 16384U
+#define TERM_CACHE_WAYS 8U
+#define TERM_CACHE_SETS (TERM_CACHE_SLOTS / TERM_CACHE_WAYS)
+#define TERM_CACHE_SET_MASK (TERM_CACHE_SETS - 1U)
+#define TERM_FNV_OFFSET_BASIS 2166136261U
+#define TERM_FNV_PRIME 16777619U
+#define TERM_TAG_LANGUAGE_SHIFT 8U
+#define TERM_TAG_EMPTY 0U
+
+static uint32_t tag_of(const narsil_term_key *key) {
+  return ((key->language_index + 1U) << TERM_TAG_LANGUAGE_SHIFT) | key->flags;
+}
+
+static int holds(const narsil_term_slot *slot, uint32_t tag, const narsil_term_key *key) {
+  return slot->tag == tag && slot->key_length == key->length &&
+         memcmp(slot->key, key->units, key->length * sizeof *key->units) == 0;
+}
+
+static int prepared(narsil_term_cache *cache) {
+  if (cache->slots != NULL) { return 1; }
+  if (cache->unavailable) { return 0; }
+  cache->slots = calloc(TERM_CACHE_SLOTS, sizeof *cache->slots);
+  cache->next_victim = calloc(TERM_CACHE_SETS, sizeof *cache->next_victim);
+  if (cache->slots == NULL || cache->next_victim == NULL) {
+    free(cache->slots);
+    free(cache->next_victim);
+    cache->slots = NULL;
+    cache->next_victim = NULL;
+    cache->unavailable = 1;
+    return 0;
+  }
+  return 1;
+}
+
+narsil_term_slot *narsil_term_cache_find(narsil_term_cache *cache, const narsil_term_key *key,
+                                         narsil_term_slot **victim) {
+  *victim = NULL;
+  if (key->length == 0 || key->length > NARSIL_TERM_CACHE_UNITS || !prepared(cache)) { return NULL; }
+  uint32_t tag = tag_of(key);
+  uint32_t hash = (TERM_FNV_OFFSET_BASIS ^ tag) * TERM_FNV_PRIME;
+  for (size_t unit = 0; unit < key->length; unit++) {
+    hash ^= key->units[unit];
+    hash *= TERM_FNV_PRIME;
+  }
+  uint32_t set = hash & TERM_CACHE_SET_MASK;
+  narsil_term_slot *ways = &cache->slots[(size_t)set * TERM_CACHE_WAYS];
+  narsil_term_slot *empty = NULL;
+  for (uint32_t way = 0; way < TERM_CACHE_WAYS; way++) {
+    if (holds(&ways[way], tag, key)) { return &ways[way]; }
+    if (empty == NULL && ways[way].tag == TERM_TAG_EMPTY) { empty = &ways[way]; }
+  }
+  if (empty == NULL) {
+    uint8_t *next = &cache->next_victim[set];
+    empty = &ways[*next];
+    *next = (uint8_t)((*next + 1U) % TERM_CACHE_WAYS);
+  }
+  *victim = empty;
+  return NULL;
+}
+
+int narsil_term_cache_store(narsil_term_slot *slot, const narsil_term_key *key, const uint16_t *value, size_t length) {
+  if (length == 0 || length > NARSIL_TERM_CACHE_UNITS) { return 0; }
+  slot->tag = tag_of(key);
+  slot->key_length = (uint16_t)key->length;
+  memcpy(slot->key, key->units, key->length * sizeof *key->units);
+  slot->value_length = (uint16_t)length;
+  memcpy(slot->value, value, length * sizeof *value);
+  return 1;
+}
+
+uint32_t narsil_term_cache_index(const narsil_term_cache *cache, const narsil_term_slot *slot) {
+  return (uint32_t)(slot - cache->slots);
+}
+
+void narsil_term_cache_release(narsil_term_cache *cache) {
+  free(cache->slots);
+  free(cache->next_victim);
+  cache->slots = NULL;
+  cache->next_victim = NULL;
+  cache->unavailable = 0;
+}