@delali/narsil-react-native 0.3.1-next.155 → 0.3.1-next.156

package/cpp/core/src/analysis/term_cache.h

@@ -0,0 +1,36 @@
+#ifndef NARSIL_ANALYSIS_TERM_CACHE_H
+#define NARSIL_ANALYSIS_TERM_CACHE_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+#define NARSIL_TERM_CACHE_UNITS 24U
+
+typedef struct {
+  uint32_t tag;
+  uint16_t key_length;
+  uint16_t value_length;
+  uint16_t key[NARSIL_TERM_CACHE_UNITS];
+  uint16_t value[NARSIL_TERM_CACHE_UNITS];
+} narsil_term_slot;
+
+typedef struct {
+  narsil_term_slot *slots;
+  uint8_t *next_victim;
+  int unavailable;
+} narsil_term_cache;
+
+typedef struct {
+  uint32_t language_index;
+  uint32_t flags;
+  const uint16_t *units;
+  size_t length;
+} narsil_term_key;
+
+narsil_term_slot *narsil_term_cache_find(narsil_term_cache *cache, const narsil_term_key *key,
+                                         narsil_term_slot **victim);
+int narsil_term_cache_store(narsil_term_slot *slot, const narsil_term_key *key, const uint16_t *value, size_t length);
+uint32_t narsil_term_cache_index(const narsil_term_cache *cache, const narsil_term_slot *slot);
+void narsil_term_cache_release(narsil_term_cache *cache);
+
+#endif

package/cpp/core/src/analysis/text.c

@@ -0,0 +1,161 @@
+#include "text.h"
+
+#include "../../include/narsil_core.h"
+
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+#define MIN_CAPACITY 16U
+
+typedef struct {
+  size_t current;
+  size_t needed;
+  size_t element_size;
+} growth;
+
+static size_t grown_capacity(growth request) {
+  size_t limit = SIZE_MAX / request.element_size;
+  size_t capacity = request.current < MIN_CAPACITY ? MIN_CAPACITY : request.current;
+  while (capacity < request.needed) {
+    if (capacity > limit / 2) { return request.needed <= limit ? request.needed : 0; }
+    capacity *= 2;
+  }
+  return capacity;
+}
+
+int narsil_text_reserve(narsil_text *text, size_t capacity) {
+  if (capacity <= text->capacity) { return NARSIL_OK; }
+  growth request = {text->capacity, capacity, sizeof(uint16_t)};
+  size_t grown = grown_capacity(request);
+  if (grown == 0) { return NARSIL_OUT_OF_MEMORY; }
+  uint16_t *units = realloc(text->units, grown * sizeof(uint16_t));
+  if (units == NULL) { return NARSIL_OUT_OF_MEMORY; }
+  text->units = units;
+  text->capacity = grown;
+  return NARSIL_OK;
+}
+
+int narsil_text_assign(narsil_text *text, const uint16_t *units, size_t length) {
+  text->length = 0;
+  return narsil_text_append(text, units, length);
+}
+
+int narsil_text_append(narsil_text *text, const uint16_t *units, size_t length) {
+  if (length == 0) { return NARSIL_OK; }
+  if (length > SIZE_MAX - text->length) { return NARSIL_OUT_OF_MEMORY; }
+  int status = narsil_text_reserve(text, text->length + length);
+  if (status != NARSIL_OK) { return status; }
+  memmove(text->units + text->length, units, length * sizeof(uint16_t));
+  text->length += length;
+  return NARSIL_OK;
+}
+
+int narsil_text_push(narsil_text *text, uint16_t unit) {
+  if (text->length == text->capacity) {
+    int status = narsil_text_reserve(text, text->length + 1);
+    if (status != NARSIL_OK) { return status; }
+  }
+  text->units[text->length++] = unit;
+  return NARSIL_OK;
+}
+
+int narsil_text_push_code_point(narsil_text *text, uint32_t code_point) {
+  if (code_point < NARSIL_SUPPLEMENTARY_FIRST) { return narsil_text_push(text, (uint16_t)code_point); }
+  uint32_t offset = code_point - NARSIL_SUPPLEMENTARY_FIRST;
+  int status =
+      narsil_text_push(text, (uint16_t)(NARSIL_HIGH_SURROGATE_FIRST + (offset >> NARSIL_SURROGATE_PAYLOAD_BITS)));
+  if (status != NARSIL_OK) { return status; }
+  return narsil_text_push(text, (uint16_t)(NARSIL_LOW_SURROGATE_FIRST + (offset & NARSIL_SURROGATE_PAYLOAD_MASK)));
+}
+
+void narsil_text_release(narsil_text *text) {
+  free(text->units);
+  text->units = NULL;
+  text->length = 0;
+  text->capacity = 0;
+}
+
+int narsil_code_points_reserve(narsil_code_points *points, size_t capacity) {
+  if (capacity <= points->capacity) { return NARSIL_OK; }
+  growth request = {points->capacity, capacity, sizeof(uint32_t)};
+  size_t grown = grown_capacity(request);
+  if (grown == 0) { return NARSIL_OUT_OF_MEMORY; }
+  uint32_t *values = realloc(points->values, grown * sizeof(uint32_t));
+  if (values == NULL) { return NARSIL_OUT_OF_MEMORY; }
+  points->values = values;
+  points->capacity = grown;
+  return NARSIL_OK;
+}
+
+int narsil_code_points_push(narsil_code_points *points, uint32_t code_point) {
+  if (points->length == points->capacity) {
+    int status = narsil_code_points_reserve(points, points->length + 1);
+    if (status != NARSIL_OK) { return status; }
+  }
+  points->values[points->length++] = code_point;
+  return NARSIL_OK;
+}
+
+void narsil_code_points_release(narsil_code_points *points) {
+  free(points->values);
+  points->values = NULL;
+  points->length = 0;
+  points->capacity = 0;
+}
+
+static int is_high_surrogate(uint32_t unit) {
+  return unit >= NARSIL_HIGH_SURROGATE_FIRST && unit < NARSIL_LOW_SURROGATE_FIRST;
+}
+
+static int is_low_surrogate(uint32_t unit) { return unit >= NARSIL_LOW_SURROGATE_FIRST && unit < NARSIL_SURROGATE_END; }
+
+static uint32_t combine_surrogates(uint32_t high, uint32_t low) {
+  return NARSIL_SUPPLEMENTARY_FIRST + ((high - NARSIL_HIGH_SURROGATE_FIRST) << NARSIL_SURROGATE_PAYLOAD_BITS) +
+         (low - NARSIL_LOW_SURROGATE_FIRST);
+}
+
+uint32_t narsil_decode_next(const uint16_t *units, size_t length, size_t *index) {
+  uint32_t unit = units[*index];
+  *index += 1;
+  if (is_high_surrogate(unit) && *index < length && is_low_surrogate(units[*index])) {
+    uint32_t low = units[*index];
+    *index += 1;
+    return combine_surrogates(unit, low);
+  }
+  return unit;
+}
+
+uint32_t narsil_decode_previous(const uint16_t *units, size_t *index) {
+  *index -= 1;
+  uint32_t unit = units[*index];
+  if (is_low_surrogate(unit) && *index > 0 && is_high_surrogate(units[*index - 1])) {
+    *index -= 1;
+    return combine_surrogates(units[*index], unit);
+  }
+  return unit;
+}
+
+int narsil_units_are_ascii(const uint16_t *units, size_t length) {
+  for (size_t i = 0; i < length; i++) {
+    if (units[i] >= NARSIL_ASCII_END) { return 0; }
+  }
+  return 1;
+}
+
+int narsil_units_equal(const uint16_t *left, size_t left_length, const uint16_t *right, size_t right_length) {
+  return left_length == right_length && (left_length == 0 || memcmp(left, right, left_length * sizeof(uint16_t)) == 0);
+}
+
+void narsil_replace_unpaired_surrogates(uint16_t *units, size_t length) {
+  for (size_t i = 0; i < length; i++) {
+    uint32_t unit = units[i];
+    if (unit < NARSIL_HIGH_SURROGATE_FIRST || unit >= NARSIL_SURROGATE_END) { continue; }
+    if (is_high_surrogate(unit) && i + 1 < length && is_low_surrogate(units[i + 1])) {
+      i++;
+      continue;
+    }
+    units[i] = (uint16_t)NARSIL_REPLACEMENT_CHARACTER;
+  }
+}

package/cpp/core/src/analysis/text.h

@@ -0,0 +1,45 @@
+#ifndef NARSIL_ANALYSIS_TEXT_H
+#define NARSIL_ANALYSIS_TEXT_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+#define NARSIL_HIGH_SURROGATE_FIRST 0xD800U
+#define NARSIL_LOW_SURROGATE_FIRST 0xDC00U
+#define NARSIL_SURROGATE_END 0xE000U
+#define NARSIL_SUPPLEMENTARY_FIRST 0x10000U
+#define NARSIL_SURROGATE_PAYLOAD_BITS 10U
+#define NARSIL_SURROGATE_PAYLOAD_MASK 0x3FFU
+#define NARSIL_ASCII_END 0x80U
+#define NARSIL_REPLACEMENT_CHARACTER 0xFFFDU
+
+typedef struct {
+  uint16_t *units;
+  size_t length;
+  size_t capacity;
+} narsil_text;
+
+typedef struct {
+  uint32_t *values;
+  size_t length;
+  size_t capacity;
+} narsil_code_points;
+
+int narsil_text_reserve(narsil_text *text, size_t capacity);
+int narsil_text_assign(narsil_text *text, const uint16_t *units, size_t length);
+int narsil_text_append(narsil_text *text, const uint16_t *units, size_t length);
+int narsil_text_push(narsil_text *text, uint16_t unit);
+int narsil_text_push_code_point(narsil_text *text, uint32_t code_point);
+void narsil_text_release(narsil_text *text);
+
+int narsil_code_points_reserve(narsil_code_points *points, size_t capacity);
+int narsil_code_points_push(narsil_code_points *points, uint32_t code_point);
+void narsil_code_points_release(narsil_code_points *points);
+
+uint32_t narsil_decode_next(const uint16_t *units, size_t length, size_t *index);
+uint32_t narsil_decode_previous(const uint16_t *units, size_t *index);
+int narsil_units_are_ascii(const uint16_t *units, size_t length);
+int narsil_units_equal(const uint16_t *left, size_t left_length, const uint16_t *right, size_t right_length);
+void narsil_replace_unpaired_surrogates(uint16_t *units, size_t length);
+
+#endif

package/cpp/core/src/analysis/transform.c

@@ -0,0 +1,103 @@
+#include "../../include/narsil_analysis.h"
+#include "../../include/narsil_core.h"
+#include "../languages/snowball/api.h"
+#include "analyser.h"
+#include "languages.h"
+#include "normalise.h"
+#include "term_cache.h"
+#include "text.h"
+
+#include <limits.h>
+#include <stddef.h>
+#include <stdint.h>
+
+static narsil_text *buffer_apart_from(narsil_analyser *analyser, narsil_unit_view current) {
+  if (current.units != NULL && current.units == analyser->step_a.units) { return &analyser->step_b; }
+  return &analyser->step_a;
+}
+
+static narsil_unit_view view_of(const narsil_text *text) {
+  narsil_unit_view view = {text->units, text->length};
+  return view;
+}
+
+static int rewrite(narsil_analyser *analyser, narsil_token_rewrite step, narsil_unit_view *current) {
+  narsil_text *target = buffer_apart_from(analyser, *current);
+  int status = step(current->units, current->length, target, &analyser->scratch);
+  if (status == NARSIL_OK) { *current = view_of(target); }
+  return status;
+}
+
+static int strip_diacritics(narsil_analyser *analyser, narsil_unit_view *current) {
+  narsil_text *target = buffer_apart_from(analyser, *current);
+  int status = narsil_strip_diacritics(current->units, current->length, target, &analyser->scratch.points);
+  if (status == NARSIL_OK) { *current = view_of(target); }
+  return status;
+}
+
+static int stem_with_snowball(narsil_analyser *analyser, const narsil_language *language, uint32_t language_index,
+                              narsil_unit_view *current) {
+  if (current->length > INT_MAX) { return NARSIL_INVALID_ARGUMENT; }
+  struct SN_env *env = analyser->snowball[language_index];
+  if (env == NULL) {
+    env = language->create_snowball();
+    if (env == NULL) { return NARSIL_OUT_OF_MEMORY; }
+    analyser->snowball[language_index] = env;
+  }
+  if (SN_set_current(env, (int)current->length, current->units) < 0) { return NARSIL_OUT_OF_MEMORY; }
+  if (language->snowball_stem(env) < 0 || env->l < 0) { return NARSIL_OUT_OF_MEMORY; }
+  current->units = env->p;
+  current->length = (size_t)env->l;
+  return NARSIL_OK;
+}
+
+static int transform_uncached(narsil_analyser *analyser, const narsil_language *language, uint32_t language_index,
+                              narsil_unit_view candidate, uint32_t flags, narsil_unit_view *result) {
+  narsil_unit_view current = candidate;
+  int status = NARSIL_OK;
+  if (language->normalise != NULL) { status = rewrite(analyser, language->normalise, &current); }
+  if (status == NARSIL_OK && (flags & NARSIL_ANALYSE_REMOVE_DIACRITICS) != 0) {
+    status = strip_diacritics(analyser, &current);
+  }
+  if (status == NARSIL_OK && (flags & NARSIL_ANALYSE_STEM) != 0) {
+    if (language->stem_by_rules != NULL) {
+      status = rewrite(analyser, language->stem_by_rules, &current);
+    } else if (language->snowball_stem != NULL) {
+      status = stem_with_snowball(analyser, language, language_index, &current);
+    }
+  }
+  if (status == NARSIL_OK) { *result = current; }
+  return status;
+}
+
+int narsil_transform_token(narsil_analyser *analyser, const narsil_language *language, uint32_t language_index,
+                           narsil_unit_view candidate, uint32_t flags, narsil_unit_view *result, uint32_t *term_slot) {
+  *term_slot = NARSIL_NO_TERM_SLOT;
+  narsil_term_key key = {language_index, flags & NARSIL_TRANSFORM_FLAGS, candidate.units, candidate.length};
+  narsil_term_slot *victim = NULL;
+  narsil_term_slot *held = narsil_term_cache_find(&analyser->terms, &key, &victim);
+  if (held != NULL) {
+    result->units = held->value;
+    result->length = held->value_length;
+    *term_slot = narsil_term_cache_index(&analyser->terms, held);
+    return NARSIL_OK;
+  }
+  int status = transform_uncached(analyser, language, language_index, candidate, key.flags, result);
+  if (status == NARSIL_OK && victim != NULL && narsil_term_cache_store(victim, &key, result->units, result->length)) {
+    *term_slot = narsil_term_cache_index(&analyser->terms, victim);
+  }
+  return status;
+}
+
+int narsil_surface_of(narsil_analyser *analyser, const narsil_language *language, narsil_unit_view candidate,
+                      narsil_unit_view *result) {
+  narsil_unit_view current = candidate;
+  if (language->normalise != NULL) {
+    int status = language->normalise(current.units, current.length, &analyser->surface_step, &analyser->scratch);
+    if (status != NARSIL_OK) { return status; }
+    current = view_of(&analyser->surface_step);
+  }
+  int status = narsil_strip_diacritics(current.units, current.length, &analyser->surface, &analyser->scratch.points);
+  if (status == NARSIL_OK) { *result = view_of(&analyser->surface); }
+  return status;
+}

package/cpp/core/src/analysis/unicode.c

@@ -0,0 +1,252 @@
+#include "unicode.h"
+
+#include "../../include/narsil_core.h"
+#include "../thread_yield.h"
+
+#include <stdatomic.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+#define BMP_SIZE 0x10000U
+#define RUN_INDEX_MASK ((1U << NARSIL_UNICODE_RUN_INDEX_BITS) - 1U)
+#define DECOMPOSITION_STRIDE 3U
+#define SPECIAL_LOWERCASE_STRIDE 3U
+#define LOWERCASE_STRIDE 2U
+#define COMPOSITION_SECOND_BITS 21U
+
+#define HANGUL_SYLLABLE_FIRST 0xAC00U
+#define HANGUL_LEADING_FIRST 0x1100U
+#define HANGUL_VOWEL_FIRST 0x1161U
+#define HANGUL_TRAILING_BEFORE_FIRST 0x11A7U
+#define HANGUL_LEADING_COUNT 19U
+#define HANGUL_VOWEL_COUNT 21U
+#define HANGUL_TRAILING_COUNT 28U
+#define HANGUL_SYLLABLES_PER_LEADING (HANGUL_VOWEL_COUNT * HANGUL_TRAILING_COUNT)
+#define HANGUL_SYLLABLE_COUNT (HANGUL_LEADING_COUNT * HANGUL_SYLLABLES_PER_LEADING)
+
+enum { PREPARE_IDLE = 0, PREPARE_BUILDING = 1, PREPARE_READY = 2 };
+
+typedef struct {
+  uint64_t key;
+  uint32_t composite;
+} composition_entry;
+
+static atomic_int prepare_state = PREPARE_IDLE;
+static uint8_t bmp_value_index[BMP_SIZE];
+static uint32_t *supplementary_starts;
+static const uint32_t *supplementary_runs;
+static uint32_t supplementary_run_count;
+static composition_entry *compositions;
+static uint32_t composition_count;
+
+static uint64_t composition_key(uint32_t first, uint32_t second) {
+  return ((uint64_t)first << COMPOSITION_SECOND_BITS) | second;
+}
+
+static int compare_compositions(const void *lhs, const void *rhs) {
+  const composition_entry *left = lhs;
+  const composition_entry *right = rhs;
+  return (left->key > right->key) - (left->key < right->key);
+}
+
+static int build_property_index(void) {
+  uint32_t start = 0;
+  uint32_t first_supplementary = narsil_unicode_property_runs_length;
+  for (uint32_t run = 0; run < narsil_unicode_property_runs_length; run++) {
+    uint32_t length = narsil_unicode_property_runs[run] >> NARSIL_UNICODE_RUN_INDEX_BITS;
+    uint32_t end = start + length;
+    for (uint32_t code_point = start; code_point < end && code_point < BMP_SIZE; code_point++) {
+      bmp_value_index[code_point] = (uint8_t)(narsil_unicode_property_runs[run] & RUN_INDEX_MASK);
+    }
+    if (end > BMP_SIZE && first_supplementary == narsil_unicode_property_runs_length) { first_supplementary = run; }
+    start = end;
+  }
+  supplementary_run_count = narsil_unicode_property_runs_length - first_supplementary;
+  if (supplementary_run_count == 0) { return NARSIL_OK; }
+  supplementary_starts = malloc((size_t)supplementary_run_count * sizeof(uint32_t));
+  if (supplementary_starts == NULL) { return NARSIL_OUT_OF_MEMORY; }
+  supplementary_runs = &narsil_unicode_property_runs[first_supplementary];
+  start = 0;
+  for (uint32_t run = 0; run < narsil_unicode_property_runs_length; run++) {
+    if (run >= first_supplementary) { supplementary_starts[run - first_supplementary] = start; }
+    start += narsil_unicode_property_runs[run] >> NARSIL_UNICODE_RUN_INDEX_BITS;
+  }
+  return NARSIL_OK;
+}
+
+static int build_compositions(void) {
+  uint32_t entries = narsil_unicode_decompositions_length / DECOMPOSITION_STRIDE;
+  composition_count = 0;
+  if (entries == 0) { return NARSIL_OK; }
+  compositions = malloc((size_t)entries * sizeof(composition_entry));
+  if (compositions == NULL) { return NARSIL_OUT_OF_MEMORY; }
+  for (uint32_t entry = 0; entry < entries; entry++) {
+    const uint32_t *row = &narsil_unicode_decompositions[(size_t)entry * DECOMPOSITION_STRIDE];
+    if (row[2] == 0 || (narsil_unicode_properties(row[0]) & NARSIL_UNICODE_COMPOSITION_EXCLUDED) != 0) { continue; }
+    compositions[composition_count].key = composition_key(row[1], row[2]);
+    compositions[composition_count].composite = row[0];
+    composition_count++;
+  }
+  qsort(compositions, composition_count, sizeof(composition_entry), compare_compositions);
+  return NARSIL_OK;
+}
+
+static void release_tables(void) {
+  free(supplementary_starts);
+  supplementary_starts = NULL;
+  free(compositions);
+  compositions = NULL;
+}
+
+int narsil_unicode_prepare(void) {
+  for (;;) {
+    int state = atomic_load_explicit(&prepare_state, memory_order_acquire);
+    if (state == PREPARE_READY) { return NARSIL_OK; }
+    if (state == PREPARE_IDLE && atomic_compare_exchange_strong_explicit(&prepare_state, &state, PREPARE_BUILDING,
+                                                                         memory_order_acq_rel, memory_order_acquire)) {
+      break;
+    }
+    if (state == PREPARE_BUILDING) { narsil_yield_thread(); }
+  }
+  int status = build_property_index();
+  if (status == NARSIL_OK) { status = build_compositions(); }
+  if (status == NARSIL_OK) {
+    atomic_store_explicit(&prepare_state, PREPARE_READY, memory_order_release);
+    return NARSIL_OK;
+  }
+  release_tables();
+  atomic_store_explicit(&prepare_state, PREPARE_IDLE, memory_order_release);
+  return status;
+}
+
+uint32_t narsil_unicode_properties(uint32_t code_point) {
+  if (code_point < BMP_SIZE) { return narsil_unicode_property_values[bmp_value_index[code_point]]; }
+  if (code_point > NARSIL_UNICODE_MAX_CODE_POINT || supplementary_run_count == 0) { return 0; }
+  uint32_t low = 0;
+  uint32_t high = supplementary_run_count;
+  while (high - low > 1) {
+    uint32_t middle = low + ((high - low) / 2);
+    if (supplementary_starts[middle] <= code_point) {
+      low = middle;
+    } else {
+      high = middle;
+    }
+  }
+  return narsil_unicode_property_values[supplementary_runs[low] & RUN_INDEX_MASK];
+}
+
+uint32_t narsil_unicode_combining_class(uint32_t code_point) {
+  return narsil_unicode_properties(code_point) & NARSIL_UNICODE_COMBINING_CLASS_MASK;
+}
+
+uint32_t narsil_unicode_script(uint32_t code_point) {
+  return (narsil_unicode_properties(code_point) >> NARSIL_UNICODE_SCRIPT_SHIFT) & NARSIL_UNICODE_SCRIPT_MASK;
+}
+
+typedef struct {
+  const uint32_t *rows;
+  uint32_t length;
+  uint32_t stride;
+} row_table;
+
+static const uint32_t *find_row(row_table table, uint32_t code_point) {
+  size_t low = 0;
+  size_t high = table.length / table.stride;
+  while (low < high) {
+    size_t middle = low + ((high - low) / 2);
+    const uint32_t *row = &table.rows[middle * table.stride];
+    if (row[0] == code_point) { return row; }
+    if (row[0] < code_point) {
+      low = middle + 1;
+    } else {
+      high = middle;
+    }
+  }
+  return NULL;
+}
+
+uint32_t narsil_unicode_lowercase_of(uint32_t code_point, uint32_t *second) {
+  *second = 0;
+  if ((narsil_unicode_properties(code_point) & NARSIL_UNICODE_LOWERCASE_MAPPING) == 0) { return code_point; }
+  row_table special_table = {narsil_unicode_special_lowercase, narsil_unicode_special_lowercase_length,
+                             SPECIAL_LOWERCASE_STRIDE};
+  const uint32_t *special = find_row(special_table, code_point);
+  if (special != NULL) {
+    *second = special[2];
+    return special[1];
+  }
+  row_table pair_table = {narsil_unicode_lowercase, narsil_unicode_lowercase_length, LOWERCASE_STRIDE};
+  const uint32_t *pair = find_row(pair_table, code_point);
+  return pair == NULL ? code_point : pair[1];
+}
+
+static const uint32_t *decomposition_of(uint32_t code_point) {
+  if ((narsil_unicode_properties(code_point) & NARSIL_UNICODE_DECOMPOSES) == 0) { return NULL; }
+  row_table table = {narsil_unicode_decompositions, narsil_unicode_decompositions_length, DECOMPOSITION_STRIDE};
+  return find_row(table, code_point);
+}
+
+static uint32_t decompose_fully(uint32_t code_point, uint32_t *out) {
+  uint32_t pending[NARSIL_UNICODE_MAX_DECOMPOSITION];
+  uint32_t pending_count = 1;
+  uint32_t written = 0;
+  pending[0] = code_point;
+  while (pending_count > 0 && written < NARSIL_UNICODE_MAX_DECOMPOSITION) {
+    uint32_t current = pending[--pending_count];
+    const uint32_t *row = decomposition_of(current);
+    if (row == NULL || pending_count + 2 > NARSIL_UNICODE_MAX_DECOMPOSITION) {
+      out[written++] = current;
+      continue;
+    }
+    if (row[2] != 0) { pending[pending_count++] = row[2]; }
+    pending[pending_count++] = row[1];
+  }
+  return written;
+}
+
+uint32_t narsil_unicode_decompose(uint32_t code_point, uint32_t *out) {
+  if (code_point >= HANGUL_SYLLABLE_FIRST && code_point < HANGUL_SYLLABLE_FIRST + HANGUL_SYLLABLE_COUNT) {
+    uint32_t index = code_point - HANGUL_SYLLABLE_FIRST;
+    uint32_t trailing = index % HANGUL_TRAILING_COUNT;
+    out[0] = HANGUL_LEADING_FIRST + (index / HANGUL_SYLLABLES_PER_LEADING);
+    out[1] = HANGUL_VOWEL_FIRST + ((index % HANGUL_SYLLABLES_PER_LEADING) / HANGUL_TRAILING_COUNT);
+    if (trailing == 0) { return 2; }
+    out[2] = HANGUL_TRAILING_BEFORE_FIRST + trailing;
+    return 3;
+  }
+  return decompose_fully(code_point, out);
+}
+
+static uint32_t compose_hangul(uint32_t first, uint32_t second) {
+  if (first >= HANGUL_LEADING_FIRST && first < HANGUL_LEADING_FIRST + HANGUL_LEADING_COUNT &&
+      second >= HANGUL_VOWEL_FIRST && second < HANGUL_VOWEL_FIRST + HANGUL_VOWEL_COUNT) {
+    return HANGUL_SYLLABLE_FIRST + ((first - HANGUL_LEADING_FIRST) * HANGUL_SYLLABLES_PER_LEADING) +
+           ((second - HANGUL_VOWEL_FIRST) * HANGUL_TRAILING_COUNT);
+  }
+  if (first >= HANGUL_SYLLABLE_FIRST && first < HANGUL_SYLLABLE_FIRST + HANGUL_SYLLABLE_COUNT &&
+      (first - HANGUL_SYLLABLE_FIRST) % HANGUL_TRAILING_COUNT == 0 && second > HANGUL_TRAILING_BEFORE_FIRST &&
+      second < HANGUL_TRAILING_BEFORE_FIRST + HANGUL_TRAILING_COUNT) {
+    return first + (second - HANGUL_TRAILING_BEFORE_FIRST);
+  }
+  return NARSIL_UNICODE_NO_COMPOSITE;
+}
+
+uint32_t narsil_unicode_compose(uint32_t first, uint32_t second) {
+  uint32_t hangul = compose_hangul(first, second);
+  if (hangul != NARSIL_UNICODE_NO_COMPOSITE) { return hangul; }
+  if ((narsil_unicode_properties(second) & NARSIL_UNICODE_COMPOSES_SECOND) == 0) { return NARSIL_UNICODE_NO_COMPOSITE; }
+  uint64_t key = composition_key(first, second);
+  uint32_t low = 0;
+  uint32_t high = composition_count;
+  while (low < high) {
+    uint32_t middle = low + ((high - low) / 2);
+    if (compositions[middle].key == key) { return compositions[middle].composite; }
+    if (compositions[middle].key < key) {
+      low = middle + 1;
+    } else {
+      high = middle;
+    }
+  }
+  return NARSIL_UNICODE_NO_COMPOSITE;
+}

package/cpp/core/src/analysis/unicode.h

@@ -0,0 +1,54 @@
+#ifndef NARSIL_ANALYSIS_UNICODE_H
+#define NARSIL_ANALYSIS_UNICODE_H
+
+#include <stdint.h>
+
+#define NARSIL_UNICODE_COMBINING_CLASS_MASK 0xFFU
+#define NARSIL_UNICODE_LETTER (1U << 8)
+#define NARSIL_UNICODE_MARK (1U << 9)
+#define NARSIL_UNICODE_NUMBER (1U << 10)
+#define NARSIL_UNICODE_FORMAT (1U << 11)
+#define NARSIL_UNICODE_CASED (1U << 12)
+#define NARSIL_UNICODE_CASE_IGNORABLE (1U << 13)
+#define NARSIL_UNICODE_LOWERCASE_MAPPING (1U << 14)
+#define NARSIL_UNICODE_DECOMPOSES (1U << 15)
+#define NARSIL_UNICODE_COMPOSES_SECOND (1U << 16)
+#define NARSIL_UNICODE_COMPOSITION_EXCLUDED (1U << 17)
+#define NARSIL_UNICODE_SCRIPT_SHIFT 20
+#define NARSIL_UNICODE_SCRIPT_MASK 0xFU
+
+#define NARSIL_UNICODE_SCRIPT_OTHER 0
+#define NARSIL_UNICODE_SCRIPT_HAN 1
+#define NARSIL_UNICODE_SCRIPT_HIRAGANA 2
+#define NARSIL_UNICODE_SCRIPT_KATAKANA 3
+#define NARSIL_UNICODE_SCRIPT_HANGUL 4
+#define NARSIL_UNICODE_SCRIPT_THAI 5
+#define NARSIL_UNICODE_SCRIPT_LAO 6
+#define NARSIL_UNICODE_SCRIPT_KHMER 7
+#define NARSIL_UNICODE_SCRIPT_MYANMAR 8
+
+#define NARSIL_UNICODE_RUN_INDEX_BITS 8
+#define NARSIL_UNICODE_MAX_CODE_POINT 0x10FFFFU
+#define NARSIL_UNICODE_MAX_DECOMPOSITION 4
+#define NARSIL_UNICODE_NO_COMPOSITE 0U
+
+extern const uint32_t narsil_unicode_property_values[];
+extern const uint32_t narsil_unicode_property_values_length;
+extern const uint32_t narsil_unicode_property_runs[];
+extern const uint32_t narsil_unicode_property_runs_length;
+extern const uint32_t narsil_unicode_lowercase[];
+extern const uint32_t narsil_unicode_lowercase_length;
+extern const uint32_t narsil_unicode_special_lowercase[];
+extern const uint32_t narsil_unicode_special_lowercase_length;
+extern const uint32_t narsil_unicode_decompositions[];
+extern const uint32_t narsil_unicode_decompositions_length;
+
+int narsil_unicode_prepare(void);
+uint32_t narsil_unicode_properties(uint32_t code_point);
+uint32_t narsil_unicode_combining_class(uint32_t code_point);
+uint32_t narsil_unicode_script(uint32_t code_point);
+uint32_t narsil_unicode_lowercase_of(uint32_t code_point, uint32_t *second);
+uint32_t narsil_unicode_decompose(uint32_t code_point, uint32_t *out);
+uint32_t narsil_unicode_compose(uint32_t first, uint32_t second);
+
+#endif

package/cpp/core/src/internal.h

@@ -1,6 +1,8 @@
 #ifndef NARSIL_CORE_INTERNAL_H
 #define NARSIL_CORE_INTERNAL_H
 
+#include "keyword/scratch.h"
+
 #include "../include/narsil_core.h"
 
 #include <stdatomic.h>
@@ -162,6 +164,7 @@ struct narsil_workspace {
   int32_t repaired[MAX_NEIGHBOURS_PER_LIST];
   double kept_distances[MAX_NEIGHBOURS_PER_LIST];
   uint8_t replacement_tried[MAX_NEIGHBOURS_PER_LIST];
+  keyword_scratch keyword;
 };
 
 typedef struct {
@@ -187,6 +190,7 @@ static inline int is_tombstoned(const walk_context *context, int32_t ordinal) {
 }
 
 uint32_t grown_capacity(uint32_t first, uint32_t needed);
+keyword_scratch *workspace_keyword_scratch(narsil_workspace *workspace);
 
 narsil_status heap_reserve(distance_heap *heap, uint32_t capacity);
 void heap_release(distance_heap *heap);