@delali/narsil-react-native 0.3.1-next.154 → 0.3.1-next.156

package/cpp/core/src/keyword/select.c

@@ -0,0 +1,155 @@
+#include "select.h"
+
+#include "pass.h"
+#include "scratch.h"
+
+#include "../../include/narsil_keyword.h"
+
+#include <stdint.h>
+
+typedef struct {
+  int32_t *ordinals;
+  double *scores;
+  uint32_t size;
+} best_heap;
+
+static int ranks_below(const best_heap *heap, uint64_t lhs, uint64_t rhs) {
+  if (heap->scores[lhs] != heap->scores[rhs]) { return heap->scores[lhs] < heap->scores[rhs]; }
+  return heap->ordinals[lhs] > heap->ordinals[rhs];
+}
+
+static void swap_places(const best_heap *heap, uint32_t lhs, uint32_t rhs) {
+  int32_t ordinal = heap->ordinals[lhs];
+  heap->ordinals[lhs] = heap->ordinals[rhs];
+  heap->ordinals[rhs] = ordinal;
+  double score = heap->scores[lhs];
+  heap->scores[lhs] = heap->scores[rhs];
+  heap->scores[rhs] = score;
+}
+
+static void sift_best_down(const best_heap *heap, uint32_t root) {
+  uint32_t node = root;
+  for (;;) {
+    uint64_t left = (2 * (uint64_t)node) + 1;
+    if (left >= heap->size) { return; }
+    uint64_t right = left + 1;
+    uint64_t lowest = node;
+    if (ranks_below(heap, left, lowest)) { lowest = left; }
+    if (right < heap->size && ranks_below(heap, right, lowest)) { lowest = right; }
+    if (lowest == node) { return; }
+    swap_places(heap, node, (uint32_t)lowest);
+    node = (uint32_t)lowest;
+  }
+}
+
+static void sort_best_first(int32_t *ordinals, double *scores, uint32_t count) {
+  best_heap heap = {ordinals, scores, count};
+  for (uint32_t root = count / 2; root > 0; root--) { sift_best_down(&heap, root - 1); }
+  for (uint32_t size = count; size > 1; size--) {
+    swap_places(&heap, 0, size - 1);
+    heap.size = size - 1;
+    sift_best_down(&heap, 0);
+  }
+}
+
+static uint32_t offer_score(double *selection, uint32_t size, uint32_t limit, double score) {
+  if (size == limit && !(score > selection[0])) { return size; }
+  uint32_t node = 0;
+  uint32_t grown = size;
+  if (size < limit) {
+    node = grown++;
+    while (node > 0) {
+      uint32_t parent = (node - 1) / 2;
+      if (!(selection[parent] > score)) { break; }
+      selection[node] = selection[parent];
+      node = parent;
+    }
+  } else {
+    for (;;) {
+      uint64_t left = (2 * (uint64_t)node) + 1;
+      if (left >= size) { break; }
+      uint64_t right = left + 1;
+      uint64_t child = right < size && selection[right] < selection[left] ? right : left;
+      if (!(selection[child] < score)) { break; }
+      selection[node] = selection[child];
+      node = (uint32_t)child;
+    }
+  }
+  selection[node] = score;
+  return grown;
+}
+
+static double limit_best_score(const keyword_pass *pass, uint32_t kept) {
+  double *selection = pass->scratch->selection;
+  const double *scores = pass->output->scores;
+  uint32_t size = 0;
+  for (uint32_t index = 0; index < kept; index++) { size = offer_score(selection, size, pass->limit, scores[index]); }
+  return selection[0];
+}
+
+double keyword_touched_best_score(const keyword_pass *pass) {
+  const keyword_scratch *scratch = pass->scratch;
+  double *selection = scratch->selection;
+  uint32_t size = 0;
+  for (uint32_t index = 0; index < scratch->touched_count; index++) {
+    size = offer_score(selection, size, pass->limit, scratch->scores[scratch->touched[index]]);
+  }
+  return selection[0];
+}
+
+void keyword_clear_matched(const keyword_pass *pass) {
+  uint32_t words = keyword_words_for(pass->capacity);
+  uint32_t *matched = pass->output->matched;
+  for (uint32_t word = 0; word < words; word++) { matched[word] = 0; }
+}
+
+uint32_t keyword_keep_best(const keyword_pass *pass, uint32_t candidates) {
+  narsil_keyword_output *output = pass->output;
+  int32_t *ordinals = output->ordinals;
+  double *scores = output->scores;
+  uint32_t count = candidates;
+  if (pass->has_limit && candidates > pass->limit) {
+    count = 0;
+    if (pass->limit > 0) {
+      double lowest = limit_best_score(pass, candidates);
+      for (uint32_t index = 0; index < candidates; index++) {
+        if (!(scores[index] >= lowest)) { continue; }
+        ordinals[count] = ordinals[index];
+        scores[count] = scores[index];
+        count++;
+      }
+    }
+  }
+  sort_best_first(ordinals, scores, count);
+  output->count = count;
+  return count;
+}
+
+uint32_t keyword_select_kept(const keyword_pass *pass) {
+  const narsil_keyword_request *request = pass->request;
+  const keyword_scratch *scratch = pass->scratch;
+  narsil_keyword_output *output = pass->output;
+  double terms = request->terms;
+  double min_score = request->min_score;
+  double required = request->term_match > 0 && terms > 0 ? request->term_match : 0;
+  uint32_t kept = 0;
+  if (required <= terms) {
+    for (uint32_t index = 0; index < scratch->touched_count; index++) {
+      uint32_t ordinal = scratch->touched[index];
+      if (required > 0 && (double)scratch->covered[ordinal] < required) { continue; }
+      double score = scratch->scores[ordinal];
+      if (min_score > 0 && !(score >= min_score)) { continue; }
+      output->ordinals[kept] = (int32_t)ordinal;
+      output->scores[kept] = score;
+      kept++;
+    }
+  }
+  if (request->collect) {
+    keyword_clear_matched(pass);
+    for (uint32_t index = 0; index < kept; index++) {
+      keyword_mark_matched(output->matched, (uint32_t)output->ordinals[index]);
+    }
+  }
+  output->total_matched = kept;
+  return keyword_keep_best(pass, kept);
+}

package/cpp/core/src/keyword/select.h

@@ -0,0 +1,17 @@
+#ifndef NARSIL_KEYWORD_SELECT_H
+#define NARSIL_KEYWORD_SELECT_H
+
+#include "pass.h"
+
+#include <stdint.h>
+
+uint32_t keyword_select_kept(const keyword_pass *pass);
+uint32_t keyword_keep_best(const keyword_pass *pass, uint32_t candidates);
+double keyword_touched_best_score(const keyword_pass *pass);
+void keyword_clear_matched(const keyword_pass *pass);
+
+static inline void keyword_mark_matched(uint32_t *matched, uint32_t ordinal) {
+  matched[ordinal >> KEYWORD_WORD_SHIFT] |= 1U << (ordinal & KEYWORD_BIT_MASK);
+}
+
+#endif

package/cpp/core/src/languages/bulgarian_stemmer.c

@@ -0,0 +1,114 @@
+#include "../../include/narsil_core.h"
+#include "../analysis/normalise.h"
+#include "../analysis/text.h"
+#include "rules.h"
+
+#include <stddef.h>
+#include <stdint.h>
+
+#define MIN_STEMMED_UNITS 4U
+#define LONGER_THAN_THREE 3U
+#define LONGER_THAN_FOUR 4U
+#define LONGER_THAN_FIVE 5U
+#define LONGER_THAN_SIX 6U
+#define NO_REPLACEMENT 0U
+#define CYRILLIC_G 0x0433U
+#define CYRILLIC_IE 0x0435U
+#define CYRILLIC_I 0x0438U
+#define CYRILLIC_SHORT_I 0x0439U
+#define CYRILLIC_KA 0x043AU
+#define CYRILLIC_EN 0x043DU
+#define CYRILLIC_HA 0x0445U
+#define CYRILLIC_HARD_SIGN 0x044AU
+#define CYRILLIC_YA 0x044FU
+#define COUNT_OF(array) (sizeof(array) / sizeof((array)[0]))
+
+typedef struct {
+  size_t longer_than;
+  const uint16_t *suffix;
+  size_t dropped;
+  uint16_t replacement;
+} suffix_rule;
+
+static const suffix_rule leading_plural_rules[] = {
+    {LONGER_THAN_FIVE, u"ища", 3, NO_REPLACEMENT},
+};
+
+static const suffix_rule definite_article_rules[] = {
+    {LONGER_THAN_SIX, u"ият", 3, NO_REPLACEMENT}, {LONGER_THAN_FIVE, u"ът", 2, NO_REPLACEMENT},
+    {LONGER_THAN_FIVE, u"то", 2, NO_REPLACEMENT}, {LONGER_THAN_FIVE, u"те", 2, NO_REPLACEMENT},
+    {LONGER_THAN_FIVE, u"та", 2, NO_REPLACEMENT}, {LONGER_THAN_FIVE, u"ия", 2, NO_REPLACEMENT},
+    {LONGER_THAN_FOUR, u"ят", 2, NO_REPLACEMENT},
+};
+
+static const suffix_rule plural_rules[] = {
+    {LONGER_THAN_SIX, u"овци", 3, NO_REPLACEMENT},  {LONGER_THAN_SIX, u"ове", 3, NO_REPLACEMENT},
+    {LONGER_THAN_SIX, u"еве", 3, CYRILLIC_SHORT_I}, {LONGER_THAN_FIVE, u"ища", 3, NO_REPLACEMENT},
+    {LONGER_THAN_FIVE, u"та", 2, NO_REPLACEMENT},   {LONGER_THAN_FIVE, u"ци", 2, CYRILLIC_KA},
+    {LONGER_THAN_FIVE, u"зи", 2, CYRILLIC_G},
+};
+
+static const suffix_rule late_plural_rules[] = {
+    {LONGER_THAN_FOUR, u"си", 2, CYRILLIC_HA},
+    {LONGER_THAN_FOUR, u"и", 1, NO_REPLACEMENT},
+};
+
+static const suffix_rule vowel_rules[] = {
+    {LONGER_THAN_THREE, u"я", 1, NO_REPLACEMENT},
+    {LONGER_THAN_THREE, u"а", 1, NO_REPLACEMENT},
+    {LONGER_THAN_THREE, u"о", 1, NO_REPLACEMENT},
+    {LONGER_THAN_THREE, u"е", 1, NO_REPLACEMENT},
+};
+
+static const suffix_rule en_rules[] = {
+    {LONGER_THAN_FOUR, u"ен", 2, CYRILLIC_EN},
+};
+
+static int apply_first(narsil_text *word, const suffix_rule *rules, size_t rule_count, int *applied) {
+  *applied = 0;
+  for (size_t i = 0; i < rule_count; i++) {
+    if (word->length <= rules[i].longer_than || !narsil_rule_ends_with(word, rules[i].suffix)) { continue; }
+    *applied = 1;
+    word->length -= rules[i].dropped;
+    return rules[i].replacement == NO_REPLACEMENT ? NARSIL_OK : narsil_text_push(word, rules[i].replacement);
+  }
+  return NARSIL_OK;
+}
+
+static int is_plural_in_e_and_i(const narsil_text *word) {
+  return word->length > LONGER_THAN_FIVE && word->units[word->length - 1] == CYRILLIC_I &&
+         word->units[word->length - 3] == CYRILLIC_IE;
+}
+
+static int plural(narsil_text *word) {
+  int applied = 0;
+  int status = apply_first(word, plural_rules, COUNT_OF(plural_rules), &applied);
+  if (status != NARSIL_OK || applied) { return status; }
+  if (is_plural_in_e_and_i(word)) {
+    uint16_t kept = word->units[word->length - 2];
+    word->length -= 3;
+    status = narsil_text_push(word, CYRILLIC_YA);
+    return status == NARSIL_OK ? narsil_text_push(word, kept) : status;
+  }
+  return apply_first(word, late_plural_rules, COUNT_OF(late_plural_rules), &applied);
+}
+
+int narsil_bulgarian_stem(const uint16_t *units, size_t length, narsil_text *out, narsil_normalise_scratch *scratch) {
+  (void)scratch;
+  int applied = 0;
+  int status = narsil_text_assign(out, units, length);
+  if (status != NARSIL_OK || length < MIN_STEMMED_UNITS) { return status; }
+  status = apply_first(out, leading_plural_rules, COUNT_OF(leading_plural_rules), &applied);
+  if (status == NARSIL_OK) {
+    status = apply_first(out, definite_article_rules, COUNT_OF(definite_article_rules), &applied);
+  }
+  if (status == NARSIL_OK) { status = plural(out); }
+  if (status == NARSIL_OK) { status = apply_first(out, vowel_rules, COUNT_OF(vowel_rules), &applied); }
+  if (status == NARSIL_OK) { status = apply_first(out, en_rules, COUNT_OF(en_rules), &applied); }
+  if (status == NARSIL_OK && out->length > LONGER_THAN_FIVE && out->units[out->length - 2] == CYRILLIC_HARD_SIGN) {
+    uint16_t last = out->units[out->length - 1];
+    out->length -= 2;
+    status = narsil_text_push(out, last);
+  }
+  return status;
+}

package/cpp/core/src/languages/data/albanian.c

@@ -0,0 +1,54 @@
+#include "../../analysis/languages.h"
+#include "declarations.h"
+
+#include <stddef.h>
+#include <stdint.h>
+
+static const uint16_t stop_word_units[321] = {
+  97, 105, 97, 106, 111, 97, 115, 97, 115, 110, 106, 235, 97, 116, 97, 97, 116, 106, 101, 97, 116, 111, 97, 116, 121,
+  98, 114, 101, 110, 100, 97, 99, 105, 108, 97, 99, 105, 108, 105, 99, 105, 108, 235, 116, 100, 101, 114, 105, 100, 104,
+  101, 100, 105, 115, 97, 100, 111, 101, 101, 100, 104, 101, 103, 106, 97, 116, 235, 103, 106, 105, 116, 104, 97, 115,
+  104, 116, 117, 103, 106, 105, 116, 104, 235, 105, 105, 109, 105, 109, 101, 105, 115, 104, 116, 101, 106, 97, 110, 235,
+  106, 97, 115, 104, 116, 235, 106, 111, 110, 235, 106, 111, 116, 101, 106, 117, 106, 117, 97, 106, 107, 97, 107, 97,
+  109, 107, 97, 110, 235, 107, 101, 107, 101, 109, 105, 107, 101, 110, 105, 107, 106, 111, 107, 117, 107, 117, 114, 107,
+  121, 107, 235, 116, 97, 107, 235, 116, 111, 107, 235, 116, 117, 108, 97, 114, 116, 109, 98, 105, 109, 101, 109, 105,
+  100, 105, 115, 109, 111, 115, 109, 235, 110, 100, 235, 114, 110, 100, 235, 114, 109, 106, 101, 116, 110, 101, 110,
+  103, 97, 110, 106, 235, 110, 117, 107, 110, 235, 110, 235, 110, 111, 115, 101, 112, 97, 112, 97, 107, 112, 97, 114,
+  97, 112, 97, 115, 112, 97, 115, 116, 97, 106, 112, 111, 114, 112, 111, 115, 104, 116, 235, 112, 114, 101, 106, 112,
+  235, 114, 113, 235, 115, 97, 115, 97, 106, 115, 104, 117, 109, 235, 115, 105, 115, 105, 112, 97, 115, 116, 97, 110,
+  105, 116, 101, 116, 101, 107, 116, 105, 116, 105, 106, 116, 106, 101, 116, 235, 114, 116, 121, 114, 101, 116, 235,
+  117, 117, 110, 235, 118, 101, 116, 101, 110, 118, 101, 116, 235, 118, 101, 116, 235, 109, 121, 116, 231, 100, 111,
+  231, 102, 97, 114, 235, 235, 115, 104, 116, 235,
+};
+
+static const uint32_t stop_word_offsets[92] = {
+  0, 2, 5, 7, 12, 15, 19, 22, 25, 31, 35, 39, 44, 48, 51, 55, 57, 58, 62, 67, 77, 83, 84, 86, 89, 94, 98, 104, 108, 112,
+  114, 118, 120, 123, 127, 129, 133, 137, 140, 142, 145, 147, 151, 155, 159, 163, 166, 168, 173, 176, 178, 182, 190,
+  192, 195, 198, 201, 203, 206, 209, 211, 214, 218, 221, 227, 230, 236, 240, 243, 245, 247, 250, 255, 257, 262, 266,
+  268, 271, 273, 276, 282, 286, 288, 289, 292, 297, 301, 306, 308, 311, 316, 321,
+};
+
+static const uint32_t keep_ranges[14] = {
+  48, 57, 65, 90, 97, 122, 199, 199, 203, 203, 231, 231, 235, 235,
+};
+
+const narsil_language narsil_language_albanian = {
+  .name = "albanian",
+  .revision = "1",
+  .stop_word_units = stop_word_units,
+  .stop_word_offsets = stop_word_offsets,
+  .stop_word_count = 91,
+  .ascii_keep = {0x03FF000000000000ULL, 0x07FFFFFE07FFFFFEULL},
+  .keep_ranges = keep_ranges,
+  .keep_range_count = 7,
+  .splits_code_points = 0,
+  .strips_possessive = 0,
+  .normalises_diacritics = 0,
+  .min_token_length = 1,
+  .ngram_size = 0,
+  .normalise = NULL,
+  .stem_by_rules = NULL,
+  .create_snowball = NULL,
+  .snowball_stem = NULL,
+  .close_snowball = NULL,
+};

package/cpp/core/src/languages/data/amharic.c

@@ -0,0 +1,93 @@
+#include "../../analysis/languages.h"
+#include "declarations.h"
+
+#include <stddef.h>
+#include <stdint.h>
+
+static const uint16_t stop_word_units[889] = {
+  4609, 4616, 4721, 4757, 4637, 4609, 4617, 4609, 4621, 4609, 4756, 4723, 4614, 4752, 4614, 4752, 4813, 4614, 4753,
+  4614, 4758, 4637, 4616, 4614, 4752, 4616, 4614, 4752, 4813, 4616, 4614, 4753, 4616, 4620, 4622, 4733, 4616, 4635,
+  4755, 4728, 4813, 4637, 4616, 4635, 4757, 4763, 4813, 4637, 4616, 4752, 4826, 4613, 4616, 4773, 4843, 4757, 4851,
+  4757, 4849, 4616, 4826, 4609, 4616, 4826, 4613, 4618, 4614, 4757, 4619, 4845, 4620, 4619, 4620, 4622, 4733, 4621,
+  4841, 4632, 4614, 4753, 4632, 4614, 4753, 4757, 4632, 4614, 4755, 4728, 4813, 4632, 4614, 4755, 4728, 4813, 4757,
+  4632, 4614, 4757, 4635, 4616, 4721, 4635, 4616, 4725, 4635, 4755, 4728, 4813, 4637, 4635, 4755, 4728, 4813, 4757,
+  4637, 4635, 4757, 4635, 4757, 4637, 4635, 4757, 4763, 4813, 4637, 4635, 4853, 4648, 4877, 4637, 4757, 4874, 4828,
+  4637, 4651, 4659, 4733, 4609, 4757, 4651, 4661, 4613, 4757, 4651, 4661, 4669, 4757, 4656, 4638, 4753, 4757, 4656,
+  4771, 4725, 4658, 4614, 4753, 4658, 4614, 4757, 4658, 4614, 4757, 4755, 4658, 4621, 4658, 4707, 4621, 4658, 4758,
+  4653, 4659, 4616, 4659, 4845, 4614, 4757, 4661, 4616, 4614, 4752, 4661, 4616, 4635, 4755, 4728, 4813, 4637, 4661,
+  4616, 4826, 4609, 4661, 4637, 4662, 4661, 4720, 4763, 4673, 4901, 4653, 4704, 4609, 4756, 4723, 4813, 4704, 4614,
+  4752, 4704, 4614, 4753, 4704, 4619, 4845, 4704, 4620, 4619, 4704, 4620, 4622, 4733, 4704, 4633, 4617, 4704, 4633,
+  4617, 4637, 4704, 4634, 4621, 4704, 4634, 4733, 4617, 4704, 4634, 4872, 4707, 4704, 4635, 4672, 4853, 4704, 4635,
+  4755, 4728, 4813, 4637, 4704, 4653, 4779, 4723, 4704, 4656, 4638, 4753, 4704, 4661, 4720, 4672, 4653, 4704, 4672,
+  4653, 4704, 4720, 4616, 4845, 4704, 4720, 4616, 4845, 4637, 4704, 4720, 4632, 4616, 4776, 4720, 4704, 4720, 4632,
+  4659, 4659, 4845, 4704, 4723, 4733, 4704, 4747, 4619, 4704, 4752, 4657, 4704, 4752, 4826, 4609, 4704, 4752, 4826,
+  4613, 4704, 4773, 4752, 4826, 4609, 4704, 4773, 4757, 4848, 4826, 4613, 4704, 4777, 4621, 4704, 4811, 4755, 4704,
+  4813, 4661, 4901, 4704, 4826, 4613, 4704, 4826, 4613, 4637, 4704, 4826, 4613, 4755, 4706, 4614, 4757, 4706, 4614,
+  4757, 4637, 4706, 4843, 4757, 4661, 4707, 4617, 4707, 4899, 4637, 4709, 4616, 4811, 4621, 4709, 4622, 4709, 4731,
+  4709, 4825, 4710, 4723, 4720, 4707, 4616, 4720, 4709, 4622, 4720, 4776, 4755, 4808, 4752, 4720, 4872, 4616, 4920,
+  4720, 4872, 4621, 4927, 4621, 4720, 4904, 4635, 4650, 4725, 4755, 4757, 4725, 4733, 4877, 4653, 4747, 4619, 4752,
+  4704, 4648, 4733, 4752, 4704, 4649, 4752, 4704, 4653, 4752, 4813, 4752, 4872, 4653, 4752, 4872, 4654, 4733, 4755,
+  4755, 4725, 4755, 4728, 4813, 4768, 4609, 4757, 4768, 4616, 4768, 4617, 4768, 4659, 4656, 4704, 4768, 4659, 4661,
+  4704, 4811, 4621, 4768, 4661, 4648, 4853, 4720, 4811, 4621, 4768, 4661, 4723, 4813, 4656, 4811, 4621, 4768, 4661,
+  4723, 4813, 4672, 4811, 4621, 4768, 4661, 4872, 4752, 4824, 4705, 4768, 4661, 4872, 4757, 4829, 4704, 4811, 4621,
+  4768, 4661, 4936, 4619, 4874, 4768, 4709, 4651, 4653, 4720, 4811, 4621, 4768, 4757, 4720, 4768, 4757, 4730, 4768,
+  4757, 4848, 4614, 4752, 4768, 4757, 4848, 4763, 4768, 4757, 4849, 4768, 4757, 4849, 4757, 4768, 4757, 4850, 4725,
+  4768, 4757, 4851, 4757, 4853, 4768, 4757, 4923, 4653, 4773, 4661, 4776, 4773, 4661, 4778, 4848, 4653, 4661, 4773,
+  4661, 4779, 4609, 4757, 4773, 4707, 4781, 4613, 4773, 4707, 4781, 4669, 4773, 4752, 4826, 4613, 4773, 4752, 4826,
+  4613, 4637, 4773, 4752, 4826, 4613, 4753, 4773, 4752, 4826, 4613, 4757, 4773, 4752, 4848, 4614, 4752, 4773, 4755,
+  4757, 4720, 4773, 4757, 4787, 4773, 4757, 4787, 4757, 4773, 4757, 4848, 4773, 4757, 4848, 4614, 4752, 4755, 4773,
+  4757, 4848, 4720, 4755, 4872, 4649, 4725, 4773, 4757, 4848, 4752, 4826, 4613, 4773, 4757, 4848, 4872, 4616, 4921,
+  4725, 4773, 4757, 4848, 4872, 4755, 4773, 4757, 4850, 4609, 4637, 4773, 4757, 4850, 4613, 4773, 4757, 4850, 4614,
+  4753, 4773, 4757, 4850, 4614, 4757, 4773, 4757, 4851, 4661, 4648, 4849, 4725, 4773, 4757, 4866, 4773, 4826, 4609,
+  4773, 4869, 4877, 4776, 4609, 4616, 4725, 4776, 4614, 4752, 4776, 4614, 4752, 4813, 4776, 4620, 4619, 4776, 4620,
+  4622, 4733, 4776, 4752, 4826, 4609, 4776, 4768, 4661, 4653, 4776, 4768, 4757, 4851, 4757, 4853, 4776, 4768, 4757,
+  4853, 4776, 4773, 4752, 4826, 4609, 4776, 4773, 4752, 4826, 4613, 4776, 4826, 4613, 4776, 4826, 4613, 4637, 4776,
+  4826, 4843, 4779, 4621, 4614, 4752, 4808, 4653, 4808, 4677, 4725, 4808, 4845, 4808, 4845, 4637, 4808, 4848, 4813,
+  4661, 4901, 4840, 4614, 4752, 4840, 4614, 4752, 4733, 4757, 4840, 4614, 4752, 4813, 4840, 4614, 4753, 4725, 4840,
+  4616, 4813, 4637, 4840, 4620, 4619, 4840, 4620, 4619, 4728, 4813, 4757, 4840, 4620, 4619, 4813, 4757, 4840, 4634,
+  4614, 4753, 4725, 4840, 4634, 4733, 4616, 4813, 4757, 4840, 4634, 4872, 4765, 4840, 4635, 4755, 4728, 4813, 4637,
+  4840, 4635, 4843, 4757, 4661, 4840, 4635, 4845, 4704, 4621, 4901, 4840, 4720, 4616, 4843, 4840, 4840, 4720, 4616,
+  4843, 4841, 4840, 4752, 4704, 4648, 4813, 4757, 4840, 4752, 4872, 4649, 4840, 4755, 4757, 4720, 4840, 4768, 4757,
+  4672, 4921, 4840, 4826, 4609, 4840, 4826, 4613, 4843, 4616, 4843, 4617, 4843, 4619, 4728, 4813, 4843, 4619, 4728,
+  4813, 4757, 4843, 4621, 4614, 4752, 4843, 4757, 4720, 4843, 4757, 4730, 4845, 4613, 4845, 4613, 4637, 4845, 4613,
+  4733, 4813, 4845, 4613, 4757, 4845, 4613, 4757, 4753, 4845, 4613, 4757, 4757, 4637, 4845, 4614, 4753, 4704, 4723,
+  4621, 4845, 4614, 4755, 4617, 4845, 4792, 4813, 4874, 4828, 4755, 4875, 4653, 4877, 4757,
+};
+
+static const uint32_t stop_word_offsets[229] = {
+  0, 5, 7, 9, 12, 14, 17, 19, 22, 25, 29, 32, 36, 42, 48, 52, 59, 62, 65, 68, 70, 72, 75, 77, 80, 84, 89, 95, 98, 101,
+  104, 109, 115, 117, 120, 125, 129, 134, 139, 143, 147, 151, 154, 157, 160, 164, 166, 169, 172, 174, 178, 182, 189,
+  193, 195, 199, 202, 207, 210, 213, 216, 219, 223, 226, 230, 233, 237, 241, 245, 251, 255, 259, 264, 267, 271, 276,
+  282, 288, 291, 294, 297, 301, 305, 310, 316, 319, 322, 326, 329, 333, 337, 340, 344, 348, 350, 353, 357, 359, 361,
+  363, 365, 368, 371, 376, 380, 385, 389, 393, 396, 398, 402, 405, 408, 410, 413, 417, 418, 420, 423, 426, 428, 430,
+  434, 440, 447, 454, 461, 467, 475, 480, 487, 490, 493, 498, 502, 505, 509, 513, 518, 522, 525, 531, 536, 540, 544,
+  548, 553, 558, 563, 568, 572, 575, 579, 582, 588, 596, 602, 609, 614, 619, 623, 628, 633, 640, 643, 646, 649, 653,
+  656, 660, 663, 667, 671, 675, 681, 685, 690, 695, 698, 702, 705, 709, 711, 714, 716, 719, 721, 724, 727, 732, 736,
+  740, 744, 747, 753, 758, 763, 769, 773, 779, 784, 790, 795, 800, 806, 810, 814, 819, 822, 825, 827, 829, 833, 838,
+  842, 845, 848, 850, 853, 857, 860, 864, 869, 875, 879, 882, 885, 887, 889,
+};
+
+static const uint32_t keep_ranges[10] = {
+  48, 57, 65, 90, 97, 122, 4608, 4959, 4969, 4988,
+};
+
+const narsil_language narsil_language_amharic = {
+  .name = "amharic",
+  .revision = "1",
+  .stop_word_units = stop_word_units,
+  .stop_word_offsets = stop_word_offsets,
+  .stop_word_count = 228,
+  .ascii_keep = {0x03FF000000000000ULL, 0x07FFFFFE07FFFFFEULL},
+  .keep_ranges = keep_ranges,
+  .keep_range_count = 5,
+  .splits_code_points = 0,
+  .strips_possessive = 0,
+  .normalises_diacritics = 0,
+  .min_token_length = 1,
+  .ngram_size = 0,
+  .normalise = NULL,
+  .stem_by_rules = NULL,
+  .create_snowball = NULL,
+  .snowball_stem = NULL,
+  .close_snowball = NULL,
+};

package/cpp/core/src/languages/data/arabic.c

@@ -0,0 +1,146 @@
+#include "../../analysis/languages.h"
+#include "declarations.h"
+#include "../snowball/arabic_stemmer.h"
+
+#include <stddef.h>
+#include <stdint.h>
+
+static const uint16_t stop_word_units[1653] = {
+  1570, 1590, 1570, 1605, 1610, 1606, 1570, 1607, 1570, 1607, 1575, 1570, 1610, 1571, 1571, 1576, 1571, 1580, 1604,
+  1571, 1580, 1605, 1593, 1571, 1582, 1571, 1582, 1584, 1571, 1589, 1576, 1581, 1571, 1590, 1581, 1609, 1571, 1601,
+  1571, 1602, 1576, 1604, 1571, 1602, 1604, 1571, 1603, 1579, 1585, 1571, 1604, 1575, 1571, 1605, 1571, 1605, 1575,
+  1571, 1605, 1575, 1605, 1603, 1571, 1605, 1587, 1609, 1571, 1606, 1571, 1606, 1575, 1571, 1606, 1578, 1571, 1606,
+  1578, 1605, 1571, 1606, 1578, 1605, 1575, 1571, 1606, 1578, 1606, 1571, 1606, 1588, 1571, 1571, 1606, 1609, 1571,
+  1608, 1571, 1608, 1588, 1603, 1571, 1608, 1604, 1574, 1603, 1571, 1608, 1604, 1574, 1603, 1605, 1571, 1608, 1604,
+  1575, 1569, 1571, 1608, 1604, 1575, 1604, 1603, 1571, 1608, 1607, 1571, 1610, 1571, 1610, 1575, 1571, 1610, 1575,
+  1606, 1571, 1610, 1590, 1575, 1571, 1610, 1606, 1571, 1610, 1606, 1605, 1575, 1573, 1584, 1573, 1584, 1575, 1573,
+  1584, 1605, 1575, 1573, 1584, 1606, 1573, 1604, 1575, 1573, 1604, 1609, 1573, 1604, 1610, 1603, 1573, 1604, 1610,
+  1603, 1605, 1573, 1604, 1610, 1603, 1605, 1575, 1573, 1604, 1610, 1603, 1606, 1573, 1605, 1575, 1573, 1606, 1573,
+  1606, 1605, 1575, 1573, 1610, 1573, 1610, 1575, 1603, 1573, 1610, 1575, 1603, 1605, 1573, 1610, 1575, 1603, 1605,
+  1575, 1573, 1610, 1575, 1603, 1606, 1573, 1610, 1575, 1606, 1575, 1573, 1610, 1575, 1607, 1573, 1610, 1575, 1607,
+  1575, 1573, 1610, 1575, 1607, 1605, 1573, 1610, 1575, 1607, 1605, 1575, 1573, 1610, 1575, 1607, 1606, 1573, 1610,
+  1575, 1610, 1573, 1610, 1607, 1575, 1575, 1576, 1578, 1583, 1571, 1575, 1579, 1585, 1575, 1580, 1604, 1575, 1581,
+  1583, 1575, 1582, 1585, 1609, 1575, 1582, 1604, 1608, 1604, 1602, 1575, 1584, 1575, 1575, 1585, 1576, 1593, 1577,
+  1575, 1585, 1578, 1583, 1575, 1587, 1578, 1581, 1575, 1604, 1575, 1591, 1575, 1585, 1575, 1593, 1575, 1583, 1577,
+  1575, 1593, 1604, 1606, 1578, 1575, 1601, 1575, 1603, 1579, 1585, 1575, 1603, 1583, 1575, 1604, 1571, 1604, 1575,
+  1569, 1575, 1604, 1571, 1604, 1609, 1575, 1604, 1575, 1575, 1604, 1575, 1582, 1610, 1585, 1577, 1575, 1604, 1575,
+  1606, 1575, 1604, 1575, 1608, 1604, 1575, 1604, 1575, 1608, 1604, 1609, 1575, 1604, 1578, 1609, 1575, 1604, 1578,
+  1610, 1575, 1604, 1579, 1575, 1606, 1610, 1575, 1604, 1579, 1575, 1606, 1610, 1577, 1575, 1604, 1584, 1575, 1578,
+  1610, 1575, 1604, 1584, 1609, 1575, 1604, 1584, 1610, 1575, 1604, 1584, 1610, 1606, 1575, 1604, 1587, 1575, 1576,
+  1602, 1575, 1604, 1601, 1575, 1604, 1604, 1575, 1574, 1610, 1575, 1604, 1604, 1575, 1578, 1610, 1575, 1604, 1604,
+  1578, 1575, 1606, 1575, 1604, 1604, 1578, 1610, 1575, 1575, 1604, 1604, 1578, 1610, 1606, 1575, 1604, 1604, 1584,
+  1575, 1606, 1575, 1604, 1604, 1584, 1610, 1606, 1575, 1604, 1604, 1608, 1575, 1578, 1610, 1575, 1604, 1605, 1575,
+  1590, 1610, 1575, 1604, 1605, 1602, 1576, 1604, 1575, 1604, 1608, 1602, 1578, 1575, 1604, 1609, 1575, 1604, 1610,
+  1608, 1605, 1575, 1605, 1575, 1575, 1605, 1575, 1605, 1575, 1605, 1587, 1575, 1606, 1575, 1606, 1576, 1585, 1609,
+  1575, 1606, 1602, 1604, 1576, 1575, 1606, 1607, 1575, 1606, 1607, 1575, 1575, 1608, 1575, 1608, 1604, 1575, 1610,
+  1575, 1610, 1575, 1585, 1575, 1610, 1575, 1605, 1575, 1610, 1590, 1575, 1576, 1576, 1574, 1587, 1576, 1575, 1578,
+  1576, 1575, 1587, 1605, 1576, 1575, 1606, 1576, 1582, 1576, 1585, 1587, 1576, 1587, 1576, 1587, 1576, 1576, 1576,
+  1588, 1603, 1604, 1576, 1590, 1593, 1576, 1591, 1570, 1606, 1576, 1593, 1583, 1576, 1593, 1590, 1576, 1603, 1576,
+  1603, 1605, 1576, 1603, 1605, 1575, 1576, 1603, 1606, 1576, 1604, 1576, 1604, 1607, 1576, 1604, 1609, 1576, 1605,
+  1575, 1576, 1605, 1575, 1584, 1575, 1576, 1605, 1606, 1576, 1606, 1576, 1606, 1575, 1576, 1607, 1576, 1607, 1575,
+  1576, 1610, 1576, 1610, 1583, 1576, 1610, 1606, 1578, 1575, 1606, 1578, 1575, 1606, 1603, 1578, 1576, 1583, 1604,
+  1578, 1580, 1575, 1607, 1578, 1581, 1578, 1578, 1581, 1608, 1604, 1578, 1604, 1602, 1575, 1569, 1578, 1604, 1603,
+  1578, 1604, 1603, 1605, 1578, 1604, 1603, 1605, 1575, 1578, 1605, 1578, 1607, 1578, 1610, 1578, 1610, 1606, 1578,
+  1610, 1606, 1603, 1579, 1604, 1575, 1579, 1577, 1579, 1605, 1579, 1605, 1577, 1580, 1593, 1604, 1580, 1604, 1604,
+  1580, 1605, 1610, 1593, 1580, 1610, 1585, 1581, 1575, 1585, 1581, 1575, 1588, 1575, 1581, 1575, 1604, 1610, 1575,
+  1581, 1575, 1610, 1581, 1576, 1584, 1575, 1581, 1578, 1609, 1581, 1584, 1575, 1585, 1581, 1585, 1609, 1581, 1587,
+  1576, 1581, 1605, 1581, 1608, 1575, 1604, 1609, 1581, 1608, 1604, 1581, 1610, 1581, 1610, 1579, 1581, 1610, 1579,
+  1605, 1575, 1581, 1610, 1606, 1582, 1604, 1575, 1582, 1604, 1575, 1604, 1583, 1608, 1606, 1583, 1608, 1606, 1603,
+  1584, 1575, 1584, 1575, 1578, 1584, 1575, 1603, 1584, 1575, 1606, 1584, 1575, 1606, 1603, 1584, 1604, 1603, 1584,
+  1604, 1603, 1605, 1584, 1604, 1603, 1605, 1575, 1584, 1604, 1603, 1606, 1584, 1607, 1584, 1608, 1584, 1608, 1575,
+  1584, 1608, 1575, 1578, 1575, 1584, 1608, 1575, 1578, 1610, 1584, 1610, 1584, 1610, 1578, 1584, 1610, 1606, 1584,
+  1610, 1606, 1603, 1585, 1575, 1581, 1585, 1576, 1585, 1580, 1593, 1585, 1608, 1610, 1583, 1603, 1585, 1610, 1579,
+  1586, 1610, 1575, 1585, 1577, 1587, 1575, 1569, 1587, 1575, 1569, 1605, 1575, 1587, 1576, 1581, 1575, 1606, 1587,
+  1585, 1593, 1575, 1606, 1587, 1606, 1577, 1587, 1606, 1608, 1575, 1578, 1587, 1608, 1601, 1587, 1608, 1609, 1588,
+  1576, 1607, 1588, 1578, 1575, 1606, 1588, 1582, 1589, 1575, 1588, 1585, 1593, 1589, 1575, 1585, 1589, 1576, 1575,
+  1581, 1589, 1601, 1585, 1589, 1607, 1590, 1583, 1590, 1605, 1606, 1591, 1575, 1602, 1591, 1575, 1604, 1605, 1575,
+  1591, 1601, 1602, 1591, 1602, 1592, 1604, 1593, 1575, 1583, 1593, 1575, 1605, 1593, 1575, 1605, 1575, 1593, 1575,
+  1605, 1577, 1593, 1576, 1585, 1593, 1583, 1575, 1593, 1583, 1577, 1593, 1583, 1583, 1593, 1583, 1587, 1593, 1583,
+  1605, 1593, 1587, 1609, 1593, 1588, 1585, 1593, 1588, 1585, 1577, 1593, 1604, 1593, 1604, 1602, 1593, 1604, 1609,
+  1593, 1604, 1610, 1603, 1593, 1604, 1610, 1607, 1593, 1604, 1610, 1607, 1575, 1593, 1605, 1575, 1593, 1606, 1593,
+  1606, 1583, 1593, 1606, 1583, 1605, 1575, 1593, 1608, 1590, 1593, 1610, 1606, 1594, 1583, 1575, 1594, 1610, 1585,
+  1601, 1601, 1573, 1606, 1601, 1575, 1606, 1601, 1602, 1591, 1601, 1604, 1575, 1606, 1601, 1608, 1601, 1608, 1602,
+  1601, 1609, 1601, 1610, 1601, 1610, 1605, 1601, 1610, 1605, 1575, 1601, 1610, 1607, 1601, 1610, 1607, 1575, 1602,
+  1575, 1604, 1602, 1575, 1605, 1602, 1576, 1604, 1602, 1583, 1602, 1591, 1602, 1604, 1605, 1575, 1602, 1608, 1577,
+  1603, 1571, 1606, 1603, 1571, 1606, 1605, 1575, 1603, 1571, 1610, 1603, 1571, 1610, 1606, 1603, 1575, 1583, 1603,
+  1575, 1606, 1603, 1575, 1606, 1578, 1603, 1582, 1603, 1584, 1575, 1603, 1584, 1604, 1603, 1603, 1585, 1576, 1603,
+  1604, 1603, 1604, 1575, 1603, 1604, 1575, 1607, 1605, 1575, 1603, 1604, 1578, 1575, 1603, 1604, 1605, 1603, 1604,
+  1605, 1575, 1603, 1604, 1610, 1603, 1605, 1575, 1603, 1604, 1610, 1607, 1605, 1575, 1603, 1605, 1603, 1605, 1575,
+  1603, 1610, 1603, 1610, 1578, 1603, 1610, 1601, 1603, 1610, 1601, 1605, 1575, 1604, 1574, 1606, 1604, 1575, 1604,
+  1575, 1578, 1604, 1575, 1587, 1610, 1605, 1575, 1604, 1583, 1606, 1604, 1583, 1609, 1604, 1587, 1578, 1604, 1587,
+  1578, 1605, 1604, 1587, 1578, 1605, 1575, 1604, 1587, 1578, 1606, 1604, 1587, 1606, 1604, 1587, 1606, 1575, 1604,
+  1593, 1604, 1604, 1593, 1605, 1585, 1604, 1602, 1575, 1569, 1604, 1603, 1604, 1603, 1605, 1604, 1603, 1605, 1575,
+  1604, 1603, 1606, 1604, 1603, 1606, 1605, 1575, 1604, 1603, 1610, 1604, 1603, 1610, 1604, 1575, 1604, 1604, 1575,
+  1605, 1605, 1604, 1605, 1604, 1605, 1575, 1604, 1605, 1575, 1584, 1575, 1604, 1606, 1604, 1606, 1575, 1604, 1607,
+  1604, 1607, 1575, 1604, 1608, 1604, 1608, 1603, 1575, 1604, 1577, 1604, 1608, 1604, 1575, 1604, 1608, 1605, 1575,
+  1604, 1610, 1604, 1610, 1578, 1604, 1610, 1587, 1604, 1610, 1587, 1575, 1604, 1610, 1587, 1578, 1604, 1610, 1587,
+  1578, 1575, 1604, 1610, 1587, 1608, 1575, 1605, 1575, 1605, 1575, 1575, 1606, 1601, 1603, 1605, 1575, 1576, 1585,
+  1581, 1605, 1575, 1583, 1575, 1605, 1605, 1575, 1584, 1575, 1605, 1575, 1586, 1575, 1604, 1605, 1575, 1601, 1578,
+  1574, 1605, 1575, 1610, 1608, 1605, 1578, 1609, 1605, 1579, 1604, 1605, 1584, 1605, 1587, 1575, 1569, 1605, 1593,
+  1605, 1593, 1575, 1584, 1605, 1602, 1575, 1576, 1604, 1605, 1603, 1575, 1606, 1603, 1605, 1603, 1575, 1606, 1603,
+  1605, 1605, 1603, 1575, 1606, 1603, 1605, 1575, 1605, 1603, 1575, 1606, 1603, 1606, 1605, 1604, 1610, 1575, 1585,
+  1605, 1604, 1610, 1608, 1606, 1605, 1605, 1575, 1605, 1605, 1606, 1605, 1606, 1605, 1606, 1584, 1605, 1606, 1607,
+  1575, 1605, 1607, 1605, 1607, 1605, 1575, 1606, 1581, 1606, 1606, 1581, 1608, 1606, 1582, 1606, 1593, 1605, 1606,
+  1593, 1605, 1575, 1606, 1601, 1587, 1606, 1601, 1587, 1607, 1606, 1607, 1575, 1610, 1577, 1607, 1572, 1604, 1575,
+  1569, 1607, 1575, 1607, 1575, 1572, 1605, 1607, 1575, 1578, 1575, 1606, 1607, 1575, 1578, 1607, 1607, 1575, 1578,
+  1610, 1607, 1575, 1578, 1610, 1606, 1607, 1575, 1603, 1607, 1575, 1607, 1606, 1575, 1607, 1576, 1607, 1580, 1607,
+  1584, 1575, 1607, 1584, 1575, 1606, 1607, 1584, 1607, 1607, 1584, 1610, 1607, 1584, 1610, 1606, 1607, 1603, 1584,
+  1575, 1607, 1604, 1607, 1604, 1575, 1607, 1604, 1605, 1607, 1605, 1607, 1605, 1575, 1607, 1606, 1607, 1606, 1575,
+  1607, 1606, 1575, 1603, 1607, 1606, 1575, 1604, 1603, 1607, 1608, 1607, 1610, 1607, 1610, 1575, 1607, 1610, 1578,
+  1607, 1610, 1607, 1575, 1578, 1608, 1608, 54, 1608, 1573, 1606, 1608, 1575, 1608, 1575, 1581, 1583, 1608, 1575, 1590,
+  1575, 1601, 1608, 1575, 1590, 1575, 1601, 1578, 1608, 1575, 1603, 1583, 1608, 1575, 1606, 1608, 1575, 1607, 1575,
+  1608, 1575, 1608, 1590, 1581, 1608, 1585, 1575, 1569, 1603, 1608, 1588, 1603, 1575, 1606, 1608, 1601, 1610, 1608,
+  1602, 1575, 1604, 1608, 1602, 1575, 1604, 1578, 1608, 1602, 1583, 1608, 1602, 1601, 1608, 1603, 1575, 1606, 1608,
+  1603, 1575, 1606, 1578, 1608, 1604, 1575, 1608, 1604, 1605, 1608, 1605, 1606, 1608, 1607, 1608, 1608, 1607, 1610,
+  1608, 1610, 1608, 1610, 1603, 1571, 1606, 1610, 1603, 1608, 1606, 1610, 1605, 1603, 1606, 1610, 1608, 1605,
+};
+
+static const uint32_t stop_word_offsets[461] = {
+  0, 2, 6, 8, 11, 13, 14, 16, 19, 23, 25, 28, 32, 36, 38, 42, 45, 49, 52, 54, 57, 62, 66, 68, 71, 74, 78, 83, 87, 91,
+  94, 96, 100, 105, 111, 116, 122, 125, 127, 130, 134, 138, 141, 146, 148, 151, 155, 158, 161, 164, 168, 173, 179, 184,
+  187, 189, 193, 195, 199, 204, 210, 215, 220, 224, 229, 234, 240, 245, 249, 252, 253, 258, 261, 264, 267, 271, 277,
+  280, 285, 289, 295, 299, 304, 309, 311, 315, 318, 324, 329, 332, 339, 343, 348, 354, 358, 362, 368, 375, 381, 385,
+  389, 394, 400, 403, 409, 415, 421, 427, 433, 439, 445, 452, 458, 464, 469, 472, 477, 480, 484, 487, 489, 494, 499,
+  502, 506, 508, 511, 513, 517, 521, 525, 526, 529, 532, 536, 539, 541, 544, 546, 550, 554, 557, 561, 564, 567, 569,
+  572, 576, 579, 581, 584, 587, 590, 595, 598, 600, 603, 605, 608, 610, 613, 616, 619, 623, 627, 631, 634, 638, 643,
+  646, 650, 655, 657, 659, 661, 664, 668, 673, 675, 678, 681, 684, 688, 691, 694, 698, 703, 706, 710, 713, 717, 720,
+  723, 725, 730, 733, 735, 738, 743, 746, 749, 753, 756, 760, 762, 765, 768, 771, 775, 778, 782, 787, 791, 793, 795,
+  798, 803, 808, 810, 813, 816, 820, 823, 825, 828, 833, 836, 841, 844, 849, 854, 859, 862, 867, 870, 873, 876, 880,
+  884, 887, 890, 894, 897, 899, 901, 904, 907, 912, 915, 917, 919, 922, 925, 929, 933, 936, 939, 942, 945, 948, 951,
+  954, 957, 961, 963, 966, 969, 973, 977, 982, 985, 987, 990, 995, 998, 1001, 1004, 1007, 1008, 1011, 1014, 1017, 1021,
+  1023, 1026, 1028, 1030, 1033, 1037, 1040, 1044, 1047, 1050, 1053, 1055, 1057, 1061, 1064, 1067, 1072, 1075, 1079,
+  1082, 1085, 1089, 1091, 1094, 1098, 1101, 1103, 1106, 1112, 1116, 1119, 1123, 1129, 1135, 1137, 1140, 1142, 1145,
+  1148, 1153, 1156, 1158, 1161, 1167, 1170, 1173, 1176, 1180, 1185, 1189, 1192, 1196, 1199, 1203, 1207, 1209, 1212,
+  1216, 1219, 1224, 1227, 1232, 1237, 1239, 1242, 1247, 1249, 1252, 1254, 1257, 1259, 1265, 1269, 1273, 1275, 1278,
+  1281, 1285, 1289, 1294, 1299, 1301, 1307, 1312, 1317, 1321, 1326, 1331, 1335, 1338, 1341, 1343, 1347, 1349, 1353,
+  1358, 1363, 1369, 1376, 1382, 1387, 1392, 1395, 1398, 1400, 1403, 1407, 1409, 1413, 1416, 1419, 1421, 1424, 1428,
+  1431, 1435, 1440, 1445, 1447, 1451, 1456, 1460, 1464, 1469, 1472, 1477, 1479, 1481, 1484, 1488, 1491, 1494, 1498,
+  1502, 1504, 1507, 1510, 1512, 1515, 1517, 1520, 1524, 1529, 1531, 1533, 1536, 1539, 1544, 1545, 1547, 1550, 1552,
+  1556, 1561, 1567, 1571, 1574, 1578, 1583, 1588, 1593, 1596, 1600, 1605, 1608, 1611, 1615, 1620, 1623, 1626, 1629,
+  1632, 1635, 1637, 1642, 1646, 1650, 1653,
+};
+
+static const uint32_t keep_ranges[12] = {
+  48, 57, 65, 90, 97, 122, 1569, 1641, 1646, 1747, 1776, 1785,
+};
+
+const narsil_language narsil_language_arabic = {
+  .name = "arabic",
+  .revision = "d9b491d016b4caf6",
+  .stop_word_units = stop_word_units,
+  .stop_word_offsets = stop_word_offsets,
+  .stop_word_count = 460,
+  .ascii_keep = {0x03FF000000000000ULL, 0x07FFFFFE07FFFFFEULL},
+  .keep_ranges = keep_ranges,
+  .keep_range_count = 6,
+  .splits_code_points = 0,
+  .strips_possessive = 0,
+  .normalises_diacritics = 0,
+  .min_token_length = 1,
+  .ngram_size = 0,
+  .normalise = NULL,
+  .stem_by_rules = NULL,
+  .create_snowball = arabic_create_env,
+  .snowball_stem = arabic_stem,
+  .close_snowball = arabic_close_env,
+};

package/cpp/core/src/languages/data/armenian.c

@@ -0,0 +1,46 @@
+#include "../../analysis/languages.h"
+#include "declarations.h"
+#include "../snowball/armenian_stemmer.h"
+
+#include <stddef.h>
+#include <stdint.h>
+
+static const uint16_t stop_word_units[126] = {
+  1377, 1397, 1380, 1377, 1397, 1388, 1377, 1397, 1398, 1377, 1397, 1405, 1380, 1400, 1410, 1380, 1400, 1410, 1412,
+  1381, 1396, 1381, 1398, 1381, 1398, 1412, 1381, 1405, 1381, 1412, 1383, 1383, 1387, 1383, 1387, 1398, 1383, 1387,
+  1398, 1412, 1383, 1387, 1408, 1383, 1387, 1412, 1383, 1408, 1384, 1405, 1407, 1385, 1387, 1387, 1398, 1387, 1405,
+  1391, 1387, 1408, 1391, 1377, 1396, 1392, 1377, 1396, 1377, 1408, 1392, 1381, 1407, 1392, 1381, 1407, 1400, 1396,
+  1381, 1398, 1412, 1396, 1381, 1403, 1396, 1387, 1398, 1398, 1377, 1398, 1377, 1415, 1398, 1408, 1377, 1398, 1408,
+  1377, 1398, 1412, 1400, 1408, 1400, 1408, 1384, 1400, 1408, 1400, 1398, 1412, 1400, 1408, 1402, 1381, 1405, 1400,
+  1410, 1400, 1410, 1396, 1402, 1387, 1407, 1387, 1406, 1408, 1377, 1415,
+};
+
+static const uint32_t stop_word_offsets[46] = {
+  0, 3, 6, 9, 12, 15, 19, 21, 23, 26, 28, 30, 31, 33, 36, 40, 43, 46, 48, 51, 52, 53, 55, 58, 60, 63, 68, 71, 75, 79,
+  82, 84, 85, 87, 90, 93, 98, 100, 103, 108, 113, 115, 118, 122, 125, 126,
+};
+
+static const uint32_t keep_ranges[10] = {
+  48, 57, 65, 90, 97, 122, 1329, 1366, 1377, 1415,
+};
+
+const narsil_language narsil_language_armenian = {
+  .name = "armenian",
+  .revision = "6d99ac3501d5d369",
+  .stop_word_units = stop_word_units,
+  .stop_word_offsets = stop_word_offsets,
+  .stop_word_count = 45,
+  .ascii_keep = {0x03FF000000000000ULL, 0x07FFFFFE07FFFFFEULL},
+  .keep_ranges = keep_ranges,
+  .keep_range_count = 5,
+  .splits_code_points = 0,
+  .strips_possessive = 0,
+  .normalises_diacritics = 0,
+  .min_token_length = 1,
+  .ngram_size = 0,
+  .normalise = NULL,
+  .stem_by_rules = NULL,
+  .create_snowball = armenian_create_env,
+  .snowball_stem = armenian_stem,
+  .close_snowball = armenian_close_env,
+};