@delali/narsil-react-native 0.3.1-next.155 → 0.3.1-next.157

package/cpp/core/src/languages/data/oromo.c

@@ -0,0 +1,51 @@
+#include "../../analysis/languages.h"
+#include "declarations.h"
+
+#include <stddef.h>
+#include <stdint.h>
+
+static const uint16_t stop_word_units[292] = {
+  97, 99, 104, 105, 97, 107, 107, 97, 97, 107, 107, 97, 109, 97, 107, 107, 97, 115, 117, 109, 97, 115, 97, 109, 109, 97,
+  97, 109, 109, 97, 115, 97, 109, 109, 111, 97, 110, 105, 97, 115, 97, 116, 105, 98, 111, 111, 100, 97, 100, 104, 97,
+  100, 117, 114, 97, 101, 101, 110, 121, 117, 101, 101, 115, 115, 97, 101, 114, 103, 97, 102, 105, 103, 97, 114, 117,
+  117, 103, 105, 100, 100, 117, 117, 104, 105, 110, 105, 108, 108, 101, 101, 105, 109, 109, 111, 111, 105, 110, 110,
+  105, 105, 114, 114, 97, 105, 114, 114, 97, 116, 116, 105, 105, 115, 97, 105, 115, 97, 97, 105, 115, 97, 97, 110, 105,
+  115, 97, 97, 110, 105, 105, 105, 115, 104, 101, 101, 105, 115, 104, 101, 101, 110, 105, 115, 105, 110, 106, 97, 108,
+  97, 106, 105, 114, 97, 106, 105, 114, 97, 110, 106, 105, 114, 116, 105, 106, 105, 114, 117, 107, 97, 110, 107, 97,
+  110, 97, 107, 97, 110, 97, 97, 102, 107, 101, 101, 107, 101, 101, 110, 121, 97, 107, 101, 101, 115, 115, 97, 107, 101,
+  101, 115, 115, 97, 110, 107, 101, 101, 115, 115, 97, 116, 116, 105, 107, 111, 111, 107, 117, 110, 109, 97, 97, 108,
+  109, 97, 108, 101, 101, 109, 105, 116, 105, 110, 105, 110, 117, 110, 117, 116, 105, 111, 115, 111, 111, 113, 111, 102,
+  97, 115, 97, 110, 97, 115, 117, 110, 116, 97, 39, 97, 110, 116, 97, 39, 101, 116, 117, 114, 101, 119, 97, 97, 110,
+  119, 97, 108, 105, 105, 110, 121, 107, 110, 121, 111, 111, 121, 111, 111, 107, 105, 105, 110, 121, 111, 111, 109,
+};
+
+static const uint32_t stop_word_offsets[67] = {
+  0, 4, 8, 13, 22, 26, 31, 35, 38, 40, 43, 48, 51, 55, 60, 65, 69, 71, 76, 82, 85, 90, 95, 99, 103, 110, 113, 117, 122,
+  129, 134, 140, 144, 148, 152, 157, 162, 166, 169, 173, 179, 182, 188, 194, 201, 210, 213, 216, 220, 225, 229, 231,
+  233, 237, 241, 245, 249, 252, 257, 261, 265, 269, 275, 278, 281, 288, 292,
+};
+
+static const uint32_t keep_ranges[8] = {
+  39, 39, 48, 57, 65, 90, 97, 122,
+};
+
+const narsil_language narsil_language_oromo = {
+  .name = "oromo",
+  .revision = "1",
+  .stop_word_units = stop_word_units,
+  .stop_word_offsets = stop_word_offsets,
+  .stop_word_count = 66,
+  .ascii_keep = {0x03FF008000000000ULL, 0x07FFFFFE07FFFFFEULL},
+  .keep_ranges = keep_ranges,
+  .keep_range_count = 4,
+  .splits_code_points = 0,
+  .strips_possessive = 0,
+  .normalises_diacritics = 0,
+  .min_token_length = 1,
+  .ngram_size = 0,
+  .normalise = NULL,
+  .stem_by_rules = NULL,
+  .create_snowball = NULL,
+  .snowball_stem = NULL,
+  .close_snowball = NULL,
+};

package/cpp/core/src/languages/data/persian.c

@@ -0,0 +1,150 @@
+#include "../../analysis/languages.h"
+#include "declarations.h"
+#include "../rules.h"
+#include "../snowball/persian_stemmer.h"
+
+#include <stddef.h>
+#include <stdint.h>
+
+static const uint16_t stop_word_units[1728] = {
+  1575, 1576, 1575, 1583, 1575, 1579, 1585, 1575, 1585, 1607, 1575, 1585, 1610, 1575, 1585, 1740, 1575, 1586, 1575,
+  1587, 1578, 1575, 1587, 1578, 1601, 1575, 1583, 1607, 1575, 1588, 1575, 1603, 1606, 1608, 1606, 1575, 1604, 1576,
+  1578, 1607, 1575, 1605, 1575, 1605, 1575, 1575, 1605, 1583, 1575, 1605, 1583, 1607, 1575, 1605, 1585, 1608, 1586,
+  1575, 1605, 1587, 1575, 1604, 1575, 1606, 1575, 1606, 1575, 1606, 1575, 1606, 1580, 1575, 1575, 1606, 1583, 1575,
+  1606, 1603, 1607, 1575, 1606, 1607, 1575, 1575, 1606, 1670, 1607, 1575, 1606, 1705, 1607, 1575, 1608, 1575, 1608,
+  1585, 1583, 1575, 1608, 1585, 1583, 1607, 1575, 1608, 1604, 1575, 1610, 1575, 1610, 1575, 1575, 1610, 1583, 1575,
+  1610, 1588, 1575, 1606, 1575, 1610, 1605, 1575, 1610, 1606, 1575, 1610, 1606, 1603, 1607, 1575, 1705, 1606, 1608,
+  1606, 1575, 1711, 1585, 1575, 1740, 1575, 1740, 1575, 1575, 1740, 1583, 1575, 1740, 1588, 1575, 1606, 1575, 1740,
+  1605, 1575, 1740, 1606, 1575, 1740, 1606, 1705, 1607, 1576, 1575, 1576, 1575, 1585, 1576, 1575, 1585, 1607, 1576,
+  1575, 1588, 1583, 1576, 1575, 1588, 1606, 1583, 1576, 1575, 1588, 1610, 1605, 1576, 1575, 1588, 1740, 1605, 1576,
+  1575, 1604, 1575, 1576, 1575, 1604, 1575, 1610, 1576, 1575, 1604, 1575, 1740, 1576, 1575, 1610, 1583, 1576, 1575,
+  1740, 1583, 1576, 1583, 1608, 1606, 1576, 1585, 1576, 1585, 1575, 1576, 1585, 1576, 1585, 1575, 1587, 1575, 1587,
+  1576, 1585, 1575, 1610, 1576, 1585, 1575, 1740, 1576, 1585, 1582, 1608, 1585, 1583, 1575, 1585, 1576, 1585, 1582,
+  1610, 1576, 1585, 1582, 1740, 1576, 1585, 1583, 1575, 1585, 1610, 1576, 1585, 1583, 1575, 1585, 1740, 1576, 1585,
+  1608, 1586, 1576, 1587, 1610, 1575, 1585, 1576, 1587, 1610, 1575, 1585, 1610, 1576, 1587, 1740, 1575, 1585, 1576,
+  1587, 1740, 1575, 1585, 1740, 1576, 1593, 1583, 1576, 1593, 1585, 1610, 1576, 1593, 1585, 1740, 1576, 1593, 1590,
+  1610, 1576, 1593, 1590, 1740, 1576, 1604, 1603, 1607, 1576, 1604, 1607, 1576, 1604, 1610, 1576, 1604, 1705, 1607,
+  1576, 1604, 1740, 1576, 1606, 1575, 1576, 1585, 1575, 1610, 1606, 1576, 1606, 1575, 1576, 1585, 1575, 1740, 1606,
+  1576, 1606, 1583, 1610, 1576, 1606, 1583, 1740, 1576, 1607, 1576, 1607, 1578, 1585, 1610, 1606, 1576, 1607, 1578,
+  1585, 1740, 1606, 1576, 1608, 1583, 1576, 1608, 1583, 1606, 1576, 1608, 1583, 1606, 1583, 1576, 1608, 1583, 1607,
+  1576, 1610, 1576, 1610, 1585, 1608, 1606, 1576, 1610, 1587, 1578, 1576, 1610, 1588, 1576, 1610, 1588, 1578, 1585,
+  1576, 1610, 1588, 1578, 1585, 1610, 1576, 1610, 1606, 1576, 1740, 1576, 1740, 1585, 1608, 1606, 1576, 1740, 1587,
+  1578, 1576, 1740, 1588, 1576, 1740, 1588, 1578, 1585, 1576, 1740, 1588, 1578, 1585, 1740, 1576, 1740, 1606, 1578,
+  1575, 1578, 1575, 1586, 1607, 1578, 1575, 1603, 1606, 1608, 1606, 1578, 1575, 1606, 1578, 1575, 1705, 1606, 1608,
+  1606, 1578, 1581, 1578, 1578, 1585, 1578, 1585, 1610, 1606, 1578, 1585, 1740, 1606, 1578, 1605, 1575, 1605, 1578,
+  1605, 1575, 1605, 1610, 1578, 1605, 1575, 1605, 1740, 1578, 1606, 1607, 1575, 1578, 1608, 1575, 1606, 1583, 1578,
+  1608, 1575, 1606, 1606, 1583, 1578, 1608, 1587, 1591, 1578, 1608, 1604, 1578, 1608, 1610, 1578, 1608, 1740, 1580,
+  1575, 1580, 1575, 1610, 1580, 1575, 1610, 1610, 1580, 1575, 1740, 1580, 1575, 1740, 1740, 1580, 1583, 1575, 1580,
+  1583, 1610, 1583, 1580, 1583, 1740, 1583, 1580, 1585, 1610, 1575, 1606, 1580, 1585, 1740, 1575, 1606, 1580, 1586,
+  1580, 1604, 1608, 1610, 1580, 1604, 1608, 1711, 1610, 1585, 1610, 1580, 1604, 1608, 1711, 1740, 1585, 1740, 1580,
+  1604, 1608, 1740, 1581, 1578, 1610, 1581, 1578, 1740, 1581, 1583, 1608, 1583, 1581, 1602, 1582, 1575, 1585, 1580,
+  1582, 1583, 1605, 1575, 1578, 1582, 1608, 1575, 1587, 1578, 1582, 1608, 1575, 1607, 1583, 1582, 1608, 1575, 1607,
+  1606, 1583, 1582, 1608, 1575, 1607, 1610, 1605, 1582, 1608, 1575, 1607, 1740, 1605, 1582, 1608, 1583, 1582, 1608,
+  1610, 1588, 1582, 1608, 1740, 1588, 1582, 1610, 1575, 1607, 1582, 1740, 1575, 1607, 1583, 1575, 1583, 1583, 1575,
+  1583, 1606, 1583, 1575, 1583, 1606, 1583, 1583, 1575, 1583, 1607, 1583, 1575, 1585, 1583, 1583, 1575, 1585, 1606,
+  1583, 1583, 1575, 1585, 1610, 1605, 1583, 1575, 1585, 1740, 1605, 1583, 1575, 1588, 1578, 1583, 1575, 1588, 1578,
+  1606, 1583, 1575, 1588, 1578, 1606, 1583, 1583, 1575, 1588, 1578, 1607, 1583, 1575, 1606, 1587, 1578, 1583, 1575,
+  1606, 1606, 1583, 1583, 1585, 1583, 1585, 1576, 1575, 1585, 1607, 1583, 1606, 1576, 1575, 1604, 1583, 1607, 1583,
+  1607, 1583, 1583, 1607, 1606, 1583, 1583, 1608, 1583, 1608, 1605, 1583, 1610, 1583, 1607, 1583, 1610, 1585, 1608,
+  1586, 1583, 1610, 1711, 1585, 1583, 1610, 1711, 1585, 1575, 1606, 1583, 1610, 1711, 1585, 1610, 1583, 1740, 1583,
+  1607, 1583, 1740, 1585, 1608, 1586, 1583, 1740, 1711, 1585, 1583, 1740, 1711, 1585, 1575, 1606, 1583, 1740, 1711,
+  1585, 1740, 1585, 1575, 1585, 1575, 1607, 1585, 1601, 1578, 1585, 1601, 1578, 1607, 1585, 1608, 1576, 1585, 1608,
+  1586, 1607, 1575, 1610, 1585, 1608, 1586, 1607, 1575, 1740, 1585, 1608, 1610, 1585, 1608, 1740, 1585, 1610, 1586,
+  1610, 1585, 1740, 1586, 1740, 1586, 1610, 1575, 1583, 1586, 1610, 1585, 1586, 1610, 1585, 1575, 1586, 1740, 1575,
+  1583, 1586, 1740, 1585, 1586, 1740, 1585, 1575, 1587, 1575, 1576, 1602, 1587, 1575, 1582, 1578, 1607, 1587, 1575,
+  1586, 1610, 1587, 1575, 1586, 1740, 1587, 1585, 1575, 1587, 1585, 1587, 1585, 1610, 1587, 1585, 1740, 1587, 1593,
+  1610, 1587, 1593, 1740, 1587, 1605, 1578, 1587, 1608, 1605, 1587, 1608, 1610, 1587, 1608, 1740, 1587, 1662, 1587,
+  1588, 1575, 1606, 1588, 1575, 1610, 1583, 1588, 1575, 1740, 1583, 1588, 1583, 1588, 1583, 1606, 1588, 1583, 1606,
+  1583, 1588, 1583, 1607, 1588, 1588, 1588, 1605, 1575, 1588, 1606, 1575, 1587, 1610, 1588, 1606, 1575, 1587, 1740,
+  1588, 1608, 1583, 1588, 1608, 1606, 1583, 1589, 1608, 1585, 1578, 1590, 1583, 1590, 1605, 1606, 1591, 1576, 1602,
+  1591, 1585, 1610, 1602, 1591, 1585, 1740, 1602, 1591, 1608, 1585, 1591, 1610, 1591, 1740, 1593, 1602, 1576, 1593,
+  1604, 1578, 1593, 1606, 1608, 1575, 1606, 1594, 1610, 1585, 1594, 1740, 1585, 1601, 1602, 1591, 1601, 1603, 1585,
+  1601, 1608, 1602, 1601, 1705, 1585, 1602, 1575, 1576, 1604, 1602, 1576, 1604, 1602, 1589, 1583, 1603, 1580, 1575,
+  1603, 1580, 1575, 1587, 1578, 1603, 1583, 1575, 1605, 1603, 1585, 1583, 1603, 1585, 1583, 1605, 1603, 1585, 1583,
+  1606, 1603, 1585, 1583, 1606, 1583, 1603, 1585, 1583, 1607, 1603, 1587, 1603, 1587, 1610, 1603, 1604, 1603, 1605,
+  1578, 1585, 1603, 1606, 1575, 1585, 1603, 1606, 1583, 1603, 1606, 1605, 1603, 1606, 1606, 1583, 1603, 1606, 1610,
+  1583, 1603, 1606, 1610, 1605, 1603, 1607, 1603, 1610, 1604, 1591, 1601, 1575, 1605, 1575, 1605, 1575, 1606, 1605,
+  1575, 1606, 1606, 1583, 1605, 1579, 1604, 1605, 1582, 1578, 1604, 1601, 1605, 1583, 1578, 1610, 1605, 1583, 1578,
+  1740, 1605, 1585, 1583, 1605, 1605, 1585, 1587, 1610, 1605, 1585, 1587, 1740, 1605, 1602, 1575, 1576, 1604, 1605,
+  1606, 1605, 1608, 1585, 1583, 1605, 1610, 1605, 1610, 1604, 1610, 1575, 1585, 1583, 1605, 1610, 1604, 1610, 1608,
+  1606, 1605, 1711, 1585, 1605, 1740, 1605, 1740, 1604, 1740, 1575, 1585, 1583, 1605, 1740, 1604, 1740, 1608, 1606,
+  1606, 1575, 1588, 1610, 1606, 1575, 1588, 1740, 1606, 1575, 1605, 1606, 1576, 1575, 1610, 1583, 1606, 1576, 1575,
+  1740, 1583, 1606, 1576, 1608, 1583, 1606, 1582, 1587, 1578, 1606, 1582, 1587, 1578, 1610, 1606, 1606, 1582, 1587,
+  1578, 1740, 1606, 1606, 1582, 1608, 1575, 1607, 1583, 1606, 1583, 1575, 1585, 1583, 1606, 1583, 1575, 1585, 1606,
+  1583, 1606, 1583, 1575, 1588, 1578, 1607, 1606, 1586, 1583, 1606, 1586, 1583, 1610, 1603, 1606, 1586, 1583, 1740,
+  1705, 1606, 1588, 1575, 1606, 1606, 1588, 1583, 1607, 1606, 1592, 1610, 1585, 1606, 1592, 1740, 1585, 1606, 1603,
+  1585, 1583, 1607, 1606, 1605, 1575, 1610, 1583, 1606, 1605, 1575, 1740, 1583, 1606, 1605, 1610, 1606, 1605, 1740,
+  1606, 1607, 1606, 1608, 1593, 1610, 1606, 1608, 1593, 1740, 1606, 1610, 1586, 1606, 1610, 1587, 1578, 1606, 1705,
+  1585, 1583, 1607, 1606, 1740, 1586, 1606, 1740, 1587, 1578, 1607, 1607, 1575, 1607, 1575, 1610, 1607, 1575, 1610,
+  1610, 1607, 1575, 1740, 1607, 1575, 1740, 1740, 1607, 1585, 1607, 1585, 1711, 1586, 1607, 1586, 1575, 1585, 1607,
+  1587, 1578, 1607, 1587, 1578, 1606, 1583, 1607, 1587, 1578, 1610, 1605, 1607, 1587, 1578, 1740, 1605, 1607, 1601,
+  1578, 1607, 1605, 1607, 1605, 1575, 1606, 1607, 1605, 1607, 1607, 1605, 1608, 1575, 1585, 1607, 1607, 1605, 1610,
+  1606, 1607, 1605, 1670, 1606, 1575, 1606, 1607, 1605, 1670, 1606, 1610, 1606, 1607, 1605, 1670, 1606, 1740, 1606,
+  1607, 1605, 1670, 1608, 1606, 1607, 1605, 1740, 1606, 1607, 1606, 1608, 1586, 1607, 1606, 1711, 1575, 1605, 1607,
+  1606, 1711, 1575, 1605, 1610, 1607, 1606, 1711, 1575, 1605, 1740, 1607, 1610, 1670, 1607, 1740, 1670, 1608, 1608,
+  1587, 1591, 1608, 1602, 1578, 1610, 1608, 1602, 1578, 1610, 1603, 1607, 1608, 1602, 1578, 1740, 1608, 1602, 1578,
+  1740, 1705, 1607, 1608, 1604, 1610, 1608, 1604, 1740, 1608, 1610, 1608, 1711, 1608, 1608, 1740, 1610, 1575, 1610,
+  1575, 1576, 1583, 1610, 1603, 1610, 1603, 1583, 1610, 1711, 1585, 1610, 1603, 1610, 1662, 1575, 1593, 1610, 1606,
+  1662, 1575, 1593, 1740, 1606, 1662, 1587, 1662, 1606, 1580, 1662, 1610, 1588, 1662, 1740, 1588, 1670, 1585, 1575,
+  1670, 1591, 1608, 1585, 1670, 1606, 1583, 1670, 1606, 1583, 1610, 1606, 1670, 1606, 1583, 1740, 1606, 1670, 1606,
+  1610, 1606, 1670, 1606, 1740, 1606, 1670, 1607, 1670, 1607, 1575, 1585, 1670, 1608, 1606, 1670, 1610, 1586, 1670,
+  1610, 1586, 1610, 1670, 1610, 1587, 1578, 1670, 1711, 1608, 1606, 1607, 1670, 1740, 1586, 1670, 1740, 1586, 1740,
+  1670, 1740, 1587, 1578, 1705, 1580, 1575, 1705, 1580, 1575, 1587, 1578, 1705, 1583, 1575, 1605, 1705, 1585, 1583,
+  1705, 1585, 1583, 1605, 1705, 1585, 1583, 1606, 1705, 1585, 1583, 1606, 1583, 1705, 1585, 1583, 1607, 1705, 1587,
+  1705, 1587, 1740, 1705, 1604, 1705, 1605, 1578, 1585, 1705, 1606, 1575, 1585, 1705, 1606, 1583, 1705, 1606, 1605,
+  1705, 1606, 1606, 1583, 1705, 1606, 1740, 1583, 1705, 1606, 1740, 1605, 1705, 1607, 1705, 1740, 1711, 1584, 1575,
+  1585, 1610, 1711, 1584, 1575, 1585, 1740, 1711, 1584, 1575, 1588, 1578, 1607, 1711, 1585, 1583, 1583, 1711, 1585,
+  1601, 1578, 1711, 1585, 1601, 1578, 1607, 1711, 1585, 1608, 1607, 1610, 1711, 1585, 1608, 1607, 1740, 1711, 1601,
+  1578, 1711, 1601, 1578, 1607, 1711, 1608, 1610, 1583, 1711, 1608, 1610, 1606, 1583, 1711, 1608, 1740, 1583, 1711,
+  1608, 1740, 1606, 1583, 1711, 1610, 1585, 1583, 1711, 1610, 1585, 1610, 1711, 1740, 1585, 1583, 1711, 1740, 1585,
+  1740, 1740, 1575, 1740, 1575, 1576, 1583, 1740, 1705, 1740, 1705, 1583, 1740, 1711, 1585, 1740, 1705, 1740,
+};
+
+static const uint32_t stop_word_offsets[442] = {
+  0, 4, 7, 10, 13, 16, 18, 21, 28, 30, 35, 40, 42, 45, 48, 52, 57, 62, 64, 68, 72, 75, 79, 83, 87, 91, 93, 97, 102, 105,
+  107, 110, 113, 118, 121, 124, 129, 134, 137, 139, 142, 145, 150, 153, 156, 161, 163, 166, 170, 174, 179, 184, 189,
+  193, 198, 203, 207, 211, 215, 217, 222, 228, 232, 236, 244, 248, 252, 258, 264, 268, 273, 279, 284, 290, 293, 297,
+  301, 305, 309, 313, 316, 319, 323, 326, 334, 342, 346, 350, 352, 358, 364, 367, 371, 376, 380, 382, 387, 391, 394,
+  399, 405, 408, 410, 415, 419, 422, 427, 433, 436, 438, 442, 448, 451, 457, 460, 462, 466, 470, 474, 479, 484, 488,
+  493, 499, 503, 506, 509, 512, 514, 517, 521, 524, 528, 531, 535, 539, 544, 549, 551, 555, 562, 569, 573, 576, 579,
+  583, 585, 589, 594, 599, 604, 610, 616, 622, 625, 629, 633, 637, 641, 644, 648, 653, 657, 661, 666, 671, 676, 680,
+  685, 691, 696, 701, 706, 708, 714, 719, 721, 724, 728, 730, 733, 737, 742, 746, 752, 757, 761, 766, 770, 776, 781,
+  783, 786, 789, 793, 796, 802, 808, 811, 814, 818, 822, 826, 829, 833, 837, 840, 844, 848, 853, 857, 861, 866, 869,
+  872, 875, 878, 881, 884, 887, 890, 893, 896, 900, 904, 906, 909, 913, 916, 918, 921, 926, 931, 934, 938, 942, 944,
+  947, 950, 954, 958, 961, 963, 965, 968, 971, 976, 979, 982, 985, 988, 991, 994, 998, 1001, 1004, 1007, 1012, 1016,
+  1019, 1023, 1027, 1032, 1036, 1038, 1041, 1043, 1047, 1051, 1054, 1057, 1061, 1065, 1069, 1071, 1073, 1077, 1079,
+  1082, 1087, 1090, 1095, 1099, 1103, 1107, 1111, 1115, 1120, 1122, 1126, 1128, 1135, 1141, 1144, 1146, 1153, 1159,
+  1163, 1167, 1170, 1175, 1180, 1184, 1188, 1194, 1200, 1206, 1211, 1217, 1223, 1226, 1231, 1236, 1240, 1244, 1248,
+  1252, 1257, 1262, 1267, 1270, 1273, 1275, 1279, 1283, 1286, 1290, 1295, 1298, 1302, 1303, 1305, 1308, 1312, 1315,
+  1319, 1321, 1325, 1329, 1332, 1337, 1342, 1347, 1350, 1352, 1356, 1359, 1365, 1369, 1375, 1381, 1387, 1392, 1396,
+  1400, 1405, 1411, 1417, 1420, 1423, 1424, 1427, 1431, 1437, 1441, 1447, 1450, 1453, 1455, 1458, 1460, 1462, 1466,
+  1468, 1474, 1477, 1482, 1487, 1489, 1492, 1495, 1498, 1501, 1505, 1508, 1513, 1518, 1522, 1526, 1528, 1532, 1535,
+  1538, 1542, 1546, 1551, 1554, 1558, 1562, 1565, 1570, 1574, 1577, 1581, 1585, 1590, 1594, 1596, 1599, 1601, 1605,
+  1609, 1612, 1615, 1619, 1623, 1627, 1629, 1631, 1636, 1641, 1647, 1651, 1655, 1660, 1665, 1670, 1673, 1677, 1681,
+  1686, 1690, 1695, 1699, 1703, 1707, 1711, 1713, 1717, 1719, 1725, 1728,
+};
+
+static const uint32_t keep_ranges[12] = {
+  48, 57, 65, 90, 97, 122, 1569, 1641, 1646, 1747, 1776, 1785,
+};
+
+const narsil_language narsil_language_persian = {
+  .name = "persian",
+  .revision = "99f39d265efbf595",
+  .stop_word_units = stop_word_units,
+  .stop_word_offsets = stop_word_offsets,
+  .stop_word_count = 441,
+  .ascii_keep = {0x03FF000000000000ULL, 0x07FFFFFE07FFFFFEULL},
+  .keep_ranges = keep_ranges,
+  .keep_range_count = 6,
+  .splits_code_points = 0,
+  .strips_possessive = 0,
+  .normalises_diacritics = 0,
+  .min_token_length = 1,
+  .ngram_size = 0,
+  .normalise = narsil_persian_normalise,
+  .stem_by_rules = NULL,
+  .create_snowball = persian_create_env,
+  .snowball_stem = persian_stem,
+  .close_snowball = persian_close_env,
+};

package/cpp/core/src/languages/data/polish.c

@@ -0,0 +1,111 @@
+#include "../../analysis/languages.h"
+#include "declarations.h"
+#include "../snowball/polish_stemmer.h"
+
+#include <stddef.h>
+#include <stdint.h>
+
+static const uint16_t stop_word_units[1414] = {
+  97, 97, 98, 121, 97, 99, 104, 97, 99, 122, 97, 99, 122, 107, 111, 108, 119, 105, 101, 107, 97, 106, 97, 108, 98, 111,
+  97, 108, 101, 97, 108, 101, 380, 97, 110, 105, 97, 380, 98, 97, 114, 100, 122, 105, 101, 106, 98, 97, 114, 100, 122,
+  111, 98, 101, 122, 98, 111, 98, 111, 119, 105, 101, 109, 98, 121, 98, 121, 108, 105, 98, 121, 109, 98, 121, 110, 97,
+  106, 109, 110, 105, 101, 106, 98, 121, 263, 98, 121, 322, 98, 121, 322, 97, 98, 121, 322, 111, 98, 121, 322, 121, 98,
+  281, 100, 122, 105, 101, 98, 281, 100, 261, 99, 97, 108, 105, 99, 97, 322, 97, 99, 97, 322, 121, 99, 104, 99, 101, 99,
+  104, 111, 263, 99, 105, 99, 105, 101, 98, 105, 101, 99, 105, 281, 99, 111, 99, 111, 107, 111, 108, 119, 105, 101, 107,
+  99, 111, 114, 97, 122, 99, 111, 347, 99, 122, 97, 115, 97, 109, 105, 99, 122, 97, 115, 101, 109, 99, 122, 101, 109,
+  117, 99, 122, 121, 99, 122, 121, 108, 105, 99, 122, 281, 115, 116, 111, 100, 97, 108, 101, 107, 111, 100, 108, 97,
+  100, 108, 97, 99, 122, 101, 103, 111, 100, 108, 97, 116, 101, 103, 111, 100, 111, 100, 111, 98, 114, 122, 101, 100,
+  111, 107, 261, 100, 100, 111, 347, 263, 100, 114, 100, 117, 380, 111, 100, 119, 97, 100, 119, 97, 106, 100, 119, 105,
+  101, 100, 119, 111, 106, 101, 100, 122, 105, 115, 105, 97, 106, 100, 122, 105, 347, 103, 100, 121, 103, 100, 121, 98,
+  121, 103, 100, 121, 380, 103, 100, 122, 105, 101, 103, 100, 122, 105, 101, 107, 111, 108, 119, 105, 101, 107, 103,
+  100, 122, 105, 101, 347, 103, 111, 103, 111, 100, 122, 104, 97, 98, 105, 105, 99, 104, 105, 105, 105, 105, 105, 105,
+  108, 101, 105, 109, 105, 110, 110, 97, 105, 110, 110, 101, 105, 110, 110, 121, 105, 110, 110, 121, 99, 104, 105, 110,
+  380, 105, 118, 105, 120, 105, 380, 106, 97, 106, 97, 107, 106, 97, 107, 97, 347, 106, 97, 107, 98, 121, 106, 97, 107,
+  105, 106, 97, 107, 105, 99, 104, 347, 106, 97, 107, 105, 101, 106, 97, 107, 105, 347, 106, 97, 107, 105, 380, 106, 97,
+  107, 107, 111, 108, 119, 105, 101, 107, 106, 97, 107, 111, 106, 97, 107, 111, 347, 106, 101, 106, 101, 100, 101, 110,
+  106, 101, 100, 110, 97, 106, 101, 100, 110, 97, 107, 106, 101, 100, 110, 97, 107, 380, 101, 106, 101, 100, 110, 111,
+  106, 101, 100, 110, 121, 109, 106, 101, 100, 121, 110, 105, 101, 106, 101, 103, 111, 106, 101, 106, 106, 101, 109,
+  117, 106, 101, 115, 116, 106, 101, 115, 116, 101, 109, 106, 101, 115, 122, 99, 122, 101, 106, 101, 347, 108, 105, 106,
+  101, 380, 101, 108, 105, 106, 117, 380, 106, 261, 107, 97, 380, 100, 121, 107, 105, 101, 100, 121, 107, 105, 101, 114,
+  117, 110, 107, 117, 107, 105, 108, 107, 97, 107, 105, 108, 107, 117, 107, 105, 109, 347, 107, 116, 111, 107, 116, 111,
+  107, 111, 108, 119, 105, 101, 107, 107, 116, 111, 347, 107, 116, 243, 114, 97, 107, 116, 243, 114, 101, 107, 116, 243,
+  114, 101, 103, 111, 107, 116, 243, 114, 101, 106, 107, 116, 243, 114, 121, 107, 116, 243, 114, 121, 99, 104, 107, 116,
+  243, 114, 121, 109, 107, 116, 243, 114, 122, 121, 107, 117, 108, 97, 116, 108, 101, 99, 122, 108, 117, 98, 109, 97,
+  109, 97, 106, 261, 109, 97, 109, 109, 97, 109, 121, 109, 97, 322, 111, 109, 103, 114, 109, 105, 109, 105, 97, 322,
+  109, 105, 109, 111, 109, 105, 281, 100, 122, 121, 109, 110, 105, 101, 109, 110, 261, 109, 111, 103, 261, 109, 111,
+  105, 109, 111, 105, 109, 109, 111, 106, 97, 109, 111, 106, 101, 109, 111, 380, 101, 109, 111, 380, 108, 105, 119, 101,
+  109, 111, 380, 110, 97, 109, 117, 109, 117, 115, 105, 109, 121, 109, 243, 106, 110, 97, 110, 97, 100, 110, 97, 109,
+  110, 97, 109, 105, 110, 97, 115, 110, 97, 115, 105, 110, 97, 115, 122, 110, 97, 115, 122, 97, 110, 97, 115, 122, 101,
+  110, 97, 115, 122, 101, 103, 111, 110, 97, 115, 122, 121, 99, 104, 110, 97, 116, 111, 109, 105, 97, 115, 116, 110, 97,
+  116, 121, 99, 104, 109, 105, 97, 115, 116, 110, 97, 119, 101, 116, 110, 105, 99, 110, 105, 99, 104, 110, 105, 101,
+  110, 105, 101, 99, 104, 110, 105, 101, 103, 111, 110, 105, 101, 106, 110, 105, 101, 109, 117, 110, 105, 103, 100, 121,
+  110, 105, 109, 110, 105, 109, 105, 110, 105, 261, 110, 105, 380, 110, 111, 110, 111, 119, 101, 110, 112, 110, 114,
+  111, 111, 98, 111, 107, 111, 100, 111, 107, 111, 107, 111, 322, 111, 111, 110, 111, 110, 97, 111, 110, 101, 111, 110,
+  105, 111, 110, 111, 111, 114, 97, 122, 111, 116, 111, 111, 119, 115, 122, 101, 109, 112, 97, 110, 112, 97, 110, 97,
+  112, 97, 110, 105, 112, 108, 112, 111, 112, 111, 100, 112, 111, 100, 99, 122, 97, 115, 112, 111, 109, 105, 109, 111,
+  112, 111, 110, 97, 100, 112, 111, 110, 105, 101, 119, 97, 380, 112, 111, 119, 105, 110, 105, 101, 110, 112, 111, 119,
+  105, 110, 110, 97, 112, 111, 119, 105, 110, 110, 105, 112, 111, 119, 105, 110, 110, 111, 112, 111, 122, 97, 112, 114,
+  97, 119, 105, 101, 112, 114, 111, 102, 112, 114, 122, 101, 99, 105, 101, 380, 112, 114, 122, 101, 100, 112, 114, 122,
+  101, 100, 101, 112, 114, 122, 101, 100, 116, 101, 109, 112, 114, 122, 101, 122, 112, 114, 122, 121, 114, 97, 122, 114,
+  97, 122, 105, 101, 114, 111, 107, 117, 114, 243, 119, 110, 105, 101, 380, 115, 97, 109, 115, 97, 109, 97, 115, 105,
+  281, 115, 107, 261, 100, 115, 111, 98, 105, 101, 115, 111, 98, 261, 115, 112, 111, 115, 243, 98, 115, 119, 111, 106,
+  101, 115, 261, 116, 97, 116, 97, 107, 116, 97, 107, 97, 116, 97, 107, 105, 116, 97, 107, 105, 99, 104, 116, 97, 107,
+  105, 101, 116, 97, 107, 380, 101, 116, 97, 109, 116, 101, 116, 101, 103, 111, 116, 101, 106, 116, 101, 108, 116, 101,
+  109, 117, 116, 101, 110, 116, 101, 114, 97, 122, 116, 101, 380, 116, 111, 116, 111, 98, 105, 101, 116, 111, 98, 261,
+  116, 111, 116, 101, 380, 116, 111, 116, 111, 98, 261, 116, 114, 122, 101, 98, 97, 116, 117, 116, 117, 116, 97, 106,
+  116, 119, 111, 105, 116, 119, 111, 105, 109, 116, 119, 111, 106, 97, 116, 119, 111, 106, 101, 116, 119, 121, 109, 116,
+  119, 243, 106, 116, 121, 116, 121, 99, 104, 116, 121, 108, 107, 111, 116, 121, 109, 116, 121, 115, 116, 122, 119, 116,
+  281, 117, 117, 108, 118, 105, 118, 105, 105, 118, 105, 105, 105, 118, 111, 108, 119, 119, 97, 109, 119, 97, 109, 105,
+  119, 97, 115, 119, 97, 115, 105, 119, 97, 115, 122, 119, 97, 115, 122, 97, 119, 97, 115, 122, 101, 119, 101, 119, 101,
+  100, 322, 117, 103, 119, 105, 101, 119, 105, 101, 108, 101, 119, 105, 101, 108, 117, 119, 105, 281, 99, 119, 105, 281,
+  99, 101, 106, 119, 115, 122, 121, 115, 99, 121, 119, 115, 122, 121, 115, 116, 107, 105, 99, 104, 119, 115, 122, 121,
+  115, 116, 107, 105, 101, 119, 115, 122, 121, 115, 116, 107, 105, 109, 119, 115, 122, 121, 115, 116, 107, 111, 119,
+  116, 101, 100, 121, 119, 119, 119, 119, 121, 119, 322, 97, 347, 110, 105, 101, 119, 347, 114, 243, 100, 120, 105, 120,
+  105, 105, 120, 105, 105, 105, 120, 105, 118, 120, 118, 122, 122, 97, 122, 97, 112, 101, 119, 110, 101, 122, 97, 119,
+  115, 122, 101, 122, 97, 347, 122, 101, 122, 101, 122, 110, 111, 119, 117, 122, 110, 111, 119, 117, 122, 110, 243, 119,
+  122, 111, 115, 116, 97, 322, 122, 322, 380, 97, 100, 101, 110, 380, 97, 100, 110, 97, 380, 97, 100, 110, 101, 380, 97,
+  100, 110, 121, 99, 104, 380, 101, 380, 101, 98, 121,
+};
+
+static const uint32_t stop_word_offsets[329] = {
+  0, 1, 4, 7, 10, 20, 22, 26, 29, 33, 36, 38, 46, 52, 55, 57, 63, 65, 69, 72, 82, 85, 88, 92, 96, 100, 106, 110, 114,
+  118, 122, 126, 130, 132, 138, 141, 143, 152, 157, 160, 167, 173, 178, 181, 186, 192, 198, 201, 209, 216, 218, 224,
+  229, 233, 235, 239, 242, 246, 250, 255, 262, 266, 269, 274, 278, 283, 295, 301, 303, 307, 310, 311, 314, 316, 319,
+  322, 324, 328, 332, 336, 342, 345, 347, 349, 351, 353, 356, 361, 366, 370, 377, 382, 387, 392, 402, 406, 411, 413,
+  418, 423, 429, 437, 442, 448, 455, 459, 462, 466, 470, 476, 483, 488, 494, 497, 499, 504, 509, 517, 522, 527, 531,
+  534, 544, 548, 553, 558, 565, 571, 576, 583, 589, 595, 597, 600, 604, 607, 609, 613, 616, 620, 624, 627, 629, 633,
+  637, 643, 647, 650, 654, 657, 661, 665, 669, 673, 680, 685, 687, 691, 693, 696, 698, 701, 704, 708, 711, 715, 719,
+  724, 729, 736, 743, 752, 763, 768, 771, 775, 778, 783, 788, 792, 797, 802, 805, 809, 812, 815, 817, 821, 823, 825,
+  826, 830, 832, 834, 839, 841, 844, 847, 850, 853, 857, 860, 866, 869, 873, 877, 879, 881, 884, 891, 897, 902, 910,
+  918, 925, 932, 939, 943, 949, 953, 961, 966, 972, 980, 985, 989, 992, 997, 1001, 1008, 1011, 1015, 1018, 1022, 1027,
+  1031, 1037, 1042, 1044, 1046, 1049, 1053, 1057, 1063, 1068, 1073, 1076, 1078, 1082, 1085, 1088, 1092, 1095, 1100,
+  1103, 1105, 1110, 1114, 1119, 1125, 1131, 1133, 1138, 1142, 1147, 1152, 1157, 1161, 1165, 1167, 1171, 1176, 1179,
+  1182, 1185, 1187, 1188, 1190, 1192, 1195, 1199, 1202, 1203, 1206, 1210, 1213, 1217, 1221, 1226, 1231, 1233, 1239,
+  1242, 1247, 1252, 1256, 1262, 1269, 1279, 1288, 1297, 1305, 1310, 1313, 1315, 1322, 1327, 1329, 1332, 1336, 1339,
+  1341, 1342, 1344, 1351, 1357, 1360, 1362, 1369, 1374, 1378, 1384, 1386, 1391, 1396, 1401, 1408, 1410, 1414,
+};
+
+static const uint32_t keep_ranges[20] = {
+  48, 57, 65, 90, 97, 122, 211, 211, 243, 243, 260, 263, 280, 281, 321, 324, 346, 347, 377, 380,
+};
+
+const narsil_language narsil_language_polish = {
+  .name = "polish",
+  .revision = "373ece41a2c5684b",
+  .stop_word_units = stop_word_units,
+  .stop_word_offsets = stop_word_offsets,
+  .stop_word_count = 328,
+  .ascii_keep = {0x03FF000000000000ULL, 0x07FFFFFE07FFFFFEULL},
+  .keep_ranges = keep_ranges,
+  .keep_range_count = 10,
+  .splits_code_points = 0,
+  .strips_possessive = 0,
+  .normalises_diacritics = 0,
+  .min_token_length = 1,
+  .ngram_size = 0,
+  .normalise = NULL,
+  .stem_by_rules = NULL,
+  .create_snowball = polish_create_env,
+  .snowball_stem = polish_stem,
+  .close_snowball = polish_close_env,
+};

package/cpp/core/src/languages/data/portuguese.c

@@ -0,0 +1,91 @@
+#include "../../analysis/languages.h"
+#include "declarations.h"
+#include "../rules.h"
+#include "../snowball/portuguese_stemmer.h"
+
+#include <stddef.h>
+#include <stdint.h>
+
+static const uint16_t stop_word_units[1028] = {
+  97, 97, 111, 97, 111, 115, 97, 113, 117, 101, 108, 97, 97, 113, 117, 101, 108, 97, 115, 97, 113, 117, 101, 108, 101,
+  97, 113, 117, 101, 108, 101, 115, 97, 113, 117, 105, 108, 111, 97, 115, 97, 116, 233, 99, 111, 109, 99, 111, 109, 111,
+  100, 97, 100, 97, 115, 100, 101, 100, 101, 108, 97, 100, 101, 108, 97, 115, 100, 101, 108, 101, 100, 101, 108, 101,
+  115, 100, 101, 112, 111, 105, 115, 100, 111, 100, 111, 115, 101, 101, 108, 97, 101, 108, 97, 115, 101, 108, 101, 101,
+  108, 101, 115, 101, 109, 101, 110, 116, 114, 101, 101, 114, 97, 101, 114, 97, 109, 101, 115, 115, 97, 101, 115, 115,
+  97, 115, 101, 115, 115, 101, 101, 115, 115, 101, 115, 101, 115, 116, 97, 101, 115, 116, 97, 109, 111, 115, 101, 115,
+  116, 97, 115, 101, 115, 116, 97, 118, 97, 101, 115, 116, 97, 118, 97, 109, 101, 115, 116, 101, 101, 115, 116, 101,
+  106, 97, 101, 115, 116, 101, 106, 97, 109, 101, 115, 116, 101, 106, 97, 109, 111, 115, 101, 115, 116, 101, 115, 101,
+  115, 116, 101, 118, 101, 101, 115, 116, 105, 118, 101, 101, 115, 116, 105, 118, 101, 109, 111, 115, 101, 115, 116,
+  105, 118, 101, 114, 101, 115, 116, 105, 118, 101, 114, 97, 101, 115, 116, 105, 118, 101, 114, 97, 109, 101, 115, 116,
+  105, 118, 101, 114, 101, 109, 101, 115, 116, 105, 118, 101, 114, 109, 111, 115, 101, 115, 116, 105, 118, 101, 115,
+  115, 101, 101, 115, 116, 105, 118, 101, 115, 115, 101, 109, 101, 115, 116, 105, 118, 233, 114, 97, 109, 111, 115, 101,
+  115, 116, 105, 118, 233, 115, 115, 101, 109, 111, 115, 101, 115, 116, 111, 117, 101, 115, 116, 225, 101, 115, 116,
+  225, 118, 97, 109, 111, 115, 101, 115, 116, 227, 111, 101, 117, 102, 111, 105, 102, 111, 109, 111, 115, 102, 111, 114,
+  102, 111, 114, 97, 102, 111, 114, 97, 109, 102, 111, 114, 101, 109, 102, 111, 114, 109, 111, 115, 102, 111, 115, 115,
+  101, 102, 111, 115, 115, 101, 109, 102, 117, 105, 102, 244, 114, 97, 109, 111, 115, 102, 244, 115, 115, 101, 109, 111,
+  115, 104, 97, 106, 97, 104, 97, 106, 97, 109, 104, 97, 106, 97, 109, 111, 115, 104, 97, 118, 101, 109, 111, 115, 104,
+  101, 105, 104, 111, 117, 118, 101, 104, 111, 117, 118, 101, 109, 111, 115, 104, 111, 117, 118, 101, 114, 104, 111,
+  117, 118, 101, 114, 97, 104, 111, 117, 118, 101, 114, 97, 109, 104, 111, 117, 118, 101, 114, 101, 105, 104, 111, 117,
+  118, 101, 114, 101, 109, 104, 111, 117, 118, 101, 114, 101, 109, 111, 115, 104, 111, 117, 118, 101, 114, 105, 97, 104,
+  111, 117, 118, 101, 114, 105, 97, 109, 104, 111, 117, 118, 101, 114, 109, 111, 115, 104, 111, 117, 118, 101, 114, 225,
+  104, 111, 117, 118, 101, 114, 227, 111, 104, 111, 117, 118, 101, 114, 237, 97, 109, 111, 115, 104, 111, 117, 118, 101,
+  115, 115, 101, 104, 111, 117, 118, 101, 115, 115, 101, 109, 104, 111, 117, 118, 233, 114, 97, 109, 111, 115, 104, 111,
+  117, 118, 233, 115, 115, 101, 109, 111, 115, 104, 225, 104, 227, 111, 105, 115, 115, 111, 105, 115, 116, 111, 106,
+  225, 108, 104, 101, 108, 104, 101, 115, 109, 97, 105, 115, 109, 97, 115, 109, 101, 109, 101, 115, 109, 111, 109, 101,
+  117, 109, 101, 117, 115, 109, 105, 110, 104, 97, 109, 105, 110, 104, 97, 115, 109, 117, 105, 116, 111, 110, 97, 110,
+  97, 115, 110, 101, 109, 110, 111, 110, 111, 115, 110, 111, 115, 115, 97, 110, 111, 115, 115, 97, 115, 110, 111, 115,
+  115, 111, 110, 111, 115, 115, 111, 115, 110, 117, 109, 110, 117, 109, 97, 110, 227, 111, 110, 243, 115, 111, 111, 115,
+  111, 117, 112, 97, 114, 97, 112, 101, 108, 97, 112, 101, 108, 97, 115, 112, 101, 108, 111, 112, 101, 108, 111, 115,
+  112, 111, 114, 113, 117, 97, 108, 113, 117, 97, 110, 100, 111, 113, 117, 101, 113, 117, 101, 109, 115, 101, 115, 101,
+  106, 97, 115, 101, 106, 97, 109, 115, 101, 106, 97, 109, 111, 115, 115, 101, 109, 115, 101, 114, 101, 105, 115, 101,
+  114, 101, 109, 111, 115, 115, 101, 114, 105, 97, 115, 101, 114, 105, 97, 109, 115, 101, 114, 225, 115, 101, 114, 227,
+  111, 115, 101, 114, 237, 97, 109, 111, 115, 115, 101, 117, 115, 101, 117, 115, 115, 111, 109, 111, 115, 115, 111, 117,
+  115, 117, 97, 115, 117, 97, 115, 115, 227, 111, 115, 243, 116, 97, 109, 98, 233, 109, 116, 101, 116, 101, 109, 116,
+  101, 109, 111, 115, 116, 101, 110, 104, 97, 116, 101, 110, 104, 97, 109, 116, 101, 110, 104, 97, 109, 111, 115, 116,
+  101, 110, 104, 111, 116, 101, 114, 101, 105, 116, 101, 114, 101, 109, 111, 115, 116, 101, 114, 105, 97, 116, 101, 114,
+  105, 97, 109, 116, 101, 114, 225, 116, 101, 114, 227, 111, 116, 101, 114, 237, 97, 109, 111, 115, 116, 101, 117, 116,
+  101, 117, 115, 116, 101, 118, 101, 116, 105, 110, 104, 97, 116, 105, 110, 104, 97, 109, 116, 105, 118, 101, 116, 105,
+  118, 101, 109, 111, 115, 116, 105, 118, 101, 114, 116, 105, 118, 101, 114, 97, 116, 105, 118, 101, 114, 97, 109, 116,
+  105, 118, 101, 114, 101, 109, 116, 105, 118, 101, 114, 109, 111, 115, 116, 105, 118, 101, 115, 115, 101, 116, 105,
+  118, 101, 115, 115, 101, 109, 116, 105, 118, 233, 114, 97, 109, 111, 115, 116, 105, 118, 233, 115, 115, 101, 109, 111,
+  115, 116, 117, 116, 117, 97, 116, 117, 97, 115, 116, 233, 109, 116, 237, 110, 104, 97, 109, 111, 115, 117, 109, 117,
+  109, 97, 118, 111, 99, 234, 118, 111, 99, 234, 115, 118, 111, 115, 224, 224, 115, 233, 114, 97, 109, 111, 115,
+};
+
+static const uint32_t stop_word_offsets[204] = {
+  0, 1, 3, 6, 12, 19, 25, 32, 38, 40, 43, 46, 50, 52, 55, 57, 61, 66, 70, 75, 81, 83, 86, 87, 90, 94, 97, 101, 103, 108,
+  111, 115, 119, 124, 128, 133, 137, 144, 149, 155, 162, 166, 172, 179, 188, 193, 199, 205, 214, 221, 229, 238, 247,
+  257, 266, 276, 287, 299, 304, 308, 317, 322, 324, 327, 332, 335, 339, 344, 349, 355, 360, 366, 369, 376, 384, 388,
+  393, 400, 407, 410, 415, 423, 429, 436, 444, 452, 460, 470, 478, 487, 496, 503, 511, 522, 530, 539, 549, 560, 562,
+  565, 569, 573, 575, 578, 582, 586, 589, 591, 596, 599, 603, 608, 614, 619, 621, 624, 627, 629, 632, 637, 643, 648,
+  654, 657, 661, 664, 667, 668, 670, 672, 676, 680, 685, 689, 694, 697, 701, 707, 710, 714, 716, 720, 725, 732, 735,
+  740, 747, 752, 758, 762, 767, 775, 778, 782, 787, 790, 793, 797, 800, 802, 808, 810, 813, 818, 823, 829, 837, 842,
+  847, 854, 859, 865, 869, 874, 882, 885, 889, 893, 898, 904, 908, 915, 920, 926, 933, 940, 948, 955, 963, 972, 982,
+  984, 987, 991, 994, 1002, 1004, 1007, 1011, 1016, 1019, 1020, 1022, 1028,
+};
+
+static const uint32_t keep_ranges[34] = {
+  48, 57, 65, 90, 97, 122, 192, 195, 199, 199, 201, 202, 205, 205, 211, 213, 218, 218, 220, 220, 224, 227, 231, 231,
+  233, 234, 237, 237, 243, 245, 250, 250, 252, 252,
+};
+
+const narsil_language narsil_language_portuguese = {
+  .name = "portuguese",
+  .revision = "c0a0c5c97ab9ca6e",
+  .stop_word_units = stop_word_units,
+  .stop_word_offsets = stop_word_offsets,
+  .stop_word_count = 203,
+  .ascii_keep = {0x03FF000000000000ULL, 0x07FFFFFE07FFFFFEULL},
+  .keep_ranges = keep_ranges,
+  .keep_range_count = 17,
+  .splits_code_points = 0,
+  .strips_possessive = 0,
+  .normalises_diacritics = 0,
+  .min_token_length = 1,
+  .ngram_size = 0,
+  .normalise = narsil_portuguese_normalise,
+  .stem_by_rules = NULL,
+  .create_snowball = portuguese_create_env,
+  .snowball_stem = portuguese_stem,
+  .close_snowball = portuguese_close_env,
+};

package/cpp/core/src/languages/data/punjabi.c

@@ -0,0 +1,80 @@
+#include "../../analysis/languages.h"
+#include "declarations.h"
+
+#include <stddef.h>
+#include <stdint.h>
+
+static const uint16_t stop_word_units[683] = {
+  2565, 2581, 2616, 2608, 2565, 2596, 2631, 2565, 2599, 2624, 2600, 2565, 2600, 2625, 2616, 2622, 2608, 2565, 2616,
+  2624, 2562, 2565, 2672, 2598, 2608, 2565, 2673, 2583, 2631, 2566, 2566, 2567, 2566, 2566, 2568, 2566, 2569, 2595,
+  2566, 2602, 2566, 2602, 2595, 2622, 2566, 2602, 2595, 2624, 2566, 2602, 2595, 2631, 2567, 2581, 2567, 2600, 2637,
+  2617, 2622, 2562, 2567, 2610, 2622, 2613, 2622, 2567, 2616, 2567, 2616, 2598, 2622, 2567, 2616, 2598, 2624, 2567,
+  2616, 2598, 2631, 2567, 2616, 2600, 2626, 2672, 2567, 2616, 2600, 2631, 2567, 2616, 2631, 2567, 2617, 2567, 2617,
+  2600, 2622, 2562, 2567, 2673, 2581, 2567, 2673, 2597, 2631, 2568, 2569, 2600, 2637, 2617, 2622, 2562, 2569, 2616,
+  2569, 2616, 2598, 2622, 2569, 2616, 2598, 2624, 2569, 2616, 2598, 2631, 2569, 2616, 2600, 2626, 2672, 2569, 2616,
+  2600, 2631, 2569, 2617, 2569, 2617, 2600, 2622, 2562, 2569, 2673, 2596, 2631, 2569, 2673, 2602, 2608, 2575, 2581,
+  2568, 2581, 2598, 2631, 2581, 2608, 2581, 2608, 2581, 2631, 2581, 2608, 2598, 2622, 2581, 2608, 2598, 2624, 2581,
+  2608, 2598, 2631, 2581, 2608, 2600, 2581, 2608, 2600, 2622, 2581, 2622, 2608, 2600, 2581, 2623, 2581, 2623, 2569,
+  2562, 2581, 2623, 2581, 2623, 2616, 2631, 2581, 2624, 2581, 2624, 2596, 2622, 2581, 2624, 2596, 2624, 2581, 2624,
+  2596, 2631, 2581, 2625, 2589, 2581, 2625, 2673, 2589, 2581, 2631, 2581, 2631, 2613, 2610, 2581, 2635, 2568, 2581,
+  2635, 2610, 2583, 2568, 2583, 2575, 2583, 2623, 2566, 2584, 2673, 2591, 2586, 2588, 2598, 2635, 2562, 2588, 2620,
+  2623, 2566, 2598, 2622, 2588, 2622, 2588, 2622, 2562, 2588, 2622, 2562, 2598, 2622, 2588, 2622, 2562, 2598, 2624,
+  2588, 2622, 2562, 2598, 2624, 2566, 2562, 2588, 2622, 2562, 2598, 2631, 2588, 2622, 2595, 2588, 2623, 2566, 2598,
+  2622, 2588, 2623, 2600, 2637, 2617, 2622, 2562, 2588, 2623, 2613, 2631, 2562, 2588, 2623, 2616, 2588, 2623, 2616,
+  2598, 2622, 2588, 2623, 2616, 2600, 2626, 2672, 2588, 2623, 2617, 2600, 2622, 2562, 2588, 2623, 2673, 2597, 2631,
+  2588, 2631, 2588, 2635, 2596, 2581, 2596, 2608, 2637, 2617, 2622, 2562, 2596, 2622, 2562, 2596, 2625, 2616, 2624,
+  2562, 2596, 2631, 2596, 2635, 2596, 2635, 2562, 2596, 2636, 2608, 2596, 2673, 2581, 2598, 2622, 2598, 2623, 2672,
+  2598, 2622, 2598, 2623, 2673, 2596, 2622, 2598, 2623, 2673, 2596, 2624, 2598, 2623, 2673, 2596, 2631, 2598, 2624,
+  2598, 2624, 2566, 2562, 2598, 2625, 2566, 2608, 2622, 2598, 2626, 2588, 2622, 2598, 2626, 2588, 2624, 2598, 2626,
+  2588, 2631, 2598, 2626, 2608, 2598, 2631, 2598, 2631, 2595, 2598, 2636, 2608, 2622, 2600, 2600, 2617, 2624, 2562,
+  2600, 2622, 2600, 2622, 2610, 2600, 2622, 2610, 2620, 2600, 2622, 2610, 2635, 2562, 2600, 2626, 2672, 2600, 2631,
+  2600, 2631, 2652, 2631, 2602, 2608, 2602, 2617, 2623, 2610, 2622, 2562, 2602, 2637, 2608, 2596, 2624, 2603, 2623,
+  2608, 2604, 2617, 2625, 2596, 2604, 2622, 2565, 2598, 2604, 2622, 2581, 2624, 2604, 2622, 2608, 2631, 2605, 2608,
+  2605, 2622, 2613, 2631, 2562, 2606, 2632, 2562, 2608, 2617, 2623, 2595, 2608, 2617, 2624, 2608, 2617, 2631, 2608,
+  2622, 2617, 2624, 2562, 2608, 2623, 2617, 2622, 2610, 2568, 2610, 2583, 2605, 2583, 2610, 2632, 2610, 2632, 2595,
+  2613, 2599, 2631, 2608, 2631, 2613, 2610, 2635, 2562, 2613, 2622, 2562, 2583, 2613, 2622, 2610, 2622, 2613, 2622,
+  2610, 2624, 2613, 2622, 2610, 2624, 2566, 2562, 2613, 2622, 2610, 2631, 2613, 2622, 2616, 2596, 2631, 2613, 2623,
+  2586, 2613, 2623, 2586, 2581, 2622, 2608, 2613, 2623, 2586, 2635, 2562, 2613, 2623, 2673, 2586, 2613, 2623, 2673,
+  2586, 2635, 2562, 2613, 2624, 2613, 2673, 2599, 2613, 2673, 2610, 2613, 2673, 2610, 2635, 2562, 2616, 2581, 2598,
+  2622, 2616, 2581, 2598, 2624, 2616, 2581, 2598, 2631, 2616, 2600, 2616, 2605, 2616, 2622, 2608, 2624, 2566, 2562,
+  2616, 2622, 2608, 2631, 2616, 2623, 2608, 2603, 2616, 2624, 2617, 2600, 2617, 2608, 2617, 2622, 2610, 2622, 2562,
+  2581, 2623, 2617, 2624, 2617, 2625, 2595, 2617, 2625, 2672, 2598, 2622, 2617, 2625, 2672, 2598, 2624, 2617, 2625,
+  2672, 2598, 2624, 2566, 2562, 2617, 2625, 2672, 2598, 2631, 2617, 2631, 2592, 2617, 2632, 2617, 2635, 2617, 2635,
+  2567, 2566, 2617, 2635, 2568, 2617, 2635, 2575, 2617, 2635, 2595, 2617, 2635, 2608, 2617, 2635, 2613, 2631,
+};
+
+static const uint32_t stop_word_offsets[186] = {
+  0, 4, 7, 11, 17, 21, 25, 29, 30, 33, 35, 38, 40, 44, 48, 52, 54, 60, 65, 67, 71, 75, 79, 84, 88, 91, 93, 98, 101, 105,
+  106, 112, 114, 118, 122, 126, 131, 135, 137, 142, 146, 150, 151, 153, 156, 158, 162, 166, 170, 174, 177, 181, 185,
+  187, 193, 197, 199, 203, 207, 211, 214, 218, 220, 224, 227, 230, 232, 234, 237, 240, 241, 245, 251, 253, 256, 261,
+  266, 273, 278, 281, 286, 293, 298, 301, 306, 312, 318, 323, 325, 327, 329, 335, 338, 343, 345, 347, 350, 353, 356,
+  358, 363, 368, 373, 378, 380, 384, 389, 393, 397, 401, 404, 406, 409, 414, 418, 420, 423, 427, 432, 435, 437, 441,
+  443, 449, 454, 457, 461, 465, 469, 473, 475, 480, 483, 487, 490, 493, 498, 502, 504, 508, 510, 513, 518, 522, 526,
+  530, 534, 540, 544, 549, 552, 558, 563, 567, 573, 575, 578, 581, 586, 590, 594, 598, 600, 602, 608, 612, 616, 618,
+  620, 622, 629, 631, 634, 639, 644, 651, 656, 659, 661, 663, 667, 670, 673, 676, 679, 683,
+};
+
+static const uint32_t keep_ranges[8] = {
+  48, 57, 65, 90, 97, 122, 2560, 2687,
+};
+
+const narsil_language narsil_language_punjabi = {
+  .name = "punjabi",
+  .revision = "1",
+  .stop_word_units = stop_word_units,
+  .stop_word_offsets = stop_word_offsets,
+  .stop_word_count = 185,
+  .ascii_keep = {0x03FF000000000000ULL, 0x07FFFFFE07FFFFFEULL},
+  .keep_ranges = keep_ranges,
+  .keep_range_count = 4,
+  .splits_code_points = 0,
+  .strips_possessive = 0,
+  .normalises_diacritics = 0,
+  .min_token_length = 1,
+  .ngram_size = 0,
+  .normalise = NULL,
+  .stem_by_rules = NULL,
+  .create_snowball = NULL,
+  .snowball_stem = NULL,
+  .close_snowball = NULL,
+};

package/cpp/core/src/languages/data/registry.c

@@ -0,0 +1,116 @@
+#include "../../analysis/languages.h"
+#include "declarations.h"
+
+#include <stdint.h>
+
+const narsil_language *const narsil_language_table[107] = {
+  &narsil_language_albanian,
+  &narsil_language_amharic,
+  &narsil_language_arabic,
+  &narsil_language_armenian,
+  &narsil_language_azerbaijani,
+  &narsil_language_bambara,
+  &narsil_language_basque,
+  &narsil_language_belarusian,
+  &narsil_language_bengali,
+  &narsil_language_bosnian,
+  &narsil_language_breton,
+  &narsil_language_bulgarian,
+  &narsil_language_burmese,
+  &narsil_language_catalan,
+  &narsil_language_chinese,
+  &narsil_language_croatian,
+  &narsil_language_czech,
+  &narsil_language_dagbani,
+  &narsil_language_danish,
+  &narsil_language_dutch,
+  &narsil_language_english,
+  &narsil_language_esperanto,
+  &narsil_language_estonian,
+  &narsil_language_ewe,
+  &narsil_language_faroese,
+  &narsil_language_fijian,
+  &narsil_language_finnish,
+  &narsil_language_french,
+  &narsil_language_ga,
+  &narsil_language_galician,
+  &narsil_language_georgian,
+  &narsil_language_german,
+  &narsil_language_greek,
+  &narsil_language_guarani,
+  &narsil_language_gujarati,
+  &narsil_language_haitian_creole,
+  &narsil_language_hausa,
+  &narsil_language_hawaiian,
+  &narsil_language_hebrew,
+  &narsil_language_hindi,
+  &narsil_language_hungarian,
+  &narsil_language_icelandic,
+  &narsil_language_igbo,
+  &narsil_language_indonesian,
+  &narsil_language_irish,
+  &narsil_language_italian,
+  &narsil_language_japanese,
+  &narsil_language_kannada,
+  &narsil_language_kazakh,
+  &narsil_language_khmer,
+  &narsil_language_kinyarwanda,
+  &narsil_language_kirundi,
+  &narsil_language_korean,
+  &narsil_language_kurmanji,
+  &narsil_language_kyrgyz,
+  &narsil_language_lao,
+  &narsil_language_latin,
+  &narsil_language_latvian,
+  &narsil_language_lingala,
+  &narsil_language_lithuanian,
+  &narsil_language_luxembourgish,
+  &narsil_language_macedonian,
+  &narsil_language_malagasy,
+  &narsil_language_malay,
+  &narsil_language_malayalam,
+  &narsil_language_maltese,
+  &narsil_language_maori,
+  &narsil_language_marathi,
+  &narsil_language_nepali,
+  &narsil_language_norwegian,
+  &narsil_language_oromo,
+  &narsil_language_persian,
+  &narsil_language_polish,
+  &narsil_language_portuguese,
+  &narsil_language_punjabi,
+  &narsil_language_romanian,
+  &narsil_language_russian,
+  &narsil_language_samoan,
+  &narsil_language_sanskrit,
+  &narsil_language_scottish_gaelic,
+  &narsil_language_serbian,
+  &narsil_language_shona,
+  &narsil_language_sinhala,
+  &narsil_language_slovak,
+  &narsil_language_slovenian,
+  &narsil_language_sorani,
+  &narsil_language_spanish,
+  &narsil_language_swahili,
+  &narsil_language_swedish,
+  &narsil_language_tagalog,
+  &narsil_language_tamil,
+  &narsil_language_tatar,
+  &narsil_language_telugu,
+  &narsil_language_thai,
+  &narsil_language_tibetan,
+  &narsil_language_tigrinya,
+  &narsil_language_tongan,
+  &narsil_language_turkish,
+  &narsil_language_twi,
+  &narsil_language_ukrainian,
+  &narsil_language_urdu,
+  &narsil_language_vietnamese,
+  &narsil_language_welsh,
+  &narsil_language_wolof,
+  &narsil_language_xhosa,
+  &narsil_language_yoruba,
+  &narsil_language_zulu,
+};
+
+const uint32_t narsil_language_table_length = 107;