axyseo 2.0.0-alpha.0.0.19 → 2.0.0-alpha.0.0.2

package/src/languageProcessing/languages/id/helpers/internal/stem.js

@@ -0,0 +1,462 @@
+import {languageProcessing} from '../../../index';
+const {flattenSortLength, buildFormRule, createRulesFromArrays} = languageProcessing;
+
+import {calculateTotalNumberOfSyllables, removeEnding, checkBeginningsList} from './stemHelpers';
+
+/**
+ * MIT License
+ *
+ * Adapted from: Copyright (c) 2013 Adinda Praditya
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software
+ * and associated documentation files (the \"Software\"), to deal in the Software without restriction,
+ * including without limitation the rights to use, copy, modify, merge, publish,  distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND,  EXPRESS OR IMPLIED,
+ * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE."
+ */
+
+/**
+ * Checks if a single syllable word has one of the suffixes/suffix combination.
+ *
+ * @param {string} word         The word to check.
+ * @param {Array} suffixesArray The array of suffixes
+ * @returns {boolean}   Whether the word ends in one of the suffixes or not.
+ */
+const checkSingleSyllableWordSuffix = function(word, suffixesArray) {
+  for (const suffix of suffixesArray) {
+    if (word.match(suffix)) {
+      return true;
+    }
+  }
+};
+
+/**
+ * Stems the prefix of the single syllable words, i.e. di-/penge-/menge-
+ *
+ * @param {string}	word			The word to check.
+ * @param {Object}	morphologyData	The Indonesian morphology data file.
+ *
+ * @returns {string} The stemmed word.
+ */
+const stemSingleSyllableWordsPrefixes = function(word, morphologyData) {
+  // If the word gets prefix di-, stem the prefix here. E.g. dicekkanlah -> cekkanlah, dibomi -> bomi
+  if (
+    word.startsWith('di') &&
+    checkBeginningsList(word, 2, morphologyData.stemming.singleSyllableWords)
+  ) {
+    return word.substring(2, word.length);
+  }
+  /*
+   * If the word gets prefix menge-/penge- and is followed by one of the words in the list, stem the prefix here.
+   * E.g. pengeboman -> boman
+   */
+  if (
+    /^[mp]enge/i.test(word) &&
+    checkBeginningsList(word, 5, morphologyData.stemming.singleSyllableWords)
+  ) {
+    return word.substring(5, word.length);
+  }
+  return word;
+};
+
+/**
+ * Stems Indonesian single syllable words. This function concerns single syllable words
+ * with this possible word format [di/penge/menge] + single syllable word + [kan/an/i] + [ku/mu/nya] + [kah/lah/pun], with [] being optional.
+ * E.g. dipel -> pel, dipelkan -> pel, dipelkanlah -> pel, pelkan -> pel, pelmulah -> pel, pengeboman -> bom, mengesahkan -> sah
+ *
+ *
+ * @param {string}	word			The word to check.
+ * @param {Object}	morphologyData	The Indonesian morphology data file.
+ *
+ * @returns {string} The stemmed word.
+ */
+const stemSingleSyllableWords = function(word, morphologyData) {
+  const singleSyllableWords = morphologyData.stemming.singleSyllableWords;
+  const suffixCombination = morphologyData.stemming.singleSyllableWordsSuffixes;
+  const inputWord = word;
+  // If the word starts with prefix di-/penge-/menge-, stem the prefix here. E.g. pengeboman -> boman, dipelkan -> pelkan
+  word = stemSingleSyllableWordsPrefixes(word, morphologyData);
+
+  // Check if a word starts with one of the words in the list, has maximum 3 syllables, and ends in one of the single syllable suffixes
+  if (
+    singleSyllableWords.some(shortWord => word.startsWith(shortWord)) &&
+    calculateTotalNumberOfSyllables(word) <= 3 &&
+    checkSingleSyllableWordSuffix(word, suffixCombination)
+  ) {
+    // If the word gets a particle suffix, stem the particle. E.g. cekkanlah -> cekkan, pelkan -> pel, vasmulah -> vasmu
+    word = removeEnding(
+      word,
+      morphologyData.stemming.regexRules.removeParticle,
+      morphologyData.stemming.doNotStemWords.doNotStemParticle,
+      morphologyData
+    );
+
+    // If the word gets a possessive pronoun suffix, stem the possessive pronoun. E.g. vasmu -> vas
+    word = removeEnding(
+      word,
+      morphologyData.stemming.regexRules.removePronoun,
+      morphologyData.stemming.doNotStemWords.doNotStemPronounSuffix,
+      morphologyData
+    );
+
+    // If the word ends in -kan/-an/-i suffix, stem the suffix. E.g. cekkan -> cek, bomi -> bom
+    const wordWithoutDerivationalSuffix = removeEnding(
+      word,
+      morphologyData.stemming.regexRules.removeSuffixes,
+      morphologyData.stemming.doNotStemWords.doNotStemSuffix,
+      morphologyData
+    );
+
+    if (singleSyllableWords.includes(wordWithoutDerivationalSuffix)) {
+      word = wordWithoutDerivationalSuffix;
+    }
+  }
+  /*
+   * We only want to stem single syllable words here.
+   * Thus, if the output word has more than one syllable, we don't stem the input word at all.
+   */
+  if (calculateTotalNumberOfSyllables(word) > 1 || word.length === 1) {
+    word = inputWord;
+  }
+  return word;
+};
+
+/**
+ * Tries stemming prefixes ke- and ter-. Ke- is always stemmed, and ter- only if it is a prefix and not part of the stem.
+ * Also if the stem of the word begins with r-, only te- is stemmed, not ter-.
+ *
+ * @param {Object}	morphologyData	The Indonesian morphology data file.
+ * @param {string}	word			The word to check.
+ *
+ * @returns {string|null}	The stem or null if the word did not start with ter/keter.
+ */
+const tryStemmingKeAndTer = function(morphologyData, word) {
+  const terException =
+    morphologyData.stemming.doNotStemWords.doNotStemPrefix.doNotStemFirstOrderPrefix.doNotStemTer;
+
+  // If prefix -ter is preceded by prefix -ke, remove it first.
+  if (word.startsWith('keter')) {
+    word = word.substring(2, word.length);
+  }
+  if (word.startsWith('ter')) {
+    // If word is on an exception list of words where -ter should not be stemmed, do not stem -ter and return the word.
+    if (terException.some(wordWithTer => word.startsWith(wordWithTer))) {
+      return word;
+    }
+    // If word (without prefixes) is on the list of words beginning with -r, remove only -te instead of -ter.
+    if (checkBeginningsList(word, 3, morphologyData.stemming.beginningModification.rBeginning)) {
+      return word.replace(/^ter/i, 'r');
+    }
+    // Otherwise, remove -ter.
+    return word.substring(3, word.length);
+  }
+};
+/**
+ * Checks whether a word has a first order prefix and whether it is on an exception list of words which require a stem mofification
+ * after removing the prefix. Returns the stem if the prefix was found and the word was matched on an exception list.
+ *
+ *
+ * @param {string}	word			The word to check.
+ * @param {Object}	morphologyData	The Indonesian morphology data file.
+ *
+ * @returns {string|null}	The stem or null if a prefix was not found, or was found but the word was not on the exception list.
+ */
+const checkFirstOrderPrefixExceptions = function(word, morphologyData) {
+  const beginningModification = morphologyData.stemming.beginningModification;
+
+  // If a word starts with "men" or "pen" and is present in the nBeginning exception list, the prefix should be replaced with "n".
+  if (/^[mp]en/i.test(word)) {
+    if (checkBeginningsList(word, 3, beginningModification.nBeginning)) {
+      return word.replace(/^[mp]en/i, 'n');
+    }
+  }
+  if (/^[mp]eng/i.test(word) && checkBeginningsList(word, 4, beginningModification.kBeginning)) {
+    return word.replace(/^[mp]eng/i, 'k');
+  }
+
+  if (/^[mp]em/i.test(word)) {
+    if (checkBeginningsList(word, 3, beginningModification.pBeginning)) {
+      return word.replace(/^(mem|pem)/i, 'p');
+    } else if (checkBeginningsList(word, 3, beginningModification.mBeginning)) {
+      return word.replace(/^(mem|pem)/i, 'm');
+    }
+  }
+  // Stem prefix ke- if found. Stem te(r)- unless the word was found on the exception list of words with stem beginning in -ter.
+  const wordAfterKeTerCheck = tryStemmingKeAndTer(morphologyData, word);
+  if (wordAfterKeTerCheck) {
+    return wordAfterKeTerCheck;
+  }
+};
+
+/**
+ * Stems the first-order prefix of a word based on regexRules. If the word is found in an exception list, implements a stem modification.
+ *
+ * @param {string} word           The word to stem.
+ * @param {Object} morphologyData The object that contains regex-based rules and exception lists for Indonesian stemming.
+ *
+ * @returns {string} The stemmed word.
+ */
+const removeFirstOrderPrefix = function(word, morphologyData) {
+  // Checks whether the word has a first order prefix and requires a stem modification.
+  const firstOrderPrefixException = checkFirstOrderPrefixExceptions(word, morphologyData);
+
+  if (firstOrderPrefixException) {
+    return firstOrderPrefixException;
+  }
+  const regex = createRulesFromArrays(morphologyData.stemming.regexRules.removeFirstOrderPrefixes);
+  const withRemovedFirstOrderPrefix = buildFormRule(word, regex);
+
+  return withRemovedFirstOrderPrefix || word;
+};
+
+/**
+ * Stems the second-order prefix of a word based on regexRules. If the word is found in an exception list, implements a stem modification.
+ *
+ * @param {string} word           The word to stem.
+ * @param {Object} morphologyData The object that contains regex-based rules and exception lists for Indonesian stemming.
+ *
+ * @returns {string} The stemmed word.
+ */
+const removeSecondOrderPrefix = function(word, morphologyData) {
+  // If a word starts with "ber" or "per" and is present in the rBeginning exception list, the prefix should be replaced with "r".
+  if (
+    (word.startsWith('ber') || word.startsWith('per')) &&
+    checkBeginningsList(word, 3, morphologyData.stemming.beginningModification.rBeginning)
+  ) {
+    return word.replace(/^(ber|per)/i, 'r');
+  }
+  // If a word starts with a first order prefix followed by peng- and is in the kBeginning exception list, then peng- should be replaced with k.
+  if (
+    /^peng/i.test(word) &&
+    checkBeginningsList(word, 4, morphologyData.stemming.beginningModification.kBeginning)
+  ) {
+    return word.replace(/^peng/i, 'k');
+  }
+  const regex = createRulesFromArrays(morphologyData.stemming.regexRules.removeSecondOrderPrefixes);
+  const withRemovedSecondOrderPrefix = buildFormRule(word, regex);
+
+  return withRemovedSecondOrderPrefix || word;
+};
+
+/**
+ * Stems derivational affixes of Indonesian words.
+ *
+ * @param {string} word           The word to stem.
+ * @param {Object} morphologyData The object that contains regex-based rules and exception lists for Indonesian stemming.
+ *
+ * @returns {string} The stemmed word.
+ */
+const stemDerivational = function(word, morphologyData) {
+  let wordLength = word.length;
+  const removeSuffixRules = morphologyData.stemming.regexRules.removeSuffixes;
+  const removeSuffixExceptions = morphologyData.stemming.doNotStemWords.doNotStemSuffix;
+  const doNotStemFirstOrderPrefix = flattenSortLength(
+    morphologyData.stemming.doNotStemWords.doNotStemPrefix.doNotStemFirstOrderPrefix
+  );
+  const doNotStemSecondOrderPrefix = flattenSortLength(
+    morphologyData.stemming.doNotStemWords.doNotStemPrefix.doNotStemSecondOrderPrefix
+  );
+
+  // If a word is in the list of words with a beginning that looks like a valid suffix, do not stem the suffix
+  if (
+    !doNotStemFirstOrderPrefix.some(wordWithPrefixLookAlike =>
+      word.startsWith(wordWithPrefixLookAlike)
+    )
+  ) {
+    /*
+     * If the word has more than 2 syllables and starts with one of first order prefixes (i.e. meng-, meny-, men-, mem-, me-,
+     * peng-, peny-, pen-, pem-, di-, ter-, ke- ), the prefix will be stemmed here. e.g. penyebaran -> sebaran, diperlebarkan -> perlebarkan
+     */
+    word = removeFirstOrderPrefix(word, morphologyData);
+  }
+
+  if (wordLength === word.length) {
+    if (
+      !doNotStemSecondOrderPrefix.some(wordWithPrefixLookAlike =>
+        word.startsWith(wordWithPrefixLookAlike)
+      )
+    ) {
+      /*
+       * If the word does not start with one of the first order prefixes but starts with one of the second order prefixes,
+       * the prefix will be stemmed here, e.g., peranakan -> anakan
+       */
+      word = removeSecondOrderPrefix(word, morphologyData);
+    }
+    // If the word has more than 2 syllables and ends in either -kan, -an, or -i suffixes, the suffix will be deleted here, e.g., anakan -> anak
+    if (calculateTotalNumberOfSyllables(word) > 2) {
+      word = removeEnding(word, removeSuffixRules, removeSuffixExceptions, morphologyData);
+    }
+  } else {
+    // If the word previously had a first order prefix, assign wordLength to the length of the word after prefix deletion.
+    wordLength = word.length;
+    /*
+     * If the word after first order prefix deletion is bigger than 2 and ends in either -kan, -an, or -i suffixes,
+     * the suffix will be stemmed here. e.g. penyebaran - sebar.
+     */
+    if (calculateTotalNumberOfSyllables(word) > 2) {
+      word = removeEnding(word, removeSuffixRules, removeSuffixExceptions, morphologyData);
+    }
+    /*
+     * If the word previously had a suffix, we check further if the word after first order prefix and suffix deletion has more than 2 syllables.
+     * If it does have more than 2 syllables and starts with one of the second order prefixes (i.e. ber-, be-, per-, pe-), the prefix will
+     * be stemmed here unless the word is in the exception list of words with a beginning that looks like a second order prefix.
+     */
+    if (wordLength !== word.length && !doNotStemSecondOrderPrefix.includes(word)) {
+      if (calculateTotalNumberOfSyllables(word) > 2) {
+        word = removeSecondOrderPrefix(word, morphologyData);
+      }
+    }
+  }
+  return word;
+};
+
+/**
+ * Stems Indonesian singular words.
+ *
+ * @param {string} word           The singular word to stem.
+ * @param {Object} morphologyData The object that contains regex-based rules and exception lists for Indonesian stemming.
+ *
+ * @returns {string} The stem of an Indonesian singular word.
+ */
+const stemSingular = function(word, morphologyData) {
+  const singleSyllableWords = stemSingleSyllableWords(word, morphologyData);
+  // Stem the single syllable words
+  word = singleSyllableWords;
+
+  const doNotStemParticle = morphologyData.stemming.doNotStemWords.doNotStemParticle;
+  const doNotStemPronoun = morphologyData.stemming.doNotStemWords.doNotStemPronounSuffix;
+
+  if (calculateTotalNumberOfSyllables(word) <= 2) {
+    return word;
+  }
+
+  // Check if a word after its derivational affixes stemmed exists in the exception list.
+  const firstDerivationalStem = stemDerivational(word, morphologyData);
+  if (
+    doNotStemParticle.includes(firstDerivationalStem) ||
+    doNotStemPronoun.includes(firstDerivationalStem)
+  ) {
+    // If it does exist in the exception list, the ending that looks like a particle or a pronoun suffix should not be stemmed.
+    return firstDerivationalStem;
+  }
+
+  /*
+   * If the word has more than 2 syllables and ends in of the particle endings (i.e. -kah, -lah, -pun), stem the particle here.
+   * e.g. bajumulah -> bajumu, bawalah -> bawa
+   */
+  word = removeEnding(
+    word,
+    morphologyData.stemming.regexRules.removeParticle,
+    doNotStemParticle,
+    morphologyData
+  );
+
+  // If the word (still) has more than 2 syllables and ends in of the possessive pronoun endings (i.e. -ku, -mu, -nya), stem the ending here.
+  if (calculateTotalNumberOfSyllables(word) > 2) {
+    // E.g. bajumu -> baju
+    word = removeEnding(
+      word,
+      morphologyData.stemming.regexRules.removePronoun,
+      doNotStemPronoun,
+      morphologyData
+    );
+  }
+
+  // If the word (still) has more than 2 syllables and has derivational affixes, the affix(es) will be stemmed here.
+  if (calculateTotalNumberOfSyllables(word) > 2) {
+    word = stemDerivational(word, morphologyData);
+  }
+  return word;
+};
+
+/**
+ * Stems Indonesian plural words.
+ *
+ * @param {string} word           The plural word to stem.
+ * @param {Object} morphologyData The object that contains regex-based rules and exception lists for Indonesian stemming.
+ *
+ * @returns {string|null} The stem of an Indonesian plural word or null if no plural was detected.
+ */
+const stemPlural = function(word, morphologyData) {
+  const hyphenIndex = word.indexOf('-');
+
+  // If there is no hyphen in the word, it can't be a reduplicated plural.
+  if (hyphenIndex === -1) {
+    return null;
+  }
+
+  const splitWord = word.split('-');
+
+  if (splitWord.length === 2) {
+    let firstPart = splitWord[0];
+    let secondPart = splitWord[1];
+
+    firstPart = stemSingular(firstPart, morphologyData);
+    secondPart = stemSingular(secondPart, morphologyData);
+
+    /*
+     * To compare the first and second part and see whether it's actually a reduplication:
+     * Trim the beginning of the word since it might be variable due to stem changes caused by prefixes.
+     * For example, in "meniru-nirukan" the singular stemmer will correctly stem the first "niru" to "tiru" because
+     * of the prefix "me". Since the second part of the word is stemmed individually, there is no "me" and hence
+     * "niru" remains "niru". To still be able to link these two forms to each other,
+     * we compare the two parts of the word after stripping the variable first or first and second letter.
+     *
+     */
+    const firstPartBeginningTrimmed = firstPart.substring(1);
+    const secondPartBeginningTrimmed =
+      secondPart.startsWith('ng') || secondPart.startsWith('ny')
+        ? secondPart.substring(2)
+        : secondPart.substring(1);
+
+    if (firstPartBeginningTrimmed === secondPartBeginningTrimmed) {
+      const nonPlurals = morphologyData.stemming.nonPluralReduplications;
+
+      // Check non-plural reduplication.
+      if (nonPlurals.includes(firstPart) && nonPlurals.includes(secondPart)) {
+        /*
+         * In words such as "mengira-ngira" prefix "me" causes a modification on both words (k->ng). This will
+         * be correctly stemmed for the first word, but not the second. Therefore, the correct base form
+         * "kira-kira" is created based on a reduplication of the correctly stemmed first part, "kira".
+         */
+        return firstPart + '-' + firstPart;
+      }
+
+      // Return the stemmed singular form of a reduplicated plural.
+      return firstPart;
+    }
+  }
+
+  return null;
+};
+
+/**
+ * Stems Indonesian words
+ *
+ * @param {string} word           The word to stem.
+ * @param {Object} morphologyData The object that contains regex-based rules and exception lists for Indonesian stemming.
+ *
+ * @returns {string} The stem of an Indonesian word.
+ */
+export default function stem(word, morphologyData) {
+  // Check words that shouldn't receive any stemming.
+  if (morphologyData.stemming.shouldNotBeStemmed.includes(word)) {
+    return word;
+  }
+
+  const stemmedPlural = stemPlural(word, morphologyData);
+
+  if (stemmedPlural) {
+    return stemmedPlural;
+  }
+
+  word = stemSingular(word, morphologyData);
+
+  return word;
+}

package/src/languageProcessing/languages/id/helpers/internal/stemHelpers.js

@@ -0,0 +1,78 @@
+import {languageProcessing} from '../../../index';
+const {buildFormRule, createRulesFromArrays} = languageProcessing;
+
+const vowelCharacters = ['a', 'e', 'i', 'o', 'u'];
+
+/**
+ * Determines if an input character is a vowel.
+ *
+ * @param {string} character    The character to check.
+ *
+ * @returns {boolean} Whether the input character is an Indonesian vowel.
+ */
+function isVowel(character) {
+  return vowelCharacters.includes(character);
+}
+
+/**
+ * Calculates the total number of syllables in the input word.
+ *
+ * @param {string} word     The word to calculate the number of syllables in.
+ *
+ * @returns {int} The total number of syllables in the word.
+ */
+export function calculateTotalNumberOfSyllables(word) {
+  let result = 0;
+
+  for (let i = 0; i < word.length; i++) {
+    if (isVowel(word[i])) {
+      result++;
+    }
+  }
+
+  return result;
+}
+
+/**
+ * Stems the ending of a word based on some regexRules after checking if the word is in the exception list.
+ *
+ * @param {string} word             The word to stem.
+ * @param {Array} regexRules        The list of regex-based rules to apply to the word in order to stem it.
+ * @param {string[]} exceptions     The list of words that should not get the ending removed.
+ * @param {Object} morphologyData   The Indonesian morphology data file
+ *
+ * @returns {string} The stemmed word.
+ */
+export function removeEnding(word, regexRules, exceptions, morphologyData) {
+  if (exceptions.includes(word)) {
+    return word;
+  }
+
+  // Check words ending in -kan whether they are words whose stem ending in -k that get suffix -an or not.
+  const wordsWithKEnding = morphologyData.stemming.doNotStemWords.doNotStemK;
+  if (word.endsWith('kan')) {
+    const wordWithoutSuffixAn = word.substring(0, word.length - 2);
+    // If a word has stem ending -k and gets suffix -an, then only stem -an here.
+    if (wordsWithKEnding.includes(wordWithoutSuffixAn)) {
+      word = wordWithoutSuffixAn;
+    }
+  }
+
+  const removePartRegex = createRulesFromArrays(regexRules);
+  const withRemovedPart = buildFormRule(word, removePartRegex);
+  return withRemovedPart || word;
+}
+
+/**
+ * Checks if the beginning of the word is present in an exception list.
+ *
+ * @param {string}   word         The word to stem.
+ * @param {int}      prefixLength The length of the prefix to be trimmed before checking in the list.
+ * @param {string[]} beginnings   The list of word beginnings that should be checked.
+ *
+ * @returns {boolean} Whether the word is found in the list with beginnings.
+ */
+export function checkBeginningsList(word, prefixLength, beginnings) {
+  const wordWithoutPrefix = word.slice(prefixLength);
+  return beginnings.some(beginning => wordWithoutPrefix.startsWith(beginning));
+}

package/src/languageProcessing/languages/id/helpers/isPassiveSentence.js

@@ -0,0 +1,39 @@
+import {languageProcessing} from '../../../index';
+const {getWords} = languageProcessing;
+
+import nonPassives from '../config/internal/nonPassiveVerbsStartingDi';
+const passivePrefix = 'di';
+
+/**
+ * Checks the passed sentence to see if it contains Indonesian passive verb-forms.
+ *
+ * @param {string} sentence     The sentence to match against.
+ *
+ * @returns {Boolean} Whether the sentence contains Indonesian passive voice.
+ */
+export default function isPassiveSentence(sentence) {
+  const words = getWords(sentence.toLowerCase());
+  let matchedPassives = words.filter(word => word.length > 4);
+  matchedPassives = matchedPassives.filter(word => word.startsWith(passivePrefix));
+  if (matchedPassives.length === 0) {
+    return false;
+  }
+
+  // Check exception list.
+  for (const nonPassive of nonPassives) {
+    matchedPassives = matchedPassives.filter(word => !word.startsWith(nonPassive));
+  }
+
+  // Check direct precedence exceptions.
+  matchedPassives = matchedPassives.filter(function(matchedPassive) {
+    let matchedPassivesShouldStay = true;
+    const passiveIndex = words.indexOf(matchedPassive);
+    const wordPrecedingPassive = words[passiveIndex - 1];
+    if (wordPrecedingPassive === 'untuk') {
+      matchedPassivesShouldStay = false;
+    }
+    return matchedPassivesShouldStay;
+  });
+
+  return matchedPassives.length !== 0;
+}

package/src/languageProcessing/languages/id/helpers/splitIntoTokensCustom.js

@@ -0,0 +1,47 @@
+import createPunctuationTokens from "../../../helpers/word/createPunctuationTokens";
+
+/*
+ * The following regex matches a word separator. A word separator is either a whitespace, a tab, a non-breaking space,
+ * an en-dash or an em-dash.
+ * Brackets are added to deal correctly with shortcodes downstream.
+ * The regex is used to split a text into tokens.
+ * Do not add punctuation marks to this regex, as they are handled separately inside `createPunctuationTokens()`.
+ * The word separator explicitly only contains characters that split two words and not a word and a space.
+ * - A space is a word separator because it separates two words if it occurs between them. For example: "foo bar"
+ * - A tab is a word separator because it separates two words if it occurs between them. For example: "foo	bar"
+ * - A non-breaking space (u00A0) is a word separator because it separates two words if it occurs between them. For example: "foo\u00A0bar"
+ * - An en-dash (u2013) and an em-dash (u2014) are word separators because they separate two words if they occur between them.
+ *   For example: "foo–bar".
+ *
+ * Hyphens are NOT word separators in Indonesian. They are most often used in a way that doesn't indicate a word boundary,
+ * for example for reduplication (e.g. 'buku' meaning 'book' and 'buku-buku' meaning books).
+ */
+const WORD_SEPARATORS_REGEX = /([\s\t\u00A0\u2013\u2014[\]])/;
+
+/**
+ * Tokenizes a text similarly to `getWords`, but in a way that's suitable for the HTML parser.
+ * 1. It does not normalize whitespace.
+ * This operation is too risky for the HTML parser because it may throw away characters and as a result, the token positions are corrupted.
+ * 2. It does not remove punctuation marks but keeps them.
+ *
+ * This algorithm splits the text by word separators: tokens that are the border between two words.
+ * This algorithm separates punctuation marks from words and keeps them as separate tokens.
+ * It only splits them off if they appear at the start or the end of a word.
+ *
+ * @param {string} text The text to tokenize.
+ *
+ * @returns {string[]} 	An array of tokens.
+ */
+function splitIntoTokensCustom( text ) {
+	if ( ! text ) {
+		return [];
+	}
+
+	// Split the sentence string into tokens. Those tokens are unrefined as they may contain punctuation.
+	const rawTokens = text.split( WORD_SEPARATORS_REGEX ).filter( x => x !== "" );
+
+	// Remove punctuation from the beginning and end of word tokens, and make them into separate tokens.
+	return createPunctuationTokens( rawTokens );
+}
+
+export default splitIntoTokensCustom;

package/src/languageProcessing/languages/it/Researcher.js

@@ -0,0 +1,48 @@
+import {languageProcessing} from '../../../index';
+const {AbstractResearcher} = languageProcessing;
+
+// All config
+import firstWordExceptions from './config/firstWordExceptions';
+import {all as functionWords} from './config/functionWords';
+import stopWords from './config/stopWords';
+import transitionWords from './config/transitionWords';
+import twoPartTransitionWords from './config/twoPartTransitionWords';
+import syllables from './config/syllables.json';
+import sentenceLength from './config/sentenceLength';
+
+// All helpers
+import getClauses from './helpers/getClauses';
+import getStemmer from './helpers/getStemmer';
+import fleschReadingScore from './helpers/calculateFleschReadingScore';
+
+/**
+ * The researcher contains all the researches, helpers, data, and config.
+ */
+export default class Researcher extends AbstractResearcher {
+  /**
+   * Constructor
+   * @param {Paper} paper The Paper object that is needed within the researches.
+   * @constructor
+   */
+  constructor(paper) {
+    super(paper);
+
+    Object.assign(this.config, {
+      language: 'it',
+      passiveConstructionType: 'periphrastic',
+      firstWordExceptions,
+      functionWords,
+      stopWords,
+      transitionWords,
+      twoPartTransitionWords,
+      syllables,
+      sentenceLength
+    });
+
+    Object.assign(this.helpers, {
+      getClauses,
+      getStemmer,
+      fleschReadingScore
+    });
+  }
+}