@fidel-tools/core 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 Liul Alemayehu
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,56 @@
+# @fidel-tools/core
+
+The core NLP pipeline and text pre-processing engine for Amharic and Ethiopic script processing.
+
+---
+
+## Features
+
+- **Normalization**: Standardizes homophones, expands labialized strings, and collapses character gemination.
+- **Tokenization**: Standard and sentence-level tokenizers with exception mapping (abbreviation expansion).
+- **Stopwords**: Morphology-aware boundary filtering that removes stopwords safely without corrupting base stems.
+- **Light Stemmer**: Prefix- and suffix-removal algorithms for root extraction.
+- **Transliteration**: Bidirectional SERA and Felig ASCII transliterators.
+
+---
+
+## Installation
+
+```bash
+pnpm add @fidel-tools/core
+```
+
+---
+
+## API & Usage
+
+### 1. Unified Pipeline
+
+```typescript
+import { Pipeline } from '@fidel-tools/core'
+import amPack from '@fidel-tools/lang-am' // Or your custom pack
+
+const nlp = new Pipeline(amPack)
+
+const normalized = nlp.normalize("ሐኪም ኀይሉ")
+const tokens = nlp.lexAnalyze("ት/ቤት እና መስሪያ ቤት")
+const stemmed = nlp.stem("ልጆቻቸውን")
+```
+
+### 2. Low-Level Component Exports
+
+```typescript
+import { normalize, sentenceTokenize, removeStopwords, stem } from '@fidel-tools/core'
+import amPack from '@fidel-tools/lang-am'
+
+// Individual functional components
+const text = normalize("ሐኪም ኀይሉ", amPack)
+const sentences = sentenceTokenize("ይህ ዓረፍተ ነገር ነው። ያኛው ደግሞ፡", amPack)
+const cleaned = removeStopwords("እና በመሆኑም ትምህርት", amPack)
+```
+
+---
+
+## License
+
+[MIT License](../../LICENSE)

package/dist/index.d.ts

@@ -0,0 +1,10 @@
+export type { LanguagePack, LanguagePackMeta, StemmerConfig, TransliterationConfig } from './types.js';
+export { Pipeline } from './pipeline.js';
+export { normalize } from './normalizer.js';
+export { sentenceTokenize } from './sentence_tokenizer.js';
+export { stem } from './stemmer.js';
+export { removeStopwords } from './stopword_remover.js';
+export { lexAnalyze } from './lexical_analyzer.js';
+export { felig_transliterate, sera_transliterate } from './transliterator.js';
+export { indexDocuments, indexQuery, indexTerms } from './indexer.js';
+export { weighTerms, weigh_terms } from './term_weighter.js';

package/dist/index.js

@@ -0,0 +1,12 @@
+// Pipeline (primary API)
+export { Pipeline } from './pipeline.js';
+// Individual functions (secondary API, for tree-shaking)
+export { normalize } from './normalizer.js';
+export { sentenceTokenize } from './sentence_tokenizer.js';
+export { stem } from './stemmer.js';
+export { removeStopwords } from './stopword_remover.js';
+export { lexAnalyze } from './lexical_analyzer.js';
+export { felig_transliterate, sera_transliterate } from './transliterator.js';
+export { indexDocuments, indexQuery, indexTerms } from './indexer.js';
+export { weighTerms, weigh_terms } from './term_weighter.js';
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAGA,yBAAyB;AACzB,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAA;AAExC,yDAAyD;AACzD,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAA;AAC3C,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAA;AAC1D,OAAO,EAAE,IAAI,EAAE,MAAM,cAAc,CAAA;AACnC,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAA;AACvD,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAClD,OAAO,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAA;AAC7E,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,cAAc,CAAA;AACrE,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAA"}

package/dist/indexer.d.ts

@@ -0,0 +1,18 @@
+import type { LanguagePack } from './types.js';
+export interface DocIndexData {
+    corpus_size: number;
+    corpus_word_count: Record<string, number>;
+    words: Record<string, Array<Record<string, number>>>;
+}
+export interface QueryIndexData {
+    corpus_size: number;
+    corpus_word_count: number;
+    words: Record<string, number>;
+}
+export declare function indexDocuments(docs: Array<{
+    id: string;
+    content: string;
+}>, pack: LanguagePack): DocIndexData;
+export declare function indexQuery(query: string, pack: LanguagePack): QueryIndexData;
+export declare function indexTerms(corpus: string[], outputIndexFilePath: string, type: "doc" | "query", pack: LanguagePack): Promise<void>;
+export default indexTerms;

package/dist/indexer.js

@@ -0,0 +1,107 @@
+import stem from "./stemmer.js";
+import lexAnalyze from "./lexical_analyzer.js";
+import rmvStopwrd from "./stopword_remover.js";
+// Pure functions — no fs dependency
+export function indexDocuments(docs, pack) {
+    const indexData = {
+        corpus_size: docs.length,
+        corpus_word_count: {},
+        words: {}
+    };
+    docs.forEach((doc) => {
+        indexData.corpus_word_count[doc.id] = doc.content.split(" ").length;
+        // preprocess
+        const unStemmedWords = rmvStopwrd(lexAnalyze(doc.content, pack), pack).split(" ");
+        const stemmedWords = unStemmedWords.map((word) => stem(word, pack));
+        const result = stemmedWords
+            .filter((e) => e)
+            .filter((e) => {
+            return e.length > 1;
+        });
+        // index
+        let wordFlag = 0;
+        result.forEach((word) => {
+            if (word in indexData.words) {
+                indexData.words[word].forEach((pathObj) => {
+                    if (doc.id in pathObj) {
+                        pathObj[doc.id]++;
+                        wordFlag = 1;
+                    }
+                });
+                if (wordFlag === 0) {
+                    indexData.words[word].push({ [doc.id]: 1 });
+                }
+                else {
+                    wordFlag = 0;
+                }
+            }
+            else {
+                indexData.words[word] = [{ [doc.id]: 1 }];
+            }
+        });
+    });
+    return indexData;
+}
+export function indexQuery(query, pack) {
+    const indexData = {
+        corpus_size: 1,
+        corpus_word_count: query.split(" ").length,
+        words: {}
+    };
+    // preprocess
+    const unStemmedWords = rmvStopwrd(lexAnalyze(query, pack), pack).split(" ");
+    const stemmedWords = unStemmedWords.map((word) => stem(word, pack));
+    const result = stemmedWords
+        .filter((e) => e)
+        .filter((e) => {
+        return e.length > 1;
+    });
+    // index
+    result.forEach((word) => {
+        if (word in indexData.words) {
+            indexData.words[word]++;
+        }
+        else {
+            indexData.words[word] = 1;
+        }
+    });
+    return indexData;
+}
+// Backwards-compat Node.js wrapper — fs lives here only
+export async function indexTerms(corpus, outputIndexFilePath, type, pack) {
+    const fs = await import("fs");
+    if (type === "doc") {
+        const docs = corpus.map(filePath => {
+            try {
+                const content = fs.readFileSync(filePath, 'utf8');
+                return { id: filePath, content };
+            }
+            catch (error) {
+                console.log(`Error reading ${filePath} file from disk:`, error);
+                return { id: filePath, content: "" };
+            }
+        });
+        const result = indexDocuments(docs, pack);
+        try {
+            fs.writeFileSync(outputIndexFilePath + '/docIndexFile.json', JSON.stringify(result, null, 2));
+            docs.forEach(doc => {
+                console.log(`Contents of ${doc.id} successfully added to index`);
+            });
+        }
+        catch (error) {
+            console.log("Index creation failed", error);
+        }
+    }
+    else {
+        try {
+            const result = indexQuery(corpus, pack);
+            fs.writeFileSync(outputIndexFilePath + '/queryIndexFile.json', JSON.stringify(result, null, 2));
+            console.log(`Contents of Query successfully added to index`);
+        }
+        catch (error) {
+            console.log("Index creation failed", error);
+        }
+    }
+}
+export default indexTerms;
+//# sourceMappingURL=indexer.js.map

package/dist/indexer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"indexer.js","sourceRoot":"","sources":["../src/indexer.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,cAAc,CAAA;AAC/B,OAAO,UAAU,MAAM,uBAAuB,CAAA;AAC9C,OAAO,UAAU,MAAM,uBAAuB,CAAA;AAgB9C,oCAAoC;AACpC,MAAM,UAAU,cAAc,CAC5B,IAA4C,EAC5C,IAAkB;IAElB,MAAM,SAAS,GAAiB;QAC9B,WAAW,EAAE,IAAI,CAAC,MAAM;QACxB,iBAAiB,EAAE,EAAE;QACrB,KAAK,EAAE,EAAE;KACV,CAAA;IAED,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;QACnB,SAAS,CAAC,iBAAiB,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAA;QAEnE,aAAa;QACb,MAAM,cAAc,GAAG,UAAU,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;QACjF,MAAM,YAAY,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAA;QACnE,MAAM,MAAM,GAAG,YAAY;aACxB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;aAChB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;YACZ,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,CAAA;QACrB,CAAC,CAAC,CAAA;QAEJ,QAAQ;QACR,IAAI,QAAQ,GAAG,CAAC,CAAA;QAChB,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;YACtB,IAAI,IAAI,IAAI,SAAS,CAAC,KAAK,EAAE,CAAC;gBAC5B,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;oBACxC,IAAI,GAAG,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC;wBACtB,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAA;wBACjB,QAAQ,GAAG,CAAC,CAAA;oBACd,CAAC;gBACH,CAAC,CAAC,CAAA;gBACF,IAAI,QAAQ,KAAK,CAAC,EAAE,CAAC;oBACnB,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAA;gBAC7C,CAAC;qBAAM,CAAC;oBACN,QAAQ,GAAG,CAAC,CAAA;gBACd,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAA;YAC3C,CAAC;QACH,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,OAAO,SAAS,CAAA;AAClB,CAAC;AAED,MAAM,UAAU,UAAU,CACxB,KAAa,EACb,IAAkB;IAElB,MAAM,SAAS,GAAmB;QAChC,WAAW,EAAE,CAAC;QACd,iBAAiB,EAAE,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM;QAC1C,KAAK,EAAE,EAAE;KACV,CAAA;IAED,aAAa;IACb,MAAM,cAAc,GAAG,UAAU,CAAC,UAAU,CAAC,KAAK,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;IAC3E,MAAM,YAAY,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAA;IACnE,MAAM,MAAM,GAAG,YAAY;SACxB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;SAChB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;QACZ,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,CAAA;IACrB,CAAC,CAAC,CAAA;IAEJ,QAAQ;IACR,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;QACtB,IAAI,IAAI,IAAI,SAAS,CAAC,KAAK,EAAE,CAAC;YAC5B,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAA;QACzB,CAAC;aAAM,CAAC;YACN,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QAC3B,CAAC;IACH,CAAC,CAAC,CAAA;IAEF,OAAO,SAAS,CAAA;AAClB,CAAC;AAED,wDAAwD;AACxD,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,MAAgB,EAChB,mBAA2B,EAC3B,IAAqB,EACrB,IAAkB;IAElB,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,CAAA;IAC7B,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;QACnB,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE;YACjC,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;gBACjD,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAA;YAClC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,GAAG,CAAC,iBAAiB,QAAQ,kBAAkB,EAAE,KAAK,CAAC,CAAA;gBAC/D,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,EAAE,EAAE,CAAA;YACtC,CAAC;QACH,CAAC,CAAC,CAAA;QACF,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;QACzC,IAAI,CAAC;YACH,EAAE,CAAC,aAAa,CAAC,mBAAmB,GAAG,oBAAoB,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAA;YAC7F,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE;gBACjB,OAAO,CAAC,GAAG,CAAC,eAAe,GAAG,CAAC,EAAE,8BAA8B,CAAC,CAAA;YAClE,CAAC,CAAC,CAAA;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,GAAG,CAAC,uBAAuB,EAAE,KAAK,CAAC,CAAA;QAC7C,CAAC;IACH,CAAC;SAAM,CAAC;QACN,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,UAAU,CAAC,MAA2B,EAAE,IAAI,CAAC,CAAA;YAC5D,EAAE,CAAC,aAAa,CAAC,mBAAmB,GAAG,sBAAsB,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAA;YAC/F,OAAO,CAAC,GAAG,CAAC,+CAA+C,CAAC,CAAA;QAC9D,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,GAAG,CAAC,uBAAuB,EAAE,KAAK,CAAC,CAAA;QAC7C,CAAC;IACH,CAAC;AACH,CAAC;AAED,eAAe,UAAU,CAAA"}

package/dist/lexical_analyzer.d.ts

@@ -0,0 +1,9 @@
+import type { LanguagePack } from './types.js';
+/**
+ * Separates words, expands abbreviations, removes numbers, breaks up hyphenated words, and removes punctuation
+ * @param corpus : Amharic text
+ * @param pack : language pack configuration
+ * @returns : Lexically analyzed Amharic text
+ */
+export declare function lexAnalyze(corpus: string, pack: LanguagePack): string;
+export default lexAnalyze;

package/dist/lexical_analyzer.js

@@ -0,0 +1,22 @@
+/**
+ * Separates words, expands abbreviations, removes numbers, breaks up hyphenated words, and removes punctuation
+ * @param corpus : Amharic text
+ * @param pack : language pack configuration
+ * @returns : Lexically analyzed Amharic text
+ */
+export function lexAnalyze(corpus, pack) {
+    // Expand exceptions (abbreviations)
+    if (pack.tokenization && pack.tokenization.exceptions) {
+        for (const key in pack.tokenization.exceptions) {
+            const expansion = pack.tokenization.exceptions[key].join(" ");
+            corpus = corpus.replaceAll(key, expansion);
+        }
+    }
+    corpus = corpus
+        .replace(/[.\?"',/#!$%^&*;:፤።{}=\-_`~()]/g, " ")
+        .replace(/[.፩፪፫፬፭፮፯፰፱፲፳፴፵፶፷፸፹፺፻0123456789]/g, " ")
+        .replace(/\s{2,}/g, " ");
+    return corpus;
+}
+export default lexAnalyze;
+//# sourceMappingURL=lexical_analyzer.js.map

package/dist/lexical_analyzer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"lexical_analyzer.js","sourceRoot":"","sources":["../src/lexical_analyzer.ts"],"names":[],"mappings":"AAGA;;;;;GAKG;AACH,MAAM,UAAU,UAAU,CAAC,MAAc,EAAE,IAAkB;IAC3D,oCAAoC;IACpC,IAAI,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,YAAY,CAAC,UAAU,EAAE,CAAC;QACtD,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,YAAY,CAAC,UAAU,EAAE,CAAC;YAC/C,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YAC7D,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,GAAG,EAAE,SAAS,CAAC,CAAA;QAC5C,CAAC;IACH,CAAC;IAED,MAAM,GAAG,MAAM;SACZ,OAAO,CAAC,iCAAiC,EAAE,GAAG,CAAC;SAC/C,OAAO,CAAC,mCAAmC,EAAE,GAAG,CAAC;SACjD,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAA;IAE1B,OAAO,MAAM,CAAA;AACf,CAAC;AAED,eAAe,UAAU,CAAA"}

package/dist/normalizer.d.ts

@@ -0,0 +1,11 @@
+import type { LanguagePack } from './types.js';
+/**
+ * Normalizes Amharic text by applying character mapping,
+ * labialized sequence normalization, and gemination collapse.
+ *
+ * @param text The input string to normalize.
+ * @param pack The language pack containing normalization configuration.
+ * @returns The normalized string.
+ */
+export declare function normalize(text: string, pack: LanguagePack): string;
+export default normalize;

package/dist/normalizer.js

@@ -0,0 +1,41 @@
+/**
+ * Normalizes Amharic text by applying character mapping,
+ * labialized sequence normalization, and gemination collapse.
+ *
+ * @param text The input string to normalize.
+ * @param pack The language pack containing normalization configuration.
+ * @returns The normalized string.
+ */
+export function normalize(text, pack) {
+    if (!pack.normalization) {
+        return text;
+    }
+    let normalized = text;
+    // 1. Apply char_map
+    const charMap = pack.normalization.char_map || {};
+    let chars = normalized.split("");
+    for (let i = 0; i < chars.length; i++) {
+        if (charMap[chars[i]] !== undefined) {
+            chars[i] = charMap[chars[i]];
+        }
+    }
+    normalized = chars.join("");
+    // 2. Apply labialized_map
+    const labializedMap = pack.normalization.labialized_map || {};
+    let chars2 = normalized.split("");
+    for (let i = 0; i < chars2.length; i++) {
+        if (labializedMap[chars2[i]] !== undefined) {
+            chars2[i] = labializedMap[chars2[i]];
+        }
+    }
+    normalized = chars2.join("");
+    // 3. Collapse gemination
+    const threshold = pack.normalization.gemination_threshold;
+    if (threshold !== undefined && threshold > 0) {
+        const regex = new RegExp(`([^\\s])\\1{${threshold},}`, 'g');
+        normalized = normalized.replace(regex, (match, p1) => p1.repeat(threshold));
+    }
+    return normalized;
+}
+export default normalize;
+//# sourceMappingURL=normalizer.js.map

package/dist/normalizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"normalizer.js","sourceRoot":"","sources":["../src/normalizer.ts"],"names":[],"mappings":"AAEA;;;;;;;GAOG;AACH,MAAM,UAAU,SAAS,CAAC,IAAY,EAAE,IAAkB;IACxD,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;QACxB,OAAO,IAAI,CAAA;IACb,CAAC;IAED,IAAI,UAAU,GAAG,IAAI,CAAA;IAErB,oBAAoB;IACpB,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,QAAQ,IAAI,EAAE,CAAA;IACjD,IAAI,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;IAChC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,SAAS,EAAE,CAAC;YACpC,KAAK,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA;QAC9B,CAAC;IACH,CAAC;IACD,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAE3B,0BAA0B;IAC1B,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC,cAAc,IAAI,EAAE,CAAA;IAC7D,IAAI,MAAM,GAAG,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;IACjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,IAAI,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,SAAS,EAAE,CAAC;YAC3C,MAAM,CAAC,CAAC,CAAC,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAA;QACtC,CAAC;IACH,CAAC;IACD,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAE5B,yBAAyB;IACzB,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,oBAAoB,CAAA;IACzD,IAAI,SAAS,KAAK,SAAS,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;QAC7C,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,eAAe,SAAS,IAAI,EAAE,GAAG,CAAC,CAAA;QAC3D,UAAU,GAAG,UAAU,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAA;IAC7E,CAAC;IAED,OAAO,UAAU,CAAA;AACnB,CAAC;AAED,eAAe,SAAS,CAAA"}

package/dist/pipeline.d.ts

@@ -0,0 +1,19 @@
+import type { LanguagePack } from './types.js';
+import type { DocIndexData, QueryIndexData } from './indexer.js';
+export declare class Pipeline {
+    private pack;
+    constructor(pack: LanguagePack);
+    normalize(text: string): string;
+    sentenceTokenize(text: string): string[];
+    stem(word: string): string;
+    removeStopwords(corpus: string): string;
+    lexAnalyze(corpus: string): string;
+    feligTransliterate(word: string, lang: "am" | "en"): string;
+    seraTransliterate(word: string, lang: "am" | "en"): string;
+    indexDocuments(docs: Array<{
+        id: string;
+        content: string;
+    }>): DocIndexData;
+    indexQuery(query: string): QueryIndexData;
+    weighTerms(index: DocIndexData | QueryIndexData, type: "doc" | "query"): Record<string, any>;
+}

package/dist/pipeline.js

@@ -0,0 +1,44 @@
+import { stem } from './stemmer.js';
+import { removeStopwords } from './stopword_remover.js';
+import { lexAnalyze } from './lexical_analyzer.js';
+import { felig_transliterate, sera_transliterate } from './transliterator.js';
+import { indexDocuments, indexQuery } from './indexer.js';
+import { weighTerms } from './term_weighter.js';
+import { normalize } from './normalizer.js';
+import { sentenceTokenize } from './sentence_tokenizer.js';
+export class Pipeline {
+    constructor(pack) {
+        this.pack = pack;
+    }
+    normalize(text) {
+        return normalize(text, this.pack);
+    }
+    sentenceTokenize(text) {
+        return sentenceTokenize(text, this.pack);
+    }
+    stem(word) {
+        return stem(word, this.pack);
+    }
+    removeStopwords(corpus) {
+        return removeStopwords(corpus, this.pack);
+    }
+    lexAnalyze(corpus) {
+        return lexAnalyze(corpus, this.pack);
+    }
+    feligTransliterate(word, lang) {
+        return felig_transliterate(word, lang, this.pack);
+    }
+    seraTransliterate(word, lang) {
+        return sera_transliterate(word, lang, this.pack);
+    }
+    indexDocuments(docs) {
+        return indexDocuments(docs, this.pack);
+    }
+    indexQuery(query) {
+        return indexQuery(query, this.pack);
+    }
+    weighTerms(index, type) {
+        return weighTerms(index, type);
+    }
+}
+//# sourceMappingURL=pipeline.js.map

package/dist/pipeline.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pipeline.js","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,cAAc,CAAA;AACnC,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAA;AACvD,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAClD,OAAO,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAA;AAC7E,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,cAAc,CAAA;AACzD,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAA;AAE/C,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAA;AAC3C,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAA;AAE1D,MAAM,OAAO,QAAQ;IACnB,YAAoB,IAAkB;QAAlB,SAAI,GAAJ,IAAI,CAAc;IAAG,CAAC;IAE1C,SAAS,CAAC,IAAY;QACpB,OAAO,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,CAAA;IACnC,CAAC;IAED,gBAAgB,CAAC,IAAY;QAC3B,OAAO,gBAAgB,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,CAAA;IAC1C,CAAC;IAED,IAAI,CAAC,IAAY;QACf,OAAO,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,CAAA;IAC9B,CAAC;IACD,eAAe,CAAC,MAAc;QAC5B,OAAO,eAAe,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,CAAA;IAC3C,CAAC;IACD,UAAU,CAAC,MAAc;QACvB,OAAO,UAAU,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,CAAA;IACtC,CAAC;IACD,kBAAkB,CAAC,IAAY,EAAE,IAAiB;QAChD,OAAO,mBAAmB,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,CAAA;IACnD,CAAC;IACD,iBAAiB,CAAC,IAAY,EAAE,IAAiB;QAC/C,OAAO,kBAAkB,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,CAAA;IAClD,CAAC;IACD,cAAc,CAAC,IAA4C;QACzD,OAAO,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,CAAA;IACxC,CAAC;IACD,UAAU,CAAC,KAAa;QACtB,OAAO,UAAU,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,CAAA;IACrC,CAAC;IACD,UAAU,CAAC,KAAoC,EAAE,IAAqB;QACpE,OAAO,UAAU,CAAC,KAAK,EAAE,IAAI,CAAC,CAAA;IAChC,CAAC;CACF"}

package/dist/sentence_tokenizer.d.ts

@@ -0,0 +1,10 @@
+import type { LanguagePack } from './types.js';
+/**
+ * Tokenizes Amharic text into sentences using configured sentence boundaries.
+ *
+ * @param text The input string to split into sentences.
+ * @param pack The language pack containing tokenization configuration.
+ * @returns An array of sentence strings.
+ */
+export declare function sentenceTokenize(text: string, pack: LanguagePack): string[];
+export default sentenceTokenize;

package/dist/sentence_tokenizer.js

@@ -0,0 +1,22 @@
+/**
+ * Tokenizes Amharic text into sentences using configured sentence boundaries.
+ *
+ * @param text The input string to split into sentences.
+ * @param pack The language pack containing tokenization configuration.
+ * @returns An array of sentence strings.
+ */
+export function sentenceTokenize(text, pack) {
+    const boundaries = pack.tokenization?.sentence_boundaries || ["።", "፡", "?", "!", "."];
+    if (boundaries.length === 0) {
+        return [text];
+    }
+    // Escape boundaries for Regex character class
+    const escaped = boundaries.map(b => b.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')).join('');
+    const regex = new RegExp(`[${escaped}]+`, 'g');
+    // Split on boundaries, trim whitespace, filter out empty sentences
+    return text.split(regex)
+        .map(s => s.trim())
+        .filter(s => s.length > 0);
+}
+export default sentenceTokenize;
+//# sourceMappingURL=sentence_tokenizer.js.map

package/dist/sentence_tokenizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sentence_tokenizer.js","sourceRoot":"","sources":["../src/sentence_tokenizer.ts"],"names":[],"mappings":"AAEA;;;;;;GAMG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAY,EAAE,IAAkB;IAC/D,MAAM,UAAU,GAAG,IAAI,CAAC,YAAY,EAAE,mBAAmB,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAA;IACtF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,OAAO,CAAC,IAAI,CAAC,CAAA;IACf,CAAC;IAED,8CAA8C;IAC9C,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,qBAAqB,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IACtF,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,IAAI,OAAO,IAAI,EAAE,GAAG,CAAC,CAAA;IAE9C,mEAAmE;IACnE,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC;SACrB,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SAClB,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;AAC9B,CAAC;AAED,eAAe,gBAAgB,CAAA"}

package/dist/stemmer.d.ts

@@ -0,0 +1,12 @@
+import type { LanguagePack } from "./types.js";
+/**
+ * Takes an Amharic word and returns the stem through affix-removal with longest match.
+ * @param word : word possibly containing one or more affix
+ * @param pack : the language pack configuration
+ * @returns : the stem of the word passed
+ *
+ * @example {stem word with affix}
+ * stem("ልጆቻቸውን", amPack) // returns "ልጅ"
+ */
+export declare function stem(word: string, pack: LanguagePack): string;
+export default stem;

package/dist/stemmer.js

@@ -0,0 +1,86 @@
+// Takes Amharic language words and produces a stem
+// ልጆች -> ልጅኦች -> ljoc -> lj -> ልጅ
+import { felig_transliterate } from "./transliterator.js";
+function escapeRegExp(str) {
+    return str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+/**
+ * Takes an Amharic word and returns the stem through affix-removal with longest match.
+ * @param word : word possibly containing one or more affix
+ * @param pack : the language pack configuration
+ * @returns : the stem of the word passed
+ *
+ * @example {stem word with affix}
+ * stem("ልጆቻቸውን", amPack) // returns "ልጅ"
+ */
+export function stem(word, pack) {
+    if (pack.stemmer.protected_words) {
+        if (pack.stemmer.protected_words.includes(word)) {
+            return word;
+        }
+        // Check if stripping any prefix yields a protected word
+        const prefixes = pack.stemmer.prefixes || [];
+        for (const prefix of prefixes) {
+            if (word.startsWith(prefix)) {
+                const stripped = word.substring(prefix.length);
+                if (pack.stemmer.protected_words.includes(stripped)) {
+                    return stripped;
+                }
+            }
+        }
+    }
+    let cv_string = felig_transliterate(word, "am", pack); // consonant-vowel string
+    const sfx_arr = [];
+    const pfx_arr = [];
+    // Prepare suffix array
+    const sarr = pack.stemmer.suffixes || [];
+    sarr.forEach((suffix) => {
+        sfx_arr.push(felig_transliterate(suffix, "am", pack));
+        if (suffix.startsWith("ዎ")) {
+            const altSuffix = "ኦ" + suffix.substring(1);
+            sfx_arr.push(felig_transliterate(altSuffix, "am", pack));
+        }
+    });
+    sfx_arr.push("Wa"); // Special case for ሯ
+    sfx_arr.sort((a, b) => b.length - a.length);
+    // Prepare prefix array
+    const parr = pack.stemmer.prefixes || [];
+    parr.forEach((prefix) => {
+        pfx_arr.push(felig_transliterate(prefix, "am", pack));
+    });
+    pfx_arr.sort((a, b) => b.length - a.length);
+    // Remove suffixes
+    sfx_arr.every(function (sfx, index) {
+        if (cv_string.endsWith(sfx)) {
+            let regex = new RegExp(`${escapeRegExp(sfx)}$`, `i`);
+            cv_string = cv_string.replace(regex, "");
+            return false;
+        }
+        else
+            return true;
+    });
+    // Remove prefixes
+    pfx_arr.every(function (pfx, index) {
+        if (cv_string.startsWith(pfx)) {
+            let regex = new RegExp(`^${escapeRegExp(pfx)}`);
+            cv_string = cv_string.replace(regex, "");
+            return false;
+        }
+        else
+            return true;
+    });
+    // Remove infixes
+    if (/.+([^aeiou])[aeiou]\1[aeiou].?/i.test(cv_string)) {
+        cv_string = cv_string.replace(/\S\S[^aeiou][aeiou]/i, cv_string[0] + cv_string[1]);
+    }
+    else if (/^(.+)a\1$/i.test(cv_string)) {
+        cv_string = cv_string.replace(/a.+/i, "");
+    }
+    const ccvMatch = cv_string.match(/[bcdfghjklmnpqrstvwxyz]{2}e/i);
+    if (ccvMatch) {
+        cv_string = cv_string.replace(/[bcdfghjklmnpqrstvwxyz]{2}e/i, ccvMatch[0].substring(0, 1) + "X" + ccvMatch[0].substring(1));
+    }
+    return felig_transliterate(cv_string, "en", pack);
+}
+export default stem;
+//# sourceMappingURL=stemmer.js.map

package/dist/stemmer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"stemmer.js","sourceRoot":"","sources":["../src/stemmer.ts"],"names":[],"mappings":"AAAA,mDAAmD;AACnD,kCAAkC;AAClC,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAA;AAGzD,SAAS,YAAY,CAAC,GAAW;IAC/B,OAAO,GAAG,CAAC,OAAO,CAAC,qBAAqB,EAAE,MAAM,CAAC,CAAA;AACnD,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,IAAI,CAAC,IAAY,EAAE,IAAkB;IACnD,IAAI,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;QACjC,IAAI,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YAChD,OAAO,IAAI,CAAA;QACb,CAAC;QACD,wDAAwD;QACxD,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAA;QAC5C,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE,CAAC;YAC9B,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC5B,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;gBAC9C,IAAI,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;oBACpD,OAAO,QAAQ,CAAA;gBACjB,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,SAAS,GAAG,mBAAmB,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAA,CAAC,yBAAyB;IAE/E,MAAM,OAAO,GAAa,EAAE,CAAA;IAC5B,MAAM,OAAO,GAAa,EAAE,CAAA;IAE5B,uBAAuB;IACvB,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAA;IACxC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE;QACtB,OAAO,CAAC,IAAI,CAAC,mBAAmB,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAA;QACrD,IAAI,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YAC3B,MAAM,SAAS,GAAG,GAAG,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAA;YAC3C,OAAO,CAAC,IAAI,CAAC,mBAAmB,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAA;QAC1D,CAAC;IACH,CAAC,CAAC,CAAA;IAEF,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA,CAAC,qBAAqB;IACxC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,CAAA;IAE3C,uBAAuB;IACvB,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAA;IACxC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE;QACtB,OAAO,CAAC,IAAI,CAAC,mBAAmB,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAA;IACvD,CAAC,CAAC,CAAA;IACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,CAAA;IAE3C,kBAAkB;IAClB,OAAO,CAAC,KAAK,CAAC,UAAU,GAAG,EAAE,KAAK;QAChC,IAAI,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAC5B,IAAI,KAAK,GAAG,IAAI,MAAM,CAAC,GAAG,YAAY,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAA;YACpD,SAAS,GAAG,SAAS,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAA;YACxC,OAAO,KAAK,CAAA;QACd,CAAC;;YAAM,OAAO,IAAI,CAAA;IACpB,CAAC,CAAC,CAAA;IAEF,kBAAkB;IAClB,OAAO,CAAC,KAAK,CAAC,UAAU,GAAG,EAAE,KAAK;QAChC,IAAI,SAAS,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YAC9B,IAAI,KAAK,GAAG,IAAI,MAAM,CAAC,IAAI,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;YAC/C,SAAS,GAAG,SAAS,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAA;YACxC,OAAO,KAAK,CAAA;QACd,CAAC;;YAAM,OAAO,IAAI,CAAA;IACpB,CAAC,CAAC,CAAA;IAEF,iBAAiB;IACjB,IAAI,iCAAiC,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QACtD,SAAS,GAAG,SAAS,CAAC,OAAO,CAC3B,sBAAsB,EACtB,SAAS,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,CAC5B,CAAA;IACH,CAAC;SAAM,IAAI,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QACxC,SAAS,GAAG,SAAS,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;IAC3C,CAAC;IAED,MAAM,QAAQ,GAAG,SAAS,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAA;IAChE,IAAI,QAAQ,EAAE,CAAC;QACb,SAAS,GAAG,SAAS,CAAC,OAAO,CAC3B,8BAA8B,EAC9B,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,GAAG,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAC7D,CAAA;IACH,CAAC;IAED,OAAO,mBAAmB,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,CAAC,CAAA;AACnD,CAAC;AAED,eAAe,IAAI,CAAA"}

package/dist/stopword_remover.d.ts

@@ -0,0 +1,12 @@
+import type { LanguagePack } from './types.js';
+/**
+ * Removes commonly occuring words that have no contribution to the semantics of the corpus.
+ * @param corpus : Amharic text
+ * @param pack : language pack configuration
+ * @returns : the corpus without stopwords listed on {@link LanguagePack.stopwords}
+ *
+ * @example {remove stopwords}
+ * removeStopwords("ይህ ሞባይል እና ኮምፒዩተር", amPack) // returns "ሞባይል ኮምፒዩተር"
+ */
+export declare function removeStopwords(corpus: string, pack: LanguagePack): string;
+export default removeStopwords;

package/dist/stopword_remover.js

@@ -0,0 +1,33 @@
+function escapeRegExp(string) {
+    return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+/**
+ * Removes commonly occuring words that have no contribution to the semantics of the corpus.
+ * @param corpus : Amharic text
+ * @param pack : language pack configuration
+ * @returns : the corpus without stopwords listed on {@link LanguagePack.stopwords}
+ *
+ * @example {remove stopwords}
+ * removeStopwords("ይህ ሞባይል እና ኮምፒዩተር", amPack) // returns "ሞባይል ኮምፒዩተር"
+ */
+export function removeStopwords(corpus, pack) {
+    let result = corpus;
+    // Sort stopwords by length descending to match longer words first
+    const sortedStopwords = [...pack.stopwords].sort((a, b) => b.length - a.length);
+    sortedStopwords.forEach((word) => {
+        // Regex that matches:
+        // Group 1: Leading boundary (non-Ge'ez char or start of string)
+        // Group 2: Optional prefix (standard prepositions)
+        // Group 3: The stopword itself
+        // Group 4: Optional suffix (standard markers)
+        // Followed by lookahead for trailing boundary (non-Ge'ez char or end of string)
+        const regex = new RegExp(`(^|[^\\u1200-\\u137F])(የ|በ|ከ|ለ|ስለ|የሚ|የማ)?(${escapeRegExp(word)})(ም|ን)?(?=[^\\u1200-\\u137F]|$)`, 'g');
+        result = result.replace(regex, (match, p1, p2, p3, p4) => {
+            // Keep boundary, prefix, and suffix, remove the stopword itself
+            return `${p1}${p2 || ''}${p4 || ''}`;
+        });
+    });
+    return result;
+}
+export default removeStopwords;
+//# sourceMappingURL=stopword_remover.js.map

package/dist/stopword_remover.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"stopword_remover.js","sourceRoot":"","sources":["../src/stopword_remover.ts"],"names":[],"mappings":"AAGA,SAAS,YAAY,CAAC,MAAc;IAClC,OAAO,MAAM,CAAC,OAAO,CAAC,qBAAqB,EAAE,MAAM,CAAC,CAAA;AACtD,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,eAAe,CAAC,MAAc,EAAE,IAAkB;IAChE,IAAI,MAAM,GAAG,MAAM,CAAA;IACnB,kEAAkE;IAClE,MAAM,eAAe,GAAG,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,CAAA;IAE/E,eAAe,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;QAC/B,sBAAsB;QACtB,gEAAgE;QAChE,mDAAmD;QACnD,+BAA+B;QAC/B,8CAA8C;QAC9C,gFAAgF;QAChF,MAAM,KAAK,GAAG,IAAI,MAAM,CACtB,6CAA6C,YAAY,CAAC,IAAI,CAAC,iCAAiC,EAChG,GAAG,CACJ,CAAA;QACD,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE;YACvD,gEAAgE;YAChE,OAAO,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,CAAA;QACtC,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,OAAO,MAAM,CAAA;AACf,CAAC;AAED,eAAe,eAAe,CAAA"}

package/dist/term_weighter.d.ts

@@ -0,0 +1,4 @@
+import type { DocIndexData, QueryIndexData } from "./indexer.js";
+export declare function weighTerms(index: DocIndexData | QueryIndexData, type: "doc" | "query"): Record<string, any>;
+export declare function weigh_terms(indexFilePath: string, outputWeightedTermsPath: string, typeOfIndex: "doc" | "query"): Promise<void>;
+export default weigh_terms;

package/dist/term_weighter.js

@@ -0,0 +1,71 @@
+// Pure function — no fs
+export function weighTerms(index, type) {
+    const weightedTerms = {};
+    if (type === "doc") {
+        const dataset = index;
+        let tf = 0;
+        let idf = 0;
+        let tf_idf = 0;
+        // calculate length normalized term frequency and inverse document frequency
+        Object.keys(dataset.words).forEach((word) => {
+            idf = Math.log2(dataset.corpus_size / dataset.words[word].length);
+            dataset.words[word].forEach((filePathObj) => {
+                let file = Object.keys(filePathObj)[0];
+                let freq = Object.values(filePathObj)[0];
+                tf = freq / dataset.corpus_word_count[file];
+                tf_idf = idf * tf;
+                // modify weighted_terms object
+                if (word in weightedTerms) {
+                    weightedTerms[word].push({ [file]: tf_idf });
+                }
+                else {
+                    weightedTerms[word] = [{ [file]: tf_idf }];
+                }
+            });
+        });
+    }
+    else if (type === "query") {
+        const dataset = index;
+        let tf = 0;
+        let idf = 1;
+        let tf_idf = 0;
+        // calculate length normalized term frequency and inverse document frequency
+        Object.keys(dataset.words).forEach((word) => {
+            let freq = dataset.words[word];
+            tf = freq / dataset.corpus_word_count;
+            tf_idf = idf * tf;
+            weightedTerms[word] = tf_idf;
+        });
+    }
+    return weightedTerms;
+}
+// Backwards-compat Node.js wrapper
+export async function weigh_terms(indexFilePath, outputWeightedTermsPath, typeOfIndex) {
+    const fs = await import("fs");
+    const weightedTermsPath = outputWeightedTermsPath + `/${typeOfIndex}WeightedTermsFile.json`;
+    // read index file
+    try {
+        const jsonString = fs.readFileSync(indexFilePath, "utf8");
+        try {
+            const dataset = JSON.parse(jsonString);
+            const result = weighTerms(dataset, typeOfIndex);
+            // output to file
+            const outJsonString = JSON.stringify(result, null, 2);
+            try {
+                fs.writeFileSync(weightedTermsPath, outJsonString);
+                console.log(`Indexed terms successfully weighted`);
+            }
+            catch (error) {
+                console.log("Weighted terms creation failed", error);
+            }
+        }
+        catch (err) {
+            console.log("Error parsing JSON string:", err);
+        }
+    }
+    catch (error) {
+        console.log(`Error reading Index file ${indexFilePath} from disk:`, error);
+    }
+}
+export default weigh_terms;
+//# sourceMappingURL=term_weighter.js.map

package/dist/term_weighter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"term_weighter.js","sourceRoot":"","sources":["../src/term_weighter.ts"],"names":[],"mappings":"AAEA,wBAAwB;AACxB,MAAM,UAAU,UAAU,CACtB,KAAoC,EACpC,IAAqB;IAErB,MAAM,aAAa,GAAwB,EAAE,CAAC;IAE9C,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;QACjB,MAAM,OAAO,GAAG,KAAqB,CAAC;QACtC,IAAI,EAAE,GAAG,CAAC,CAAC;QACX,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,IAAI,MAAM,GAAG,CAAC,CAAC;QAEf,4EAA4E;QAC5E,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;YACxC,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC;YAElE,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,OAAO,CACvB,CAAC,WAAmC,EAAE,EAAE;gBACpC,IAAI,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;gBACvC,IAAI,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;gBAEzC,EAAE,GAAG,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC;gBAC5C,MAAM,GAAG,GAAG,GAAG,EAAE,CAAC;gBAElB,+BAA+B;gBAC/B,IAAI,IAAI,IAAI,aAAa,EAAE,CAAC;oBACxB,aAAa,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;gBACjD,CAAC;qBAAM,CAAC;oBACJ,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;gBAC/C,CAAC;YACL,CAAC,CACJ,CAAC;QACN,CAAC,CAAC,CAAC;IACP,CAAC;SAAM,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;QAC1B,MAAM,OAAO,GAAG,KAAuB,CAAC;QACxC,IAAI,EAAE,GAAG,CAAC,CAAC;QACX,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,IAAI,MAAM,GAAG,CAAC,CAAC;QAEf,4EAA4E;QAC5E,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;YACxC,IAAI,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAE/B,EAAE,GAAG,IAAI,GAAG,OAAO,CAAC,iBAAiB,CAAC;YACtC,MAAM,GAAG,GAAG,GAAG,EAAE,CAAC;YAElB,aAAa,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC;QACjC,CAAC,CAAC,CAAC;IACP,CAAC;IAED,OAAO,aAAa,CAAC;AACzB,CAAC;AAED,mCAAmC;AACnC,MAAM,CAAC,KAAK,UAAU,WAAW,CAC7B,aAAqB,EACrB,uBAA+B,EAC/B,WAA4B;IAE5B,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;IAC9B,MAAM,iBAAiB,GACnB,uBAAuB,GAAG,IAAI,WAAW,wBAAwB,CAAC;IAEtE,kBAAkB;IAClB,IAAI,CAAC;QACD,MAAM,UAAU,GAAG,EAAE,CAAC,YAAY,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;QAC1D,IAAI,CAAC;YACD,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YACvC,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;YAEhD,iBAAiB;YACjB,MAAM,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;YAEtD,IAAI,CAAC;gBACD,EAAE,CAAC,aAAa,CAAC,iBAAiB,EAAE,aAAa,CAAC,CAAC;gBACnD,OAAO,CAAC,GAAG,CAAC,qCAAqC,CAAC,CAAC;YACvD,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACb,OAAO,CAAC,GAAG,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;YACzD,CAAC;QACL,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,OAAO,CAAC,GAAG,CAAC,4BAA4B,EAAE,GAAG,CAAC,CAAC;QACnD,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACb,OAAO,CAAC,GAAG,CACP,4BAA4B,aAAa,aAAa,EACtD,KAAK,CACR,CAAC;IACN,CAAC;AACL,CAAC;AAED,eAAe,WAAW,CAAC"}

package/dist/transliterator.d.ts

@@ -0,0 +1,20 @@
+import type { LanguagePack } from './types.js';
+/**
+ *@deprecated Use {@link felig_transliterate} function instead.
+ */
+export declare function sera_transliterate(word: string, lang: "am" | "en", pack: LanguagePack): string;
+/**
+ * Transliterates between Amharic and English
+ * @param word : English or Amharic word
+ * @param lang : language to transliterate form
+ * @returns : a transliterated string
+ *
+ * @example{ transliterate Amharic word to English}
+ * flig_transliterate("ወንበር","am") // returns "wenber"
+ */
+export declare function felig_transliterate(word: string, lang: "am" | "en", pack: LanguagePack): string;
+declare const transliterate: {
+    sera_transliterate: typeof sera_transliterate;
+    felig_transliterate: typeof felig_transliterate;
+};
+export default transliterate;

package/dist/transliterator.js

@@ -0,0 +1,88 @@
+/**
+ *@deprecated Use {@link felig_transliterate} function instead.
+ */
+export function sera_transliterate(word, lang, pack) {
+    let trans_word = "";
+    const sera_transliteration_lookup_table = pack.transliteration.sera.map;
+    if (lang === "am") {
+        let tokens = word.split("");
+        tokens.forEach((letter) => {
+            if (sera_transliteration_lookup_table[letter] !== undefined) {
+                trans_word += sera_transliteration_lookup_table[letter];
+            }
+        });
+    }
+    else if (lang === "en") {
+        let tokens = word.match(/.{1,2}/g);
+        if (tokens) {
+            tokens.forEach((letter) => {
+                let en_letter = Object.keys(sera_transliteration_lookup_table).find((key) => sera_transliteration_lookup_table[key] === letter);
+                if (en_letter !== undefined) {
+                    trans_word += en_letter;
+                }
+            });
+        }
+    }
+    return trans_word;
+}
+/**
+ * Transliterates between Amharic and English
+ * @param word : English or Amharic word
+ * @param lang : language to transliterate form
+ * @returns : a transliterated string
+ *
+ * @example{ transliterate Amharic word to English}
+ * flig_transliterate("ወንበር","am") // returns "wenber"
+ */
+export function felig_transliterate(word, lang, pack) {
+    let trans_word = "";
+    const felig_transliteration_lookup_table = pack.transliteration.felig.map;
+    if (lang === "am") {
+        let tokens = word.split("");
+        tokens.forEach((letter) => {
+            if (felig_transliteration_lookup_table[letter] !== undefined) {
+                trans_word += felig_transliteration_lookup_table[letter];
+            }
+        });
+    }
+    else if (lang === "en") {
+        let tokens = word.match(/.{1,2}/g);
+        if (tokens === null) {
+            return "";
+        }
+        tokens.forEach((letter) => {
+            if (/[^aeiou][aeiou]/i.test(letter)) {
+                let am_letter = "";
+                if (/[W][a]/g.test(letter)) {
+                    am_letter = Object.keys(felig_transliteration_lookup_table).find((key) => felig_transliteration_lookup_table[key] === letter.toLowerCase());
+                }
+                else {
+                    am_letter = Object.keys(felig_transliteration_lookup_table).find((key) => felig_transliteration_lookup_table[key] === letter);
+                }
+                if (am_letter !== undefined) {
+                    trans_word += am_letter;
+                }
+            }
+            else {
+                let ltrs = letter.split("");
+                let am_letter = "";
+                ltrs.forEach((ltr) => {
+                    const found = Object.keys(felig_transliteration_lookup_table).find((key) => felig_transliteration_lookup_table[key] === ltr);
+                    if (found !== undefined) {
+                        am_letter += found;
+                    }
+                });
+                if (am_letter !== "" && am_letter !== "ኧ") {
+                    trans_word += am_letter;
+                }
+            }
+        });
+    }
+    return trans_word;
+}
+const transliterate = {
+    sera_transliterate,
+    felig_transliterate,
+};
+export default transliterate;
+//# sourceMappingURL=transliterator.js.map

package/dist/transliterator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"transliterator.js","sourceRoot":"","sources":["../src/transliterator.ts"],"names":[],"mappings":"AAGA;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAY,EAAE,IAAiB,EAAE,IAAkB;IACpF,IAAI,UAAU,GAAG,EAAE,CAAA;IACnB,MAAM,iCAAiC,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,GAAG,CAAA;IAEvE,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;QAClB,IAAI,MAAM,GAAa,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;QACrC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE;YACxB,IAAI,iCAAiC,CAAC,MAAM,CAAC,KAAK,SAAS,EAAE,CAAC;gBAC5D,UAAU,IAAI,iCAAiC,CAAC,MAAM,CAAC,CAAA;YACzD,CAAC;QACH,CAAC,CAAC,CAAA;IACJ,CAAC;SAAM,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;QACzB,IAAI,MAAM,GAAoB,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;QACnD,IAAI,MAAM,EAAE,CAAC;YACX,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE;gBACxB,IAAI,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC,IAAI,CACjE,CAAC,GAAG,EAAE,EAAE,CAAC,iCAAiC,CAAC,GAAG,CAAC,KAAK,MAAM,CAC3D,CAAA;gBACD,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;oBAC5B,UAAU,IAAI,SAAS,CAAA;gBACzB,CAAC;YACH,CAAC,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;IAED,OAAO,UAAU,CAAA;AACnB,CAAC;AAED;;;;;;;;GAQG;AAEH,MAAM,UAAU,mBAAmB,CAAC,IAAY,EAAE,IAAiB,EAAE,IAAkB;IACrF,IAAI,UAAU,GAAG,EAAE,CAAA;IACnB,MAAM,kCAAkC,GAAG,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,GAAG,CAAA;IAEzE,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;QAClB,IAAI,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;QAC3B,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE;YACxB,IAAI,kCAAkC,CAAC,MAAM,CAAC,KAAK,SAAS,EAAE,CAAC;gBAC7D,UAAU,IAAI,kCAAkC,CAAC,MAAM,CAAC,CAAA;YAC1D,CAAC;QACH,CAAC,CAAC,CAAA;IACJ,CAAC;SAAM,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;QACzB,IAAI,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;QAElC,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;YACpB,OAAO,EAAE,CAAA;QACX,CAAC;QAED,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE;YACxB,IAAI,kBAAkB,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;gBACpC,IAAI,SAAS,GAAW,EAAE,CAAA;gBAE1B,IAAI,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;oBAC3B,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC,IAAI,CAC9D,CAAC,GAAG,EAAE,EAAE,CACN,kCAAkC,CAAC,GAAG,CAAC,KAAK,MAAM,CAAC,WAAW,EAAE,CAClE,CAAA;gBACJ,CAAC;qBAAM,CAAC;oBACN,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC,IAAI,CAC9D,CAAC,GAAG,EAAE,EAAE,CAAC,kCAAkC,CAAC,GAAG,CAAC,KAAK,MAAM,CAC3D,CAAA;gBACJ,CAAC;gBAED,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;oBAC5B,UAAU,IAAI,SAAS,CAAA;gBACzB,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,IAAI,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;gBAC3B,IAAI,SAAS,GAAG,EAAE,CAAA;gBAClB,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;oBACnB,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC,IAAI,CAChE,CAAC,GAAG,EAAE,EAAE,CAAC,kCAAkC,CAAC,GAAG,CAAC,KAAK,GAAG,CACzD,CAAA;oBACD,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;wBACxB,SAAS,IAAI,KAAK,CAAA;oBACpB,CAAC;gBACH,CAAC,CAAC,CAAA;gBAEF,IAAI,SAAS,KAAK,EAAE,IAAI,SAAS,KAAK,GAAG,EAAE,CAAC;oBAC1C,UAAU,IAAI,SAAS,CAAA;gBACzB,CAAC;YACH,CAAC;QACH,CAAC,CAAC,CAAA;IACJ,CAAC;IAED,OAAO,UAAU,CAAA;AACnB,CAAC;AAED,MAAM,aAAa,GAAG;IACpB,kBAAkB;IAClB,mBAAmB;CACpB,CAAA;AAED,eAAe,aAAa,CAAA"}

package/dist/types.d.ts

@@ -0,0 +1,54 @@
+export interface LanguagePackMeta {
+    code: string;
+    name: string;
+    native_name?: string;
+    script: string;
+    version?: string;
+    authors?: string[];
+}
+export interface NormalizationConfig {
+    char_map: Record<string, string>;
+    labialized_map: Record<string, string>;
+    gemination_threshold: number;
+}
+export interface TokenizationConfig {
+    split_on_spaces: boolean;
+    sentence_boundaries: string[];
+    punctuation: string[];
+    exceptions: Record<string, string[]>;
+}
+export interface StemmerConfig {
+    prefixes: string[];
+    suffixes: string[];
+    protected_words: string[];
+}
+export interface TransliterationSchemeConfig {
+    scheme: string;
+    map: Record<string, string>;
+}
+export interface TransliterationConfig {
+    sera: TransliterationSchemeConfig;
+    felig: TransliterationSchemeConfig;
+}
+export interface NumbersConfig {
+    ethiopic_to_arabic: Record<string, string>;
+}
+export interface SentimentConfig {
+    model: string;
+    lexicon: string;
+}
+export interface NERConfig {
+    model: string;
+    name_lists: string[];
+}
+export interface LanguagePack {
+    meta: LanguagePackMeta;
+    normalization?: NormalizationConfig;
+    tokenization?: TokenizationConfig;
+    stopwords: string[];
+    stemmer: StemmerConfig;
+    transliteration: TransliterationConfig;
+    numbers?: NumbersConfig;
+    sentiment?: SentimentConfig;
+    ner?: NERConfig;
+}

package/dist/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":""}

package/package.json

@@ -0,0 +1,44 @@
+{
+  "name": "@fidel-tools/core",
+  "version": "0.1.0",
+  "description": "Amharic Language Pre-processor toolkit",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "type": "module",
+  "files": [
+    "dist"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "keywords": [
+    "Amharic",
+    "Stemmer",
+    "Affixes",
+    "Amharic",
+    "Stopword",
+    "Amharic",
+    "Sentiment",
+    "analysis"
+  ],
+  "author": "Fidel Tools Team",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/Yehonatal/fidel-tools.git"
+  },
+  "homepage": "https://fidel-tools.vercel.app/",
+  "devDependencies": {
+    "@babel/core": "^7.19.0",
+    "@babel/plugin-transform-modules-commonjs": "^7.18.6",
+    "@types/node": "^20.19.42",
+    "jest": "^28.1.3",
+    "typescript": "^5.0.4"
+  },
+  "dependencies": {},
+  "scripts": {
+    "start": "node dist/index.js",
+    "build": "tsc",
+    "test": "pnpm jest --coverage"
+  }
+}