@gabrielrufino/cerebrum 1.3.0 → 1.4.0

package/.github/workflows/cd.yml

@@ -13,8 +13,8 @@ jobs:
     name: Node package CD
     runs-on: ubuntu-latest
     steps:
-      - uses: gabrielrufino/check-ci@main
+      - uses: actalog/check-ci@main
       - uses: actions/checkout@v4
-      - uses: gabrielrufino/node-pkg-cd@v1
+      - uses: actalog/node-pkg-cd@v1
         with:
           node-auth-token: ${{ secrets.NODE_AUTH_TOKEN }}

package/.github/workflows/ci.yml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: gabrielrufino/node-ci@v3
+      - uses: actalog/node-ci@v3
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.sha }}

package/dist/NLP/Tokenizer.js

@@ -0,0 +1,26 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Tokenizer = void 0;
+class Tokenizer {
+    constructor(text = '', ignore = []) {
+        this.text = text;
+        this.ignore = ignore;
+    }
+    setText(text) {
+        this.text = text;
+        return this;
+    }
+    setIgnore(ignore) {
+        this.ignore = ignore;
+        return this;
+    }
+    execute() {
+        const ignore = this.ignore.map(item => item.toLocaleLowerCase());
+        const punctuation = /[.,;:!?"]/g;
+        return this.text
+            .replace(punctuation, '')
+            .split(/\s+/)
+            .filter(token => token && !ignore.includes(token.toLocaleLowerCase()));
+    }
+}
+exports.Tokenizer = Tokenizer;

package/dist/NLP/Tokenizer.test.js

@@ -0,0 +1,49 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const vitest_1 = require("vitest");
+const Tokenizer_1 = require("./Tokenizer");
+(0, vitest_1.describe)('Tokenizer', () => {
+    (0, vitest_1.it)('should tokenize a simple text', () => {
+        const tokens = new Tokenizer_1.Tokenizer()
+            .setText("Hello world")
+            .execute();
+        (0, vitest_1.expect)(tokens).toEqual(['Hello', 'world']);
+    });
+    (0, vitest_1.it)('should remove punctuation', () => {
+        const tokens = new Tokenizer_1.Tokenizer()
+            .setText("Hello, world!")
+            .execute();
+        (0, vitest_1.expect)(tokens).toEqual(['Hello', 'world']);
+    });
+    (0, vitest_1.it)('should handle multiple spaces between words', () => {
+        const tokens = new Tokenizer_1.Tokenizer()
+            .setText("Hello     world")
+            .execute();
+        (0, vitest_1.expect)(tokens).toEqual(['Hello', 'world']);
+    });
+    (0, vitest_1.it)('should ignore specified words', () => {
+        const tokens = new Tokenizer_1.Tokenizer()
+            .setText("Hello, world! Let's tokenize this text with attention to hyphenated-words.")
+            .setIgnore(['this', 'with', 'to'])
+            .execute();
+        (0, vitest_1.expect)(tokens).toEqual(['Hello', 'world', "Let's", 'tokenize', 'text', 'attention', 'hyphenated-words']);
+    });
+    (0, vitest_1.it)('should return an empty array for empty text', () => {
+        const tokens = new Tokenizer_1.Tokenizer()
+            .execute();
+        (0, vitest_1.expect)(tokens).toEqual([]);
+    });
+    (0, vitest_1.it)('should allow setting text and ignore list after instantiation', () => {
+        const tokenizer = new Tokenizer_1.Tokenizer();
+        tokenizer.setText("Testing tokenization").setIgnore(['Testing']);
+        const tokens = tokenizer.execute();
+        (0, vitest_1.expect)(tokens).toEqual(['tokenization']);
+    });
+    (0, vitest_1.it)('should be case insensitive when filtering ignored words', () => {
+        const tokens = new Tokenizer_1.Tokenizer()
+            .setText("Tokenize and ignore some Words")
+            .setIgnore(['and', 'some', 'words'])
+            .execute();
+        (0, vitest_1.expect)(tokens).toEqual(['Tokenize', 'ignore']);
+    });
+});

package/dist/NLP/index.js

@@ -0,0 +1,17 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./Tokenizer"), exports);

package/dist/index.js

@@ -10,13 +10,37 @@ var __createBinding = (this && this.__createBinding) || (Object.create ? (functi
     if (k2 === undefined) k2 = k;
     o[k2] = m[k];
 }));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
 var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.NLP = void 0;
 __exportStar(require("./Math/Fibonacci"), exports);
 __exportStar(require("./Math/LinearRegression"), exports);
 __exportStar(require("./Math/TwoSum"), exports);
+exports.NLP = __importStar(require("./NLP"));
 __exportStar(require("./Search/BinarySearch"), exports);
 __exportStar(require("./Search/LinearSearch"), exports);
 __exportStar(require("./Sort/BubbleSort"), exports);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gabrielrufino/cerebrum",
-  "version": "1.3.0",
+  "version": "1.4.0",
   "description": "Algorithms made in TypeScript",
   "main": "dist/index.js",
   "type": "module",
@@ -15,14 +15,14 @@
   "author": "Gabriel Rufino <contato@gabrielrufino.com>",
   "license": "UNLICENSED",
   "devDependencies": {
-    "@commitlint/cli": "^19.5.0",
-    "@commitlint/config-conventional": "^19.5.0",
+    "@commitlint/cli": "^19.6.0",
+    "@commitlint/config-conventional": "^19.6.0",
     "@faker-js/faker": "^8.4.1",
     "@gabrielrufino/eslint-config": "^1.6.0",
-    "@vitest/coverage-v8": "^2.1.4",
+    "@vitest/coverage-v8": "^2.1.5",
     "eslint": "^8.57.1",
-    "husky": "^9.1.6",
-    "typescript": "^5.6.3",
+    "husky": "^9.1.7",
+    "typescript": "^5.7.2",
     "vitest": "^2.1.0"
   },
   "funding": [

package/src/NLP/Tokenizer.test.ts

@@ -0,0 +1,60 @@
+import { describe, it, expect } from 'vitest'
+import { Tokenizer } from './Tokenizer'
+
+describe('Tokenizer', () => {
+  it('should tokenize a simple text', () => {
+    const tokens = new Tokenizer()
+      .setText("Hello world")
+      .execute()
+
+    expect(tokens).toEqual(['Hello', 'world'])
+  })
+
+  it('should remove punctuation', () => {
+    const tokens = new Tokenizer()
+        .setText("Hello, world!")
+        .execute()
+
+    expect(tokens).toEqual(['Hello', 'world'])
+  })
+
+  it('should handle multiple spaces between words', () => {
+    const tokens = new Tokenizer()
+      .setText("Hello     world")
+      .execute()
+
+    expect(tokens).toEqual(['Hello', 'world'])
+  })
+
+  it('should ignore specified words', () => {
+    const tokens = new Tokenizer()
+      .setText("Hello, world! Let's tokenize this text with attention to hyphenated-words.")
+      .setIgnore(['this', 'with', 'to'])
+      .execute()
+
+    expect(tokens).toEqual(['Hello', 'world', "Let's", 'tokenize', 'text', 'attention', 'hyphenated-words'])
+  })
+
+  it('should return an empty array for empty text', () => {
+    const tokens = new Tokenizer()
+      .execute()
+
+    expect(tokens).toEqual([])
+  })
+
+  it('should allow setting text and ignore list after instantiation', () => {
+    const tokenizer = new Tokenizer()
+    tokenizer.setText("Testing tokenization").setIgnore(['Testing'])
+    const tokens = tokenizer.execute()
+    expect(tokens).toEqual(['tokenization'])
+  })
+  
+  it('should be case insensitive when filtering ignored words', () => {
+    const tokens = new Tokenizer()
+      .setText("Tokenize and ignore some Words")
+      .setIgnore(['and', 'some', 'words'])
+      .execute()
+
+    expect(tokens).toEqual(['Tokenize', 'ignore'])
+  })
+})

package/src/NLP/Tokenizer.ts

@@ -0,0 +1,26 @@
+export class Tokenizer {
+  constructor (
+    private text: string = '',
+    private ignore: string[] = []
+  ) {}
+
+  public setText(text: string): this {
+    this.text = text
+    return this
+  }
+
+  public setIgnore(ignore: string[]): this {
+    this.ignore = ignore
+    return this
+  }
+
+  public execute(): string[] {
+    const ignore = this.ignore.map(item => item.toLocaleLowerCase())
+    const punctuation = /[.,;:!?"]/g
+
+    return this.text
+      .replace(punctuation, '')
+      .split(/\s+/)
+      .filter(token => token && !ignore.includes(token.toLocaleLowerCase()))
+  }
+}

package/src/NLP/index.ts

@@ -0,0 +1 @@
+export * from './Tokenizer'

package/src/index.ts

@@ -2,6 +2,8 @@ export * from './Math/Fibonacci';
 export * from './Math/LinearRegression';
 export * from './Math/TwoSum';
 
+export * as NLP from './NLP';
+
 export * from './Search/BinarySearch';
 export * from './Search/LinearSearch';