nlptoolkit-postagger 1.0.0

package/source/PosTaggedCorpus.ts

@@ -0,0 +1,62 @@
+import {Corpus} from "nlptoolkit-corpus/dist/Corpus";
+import {CounterHashMap} from "nlptoolkit-datastructure/dist/CounterHashMap";
+import {Sentence} from "nlptoolkit-corpus/dist/Sentence";
+import * as fs from "fs";
+import {PosTaggedWord} from "./PosTaggedWord";
+
+export class PosTaggedCorpus extends Corpus{
+
+    private tagList: CounterHashMap<string> = new CounterHashMap<string>()
+
+    /**
+     * A constructor of {@link PosTaggedCorpus} which initializes the sentences of the corpus, the word list of
+     * the corpus, and all possible tags.
+     */
+    constructor(fileName?: string) {
+        super();
+        if (fileName != undefined) {
+            let newSentence = new Sentence();
+            let data = fs.readFileSync(fileName, 'utf8')
+            let lines = data.split("\n")
+            for (let line of lines) {
+                let words = line.split(/\s/);
+                for (let word of words){
+                    if (word != ""){
+                        if (word.includes("/")){
+                            let name = word.substring(0, word.lastIndexOf('/'));
+                            let tag = word.substring(word.lastIndexOf('/') + 1);
+                            let shortTag
+                            if (tag.includes("+")){
+                                shortTag = tag.substring(0, tag.indexOf("+"));
+                            } else {
+                                if (tag.includes("-")){
+                                    shortTag = tag.substring(0, tag.indexOf("-"));
+                                } else {
+                                    shortTag = tag;
+                                }
+                            }
+                            this.tagList.put(shortTag);
+                            newSentence.addWord(new PosTaggedWord(name, shortTag));
+                            if (tag == "."){
+                                this.addSentence(newSentence);
+                                newSentence = new Sentence();
+                            }
+                        }
+                    }
+                }
+            }
+            if (newSentence.wordCount() > 0){
+                this.addSentence(newSentence);
+            }
+        }
+    }
+
+    /**
+     * getTagList returns all possible tags as a set.
+     *
+     * @return Set of all possible tags.
+     */
+    getTagList(): IterableIterator<string>{
+        return this.tagList.keys()
+    }
+}

package/source/PosTaggedWord.ts

@@ -0,0 +1,25 @@
+import {Word} from "nlptoolkit-dictionary/dist/Dictionary/Word";
+
+export class PosTaggedWord extends Word{
+
+    private tag: string
+
+    /**
+     * A constructor of {@link PosTaggedWord} which takes name and tag as input and sets the corresponding attributes
+     * @param name Name of the word
+     * @param tag Tag of the word
+     */
+    constructor(name: string, tag: string) {
+        super(name);
+        this.tag = tag
+    }
+
+    /**
+     * Accessor method for tag attribute.
+     *
+     * @return Tag of the word.
+     */
+    getTag(): string{
+        return this.tag
+    }
+}

package/source/PosTagger.ts

@@ -0,0 +1,9 @@
+import {Sentence} from "nlptoolkit-corpus/dist/Sentence";
+import {PosTaggedCorpus} from "./PosTaggedCorpus";
+
+export interface PosTagger {
+
+    train(corpus: PosTaggedCorpus): void;
+    posTag(sentence: Sentence): Sentence;
+
+}

package/source/tsconfig.json

@@ -0,0 +1,13 @@
+{
+  "compilerOptions": {
+    "outDir": "../dist",
+    "module": "umd",
+    "target": "es6",
+    "sourceMap": true,
+    "noImplicitAny": true,
+    "strictNullChecks": false,
+    "removeComments": false,
+    "moduleResolution": "node",
+    "declaration": true
+  }
+}

package/tests/DummyPosTaggerTest.ts

@@ -0,0 +1,26 @@
+import * as assert from "assert";
+import {DummyPosTagger} from "../dist/DummyPosTagger";
+import {PosTaggedCorpus} from "../dist/PosTaggedCorpus";
+import {PosTaggedWord} from "../dist/PosTaggedWord";
+
+describe('DummyPosTaggerTest', function() {
+    describe('DummyPosTaggerTest', function() {
+        it('testPosTag', function() {
+            let posTagger = new DummyPosTagger();
+            let posTaggedCorpus = new PosTaggedCorpus("brown.txt");
+            posTagger.train(posTaggedCorpus);
+            let correct = 0, incorrect = 0;
+            for (let i = 0; i < posTaggedCorpus.sentenceCount(); i++){
+                let taggedSentence = posTagger.posTag(posTaggedCorpus.getSentence(i));
+                for (let j = 0; j < taggedSentence.wordCount(); j++){
+                    if ((<PosTaggedWord>posTaggedCorpus.getSentence(i).getWord(j)).getTag() == (<PosTaggedWord>taggedSentence.getWord(j)).getTag()){
+                        correct++;
+                    } else {
+                        incorrect++;
+                    }
+                }
+            }
+            assert.ok(Math.abs(100 * correct / (correct + incorrect) - 0.88) < 0.01);
+        });
+    });
+});

package/tests/HmmPosTaggerTest.ts

@@ -0,0 +1,26 @@
+import * as assert from "assert";
+import {PosTaggedCorpus} from "../dist/PosTaggedCorpus";
+import {PosTaggedWord} from "../dist/PosTaggedWord";
+import {HmmPosTagger} from "../dist/HmmPosTagger";
+
+describe('HmmPosTaggerTest', function() {
+    describe('HmmPosTaggerTest', function() {
+        it('testPosTag', function() {
+            let posTagger = new HmmPosTagger();
+            let posTaggedCorpus = new PosTaggedCorpus("brown.txt");
+            posTagger.train(posTaggedCorpus);
+            let correct = 0, incorrect = 0;
+            for (let i = 0; i < posTaggedCorpus.sentenceCount(); i++){
+                let taggedSentence = posTagger.posTag(posTaggedCorpus.getSentence(i));
+                for (let j = 0; j < taggedSentence.wordCount(); j++){
+                    if ((<PosTaggedWord>posTaggedCorpus.getSentence(i).getWord(j)).getTag() == (<PosTaggedWord>taggedSentence.getWord(j)).getTag()){
+                        correct++;
+                    } else {
+                        incorrect++;
+                    }
+                }
+            }
+            assert.ok(Math.abs(100 * correct / (correct + incorrect) - 97.59) < 0.01);
+        });
+    });
+});

package/tests/NaivePosTaggerTest.ts

@@ -0,0 +1,26 @@
+import * as assert from "assert";
+import {PosTaggedCorpus} from "../dist/PosTaggedCorpus";
+import {PosTaggedWord} from "../dist/PosTaggedWord";
+import {NaivePosTagger} from "../dist/NaivePosTagger";
+
+describe('NaivePosTaggerTest', function() {
+    describe('NaivePosTaggerTest', function() {
+        it('testPosTag', function() {
+            let posTagger = new NaivePosTagger();
+            let posTaggedCorpus = new PosTaggedCorpus("brown.txt");
+            posTagger.train(posTaggedCorpus);
+            let correct = 0, incorrect = 0;
+            for (let i = 0; i < posTaggedCorpus.sentenceCount(); i++){
+                let taggedSentence = posTagger.posTag(posTaggedCorpus.getSentence(i));
+                for (let j = 0; j < taggedSentence.wordCount(); j++){
+                    if ((<PosTaggedWord>posTaggedCorpus.getSentence(i).getWord(j)).getTag() == (<PosTaggedWord>taggedSentence.getWord(j)).getTag()){
+                        correct++;
+                    } else {
+                        incorrect++;
+                    }
+                }
+            }
+            assert.ok(Math.abs(100 * correct / (correct + incorrect) - 93.69) < 0.01);
+        });
+    });
+});

package/tsconfig.json

@@ -0,0 +1,15 @@
+{
+    "compilerOptions": {
+        "module": "commonjs",
+        "target": "es5",
+        "sourceMap": true,
+        "noImplicitAny": true,
+        "removeComments": false,
+        "moduleResolution": "node"
+    },
+    "exclude": [
+        "source",
+        "node_modules",
+        "dist"
+    ]
+}