abstractor 1.0.8

data/lib/lingscope/nbproject/genfiles.properties

@@ -0,0 +1,8 @@
+build.xml.data.CRC32=f6f84b20
+build.xml.script.CRC32=f044596d
+build.xml.stylesheet.CRC32=28e38971@1.38.2.45
+# This file is used by a NetBeans-based IDE to track changes in generated files such as build-impl.xml.
+# Do not edit this file. You may delete it but then the IDE will never regenerate such files for you.
+nbproject/build-impl.xml.data.CRC32=f6f84b20
+nbproject/build-impl.xml.script.CRC32=46d3adf7
+nbproject/build-impl.xml.stylesheet.CRC32=5183f3c0@1.51.1.46

data/lib/lingscope/nbproject/private/private.properties

@@ -0,0 +1,10 @@
+compile.on.save=true
+do.depend=false
+do.jar=true
+file.reference.abner.jar=/Users/shashank/Documents/libraries/abner.jar
+file.reference.commons-lang-2.4.jar=/Users/shashank/Documents/libraries/commons-lang-2.4/commons-lang-2.4.jar
+file.reference.sptoolkit.jar=/Users/shashank/Documents/libraries/sptoolkit.jar
+file.reference.stanford-postagger.jar=/Users/shashank/Documents/libraries/stanford-postagger-full-2008-09-28/stanford-postagger.jar
+javac.debug=true
+javadoc.preview=true
+user.properties.file=/Users/shashank/Library/Application Support/NetBeans/7.2beta/build.properties

data/lib/lingscope/nbproject/private/private.xml

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project-private xmlns="http://www.netbeans.org/ns/project-private/1">
+    <editor-bookmarks xmlns="http://www.netbeans.org/ns/editor-bookmarks/1"/>
+    <editor-bookmarks xmlns="http://www.netbeans.org/ns/editor-bookmarks/2" lastBookmarkId="0"/>
+</project-private>

data/lib/lingscope/nbproject/project.properties

@@ -0,0 +1,85 @@
+annotation.processing.enabled=true
+annotation.processing.enabled.in.editor=false
+annotation.processing.run.all.processors=true
+annotation.processing.source.output=${build.generated.sources.dir}/ap-source-output
+application.title=lingscope
+application.vendor=shashank
+build.classes.dir=${build.dir}/classes
+build.classes.excludes=**/*.java,**/*.form
+# This directory is removed when the project is cleaned:
+build.dir=build
+build.generated.dir=${build.dir}/generated
+build.generated.sources.dir=${build.dir}/generated-sources
+# Only compile against the classpath explicitly listed here:
+build.sysclasspath=ignore
+build.test.classes.dir=${build.dir}/test/classes
+build.test.results.dir=${build.dir}/test/results
+# Uncomment to specify the preferred debugger connection transport:
+#debug.transport=dt_socket
+debug.classpath=\
+    ${run.classpath}
+debug.test.classpath=\
+    ${run.test.classpath}
+# This directory is removed when the project is cleaned:
+dist.dir=dist
+dist.jar=${dist.dir}/lingscope.jar
+dist.javadoc.dir=${dist.dir}/javadoc
+endorsed.classpath=
+excludes=
+file.reference.abner.jar=..\\..\\libraries\\abner.jar
+file.reference.commons-lang-2.4.jar=..\\..\\libraries\\commons-lang-2.4\\commons-lang-2.4.jar
+file.reference.java_utils.jar=..\\java_utils\\dist\\java_utils.jar
+file.reference.sptoolkit.jar=..\\..\\libraries\\sptoolkit.jar
+file.reference.stanford-postagger.jar=..\\..\\libraries\\stanford-postagger-full-2008-09-28\\stanford-postagger.jar
+includes=**
+jar.compress=false
+javac.classpath=\
+    ${file.reference.java_utils.jar}:\
+    ${file.reference.sptoolkit.jar}:\
+    ${file.reference.abner.jar}:\
+    ${file.reference.commons-lang-2.4.jar}:\
+    ${file.reference.stanford-postagger.jar}
+# Space-separated list of extra javac options
+javac.compilerargs=
+javac.deprecation=false
+javac.processorpath=\
+    ${javac.classpath}
+javac.source=1.5
+javac.target=1.5
+javac.test.classpath=\
+    ${javac.classpath}:\
+    ${build.classes.dir}:\
+    ${libs.junit.classpath}:\
+    ${libs.junit_4.classpath}
+javac.test.processorpath=\
+    ${javac.test.classpath}
+javadoc.additionalparam=
+javadoc.author=false
+javadoc.encoding=${source.encoding}
+javadoc.noindex=false
+javadoc.nonavbar=false
+javadoc.notree=false
+javadoc.private=false
+javadoc.reference.commons-lang-2.4.jar=../../libraries/commons-lang-2.4/commons-lang-2.4-javadoc.jar
+javadoc.splitindex=true
+javadoc.use=true
+javadoc.version=false
+javadoc.windowtitle=
+main.class=lingscope.Main
+manifest.file=manifest.mf
+meta.inf.dir=${src.dir}/META-INF
+platform.active=default_platform
+run.classpath=\
+    ${javac.classpath}:\
+    ${build.classes.dir}
+# Space-separated list of JVM arguments used when running the project
+# (you may also define separate properties like run-sys-prop.name=value instead of -Dname=value
+# or test-sys-prop.name=value to set system properties for unit tests):
+run.jvmargs=
+run.test.classpath=\
+    ${javac.test.classpath}:\
+    ${build.test.classes.dir}
+source.encoding=UTF-8
+source.reference.commons-lang-2.4.jar=../../libraries/commons-lang-2.4/commons-lang-2.4-sources.jar
+src.dir=src
+test.src.dir=test

data/lib/lingscope/nbproject/project.xml

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://www.netbeans.org/ns/project/1">
+    <type>org.netbeans.modules.java.j2seproject</type>
+    <configuration>
+        <data xmlns="http://www.netbeans.org/ns/j2se-project/3">
+            <name>lingscope</name>
+            <source-roots>
+                <root id="src.dir"/>
+            </source-roots>
+            <test-roots>
+                <root id="test.src.dir"/>
+            </test-roots>
+        </data>
+    </configuration>
+</project>

data/lib/lingscope/negation_models/baseline_cue_all_both.model

@@ -0,0 +1,49 @@
+absence of
+loss
+lacks
+fails
+no
+failed
+absent
+impossible
+favored over
+missing
+over
+rather than
+fail
+with the exception of
+favored
+none
+no longer
+( not
+( no
+except
+may
+lacked
+unable
+rule out
+or
+absence
+exclude
+lack
+without
+cannot
+lacking
+excluding
+neither
+not
+failure
+never
+nor
+either
+can not
+instead of
+instead
+could not
+negative
+with the notable exception of
+denied
+denies
+allergy
+allergies
+allergic

data/lib/lingscope/negation_models/baseline_cue_all_clinical.model

@@ -0,0 +1,19 @@
+absence of
+may
+rule out
+no
+exclude
+lack
+without
+cannot
+not
+favored over
+neither
+either
+over
+nor
+rather than
+favored
+none
+negative
+no longer

data/lib/lingscope/src/lingscope/algorithms/AbnerTokenizer.java

@@ -0,0 +1,58 @@
+/*
+ * To change this template, choose Tools | Templates
+ * and open the template in the editor.
+ */
+
+package lingscope.algorithms;
+
+import abner.Scanner;
+import java.io.StringReader;
+
+/**
+ *
+ * @author shashank
+ */
+public class AbnerTokenizer {
+    ////////////////////////////////////////////////////////////////
+    /**
+       <p>Take raw text apply ABNER's built-in tokenization on it.
+     */
+    public static String tokenize(String s) {
+	StringBuffer sb = new StringBuffer();
+	try {
+	    Scanner scanner = new Scanner(new StringReader(s));
+	    String t;
+	    while ((t = scanner.nextToken()) != null) {
+		sb.append(t+" ");
+		if (t.toString().matches("[?!\\.]"))
+		    sb.append("\n");
+	    }
+	    return sb.toString();
+	} catch (Exception e) {
+	    System.err.println(e);
+	}
+	return sb.toString();
+    }
+
+    /**
+     * Takes an input and splits the sentence by punctuations and spaces, then
+     * stitches it back together with a space and returns
+     * @param input the input string to process
+     * @return processed input string, where all words and punctuations are
+     * seperated by space
+     */
+    public static String splitTermsByPunctuation(String input) {
+        if (input.isEmpty()) {
+            return "";
+        }
+        input = input.replaceAll("\\n", " ");
+        String ret = tokenize(input).trim();
+        if (ret.matches(".*\\w\\.$")) { // If a space is not put between the period in the end, then introduce one
+            ret += " .";
+        }
+        if (input.endsWith(".") && (!ret.endsWith("."))) {
+            ret += " .";
+        }
+        return ret;
+    }
+}

data/lib/lingscope/src/lingscope/algorithms/AnnotationComparer.java

@@ -0,0 +1,207 @@
+package lingscope.algorithms;
+
+import generalutils.Statistics;
+import java.util.List;
+import lingscope.io.AnnotatedSentencesIO;
+import lingscope.structures.AnnotatedSentence;
+
+/**
+ * Compares two annotations or two annotation files
+ * @author shashank
+ */
+public class AnnotationComparer {
+
+    private double tp;
+    private double fp;
+    private double fn;
+    private double tn;
+    private double perfectMatches;
+    private int totalSentences;
+    private int numFolds;
+    private double[] tpFolds;
+    private double[] fpFolds;
+    private double[] fnFolds;
+    private double[] tnFolds;
+    private double[] perfectMatchesFolds;
+    private double[] totalSentencesFolds;
+
+    public AnnotationComparer(int numFolds) {
+        this.numFolds = numFolds;
+        reset();
+    }
+
+    /**
+     * Resets the values for the comparer
+     */
+    public final void reset() {
+        tp = 0;
+        fp = 0;
+        fn = 0;
+        tn = 0;
+        perfectMatches = 0;
+        totalSentences = 0;
+        tpFolds = new double[numFolds];
+        resetFold(tpFolds, numFolds);
+        fpFolds = new double[numFolds];
+        resetFold(fpFolds, numFolds);
+        fnFolds = new double[numFolds];
+        resetFold(fnFolds, numFolds);
+        tnFolds = new double[numFolds];
+        resetFold(tnFolds, numFolds);
+        perfectMatchesFolds = new double[numFolds];
+        resetFold(perfectMatchesFolds, numFolds);
+        totalSentencesFolds = new double[numFolds];
+        resetFold(totalSentencesFolds, numFolds);
+    }
+
+    private void resetFold(double[] folds, int numFolds) {
+        for (int i = 0; i < numFolds; ++i) {
+            folds[i] = 0;
+        }
+    }
+
+    public void compareAnnotationFiles(String goldFile, String testFile) {
+        List<AnnotatedSentence> goldSentences = AnnotatedSentencesIO.read(goldFile);
+        List<AnnotatedSentence> testSentences = AnnotatedSentencesIO.read(testFile);
+        if (goldSentences.size() != testSentences.size()) {
+            throw new RuntimeException("Number of sentences in gold and test file are not same");
+        }
+        for (int i = 0; i < goldSentences.size(); ++i) {
+            AnnotatedSentence goldSentence = goldSentences.get(i);
+            AnnotatedSentence testSentence = testSentences.get(i);
+            compareAnnotations(goldSentence, testSentence);
+        }
+    }
+
+    public void compareAnnotations(AnnotatedSentence goldSentence, AnnotatedSentence testSentence) {
+        int localTp = 0;
+        int localFp = 0;
+        int localFn = 0;
+        int localTn = 0;
+        if (goldSentence.getIsAnnotatedTags().size() != testSentence.getIsAnnotatedTags().size()) {
+            System.err.println("Size mismatch GOLD: " + goldSentence.getRawText());
+            System.err.println("Size mismatch TEST: " + testSentence.getRawText());
+        }
+        int numTags = Math.min(goldSentence.getIsAnnotatedTags().size(), testSentence.getIsAnnotatedTags().size());
+        for (int i = 0; i < numTags; ++i) {
+            boolean goldTag = goldSentence.getIsAnnotatedTags().get(i);
+            boolean testTag = testSentence.getIsAnnotatedTags().get(i);
+            if (goldTag && testTag) {
+                ++localTp;
+            } else if (goldTag && (!testTag)) {
+                ++localFn;
+            } else if ((!goldTag) && testTag) {
+                ++localFp;
+            } else if ((!goldTag) && (!testTag)) {
+                ++localTn;
+            }
+        }
+        int foldNum = totalSentences % numFolds;
+        tp += localTp;
+        fp += localFp;
+        fn += localFn;
+        tn += localTn;
+        tpFolds[foldNum] += localTp;
+        fpFolds[foldNum] += localFp;
+        fnFolds[foldNum] += localFn;
+        tnFolds[foldNum] += localTn;
+        if (localFp != 0) {
+            System.out.println("FP Gold: " + goldSentence.getRawText());
+            System.out.println("FP Test: " + testSentence.getRawText());
+        }
+        if (localFn != 0) {
+            System.out.println("FN Gold: " + goldSentence.getRawText());
+            System.out.println("FN Test: " + testSentence.getRawText());
+        }
+        if (localFp == 0 && localFn == 0) {
+            ++perfectMatches;
+            ++perfectMatchesFolds[foldNum];
+        }
+        ++totalSentences;
+        ++totalSentencesFolds[foldNum];
+    }
+
+    public void printStats() {
+        System.out.println("TP: " + tp);
+        System.out.println("FP: " + fp);
+        System.out.println("FN: " + fn);
+        System.out.println("TN: " + tn);
+        double precision = getPrecision(tp, fp);
+        double recall = getRecall(tp, fn);
+        System.out.println("Overall precision: " + precision);
+        System.out.println("Overall recall: " + recall);
+        System.out.println("Overall F1-score: " + getFScore(recall, precision));
+        System.out.println("Overall Accuracy: " + getAccuracy(tp, fp, fn, tn));
+        System.out.println("Overall Perfect : " + getPerfectAccuracy(perfectMatches, totalSentences));
+        double[] recalls = getRecalls();
+        double[] precisions = getPrecisions();
+        double[] fScores = getFScores(recalls, precisions);
+        double[] accuracies = getAccuracies();
+        double[] perfectAccuracies = getPerfectAccuracies();
+        System.out.println("Average precision: " + Statistics.mean(precisions) + " +/- " + Statistics.stdDev(precisions));
+        System.out.println("Average recall: " + Statistics.mean(recalls) + " +/- " + Statistics.stdDev(recalls));
+        System.out.println("Average F1-score: " + Statistics.mean(fScores) + " +/- " + Statistics.stdDev(fScores));
+        System.out.println("Average Accuracy: " + Statistics.mean(accuracies) + " +/- " + Statistics.stdDev(accuracies));
+        System.out.println("Average Perfect: " + Statistics.mean(perfectAccuracies) + " +/- " + Statistics.stdDev(perfectAccuracies));
+    }
+
+    public double[] getRecalls() {
+        double[] recalls = new double[numFolds];
+        for (int i = 0; i < numFolds; ++i) {
+            recalls[i] = getRecall(tpFolds[i], fnFolds[i]);
+        }
+        return recalls;
+    }
+
+    public double[] getPrecisions() {
+        double[] precisions = new double[numFolds];
+        for (int i = 0; i < numFolds; ++i) {
+            precisions[i] = getPrecision(tpFolds[i], fpFolds[i]);
+        }
+        return precisions;
+    }
+
+    public double[] getFScores(double[] recalls, double[] precisions) {
+        double[] fScores = new double[numFolds];
+        for (int i = 0; i < numFolds; ++i) {
+            fScores[i] = getFScore(recalls[i], precisions[i]);
+        }
+        return fScores;
+    }
+
+    public double[] getAccuracies() {
+        double[] accuracies = new double[numFolds];
+        for (int i = 0; i < numFolds; ++i) {
+            accuracies[i] = getAccuracy(tpFolds[i], fpFolds[i], fnFolds[i], tnFolds[i]);
+        }
+        return accuracies;
+    }
+
+    public double[] getPerfectAccuracies() {
+        double[] pcss = new double[numFolds];
+        for (int i = 0; i < numFolds; ++i) {
+            pcss[i] = getPerfectAccuracy(perfectMatchesFolds[i], totalSentencesFolds[i]);
+        }
+        return pcss;
+    }
+
+    public double getPerfectAccuracy(double trues, double total) {
+        return trues / total;
+    }
+
+    public double getAccuracy(double tp, double fp, double fn, double tn) {
+        return (tp + tn) / (tp + fp + fn + tn);
+    }
+
+    public double getRecall(double tp, double fn) {
+        return tp / (tp + fn);
+    }
+
+    public double getPrecision(double tp, double fp) {
+        return tp / (tp + fp);
+    }
+
+    public double getFScore(double recall, double precision) {
+        return 2 * precision * recall / (recall + precision);
+    }
+}