opener-opinion-detector-base 2.0.0

data/core/vendor/src/crfsuite/lib/crf/src/dataset.c

@@ -0,0 +1,106 @@
+/*
+ *      Implementation for data sets (dataset_t).
+ *
+ * Copyright (c) 2007-2010, Naoaki Okazaki
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the names of the authors nor the names of its contributors
+ *       may be used to endorse or promote products derived from this
+ *       software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
+ * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* $Id$ */
+
+#ifdef    HAVE_CONFIG_H
+#include <config.h>
+#endif/*HAVE_CONFIG_H*/
+
+#include <os.h>
+
+#include <stdlib.h>
+#include <crfsuite.h>
+#include "crfsuite_internal.h"
+
+void dataset_init_trainset(dataset_t *ds, crfsuite_data_t *data, int holdout)
+{
+    int i, n = 0;
+
+    for (i = 0;i < data->num_instances;++i) {
+        if (data->instances[i].group != holdout) {
+            ++n;
+        }
+    }
+
+    ds->data = data;
+    ds->num_instances = n;
+    ds->perm = (int*)malloc(sizeof(int) * n);
+
+    n = 0;
+    for (i = 0;i < data->num_instances;++i) {
+        if (data->instances[i].group != holdout) {
+            ds->perm[n++] = i;
+        }
+    }    
+}
+
+void dataset_init_testset(dataset_t *ds, crfsuite_data_t *data, int holdout)
+{
+    int i, n = 0;
+
+    for (i = 0;i < data->num_instances;++i) {
+        if (data->instances[i].group == holdout) {
+            ++n;
+        }
+    }
+
+    ds->data = data;
+    ds->num_instances = n;
+    ds->perm = (int*)malloc(sizeof(int) * n);
+
+    n = 0;
+    for (i = 0;i < data->num_instances;++i) {
+        if (data->instances[i].group == holdout) {
+            ds->perm[n++] = i;
+        }
+    }
+}
+
+void dataset_finish(dataset_t *ds)
+{
+    free(ds->perm);
+}
+
+void dataset_shuffle(dataset_t *ds)
+{
+    int i;
+    for (i = 0;i < ds->num_instances;++i) {
+        int j = rand() % ds->num_instances;
+        int tmp = ds->perm[j];
+        ds->perm[j] = ds->perm[i];
+        ds->perm[i] = tmp;
+    }
+}
+
+crfsuite_instance_t *dataset_get(dataset_t *ds, int i)
+{
+    return &ds->data->instances[ds->perm[i]];
+}

data/core/vendor/src/crfsuite/lib/crf/src/dictionary.c

@@ -0,0 +1,118 @@
+/*
+ *      Implementation of dictionary.
+ *
+ * Copyright (c) 2007-2010, Naoaki Okazaki
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the names of the authors nor the names of its contributors
+ *       may be used to endorse or promote products derived from this
+ *       software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
+ * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* $Id$ */
+
+#include <os.h>
+
+#include <stdlib.h>
+#include <string.h>
+
+#include <crfsuite.h>
+#include "quark.h"
+
+static int dictionary_addref(crfsuite_dictionary_t* dic)
+{
+    return crfsuite_interlocked_increment(&dic->nref);
+}
+
+static int dictionary_release(crfsuite_dictionary_t* dic)
+{
+    int count = crfsuite_interlocked_decrement(&dic->nref);
+    if (count == 0) {
+        quark_t *qrk = (quark_t*)dic->internal;
+        quark_delete(qrk);
+        free(dic);
+    }
+    return count;
+}
+
+static int dictionary_get(crfsuite_dictionary_t* dic, const char *str)
+{
+    quark_t *qrk = (quark_t*)dic->internal;
+    return quark_get(qrk, str);
+}
+
+static int dictionary_to_id(crfsuite_dictionary_t* dic, const char *str)
+{
+    quark_t *qrk = (quark_t*)dic->internal;
+    return quark_to_id(qrk, str);    
+}
+
+static int dictionary_to_string(crfsuite_dictionary_t* dic, int id, char const **pstr)
+{
+    quark_t *qrk = (quark_t*)dic->internal;
+    const char *str = quark_to_string(qrk, id);
+    if (str != NULL) {
+        char *dst = (char*)malloc(strlen(str)+1);
+        if (dst) {
+            strcpy(dst, str);
+            *pstr = dst;
+            return 0;
+        }
+    }
+    return 1;
+}
+
+static int dictionary_num(crfsuite_dictionary_t* dic)
+{
+    quark_t *qrk = (quark_t*)dic->internal;
+    return quark_num(qrk);
+}
+
+static void dictionary_free(crfsuite_dictionary_t* dic, const char *str)
+{
+    free((char*)str);
+}
+
+int crfsuite_dictionary_create_instance(const char *interface, void **ptr)
+{
+    if (strcmp(interface, "dictionary") == 0) {
+        crfsuite_dictionary_t* dic = (crfsuite_dictionary_t*)calloc(1, sizeof(crfsuite_dictionary_t));
+
+        if (dic != NULL) {
+            dic->internal = quark_new();
+            dic->nref = 1;
+            dic->addref = dictionary_addref;
+            dic->release = dictionary_release;
+            dic->get = dictionary_get;
+            dic->to_id = dictionary_to_id;
+            dic->to_string = dictionary_to_string;
+            dic->num = dictionary_num;
+            dic->free = dictionary_free;
+            *ptr = dic;
+            return 0;
+        } else {
+            return -1;
+        }
+    } else {
+        return 1;
+    }
+}

data/core/vendor/src/crfsuite/lib/crf/src/holdout.c

@@ -0,0 +1,80 @@
+/*
+ *      Holdout evaluation.
+ *
+ * Copyright (c) 2007-2010, Naoaki Okazaki
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the names of the authors nor the names of its contributors
+ *       may be used to endorse or promote products derived from this
+ *       software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
+ * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* $Id$ */
+
+#ifdef    HAVE_CONFIG_H
+#include <config.h>
+#endif/*HAVE_CONFIG_H*/
+
+#include <os.h>
+
+#include <stdlib.h>
+#include <crfsuite.h>
+#include "crfsuite_internal.h"
+#include "logging.h"
+
+void holdout_evaluation(
+    encoder_t *gm,
+    dataset_t *ds,
+    const floatval_t *w,
+    logging_t *lg
+    )
+{
+    int i;
+    crfsuite_evaluation_t eval;
+    const int N = ds->num_instances;
+    int *viterbi = NULL;
+    int max_length = 0;
+
+    /* Initialize the evaluation table. */
+    crfsuite_evaluation_init(&eval, ds->data->labels->num(ds->data->labels));
+
+    gm->set_weights(gm, w, 1.);
+
+    for (i = 0;i < N;++i) {
+        floatval_t score;
+        const crfsuite_instance_t *inst = dataset_get(ds, i);
+
+        if (max_length < inst->num_items) {
+            free(viterbi);
+            viterbi = (int*)malloc(sizeof(int) * inst->num_items);
+        }
+
+        gm->set_instance(gm, inst);
+        gm->viterbi(gm, viterbi, &score);
+
+        crfsuite_evaluation_accmulate(&eval, inst->labels, viterbi, inst->num_items);
+    }
+
+    /* Report the performance. */
+    crfsuite_evaluation_finalize(&eval);
+    crfsuite_evaluation_output(&eval, ds->data->labels, lg->func, lg->instance);
+}

data/core/vendor/src/crfsuite/lib/crf/src/logging.c

@@ -0,0 +1,91 @@
+/*
+ *      Logging utility.
+ *
+ * Copyright (c) 2007-2010, Naoaki Okazaki
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the names of the authors nor the names of its contributors
+ *       may be used to endorse or promote products derived from this
+ *       software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
+ * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* $Id$ */
+
+#include <os.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+
+#include <crfsuite.h>
+#include "logging.h"
+
+void logging(logging_t* lg, const char *format, ...)
+{
+    va_list args;
+    va_start(args, format);
+
+    if (lg != NULL && lg->func != NULL) {
+        lg->func(lg->instance, format, args);
+    }
+}
+
+void logging_timestamp(logging_t* lg, const char *format)
+{
+    time_t ts;
+    char timestamp[80];
+
+    time(&ts);
+    strftime(
+        timestamp, sizeof(timestamp),
+        "%Y-%m-%dT%H:%M:%SZ",
+        gmtime(&ts)
+        );
+    logging(lg, format, timestamp);
+}
+
+void logging_progress_start(logging_t* lg)
+{
+    lg->percent = 0;
+    logging(lg, "0");
+}
+
+void logging_progress(logging_t* lg, int percent)
+{
+    while (lg->percent < percent) {
+        ++lg->percent;
+        if (lg->percent % 2 == 0) {
+            if (lg->percent % 10 == 0) {
+                logging(lg, "%d", lg->percent / 10);
+            } else {
+                logging(lg, ".");
+            }
+        }
+    }
+}
+
+void logging_progress_end(logging_t* lg)
+{
+    logging_progress(lg, 100);
+    logging(lg, "\n");
+}

data/core/vendor/src/crfsuite/lib/crf/src/logging.h

@@ -0,0 +1,48 @@
+/*
+ *      Logging utility.
+ *
+ * Copyright (c) 2007-2010, Naoaki Okazaki
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the names of the authors nor the names of its contributors
+ *       may be used to endorse or promote products derived from this
+ *       software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
+ * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* $Id$ */
+
+#ifndef    __LOGGING_H__
+#define    __LOGGING_H__
+
+typedef struct {
+    void *instance;
+    crfsuite_logging_callback func;
+    int percent;
+} logging_t;
+
+void logging(logging_t* lg, const char *format, ...);
+void logging_timestamp(logging_t* lg, const char *format);
+void logging_progress_start(logging_t* lg);
+void logging_progress(logging_t* lg, int percent);
+void logging_progress_end(logging_t* lg);
+
+#endif/*__LOGGING_H__*/