opener-opinion-detector-basic 1.0.0

data/core/vendor/src/crfsuite/example/chunking.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python
+
+"""
+A feature extractor for chunking.
+Copyright 2010,2011 Naoaki Okazaki.
+"""
+
+# Separator of field values.
+separator = ' '
+
+# Field names of the input data.
+fields = 'w pos y'
+
+# Attribute templates.
+templates = (
+    (('w', -2), ),
+    (('w', -1), ),
+    (('w',  0), ),
+    (('w',  1), ),
+    (('w',  2), ),
+    (('w', -1), ('w',  0)),
+    (('w',  0), ('w',  1)),
+    (('pos', -2), ),
+    (('pos', -1), ),
+    (('pos',  0), ),
+    (('pos',  1), ),
+    (('pos',  2), ),
+    (('pos', -2), ('pos', -1)),
+    (('pos', -1), ('pos',  0)),
+    (('pos',  0), ('pos',  1)),
+    (('pos',  1), ('pos',  2)),
+    (('pos', -2), ('pos', -1), ('pos',  0)),
+    (('pos', -1), ('pos',  0), ('pos',  1)),
+    (('pos',  0), ('pos',  1), ('pos',  2)),
+    )
+
+
+import crfutils
+
+def feature_extractor(X):
+    # Apply attribute templates to obtain features (in fact, attributes)
+    crfutils.apply_templates(X, templates)
+    if X:
+	# Append BOS and EOS features manually
+        X[0]['F'].append('__BOS__')     # BOS feature
+        X[-1]['F'].append('__EOS__')    # EOS feature
+
+if __name__ == '__main__':
+    crfutils.main(feature_extractor, fields=fields, sep=separator)

data/core/vendor/src/crfsuite/example/crfutils.py

@@ -0,0 +1,179 @@
+"""
+A miscellaneous utility for sequential labeling.
+Copyright 2010,2011 Naoaki Okazaki.
+"""
+
+import optparse
+import sys
+
+def apply_templates(X, templates):
+    """
+    Generate features for an item sequence by applying feature templates.
+    A feature template consists of a tuple of (name, offset) pairs,
+    where name and offset specify a field name and offset from which
+    the template extracts a feature value. Generated features are stored
+    in the 'F' field of each item in the sequence.
+
+    @type   X:      list of mapping objects
+    @param  X:      The item sequence.
+    @type   template:   tuple of (str, int)
+    @param  template:   The feature template.
+    """
+    for template in templates:
+        name = '|'.join(['%s[%d]' % (f, o) for f, o in template])
+        for t in range(len(X)):
+            values = []
+            for field, offset in template:
+                p = t + offset
+                if p not in range(len(X)):
+                    values = []
+                    break
+                values.append(X[p][field])
+            if values:
+                X[t]['F'].append('%s=%s' % (name, '|'.join(values)))
+
+def readiter(fi, names, sep=' '):
+    """
+    Return an iterator for item sequences read from a file object.
+    This function reads a sequence from a file object L{fi}, and
+    yields the sequence as a list of mapping objects. Each line
+    (item) from the file object is split by the separator character
+    L{sep}. Separated values of the item are named by L{names},
+    and stored in a mapping object. Every item has a field 'F' that
+    is reserved for storing features.
+
+    @type   fi:     file
+    @param  fi:     The file object.
+    @type   names:  tuple
+    @param  names:  The list of field names.
+    @type   sep:    str
+    @param  sep:    The separator character.
+    @rtype          list of mapping objects
+    @return         An iterator for sequences.
+    """
+    X = []
+    for line in fi:
+        line = line.strip('\n')
+        if not line:
+            yield X
+            X = []
+        else:
+            fields = line.split(sep)
+            if len(fields) < len(names):
+                raise ValueError(
+                    'Too few fields (%d) for %r\n%s' % (len(fields), names, line))
+            item = {'F': []}    # 'F' is reserved for features.
+            for i in range(len(names)):
+                item[names[i]] = fields[i]
+            X.append(item)
+
+def escape(src):
+    """
+    Escape colon characters from feature names.
+
+    @type   src:    str
+    @param  src:    A feature name
+    @rtype          str
+    @return         The feature name escaped.
+    """
+    return src.replace(':', '__COLON__')
+
+def output_features(fo, X, field=''):
+    """
+    Output features (and reference labels) of a sequence in CRFSuite
+    format. For each item in the sequence, this function writes a
+    reference label (if L{field} is a non-empty string) and features.
+
+    @type   fo:     file
+    @param  fo:     The file object.
+    @type   X:      list of mapping objects
+    @param  X:      The sequence.
+    @type   field:  str
+    @param  field:  The field name of reference labels.
+    """
+    for t in range(len(X)):
+        if field:
+            fo.write('%s' % X[t][field])
+        for a in X[t]['F']:
+            if isinstance(a, str):
+                fo.write('\t%s' % escape(a))
+            else:
+                fo.write('\t%s:%f' % (escape(a[0]), a[1]))
+        fo.write('\n')
+    fo.write('\n')
+
+def to_crfsuite(X):
+    """
+    Convert an item sequence into an object compatible with crfsuite
+    Python module.
+
+    @type   X:      list of mapping objects
+    @param  X:      The sequence.
+    @rtype          crfsuite.ItemSequence
+    @return        The same sequence in crfsuite.ItemSequence type.
+    """
+    import crfsuite
+    xseq = crfsuite.ItemSequence()
+    for x in X:
+        item = crfsuite.Item()
+        for f in x['F']:
+            if isinstance(f, str):
+                item.append(crfsuite.Attribute(escape(f)))
+            else:
+                item.append(crfsuite.Attribute(escape(f[0]), f[1]))
+        xseq.append(item)
+    return xseq
+
+def main(feature_extractor, fields='w pos y', sep=' '):
+    fi = sys.stdin
+    fo = sys.stdout
+
+    # Parse the command-line arguments.
+    parser = optparse.OptionParser(usage="""usage: %prog [options]
+This utility reads a data set from STDIN, and outputs attributes to STDOUT.
+Each line of a data set must consist of field values separated by SEPARATOR
+characters. The names and order of field values can be specified by -f option.
+The separator character can be specified with -s option. Instead of outputting
+attributes, this utility tags the input data when a model file is specified by
+-t option (CRFsuite Python module must be installed)."""
+        )
+    parser.add_option(
+        '-t', dest='model',
+        help='tag the input using the model (requires "crfsuite" module)'
+        )
+    parser.add_option(
+        '-f', dest='fields', default=fields,
+        help='specify field names of input data [default: "%default"]'
+        )
+    parser.add_option(
+        '-s', dest='separator', default=sep,
+        help='specify the separator of columns of input data [default: "%default"]'
+        )
+    (options, args) = parser.parse_args()
+
+    # The fields of input: ('w', 'pos', 'y) by default.
+    F = options.fields.split(' ')
+
+    if not options.model:
+        # The generator function readiter() reads a sequence from a 
+        for X in readiter(fi, F, options.separator):
+            feature_extractor(X)
+            output_features(fo, X, 'y')
+
+    else:
+        # Create a tagger with an existing model.
+        import crfsuite
+        tagger = crfsuite.Tagger()
+        tagger.open(options.model)
+
+        # For each sequence from STDIN.
+        for X in readiter(fi, F, options.separator):
+            # Obtain features.
+            feature_extractor(X)
+            xseq = to_crfsuite(X)
+            yseq = tagger.tag(xseq)
+            for t in range(len(X)):
+                v = X[t]
+                fo.write('\t'.join([v[f] for f in F]))
+                fo.write('\t%s\n' % yseq[t])
+            fo.write('\n')

data/core/vendor/src/crfsuite/example/ner.py

@@ -0,0 +1,270 @@
+#!/usr/bin/env python
+
+"""
+A feature extractor for named eneity recognition (NER).
+Copyright 2010,2011 Naoaki Okazaki.
+"""
+
+# Separator of field values.
+separator = ' '
+
+# Field names of the input data.
+fields = 'y w pos chk'
+
+
+import crfutils
+
+def get_shape(token):
+    r = ''
+    for c in token:
+        if c.isupper():
+            r += 'U'
+        elif c.islower():
+            r += 'L'
+        elif c.isdigit():
+            r += 'D'
+        elif c in ('.', ','):
+            r += '.'
+        elif c in (';', ':', '?', '!'):
+            r += ';'
+        elif c in ('+', '-', '*', '/', '=', '|', '_'):
+            r += '-'
+        elif c in ('(', '{', '[', '<'):
+            r += '('
+        elif c in (')', '}', ']', '>'):
+            r += ')'
+        else:
+            r += c
+    return r
+
+def degenerate(src):
+    dst = ''
+    for c in src:
+        if not dst or dst[-1] != c:
+            dst += c
+    return dst
+
+def get_type(token):
+    T = (
+        'AllUpper', 'AllDigit', 'AllSymbol',
+        'AllUpperDigit', 'AllUpperSymbol', 'AllDigitSymbol',
+        'AllUpperDigitSymbol',
+        'InitUpper',
+        'AllLetter',
+        'AllAlnum',
+        )
+    R = set(T)
+    if not token:
+        return 'EMPTY'
+
+    for i in range(len(token)):
+        c = token[i]
+        if c.isupper():
+            R.discard('AllDigit')
+            R.discard('AllSymbol')
+            R.discard('AllDigitSymbol')
+        elif c.isdigit() or c in (',', '.'):
+            R.discard('AllUpper')
+            R.discard('AllSymbol')
+            R.discard('AllUpperSymbol')
+            R.discard('AllLetter')
+        elif c.islower():
+            R.discard('AllUpper')
+            R.discard('AllDigit')
+            R.discard('AllSymbol')
+            R.discard('AllUpperDigit')
+            R.discard('AllUpperSymbol')
+            R.discard('AllDigitSymbol')
+            R.discard('AllUpperDigitSymbol')
+        else:
+            R.discard('AllUpper')
+            R.discard('AllDigit')
+            R.discard('AllUpperDigit')
+            R.discard('AllLetter')
+            R.discard('AllAlnum')
+
+        if i == 0 and not c.isupper():
+            R.discard('InitUpper')
+
+    for tag in T:
+        if tag in R:
+            return tag
+    return 'NO'
+
+def get_2d(token):
+    return len(token) == 2 and token.isdigit()
+
+def get_4d(token):
+    return len(token) == 4 and token.isdigit()
+
+def get_da(token):
+    bd = False
+    ba = False
+    for c in token:
+        if c.isdigit():
+            bd = True
+        elif c.isalpha():
+            ba = True
+        else:
+            return False
+    return bd and ba
+
+def get_dand(token, p):
+    bd = False
+    bdd = False
+    for c in token:
+        if c.isdigit():
+            bd = True
+        elif c == p:
+            bdd = True
+        else:
+            return False
+    return bd and bdd
+
+def get_all_other(token):
+    for c in token:
+        if c.isalnum():
+            return False
+    return True
+
+def get_capperiod(token):
+    return len(token) == 2 and token[0].isupper() and token[1] == '.'
+
+def contains_upper(token):
+    b = False
+    for c in token:
+        b |= c.isupper()
+    return b
+
+def contains_lower(token):
+    b = False
+    for c in token:
+        b |= c.islower()
+    return b
+
+def contains_alpha(token):
+    b = False
+    for c in token:
+        b |= c.isalpha()
+    return b
+
+def contains_digit(token):
+    b = False
+    for c in token:
+        b |= c.isdigit()
+    return b
+
+def contains_symbol(token):
+    b = False
+    for c in token:
+        b |= ~c.isalnum()
+    return b
+
+def b(v):
+    return 'yes' if v else 'no'
+
+def observation(v, defval=''):
+    # Lowercased token.
+    v['wl'] = v['w'].lower()
+    # Token shape.
+    v['shape'] = get_shape(v['w'])
+    # Token shape degenerated.
+    v['shaped'] = degenerate(v['shape'])
+    # Token type.
+    v['type'] = get_type(v['w'])
+
+    # Prefixes (length between one to four).
+    v['p1'] = v['w'][0] if len(v['w']) >= 1 else defval
+    v['p2'] = v['w'][:2] if len(v['w']) >= 2 else defval
+    v['p3'] = v['w'][:3] if len(v['w']) >= 3 else defval
+    v['p4'] = v['w'][:4] if len(v['w']) >= 4 else defval
+
+    # Suffixes (length between one to four).
+    v['s1'] = v['w'][-1] if len(v['w']) >= 1 else defval
+    v['s2'] = v['w'][-2:] if len(v['w']) >= 2 else defval
+    v['s3'] = v['w'][-3:] if len(v['w']) >= 3 else defval
+    v['s4'] = v['w'][-4:] if len(v['w']) >= 4 else defval
+
+    # Two digits
+    v['2d'] = b(get_2d(v['w']))
+    # Four digits.
+    v['4d'] = b(get_4d(v['w']))
+    # Alphanumeric token.
+    v['d&a'] = b(get_da(v['w']))
+    # Digits and '-'.
+    v['d&-'] = b(get_dand(v['w'], '-'))
+    # Digits and '/'.
+    v['d&/'] = b(get_dand(v['w'], '/'))
+    # Digits and ','.
+    v['d&,'] = b(get_dand(v['w'], ','))
+    # Digits and '.'.
+    v['d&.'] = b(get_dand(v['w'], '.'))
+    # A uppercase letter followed by '.'
+    v['up'] = b(get_capperiod(v['w']))
+
+    # An initial uppercase letter.
+    v['iu'] = b(v['w'] and v['w'][0].isupper())
+    # All uppercase letters.
+    v['au'] = b(v['w'].isupper())
+    # All lowercase letters.
+    v['al'] = b(v['w'].islower())
+    # All digit letters.
+    v['ad'] = b(v['w'].isdigit())
+    # All other (non-alphanumeric) letters.
+    v['ao'] = b(get_all_other(v['w']))
+
+    # Contains a uppercase letter.
+    v['cu'] = b(contains_upper(v['w']))
+    # Contains a lowercase letter.
+    v['cl'] = b(contains_lower(v['w']))
+    # Contains a alphabet letter.
+    v['ca'] = b(contains_alpha(v['w']))
+    # Contains a digit.
+    v['cd'] = b(contains_digit(v['w']))
+    # Contains a symbol.
+    v['cs'] = b(contains_symbol(v['w']))
+
+def disjunctive(X, t, field, begin, end):
+    name = '%s[%d..%d]' % (field, begin, end)
+    for offset in range(begin, end+1):
+        p = t + offset
+        if p not in range(0, len(X)):
+            continue
+        X[t]['F'].append('%s=%s' % (name, X[p][field]))
+
+U = [
+    'w', 'wl', 'pos', 'chk', 'shape', 'shaped', 'type',
+    'p1', 'p2', 'p3', 'p4',
+    's1', 's2', 's3', 's4',
+    '2d', '4d', 'd&a', 'd&-', 'd&/', 'd&,', 'd&.', 'up',
+    'iu', 'au', 'al', 'ad', 'ao',
+    'cu', 'cl', 'ca', 'cd', 'cs',
+    ]
+B = ['w', 'pos', 'chk', 'shaped', 'type']
+
+templates = []
+for name in U:
+    templates += [((name, i),) for i in range(-2, 3)]
+for name in B:
+    templates += [((name, i), (name, i+1)) for i in range(-2, 2)]
+
+def feature_extractor(X):
+    # Append observations.
+    for x in X:
+        observation(x)
+
+    # Apply the feature templates.
+    crfutils.apply_templates(X, templates)
+
+    # Append disjunctive features.
+    for t in range(len(X)):
+        disjunctive(X, t, 'w', -4, -1)
+        disjunctive(X, t, 'w', 1, 4)
+
+    # Append BOS and EOS features.
+    if X:
+        X[0]['F'].append('__BOS__')
+        X[-1]['F'].append('__EOS__')
+
+if __name__ == '__main__':
+    crfutils.main(feature_extractor, fields=fields, sep=separator)