@ccmaymay/concrete 4.15.0

package/summarization_types.js

@@ -0,0 +1,433 @@
+//
+// Autogenerated by Thrift Compiler (0.15.0)
+//
+// DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+//
+"use strict";
+
+const thrift = require('thrift');
+const Thrift = thrift.Thrift;
+const Int64 = require('node-int64');
+
+const communication_ttypes = require('./communication_types');
+const services_ttypes = require('./services_types');
+const structure_ttypes = require('./structure_types');
+const uuid_ttypes = require('./uuid_types');
+
+
+const ttypes = module.exports = {};
+ttypes.SummarySourceType = {
+  'DOCUMENT' : 0,
+  'TOKENIZATION' : 1,
+  'ENTITY' : 2
+};
+const SummarizationRequest = module.exports.SummarizationRequest = class {
+  constructor(args) {
+    this.queryTerms = null;
+    this.maximumTokens = null;
+    this.maximumCharacters = null;
+    this.sourceType = null;
+    this.sourceIds = null;
+    this.sourceCommunication = null;
+    if (args) {
+      if (args.queryTerms !== undefined && args.queryTerms !== null) {
+        this.queryTerms = Thrift.copyList(args.queryTerms, [null]);
+      }
+      if (args.maximumTokens !== undefined && args.maximumTokens !== null) {
+        this.maximumTokens = args.maximumTokens;
+      }
+      if (args.maximumCharacters !== undefined && args.maximumCharacters !== null) {
+        this.maximumCharacters = args.maximumCharacters;
+      }
+      if (args.sourceType !== undefined && args.sourceType !== null) {
+        this.sourceType = args.sourceType;
+      }
+      if (args.sourceIds !== undefined && args.sourceIds !== null) {
+        this.sourceIds = Thrift.copyList(args.sourceIds, [uuid_ttypes.UUID]);
+      }
+      if (args.sourceCommunication !== undefined && args.sourceCommunication !== null) {
+        this.sourceCommunication = new communication_ttypes.Communication(args.sourceCommunication);
+      }
+    }
+  }
+
+  read (input) {
+    input.readStructBegin();
+    while (true) {
+      const ret = input.readFieldBegin();
+      const ftype = ret.ftype;
+      const fid = ret.fid;
+      if (ftype == Thrift.Type.STOP) {
+        break;
+      }
+      switch (fid) {
+        case 1:
+        if (ftype == Thrift.Type.LIST) {
+          this.queryTerms = [];
+          const _rtmp31 = input.readListBegin();
+          const _size0 = _rtmp31.size || 0;
+          for (let _i2 = 0; _i2 < _size0; ++_i2) {
+            let elem3 = null;
+            elem3 = input.readString();
+            this.queryTerms.push(elem3);
+          }
+          input.readListEnd();
+        } else {
+          input.skip(ftype);
+        }
+        break;
+        case 2:
+        if (ftype == Thrift.Type.I32) {
+          this.maximumTokens = input.readI32();
+        } else {
+          input.skip(ftype);
+        }
+        break;
+        case 3:
+        if (ftype == Thrift.Type.I32) {
+          this.maximumCharacters = input.readI32();
+        } else {
+          input.skip(ftype);
+        }
+        break;
+        case 4:
+        if (ftype == Thrift.Type.I32) {
+          this.sourceType = input.readI32();
+        } else {
+          input.skip(ftype);
+        }
+        break;
+        case 5:
+        if (ftype == Thrift.Type.LIST) {
+          this.sourceIds = [];
+          const _rtmp35 = input.readListBegin();
+          const _size4 = _rtmp35.size || 0;
+          for (let _i6 = 0; _i6 < _size4; ++_i6) {
+            let elem7 = null;
+            elem7 = new uuid_ttypes.UUID();
+            elem7.read(input);
+            this.sourceIds.push(elem7);
+          }
+          input.readListEnd();
+        } else {
+          input.skip(ftype);
+        }
+        break;
+        case 6:
+        if (ftype == Thrift.Type.STRUCT) {
+          this.sourceCommunication = new communication_ttypes.Communication();
+          this.sourceCommunication.read(input);
+        } else {
+          input.skip(ftype);
+        }
+        break;
+        default:
+          input.skip(ftype);
+      }
+      input.readFieldEnd();
+    }
+    input.readStructEnd();
+    return;
+  }
+
+  write (output) {
+    output.writeStructBegin('SummarizationRequest');
+    if (this.queryTerms !== null && this.queryTerms !== undefined) {
+      output.writeFieldBegin('queryTerms', Thrift.Type.LIST, 1);
+      output.writeListBegin(Thrift.Type.STRING, this.queryTerms.length);
+      for (let iter8 in this.queryTerms) {
+        if (this.queryTerms.hasOwnProperty(iter8)) {
+          iter8 = this.queryTerms[iter8];
+          output.writeString(iter8);
+        }
+      }
+      output.writeListEnd();
+      output.writeFieldEnd();
+    }
+    if (this.maximumTokens !== null && this.maximumTokens !== undefined) {
+      output.writeFieldBegin('maximumTokens', Thrift.Type.I32, 2);
+      output.writeI32(this.maximumTokens);
+      output.writeFieldEnd();
+    }
+    if (this.maximumCharacters !== null && this.maximumCharacters !== undefined) {
+      output.writeFieldBegin('maximumCharacters', Thrift.Type.I32, 3);
+      output.writeI32(this.maximumCharacters);
+      output.writeFieldEnd();
+    }
+    if (this.sourceType !== null && this.sourceType !== undefined) {
+      output.writeFieldBegin('sourceType', Thrift.Type.I32, 4);
+      output.writeI32(this.sourceType);
+      output.writeFieldEnd();
+    }
+    if (this.sourceIds !== null && this.sourceIds !== undefined) {
+      output.writeFieldBegin('sourceIds', Thrift.Type.LIST, 5);
+      output.writeListBegin(Thrift.Type.STRUCT, this.sourceIds.length);
+      for (let iter9 in this.sourceIds) {
+        if (this.sourceIds.hasOwnProperty(iter9)) {
+          iter9 = this.sourceIds[iter9];
+          iter9.write(output);
+        }
+      }
+      output.writeListEnd();
+      output.writeFieldEnd();
+    }
+    if (this.sourceCommunication !== null && this.sourceCommunication !== undefined) {
+      output.writeFieldBegin('sourceCommunication', Thrift.Type.STRUCT, 6);
+      this.sourceCommunication.write(output);
+      output.writeFieldEnd();
+    }
+    output.writeFieldStop();
+    output.writeStructEnd();
+    return;
+  }
+
+};
+const SummaryConcept = module.exports.SummaryConcept = class {
+  constructor(args) {
+    this.tokens = null;
+    this.concept = null;
+    this.confidence = 1;
+    this.utility = 1;
+    if (args) {
+      if (args.tokens !== undefined && args.tokens !== null) {
+        this.tokens = new structure_ttypes.TokenRefSequence(args.tokens);
+      }
+      if (args.concept !== undefined && args.concept !== null) {
+        this.concept = args.concept;
+      }
+      if (args.confidence !== undefined && args.confidence !== null) {
+        this.confidence = args.confidence;
+      }
+      if (args.utility !== undefined && args.utility !== null) {
+        this.utility = args.utility;
+      }
+    }
+  }
+
+  read (input) {
+    input.readStructBegin();
+    while (true) {
+      const ret = input.readFieldBegin();
+      const ftype = ret.ftype;
+      const fid = ret.fid;
+      if (ftype == Thrift.Type.STOP) {
+        break;
+      }
+      switch (fid) {
+        case 1:
+        if (ftype == Thrift.Type.STRUCT) {
+          this.tokens = new structure_ttypes.TokenRefSequence();
+          this.tokens.read(input);
+        } else {
+          input.skip(ftype);
+        }
+        break;
+        case 2:
+        if (ftype == Thrift.Type.STRING) {
+          this.concept = input.readString();
+        } else {
+          input.skip(ftype);
+        }
+        break;
+        case 3:
+        if (ftype == Thrift.Type.DOUBLE) {
+          this.confidence = input.readDouble();
+        } else {
+          input.skip(ftype);
+        }
+        break;
+        case 4:
+        if (ftype == Thrift.Type.DOUBLE) {
+          this.utility = input.readDouble();
+        } else {
+          input.skip(ftype);
+        }
+        break;
+        default:
+          input.skip(ftype);
+      }
+      input.readFieldEnd();
+    }
+    input.readStructEnd();
+    return;
+  }
+
+  write (output) {
+    output.writeStructBegin('SummaryConcept');
+    if (this.tokens !== null && this.tokens !== undefined) {
+      output.writeFieldBegin('tokens', Thrift.Type.STRUCT, 1);
+      this.tokens.write(output);
+      output.writeFieldEnd();
+    }
+    if (this.concept !== null && this.concept !== undefined) {
+      output.writeFieldBegin('concept', Thrift.Type.STRING, 2);
+      output.writeString(this.concept);
+      output.writeFieldEnd();
+    }
+    if (this.confidence !== null && this.confidence !== undefined) {
+      output.writeFieldBegin('confidence', Thrift.Type.DOUBLE, 3);
+      output.writeDouble(this.confidence);
+      output.writeFieldEnd();
+    }
+    if (this.utility !== null && this.utility !== undefined) {
+      output.writeFieldBegin('utility', Thrift.Type.DOUBLE, 4);
+      output.writeDouble(this.utility);
+      output.writeFieldEnd();
+    }
+    output.writeFieldStop();
+    output.writeStructEnd();
+    return;
+  }
+
+};
+const Summary = module.exports.Summary = class {
+  constructor(args) {
+    this.summaryCommunication = null;
+    this.concepts = null;
+    if (args) {
+      if (args.summaryCommunication !== undefined && args.summaryCommunication !== null) {
+        this.summaryCommunication = new communication_ttypes.Communication(args.summaryCommunication);
+      }
+      if (args.concepts !== undefined && args.concepts !== null) {
+        this.concepts = Thrift.copyList(args.concepts, [ttypes.SummaryConcept]);
+      }
+    }
+  }
+
+  read (input) {
+    input.readStructBegin();
+    while (true) {
+      const ret = input.readFieldBegin();
+      const ftype = ret.ftype;
+      const fid = ret.fid;
+      if (ftype == Thrift.Type.STOP) {
+        break;
+      }
+      switch (fid) {
+        case 1:
+        if (ftype == Thrift.Type.STRUCT) {
+          this.summaryCommunication = new communication_ttypes.Communication();
+          this.summaryCommunication.read(input);
+        } else {
+          input.skip(ftype);
+        }
+        break;
+        case 2:
+        if (ftype == Thrift.Type.LIST) {
+          this.concepts = [];
+          const _rtmp311 = input.readListBegin();
+          const _size10 = _rtmp311.size || 0;
+          for (let _i12 = 0; _i12 < _size10; ++_i12) {
+            let elem13 = null;
+            elem13 = new ttypes.SummaryConcept();
+            elem13.read(input);
+            this.concepts.push(elem13);
+          }
+          input.readListEnd();
+        } else {
+          input.skip(ftype);
+        }
+        break;
+        default:
+          input.skip(ftype);
+      }
+      input.readFieldEnd();
+    }
+    input.readStructEnd();
+    return;
+  }
+
+  write (output) {
+    output.writeStructBegin('Summary');
+    if (this.summaryCommunication !== null && this.summaryCommunication !== undefined) {
+      output.writeFieldBegin('summaryCommunication', Thrift.Type.STRUCT, 1);
+      this.summaryCommunication.write(output);
+      output.writeFieldEnd();
+    }
+    if (this.concepts !== null && this.concepts !== undefined) {
+      output.writeFieldBegin('concepts', Thrift.Type.LIST, 2);
+      output.writeListBegin(Thrift.Type.STRUCT, this.concepts.length);
+      for (let iter14 in this.concepts) {
+        if (this.concepts.hasOwnProperty(iter14)) {
+          iter14 = this.concepts[iter14];
+          iter14.write(output);
+        }
+      }
+      output.writeListEnd();
+      output.writeFieldEnd();
+    }
+    output.writeFieldStop();
+    output.writeStructEnd();
+    return;
+  }
+
+};
+const SummarizationCapability = module.exports.SummarizationCapability = class {
+  constructor(args) {
+    this.type = null;
+    this.lang = null;
+    if (args) {
+      if (args.type !== undefined && args.type !== null) {
+        this.type = args.type;
+      } else {
+        throw new Thrift.TProtocolException(Thrift.TProtocolExceptionType.UNKNOWN, 'Required field type is unset!');
+      }
+      if (args.lang !== undefined && args.lang !== null) {
+        this.lang = args.lang;
+      } else {
+        throw new Thrift.TProtocolException(Thrift.TProtocolExceptionType.UNKNOWN, 'Required field lang is unset!');
+      }
+    }
+  }
+
+  read (input) {
+    input.readStructBegin();
+    while (true) {
+      const ret = input.readFieldBegin();
+      const ftype = ret.ftype;
+      const fid = ret.fid;
+      if (ftype == Thrift.Type.STOP) {
+        break;
+      }
+      switch (fid) {
+        case 1:
+        if (ftype == Thrift.Type.I32) {
+          this.type = input.readI32();
+        } else {
+          input.skip(ftype);
+        }
+        break;
+        case 2:
+        if (ftype == Thrift.Type.STRING) {
+          this.lang = input.readString();
+        } else {
+          input.skip(ftype);
+        }
+        break;
+        default:
+          input.skip(ftype);
+      }
+      input.readFieldEnd();
+    }
+    input.readStructEnd();
+    return;
+  }
+
+  write (output) {
+    output.writeStructBegin('SummarizationCapability');
+    if (this.type !== null && this.type !== undefined) {
+      output.writeFieldBegin('type', Thrift.Type.I32, 1);
+      output.writeI32(this.type);
+      output.writeFieldEnd();
+    }
+    if (this.lang !== null && this.lang !== undefined) {
+      output.writeFieldBegin('lang', Thrift.Type.STRING, 2);
+      output.writeString(this.lang);
+      output.writeFieldEnd();
+    }
+    output.writeFieldStop();
+    output.writeStructEnd();
+    return;
+  }
+
+};

package/tokenization_fu.js

@@ -0,0 +1,33 @@
+/**
+ * @class Tokenization
+ * @classdesc concrete.js extensions to the Tokenization class
+ */
+const Tokenization = module.exports = require('./structure_types').Tokenization;
+  
+/**
+ * Add a TokenTagging to this Tokenization
+ * @param {TokenTagging} tokenTagging
+ */
+Tokenization.prototype.addTokenTagging = function(tokenTagging) {
+  if (!this.tokenTaggingList) {
+    this.tokenTaggingList = [];
+  }
+  this.tokenTaggingList.push(tokenTagging);
+};
+
+/**
+ * Get all TokenTaggings with the specified taggingType
+ * @param {String} taggingType - A string specifying a TokenTagging.taggingType
+ * @returns {Array} A (possibly empty) array of TokenTagging objects
+ */
+Tokenization.prototype.getTokenTaggingsOfType = function(taggingType) {
+  var tokenTaggings = [];
+
+  for (var tokenTaggingIndex in this.tokenTaggingList) {
+    if (this.tokenTaggingList[tokenTaggingIndex].taggingType === taggingType) {
+      tokenTaggings.push(this.tokenTaggingList[tokenTaggingIndex]);
+    }
+  }
+
+  return tokenTaggings;
+};

package/tokentagging_fu.js

@@ -0,0 +1,241 @@
+/**
+ * @class TokenTagging
+ * @classdesc concrete.js extensions to the TokenTagging class
+ */
+const concrete = {};
+concrete.structure = require('./structure_types');
+const TokenTagging = module.exports = concrete.structure.TokenTagging;
+const TaggedToken = concrete.structure.TaggedToken;
+concrete.metadata = require('./metadata_types');
+const AnnotationMetadata = concrete.metadata.AnnotationMetadata;
+
+const jQuery = {extend: function(){throw new Error("not implemented");}};
+const $ = jQuery;
+
+/**
+ * Create a valid TokenTagging with required fields AnnotationMetadata and UUID
+ *
+ * Example usage:
+ *
+ *     tt = TokenTagging.create({taggingType: 'NER'}, {tool: 'HIT'})
+ *
+ * @param {Object} options - Override default TokenTagging fields (except metadata)
+ * @param {Object} metadataOptions - Override default tokenTagging.metadata fields
+ */
+TokenTagging.create = function(options, metadataOptions) {
+  var tokenTagging = new TokenTagging();
+  tokenTagging.metadata = new AnnotationMetadata();
+  tokenTagging.metadata.timestamp = Math.floor(Date.now()/1000);
+  tokenTagging.metadata.tool = 'concrete.js - TokenTagging.create()';
+  tokenTagging.taggedTokenList = [];
+  tokenTagging.taggingType = '';
+  tokenTagging.uuid = concrete.util.generateUUID();
+
+  tokenTagging = $.extend({}, tokenTagging, options);
+  tokenTagging.metadata = $.extend({}, tokenTagging.metadata, metadataOptions);
+  return tokenTagging;
+};
+
+/**
+ * Get BIO value for TaggedToken at tokenIndex
+ *
+ * @param {Integer] tokenIndex
+ * @returns {String|null} - 'B', 'I', 'O' or null
+ */
+TokenTagging.prototype.bioGetBIOValue = function(tokenIndex) {
+  var taggedToken = this.getTaggedTokenWithTokenIndex(tokenIndex);
+  if (taggedToken && taggedToken.tag) {
+    var firstChar = taggedToken.tag.charAt(0);
+    if (firstChar === 'B' || firstChar === 'I' || firstChar === 'O' ) {
+      return firstChar;
+    }
+  }
+  return null;
+};
+
+/**
+ * Get tag value (stripped of BIO tag and separator) for TaggedToken at tokenIndex
+ *
+ * @param {Integer] tokenIndex
+ * @returns {String|null} - 'B', 'I', 'O' or null
+ *
+ */
+TokenTagging.prototype.bioGetTagValue = function(tokenIndex) {
+  var taggedToken = this.getTaggedTokenWithTokenIndex(tokenIndex);
+  if (taggedToken && taggedToken.tag) {
+    return taggedToken.tag.substring(2);
+  }
+  return null;
+};
+
+/**
+ * Returns separator character for BIO TokenTaggings.
+ *
+ * If the separator character had not been set before this function was called,
+ * the separator character will be set to '-'.
+ *
+ * @returns {String} - Separator character for BIO TokenTaggings
+ */
+TokenTagging.prototype.bioGetTagSeparator = function() {
+  if (this.bioTagSeparator === undefined) {
+    this.bioTagSeparator = '-';
+  }
+  return this.bioTagSeparator;
+};
+
+/**
+ * Returns token index of 'B' tag for the (possibly multi-token) 'BI'
+ * tagging at the specified tokenIndex.
+ *
+ * If the tag at tokenIndex is a 'B' tag, return tokenIndex.  If the
+ * tag at tokenIndex is an 'I' tag, find the index of the 'B' tag for
+ * this 'I' tag.
+ *
+ * @param {Number} tokenIndex -
+ * @returns {Number} - Token index of "B" tag
+ * @throws {TypeError} Thrown if the tag at TokenIndex is not a 'B' or
+ *                     'I' tag.  Also thrown if the tag at TokenIndex
+ *                     is a valid 'I' tag, but not part of a valid 'BI*'
+ *                     multi-token tagging.
+ */
+TokenTagging.prototype.bioGetTokenIndexForB = function(tokenIndex) {
+  if (this.bioGetBIOValue(tokenIndex) !== 'B' && this.bioGetBIOValue(tokenIndex) !== 'I') {
+    throw new TypeError("TokenTagging.getBIOTokenIndexForB expected a 'B' or 'I' tag at tokenIndex " + tokenIndex);
+  }
+
+  var bTokenIndex = tokenIndex;
+  while (this.bioGetBIOValue(bTokenIndex) === 'I') {
+    bTokenIndex -= 1;
+  }
+  if (this.bioGetBIOValue(bTokenIndex) !== 'B') {
+    throw new TypeError("TokenTagging.getBIOTokenIndex expected a 'B' tag at tokenIndex " +
+                        bTokenIndex + ', but tag was "' +
+                        this.getTaggedTokenWithTokenIndex(bTokenIndex) + "'");
+  }
+  return bTokenIndex;
+};
+
+/**
+ * Set BIO TaggedToken tag
+ *
+ * @param {String} bioValue - Should be 'B', 'I' or 'O'
+ * @param {String} tagText
+ * @param {Number} tokenIndex
+ * @throws {TypeError} Thrown if bioValue is not 'B'|'I'|'O'.  Also thrown
+ *                     if bioValue is 'I', but not part of a valid 'BI*'
+ *                     multi-token tagging.
+ */
+TokenTagging.prototype.bioSetTaggedTokenTag = function(bioValue, tagText, tokenIndex) {
+  if (bioValue !== 'B' && bioValue !== 'I' && bioValue !== 'O') {
+    throw new TypeError("TokenTagging.bioSetTaggedTokenTag() expected bioValue to be 'B', 'I' or 'O', " +
+                        "but instead it was '" + bioValue + "'");
+  }
+
+  var bioTagText;
+  if (bioValue === 'B') {
+    this.setTaggedTokenTag(bioValue + this.bioGetTagSeparator() + tagText, tokenIndex);
+  }
+  else if (bioValue === 'I') {
+    var bioPreviousValue = this.bioGetBIOValue(tokenIndex-1);
+    if (bioPreviousValue === 'B' || bioPreviousValue === 'I') {
+      // Get tag value from previous tag, ignore 'tagText' passed into function
+      this.setTaggedTokenTag(bioValue + this.bioGetTagSeparator() + this.bioGetTagValue(tokenIndex-1), tokenIndex);
+    }
+    else {
+      throw new TypeError("TokenTagging.bioSetTaggedTokenTag() encountered inconsistent BIO tagging " +
+                          "at tokenIndex " + (tokenIndex-1));
+    }
+  }
+  else {
+    this.setTaggedTokenTag('O', tokenIndex);
+  }
+
+  var bioValueNext = this.bioGetBIOValue(tokenIndex+1);
+  var tagTextNext = this.bioGetTagValue(tokenIndex+1);
+  if (bioValueNext === 'I') {
+    if (bioValue === 'O') {
+      this.bioSetTaggedTokenTag('B', tagTextNext, tokenIndex+1);
+    }
+    else {
+      if (tagText !== tagTextNext) {
+        // Update tagText for all following 'I' tokens
+        this.bioSetTaggedTokenTag('I', tagText, tokenIndex+1);
+      }
+    }
+  }
+};
+
+/**
+ * For BIO TokenTaggings, sets separator character to be used between
+ * B/I/O character and rest of tag
+ *
+ * @param {String} separator - String used as separator character
+ */
+TokenTagging.prototype.bioSetTagSeparator = function(separator) {
+  this.bioTagSeparator = separator;
+};
+
+/**
+ * Return a deep copy of this TokenTagging's taggedTokenList.
+ *
+ * @returns {TaggedToken[]}
+ */
+TokenTagging.prototype.deepCopyTaggedTokenList = function() {
+  var taggedTokenListCopy = [];
+  for (var i = 0; i < this.taggedTokenList.length; i++) {
+    var taggedToken = new TaggedToken();
+    taggedTokenListCopy.push(jQuery.extend(true, taggedToken, this.taggedTokenList[i]));
+  }
+  return taggedTokenListCopy;
+};
+
+/**
+ * Return the TaggedToken (or null) with the specified tokenIndex
+ *
+ * @param {Number} tokenIndex
+ * @returns {TaggedToken|null}
+ */
+TokenTagging.prototype.getTaggedTokenWithTokenIndex = function(tokenIndex) {
+  for (var i = 0; i < this.taggedTokenList.length; i++) {
+    if (this.taggedTokenList[i].tokenIndex === tokenIndex) {
+      return this.taggedTokenList[i];
+    }
+  }
+  return null;
+};
+
+/**
+ * Set taggedTokenList to a list of TaggedTokens (one per token) with identical tags
+ *
+ * @param {Tokenization} tokenization - Used to determine # of TokenTags
+ * @param {String} tagText - Value for each TaggedToken's "tag" field
+ */
+TokenTagging.prototype.setAllTaggedTokenTags = function(tokenization, tagText) {
+  // Discard the contents of the existing taggedTokenList
+  this.taggedTokenList = [];
+
+  for (var i = 0; i < tokenization.tokenList.tokenList.length; i++) {
+    var taggedToken = new TaggedToken();
+    taggedToken.tag = tagText;
+    taggedToken.tokenIndex = i;
+    this.taggedTokenList.push(taggedToken);
+  }
+};
+
+/**
+ * Sets the tag of the TaggedToken with the specified tokenIndex.
+ * If a TaggedToken with the specified tokenIndex does not exist,
+ * than it will be created.
+ *
+ * @param {String} tagText
+ * @param {Number} tokenIndex
+ */
+TokenTagging.prototype.setTaggedTokenTag = function(tagText, tokenIndex) {
+  var taggedToken = this.getTaggedTokenWithTokenIndex(tokenIndex);
+  if (!taggedToken) {
+    taggedToken = new TaggedToken();
+    taggedToken.tokenIndex = tokenIndex;
+    this.taggedTokenList.push(taggedToken);
+  }
+  taggedToken.tag = tagText;
+};