@korap/cmc-tagger 1.1.0

package/src/index.js

@@ -0,0 +1,206 @@
+#!/usr/bin/env node
+
+const packageVersion = require('../package.json').version;
+
+const emoticonRegex = /^(\:\w+\:|\<[\/\\]?3|[\(\)\\\D|\*\$][\-\^]?[\:\;\=]|[\:\;\=B8][\-\^]?[3DOPp\@\$\*\\\)\(\/\|])(?=\s|[\!\.\?]|$)/;
+const hashtagRegex = /^#(?=.*\p{L})[\p{L}\p{M}\p{N}]+$/u;
+const urlRegex = /^(ftp|http)s?:\/\/[^\s]+/;
+const emailRegex = /^\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b/;
+const addressRegex = /^@[a-zA-Z0-9]+/;
+const actionWordRegex = /^:[^:]+:$/;
+const wikiEmojiRegex = /^\[_EMOJI:[^\]]+\]$/;
+
+// Load emoji data
+let emojiData = {};
+try {
+  emojiData = require('./emoji_data.json');
+} catch (e) {
+  // Silent fallback if file doesn't exist (e.g. during initial setup before script run)
+}
+
+// Function to strip emoji modifiers and zero-width joiners to get base emoji
+function getBaseEmoji(emoji) {
+  const stripped = emoji
+    // Remove skin tone modifiers (U+1F3FB-U+1F3FF)
+    .replace(/[\u{1F3FB}-\u{1F3FF}]/gu, '')
+    // Remove zero-width joiners (U+200D)
+    .replace(/\u200D/g, '')
+    // Remove variation selectors (U+FE0F, U+FE0E)
+    .replace(/[\uFE0E\uFE0F]/g, '');
+
+  // Extract the first emoji character using Array spread to handle multi-byte emoji
+  const chars = [...stripped];
+  return chars.length > 0 ? chars[0] : stripped;
+}
+
+const optionDefinitions = [
+  { name: 'sparse', alias: 's', type: Boolean, description: 'Print only the files, lines that have POS annotations.' },
+  { name: 'version', alias: 'V', type: Boolean, description: 'Print the current package version.' },
+  { name: 'help', alias: 'h', type: Boolean, description: 'Print this usage guide.' },
+]
+
+const sections = [
+  {
+    header: 'conllu-cmc',
+    content: 'Reads CoNLL-U format from stdin and annotates emojis, emoticons, hashtags, URLs, email addresses, @addresses, and action words. Writes CoNLL-U format to stdout.'
+  },
+  {
+    header: 'Synopsis',
+    content: '$ conllu-cmc [-s] < input.conllu > output.conllu\n$ cmc-tagger [-s] < input.conllu > output.conllu\n$ conllu-cmc -V\n$ cmc-tagger -V'
+  },
+  {
+    header: 'Options',
+    optionList: optionDefinitions
+  }
+]
+
+const getUsage = require('command-line-usage')
+const commandLineArgs = require('command-line-args')
+
+var options;
+try {
+  options = commandLineArgs(optionDefinitions)
+} catch (e) {
+  console.error(e.message);
+  options = { help: true };
+}
+
+if (options.help) {
+  const usage = getUsage(sections);
+  console.log(usage);
+  process.exit(0);
+}
+
+if (options.version) {
+  console.log(packageVersion);
+  process.exit(0);
+}
+
+const EmojiRegex = require('emoji-regex');
+const emojiRegex = EmojiRegex();
+const { once } = require('events');
+const readline = require('readline');
+global.header = '';
+global.fileheader = '';
+global.standalone = false
+
+
+const rl = readline.createInterface({
+  input: process.stdin,
+  terminal: false,
+
+});
+
+
+async function writeOutput(text) {
+  if (!process.stdout.write(text)) {
+    await once(process.stdout, 'drain');
+  }
+}
+
+
+async function parseConllu(line) {
+  if (line.match('#\\s*foundry')) {
+    if (line.match('=\\s*base')) {
+      if (options.sparse) {
+        global.standalone = true
+      }
+      await writeOutput("# foundry = cmc\n");
+    } else {
+      await writeOutput(`${line}\n`);
+    }
+    return
+  }
+
+  if (global.standalone) {
+    if (line.match('^#\\s*filename')) {
+      global.fileheader = `${line}\n`;
+      return;
+    } else if (line.match('^#\\s*text_id')) {
+      global.fileheader += `${line}\n`;
+      return;
+    } else if (line.match('^#\\s*eo[ft]')) {
+      await writeOutput(`${line}\n`);
+      return;
+    } else if (line.match('^#')) {
+      global.header += `${line}\n`;
+      return;
+    } else if (line.trim().match('^$')) {
+      if (global.header == "") {
+        await writeOutput("\n");
+      }
+      global.header = '';
+      return
+    }
+  } else {
+    if (!line.match('^\\d+')) {
+      await writeOutput(`${line}\n`);
+      return;
+    }
+  }
+
+  const columns = line.trim().split('\t');
+
+  const word = columns[1];
+  // Guard clause: if word is undefined, just output the line as-is
+  if (!word) {
+    await writeOutput(`${line}\n`);
+    return;
+  }
+
+  var new_tag = null;
+  if (word.match(wikiEmojiRegex)) {
+    new_tag = 'EMOWIKI';
+  } else if (word.match(emojiRegex)) {
+    new_tag = 'EMOIMG';
+  } else if (word.match(actionWordRegex)) {
+    new_tag = 'AKW';
+  } else if (word.match(emoticonRegex)) {
+    new_tag = 'EMOASC';
+  } else if (word.match(hashtagRegex)) {
+    new_tag = 'HST';
+  } else if (word.match(urlRegex)) {
+    new_tag = 'URL';
+  } else if (word.match(emailRegex)) {
+    new_tag = 'EML';
+  } else if (word.match(addressRegex)) {
+    new_tag = 'ADR';
+  }
+  if (new_tag) {
+    columns[4] = new_tag;
+    columns[5] = '_';
+    // For EMOIMG tokens, set lemma to the base emoji (without modifiers)
+    if (new_tag === 'EMOIMG') {
+      const base = getBaseEmoji(word);
+      columns[2] = base;
+
+      // Look up emoji metadata
+      // Try exact match first, then base emoji
+      const data = emojiData[word] || emojiData[base];
+      if (data) {
+        // g=group|s=subgroup|q=qualified|v=version|n=name
+        columns[5] = `g=${data.g}|s=${data.s}|q=${data.q}|v=${data.v}|n=${data.n}`;
+      }
+    }
+    if (global.standalone) {
+      await writeOutput(fileheader);
+      await writeOutput(header);
+      new_tag = null;
+      header = fileheader = '';
+    }
+    await writeOutput(columns.join('\t') + '\n');
+  } else if (!global.standalone) {
+    await writeOutput(`${line}\n`);
+  }
+}
+
+async function main() {
+  for await (const line of rl) {
+    await parseConllu(line);
+  }
+}
+
+main().catch((error) => {
+  console.error(error);
+  process.exit(1);
+});