melisa 0.1.0

data/ext/marisa/tools/marisa-build.cc

@@ -0,0 +1,206 @@
+#ifdef _WIN32
+ #include <fcntl.h>
+ #include <io.h>
+ #include <stdio.h>
+#endif  // _WIN32
+
+#include <cstdlib>
+#include <fstream>
+#include <iostream>
+#include <string>
+
+#include <marisa.h>
+
+#include "cmdopt.h"
+
+namespace {
+
+int param_num_tries = MARISA_DEFAULT_NUM_TRIES;
+marisa::TailMode param_tail_mode = MARISA_DEFAULT_TAIL;
+marisa::NodeOrder param_node_order = MARISA_DEFAULT_ORDER;
+marisa::CacheLevel param_cache_level = MARISA_DEFAULT_CACHE;
+const char *output_filename = NULL;
+
+void print_help(const char *cmd) {
+  std::cerr << "Usage: " << cmd << " [OPTION]... [FILE]...\n\n"
+      "Options:\n"
+      "  -n, --num-tries=[N]  limit the number of tries"
+      " [" << MARISA_MIN_NUM_TRIES << ", " << MARISA_MAX_NUM_TRIES
+      << "] (default: 3)\n"
+      "  -t, --text-tail      build a dictionary with text TAIL (default)\n"
+      "  -b, --binary-tail    build a dictionary with binary TAIL\n"
+      "  -w, --weight-order   arrange siblings in weight order (default)\n"
+      "  -l, --label-order    arrange siblings in label order\n"
+      "  -c, --cache-level=[N]    specify the cache size"
+      " [1, 5] (default: 3)\n"
+      "  -o, --output=[FILE]  write tries to FILE (default: stdout)\n"
+      "  -h, --help           print this help\n"
+      << std::endl;
+}
+
+void read_keys(std::istream &input, marisa::Keyset *keyset) {
+  std::string line;
+  while (std::getline(input, line)) {
+    const std::string::size_type delim_pos = line.find_last_of('\t');
+    float weight = 1.0F;
+    if (delim_pos != line.npos) {
+      char *end_of_value;
+      weight = (float)std::strtod(&line[delim_pos + 1], &end_of_value);
+      if (*end_of_value == '\0') {
+        line.resize(delim_pos);
+      }
+    }
+    keyset->push_back(line.c_str(), line.length(), weight);
+  }
+}
+
+int build(const char * const *args, std::size_t num_args) {
+  marisa::Keyset keyset;
+  if (num_args == 0) try {
+    read_keys(std::cin, &keyset);
+  } catch (const marisa::Exception &ex) {
+    std::cerr << ex.what() << ": failed to read keys" << std::endl;
+    return 10;
+  }
+
+  for (std::size_t i = 0; i < num_args; ++i) try {
+    std::ifstream input_file(args[i], std::ios::binary);
+    if (!input_file) {
+      std::cerr << "error: failed to open: " << args[i] << std::endl;
+      return 11;
+    }
+    read_keys(input_file, &keyset);
+  } catch (const marisa::Exception &ex) {
+    std::cerr << ex.what() << ": failed to read keys" << std::endl;
+    return 12;
+  }
+
+  marisa::Trie trie;
+  try {
+    trie.build(keyset, param_num_tries | param_tail_mode | param_node_order |
+        param_cache_level);
+  } catch (const marisa::Exception &ex) {
+    std::cerr << ex.what() << ": failed to build a dictionary" << std::endl;
+    return 20;
+  }
+
+  std::cerr << "#keys: " << trie.num_keys() << std::endl;
+  std::cerr << "#nodes: " << trie.num_nodes() << std::endl;
+  std::cerr << "size: " << trie.io_size() << std::endl;
+
+  if (output_filename != NULL) {
+    try {
+      trie.save(output_filename);
+    } catch (const marisa::Exception &ex) {
+      std::cerr << ex.what() << ": failed to write a dictionary to file: "
+          << output_filename << std::endl;
+      return 30;
+    }
+  } else {
+#ifdef _WIN32
+    const int stdout_fileno = ::_fileno(stdout);
+    if (stdout_fileno < 0) {
+      std::cerr << "error: failed to get the file descriptor of "
+          "standard output" << std::endl;
+      return 31;
+    }
+    if (::_setmode(stdout_fileno, _O_BINARY) == -1) {
+      std::cerr << "error: failed to set binary mode" << std::endl;
+      return 32;
+    }
+#endif  // _WIN32
+    try {
+      std::cout << trie;
+    } catch (const marisa::Exception &ex) {
+      std::cerr << ex.what()
+          << ": failed to write a dictionary to standard output" << std::endl;
+      return 33;
+    }
+  }
+  return 0;
+}
+
+}  // namespace
+
+int main(int argc, char *argv[]) {
+  std::ios::sync_with_stdio(false);
+
+  ::cmdopt_option long_options[] = {
+    { "max-num-tries", 1, NULL, 'n' },
+    { "text-tail", 0, NULL, 't' },
+    { "binary-tail", 0, NULL, 'b' },
+    { "weight-order", 0, NULL, 'w' },
+    { "label-order", 0, NULL, 'l' },
+    { "cache-level", 1, NULL, 'c' },
+    { "output", 1, NULL, 'o' },
+    { "help", 0, NULL, 'h' },
+    { NULL, 0, NULL, 0 }
+  };
+  ::cmdopt_t cmdopt;
+  ::cmdopt_init(&cmdopt, argc, argv, "n:tbwlc:o:h", long_options);
+  int label;
+  while ((label = ::cmdopt_get(&cmdopt)) != -1) {
+    switch (label) {
+      case 'n': {
+        char *end_of_value;
+        const long value = std::strtol(cmdopt.optarg, &end_of_value, 10);
+        if ((*end_of_value != '\0') || (value <= 0) ||
+            (value > MARISA_MAX_NUM_TRIES)) {
+          std::cerr << "error: option `-n' with an invalid argument: "
+              << cmdopt.optarg << std::endl;
+          return 1;
+        }
+        param_num_tries = (int)value;
+        break;
+      }
+      case 't': {
+        param_tail_mode = MARISA_TEXT_TAIL;
+        break;
+      }
+      case 'b': {
+        param_tail_mode = MARISA_BINARY_TAIL;
+        break;
+      }
+      case 'w': {
+        param_node_order = MARISA_WEIGHT_ORDER;
+        break;
+      }
+      case 'l': {
+        param_node_order = MARISA_LABEL_ORDER;
+        break;
+      }
+      case 'c': {
+        char *end_of_value;
+        const long value = std::strtol(cmdopt.optarg, &end_of_value, 10);
+        if ((*end_of_value != '\0') || (value < 1) || (value > 5)) {
+          std::cerr << "error: option `-c' with an invalid argument: "
+              << cmdopt.optarg << std::endl;
+          return 2;
+        } else if (value == 1) {
+          param_cache_level = MARISA_TINY_CACHE;
+        } else if (value == 2) {
+          param_cache_level = MARISA_SMALL_CACHE;
+        } else if (value == 3) {
+          param_cache_level = MARISA_NORMAL_CACHE;
+        } else if (value == 4) {
+          param_cache_level = MARISA_LARGE_CACHE;
+        } else if (value == 5) {
+          param_cache_level = MARISA_HUGE_CACHE;
+        }
+        break;
+      }
+      case 'o': {
+        output_filename = cmdopt.optarg;
+        break;
+      }
+      case 'h': {
+        print_help(argv[0]);
+        return 0;
+      }
+      default: {
+        return 1;
+      }
+    }
+  }
+  return build(cmdopt.argv + cmdopt.optind, cmdopt.argc - cmdopt.optind);
+}

data/ext/marisa/tools/marisa-common-prefix-search.cc

@@ -0,0 +1,143 @@
+#include <cstdlib>
+#include <iostream>
+#include <string>
+
+#include <marisa.h>
+
+#include "cmdopt.h"
+
+namespace {
+
+std::size_t max_num_results = 10;
+bool mmap_flag = true;
+
+void print_help(const char *cmd) {
+  std::cerr << "Usage: " << cmd << " [OPTION]... DIC\n\n"
+      "Options:\n"
+      "  -n, --max-num-results=[N]  limit the number of results to N"
+      " (default: 10)\n"
+      "                             0: no limit\n"
+      "  -m, --mmap-dictionary  use memory-mapped I/O to load a dictionary"
+      " (default)\n"
+      "  -r, --read-dictionary  read an entire dictionary into memory\n"
+      "  -h, --help             print this help\n"
+      << std::endl;
+}
+
+int common_prefix_search(const char * const *args, std::size_t num_args) {
+  if (num_args == 0) {
+    std::cerr << "error: dictionary is not specified" << std::endl;
+    return 10;
+  } else if (num_args > 1) {
+    std::cerr << "error: more than one dictionaries are specified"
+        << std::endl;
+    return 11;
+  }
+
+  marisa::Trie trie;
+  if (mmap_flag) {
+    try {
+      trie.mmap(args[0]);
+    } catch (const marisa::Exception &ex) {
+      std::cerr << ex.what() << ": failed to mmap a dictionary file: "
+          << args[0] << std::endl;
+      return 20;
+    }
+  } else {
+    try {
+      trie.load(args[0]);
+    } catch (const marisa::Exception &ex) {
+      std::cerr << ex.what() << ": failed to load a dictionary file: "
+          << args[0] << std::endl;
+      return 21;
+    }
+  }
+
+  marisa::Agent agent;
+  marisa::Keyset keyset;
+  std::string str;
+  while (std::getline(std::cin, str)) {
+    try {
+      agent.set_query(str.c_str(), str.length());
+      while (trie.common_prefix_search(agent)) {
+        keyset.push_back(agent.key());
+      }
+      if (keyset.empty()) {
+        std::cout << "not found" << std::endl;
+      } else {
+        std::cout << keyset.size() << " found" << std::endl;
+        const std::size_t end = std::min(max_num_results, keyset.size());
+        for (std::size_t i = 0; i < end; ++i) {
+          std::cout << keyset[i].id() << '\t';
+          std::cout.write(keyset[i].ptr(), keyset[i].length()) << '\t';
+          std::cout << str << '\n';
+        }
+      }
+      keyset.reset();
+    } catch (const marisa::Exception &ex) {
+      std::cerr << ex.what() << ": common_prefix_search() failed: "
+          << str << std::endl;
+      return 30;
+    }
+
+    if (!std::cout) {
+      std::cerr << "error: failed to write results to standard output"
+          << std::endl;
+      return 31;
+    }
+  }
+
+  return 0;
+}
+
+}  // namespace
+
+int main(int argc, char *argv[]) {
+  std::ios::sync_with_stdio(false);
+
+  ::cmdopt_option long_options[] = {
+    { "max-num-results", 1, NULL, 'n' },
+    { "mmap-dictionary", 0, NULL, 'm' },
+    { "read-dictionary", 0, NULL, 'r' },
+    { "help", 0, NULL, 'h' },
+    { NULL, 0, NULL, 0 }
+  };
+  ::cmdopt_t cmdopt;
+  ::cmdopt_init(&cmdopt, argc, argv, "n:mrh", long_options);
+  int label;
+  while ((label = ::cmdopt_get(&cmdopt)) != -1) {
+    switch (label) {
+      case 'n': {
+        char *end_of_value;
+        const long value = std::strtol(cmdopt.optarg, &end_of_value, 10);
+        if ((*end_of_value != '\0') || (value < 0)) {
+          std::cerr << "error: option `-n' with an invalid argument: "
+              << cmdopt.optarg << std::endl;
+        }
+        if ((value == 0) || ((unsigned long long)value > MARISA_SIZE_MAX)) {
+          max_num_results = MARISA_SIZE_MAX;
+        } else {
+          max_num_results = (std::size_t)value;
+        }
+        break;
+      }
+      case 'm': {
+        mmap_flag = true;
+        break;
+      }
+      case 'r': {
+        mmap_flag = false;
+        break;
+      }
+      case 'h': {
+        print_help(argv[0]);
+        return 0;
+      }
+      default: {
+        return 1;
+      }
+    }
+  }
+  return common_prefix_search(cmdopt.argv + cmdopt.optind,
+      cmdopt.argc - cmdopt.optind);
+}

data/ext/marisa/tools/marisa-dump.cc

@@ -0,0 +1,151 @@
+#ifdef _WIN32
+ #include <fcntl.h>
+ #include <io.h>
+ #include <stdio.h>
+#endif  // _WIN32
+
+#include <cstdlib>
+#include <iostream>
+#include <string>
+
+#include <marisa.h>
+
+#include "cmdopt.h"
+
+namespace {
+
+const char *delimiter = "\n";
+bool mmap_flag = true;
+
+void print_help(const char *cmd) {
+  std::cerr << "Usage: " << cmd << " [OPTION]... DIC...\n\n"
+      "Options:\n"
+      "  -d, --delimiter=[S]    specify the delimier (default: \"\\n\")\n"
+      "  -m, --mmap-dictionary  use memory-mapped I/O to load a dictionary"
+      " (default)\n"
+      "  -r, --read-dictionary  read an entire dictionary into memory\n"
+      "  -h, --help             print this help\n"
+      << std::endl;
+}
+
+int dump(const marisa::Trie &trie) {
+  std::size_t num_keys = 0;
+  marisa::Agent agent;
+  agent.set_query("");
+  try {
+    while (trie.predictive_search(agent)) {
+      std::cout.write(agent.key().ptr(), agent.key().length()) << delimiter;
+      if (!std::cout) {
+        std::cerr << "error: failed to write results to standard output"
+            << std::endl;
+        return 20;
+      }
+      ++num_keys;
+    }
+  } catch (const marisa::Exception &ex) {
+    std::cerr << ex.what() << ": predictive_search() failed" << std::endl;
+    return 21;
+  }
+  std::cerr << "#keys: " << num_keys << std::endl;
+  return 0;
+}
+
+int dump(const char *filename) {
+  marisa::Trie trie;
+  if (filename != NULL) {
+    std::cerr << "input: " << filename << std::endl;
+    if (mmap_flag) {
+      try {
+        trie.mmap(filename);
+      } catch (const marisa::Exception &ex) {
+        std::cerr << ex.what() << ": failed to mmap a dictionary file: "
+            << filename << std::endl;
+        return 10;
+      }
+    } else {
+      try {
+        trie.load(filename);
+      } catch (const marisa::Exception &ex) {
+        std::cerr << ex.what() << ": failed to load a dictionary file: "
+            << filename << std::endl;
+        return 11;
+      }
+    }
+  } else {
+    std::cerr << "input: <stdin>" << std::endl;
+#ifdef _WIN32
+    const int stdin_fileno = ::_fileno(stdin);
+    if (stdin_fileno < 0) {
+      std::cerr << "error: failed to get the file descriptor of "
+          "standard input" << std::endl;
+      return 20;
+    }
+    if (::_setmode(stdin_fileno, _O_BINARY) == -1) {
+      std::cerr << "error: failed to set binary mode" << std::endl;
+      return 21;
+    }
+#endif  // _WIN32
+    try {
+      std::cin >> trie;
+    } catch (const marisa::Exception &ex) {
+      std::cerr << ex.what()
+          << ": failed to read a dictionary from standard input" << std::endl;
+      return 22;
+    }
+  }
+  return dump(trie);
+}
+
+int dump(const char * const *args, std::size_t num_args) {
+  if (num_args == 0) {
+    return dump(NULL);
+  }
+  for (std::size_t i = 0; i < num_args; ++i) {
+    const int result = dump(args[i]);
+    if (result != 0) {
+      return result;
+    }
+  }
+  return 0;
+}
+
+}  // namespace
+
+int main(int argc, char *argv[]) {
+  std::ios::sync_with_stdio(false);
+
+  ::cmdopt_option long_options[] = {
+    { "delimiter", 1, NULL, 'd' },
+    { "mmap-dictionary", 0, NULL, 'm' },
+    { "read-dictionary", 0, NULL, 'r' },
+    { "help", 0, NULL, 'h' },
+    { NULL, 0, NULL, 0 }
+  };
+  ::cmdopt_t cmdopt;
+  ::cmdopt_init(&cmdopt, argc, argv, "d:mrh", long_options);
+  int label;
+  while ((label = ::cmdopt_get(&cmdopt)) != -1) {
+    switch (label) {
+      case 'd': {
+        delimiter = cmdopt.optarg;
+        break;
+      }
+      case 'm': {
+        mmap_flag = true;
+        break;
+      }
+      case 'r': {
+        mmap_flag = false;
+        break;
+      }
+      case 'h': {
+        print_help(argv[0]);
+        return 0;
+      }
+      default: {
+        return 1;
+      }
+    }
+  }
+  return dump(cmdopt.argv + cmdopt.optind, cmdopt.argc - cmdopt.optind);
+}