@pulsar-edit/fuzzy-native 1.2.4

package/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Hanson Wang
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,78 @@
+# fuzzy-native
+
+[![Build Status](https://travis-ci.org/hansonw/fuzzy-native.svg?branch=master)](https://travis-ci.org/hansonw/fuzzy-native)
+
+Fuzzy string matching library package for Node. Implemented natively in C++ for speed with support for multithreading.
+
+The scoring algorithm is heavily tuned for file paths, but should work for general strings - it also supports the same algorithm that was implemented in Fuzzaldrin library, the one that powers command-pallette and other fuzzy-finders in Pulsar
+
+## API
+
+```ts
+export type MatcherOptions = {
+  // Default: false
+  caseSensitive?: boolean,
+
+  // Default: infinite
+  maxResults?: number,
+
+  // Maximum gap to allow between consecutive letters in a match.
+  // Provide a smaller maxGap to speed up query results.
+  // Default: unlimited
+  maxGap?: number;
+
+  // Default: 1
+  numThreads?: number,
+
+  // Default: false
+  recordMatchIndexes?: boolean,
+
+  // can be either "fuzzaldrin" or anything else to use the file-based option
+  algorithm?: string
+}
+
+export type MatchResult = {
+  id: number,
+  value: string,
+
+  // A number in the range (0-1]. Higher scores are more relevant.
+  // 0 denotes "no match" and will never be returned.
+  score: number,
+
+  // Matching character index in `value` for each character in `query`.
+  // This can be costly, so this is only returned if `recordMatchIndexes` was set in `options`.
+  matchIndexes?: Array<number>,
+}
+
+export class Matcher {
+  constructor(candidates: Array<string>) {}
+
+  // Returns all matching candidates (subject to `options`).
+  // Will be ordered by score, descending.
+  match: (query: string, options?: MatcherOptions) => Array<MatchResult>;
+
+  addCandidates: (ids: Array<number>, candidates: Array<string>) => void;
+  removeCandidates: (ids: Array<number>) => void;
+  setCandidates: (ids: Array<number>, candidates: Array<string>) => void;
+}
+```
+
+See also the [spec](spec/fuzzy-native-spec.js) for basic usage.
+
+## Scoring algorithm
+
+### Default
+The _default scoring_ algorithm is mostly borrowed from @wincent's excellent [command-t](https://github.com/wincent/command-t) vim plugin; most of the code is from [his implementation in  match.c](https://github.com/wincent/command-t/blob/master/ruby/command-t/match.c).
+
+Read [the source code](src/score_match.cpp) for a quick overview of how it works (the function `recursive_match`).
+
+NB: [score_match.cpp](src/score_match.cpp) and [score_match.h](src/score_match.h) have no dependencies besides the C/C++ stdlib and can easily be reused for other purposes.
+
+There are a few notable additional optimizations:
+
+- Before running the recursive matcher, we first do a backwards scan through the haystack to see if the needle exists at all. At the same time, we compute the right-most match for each character in the needle to prune the search space.
+- For each candidate string, we pre-compute and store a bitmask of its letters in `MatcherBase`. We then compare this the "letter bitmask" of the query to quickly prune out non-matches.
+
+### Fuzzaldrin
+
+Ported from Atom's Fuzzaldrin app - it's easier to read the original version than to read the C++ one, they are basically identic: see [scorer.coffee](https://github.com/atom/fuzzaldrin/blob/master/src/scorer.coffee) from Atom's archived repository.

package/binding.gyp

@@ -0,0 +1,40 @@
+{
+  'targets': [
+    {
+      'target_name': 'fuzzy-native',
+      'include_dirs': [ '<!(node -e "require(\'nan\')")' ],
+      'cflags': [
+        '-std=c++17',
+        '-O3',
+      ],
+      'xcode_settings': {
+        'OTHER_CPLUSPLUSFLAGS': [
+          '-std=c++17',
+          '-O3',
+          '-stdlib=libc++',
+        ],
+        'MACOSX_DEPLOYMENT_TARGET': '10.7',
+      },
+      'sources': [
+        'src/binding.cpp',
+        'src/fuzzaldrin_score.cpp',
+        'src/score_match.cpp',
+        'src/MatcherBase.cpp',
+      ],
+      'conditions': [
+        ['OS == "win"', {
+          'defines': ['_HAS_EXCEPTIONS'],
+          'msvs_disabled_warnings': [
+            4267,  # conversion from 'size_t' to 'int', possible loss of data
+            4530,  # exception unwinding
+          ],
+        }],
+        ['OS == "linux"', {
+          'ldflags': [
+            '-s',
+          ],
+        }],
+      ],
+    }
+  ]
+}

package/lib/main.js

@@ -0,0 +1 @@
+module.exports = require('../build/Release/fuzzy-native.node')

package/lib/main.js.flow

@@ -0,0 +1,45 @@
+/* @flow */
+
+export type MatcherOptions = {
+  // Default: false
+  caseSensitive?: boolean,
+
+  // Default: infinite
+  maxResults?: number,
+
+  // Maximum gap to allow between consecutive letters in a match.
+  // Provide a smaller maxGap to speed up query results.
+  // Default: unlimited
+  maxGap?: number;
+
+  // Default: 1
+  numThreads?: number,
+
+  // Default: false
+  recordMatchIndexes?: boolean,
+}
+
+export type MatchResult = {
+  id: number,
+  value: string,
+
+  // A number in the range (0-1]. Higher scores are more relevant.
+  // 0 denotes "no match" and will never be returned.
+  score: number,
+
+  // Matching character index in `value` for each character in `query`.
+  // This can be costly, so this is only returned if `recordMatchIndexes` was set in `options`.
+  matchIndexes?: Array<number>,
+}
+
+export class Matcher {
+  constructor(candidates: Array<string>) {}
+
+  // Returns all matching candidates (subject to `options`).
+  // Will be ordered by score, descending.
+  match: (query: string, options?: MatcherOptions) => Array<MatchResult>;
+
+  addCandidates: (ids: Array<number>, candidates: Array<string>) => void;
+  removeCandidates: (ids: Array<number>) => void;
+  setCandidates: (ids: Array<number>, candidates: Array<string>) => void;
+}

package/package.json

@@ -0,0 +1,30 @@
+{
+  "name": "@pulsar-edit/fuzzy-native",
+  "version": "1.2.4",
+  "description": "Native C++ implementation of a fuzzy string matcher.",
+  "main": "lib/main.js",
+  "scripts": {
+    "test": "jasmine-node --captureExceptions spec"
+  },
+  "files": [
+    "binding.gyp",
+    "lib",
+    "src"
+  ],
+  "keywords": [
+    "fuzzy",
+    "native",
+    "matcher",
+    "string"
+  ],
+  "author": "Hanson Wang (hanson.wng@gmail.com)",
+  "repository": "https://github.com/pulsar-edit/fuzzy-native",
+  "license": "MIT",
+  "dependencies": {
+    "nan": "2.17.0"
+  },
+  "devDependencies": {
+    "jasmine-node": "^1.14.5",
+    "rimraf": "^2.5.2"
+  }
+}

package/src/MatcherBase.cpp

@@ -0,0 +1,329 @@
+#include "MatcherBase.h"
+#include "score_match.h"
+#include "fuzzaldrin_score.h"
+
+#include <algorithm>
+#include <atomic>
+#include <queue>
+#include <thread>
+
+using namespace std;
+
+typedef priority_queue<MatchResult> ResultHeap;
+
+inline uint64_t letter_bitmask(const std::string &str) {
+  uint64_t result = 0;
+  for (char c : str) {
+    if (c >= 'a' && c <= 'z') {
+      int index = c - 'a';
+      uint64_t count_bit = (result >> (index * 2));
+      // "Increment" the count_bit:
+      // 00 -> 01
+      // 01 -> 11
+      // 11 -> 11
+      count_bit = ((count_bit << 1) | 1) & 3;
+      result |= count_bit << (index * 2);
+    } else if (c == '-') {
+      result |= (1UL << 52);
+    } else if (c >= '0' && c <= '9') {
+      result |= (1UL << (c - '0' + 54));
+    }
+  }
+  return result;
+}
+
+inline string str_to_lower(const std::string &s) {
+  string lower(s);
+  for (auto& c : lower) {
+    if (c >= 'A' && c <= 'Z') {
+      c += 'a' - 'A';
+    }
+  }
+  return lower;
+}
+
+bool is_slash(char c) {
+  return c == '/' || c == '\\';
+}
+
+int num_dirs(const std::string &path) {
+  int num = 0;
+  for (size_t i = 0; i < path.length(); i++) {
+    if (is_slash(path[i])) {
+      num++;
+    }
+  }
+  return num;
+}
+
+int score_based_root_path(const MatchOptions &options,
+                          const MatcherBase::CandidateData &candidate) {
+  const std::string &root = options.root_path;
+  if (root.length() == 0) {
+    return 0;
+  }
+
+  const std::string &value = candidate.value;
+
+  size_t num_common_dirs = 0;
+  size_t i = 0;
+
+  // Count number of common directories
+  for (; i < root.length() && i < value.length(); i++) {
+    if (root[i] != value[i]) {
+      break;
+    }
+    if (is_slash(root[i])) {
+      num_common_dirs++;
+    }
+  }
+
+  if (i == root.length() && i < value.length() && is_slash(value[i])) {
+    num_common_dirs++;
+  }
+
+  return 1000 * num_common_dirs - candidate.num_dirs;
+}
+
+// Push a new entry on the heap while ensuring size <= max_results.
+void push_heap(ResultHeap &heap,
+               float score,
+               int score_based_root_path,
+               uint32_t id,
+               const std::string *value,
+               size_t max_results) {
+  MatchResult result(score, score_based_root_path, id, value);
+  if (heap.size() < max_results || result < heap.top()) {
+    heap.push(std::move(result));
+    if (heap.size() > max_results) {
+      heap.pop();
+    }
+  }
+}
+
+vector<MatchResult> finalize(const string &query,
+                             const string &query_case,
+                             const MatchOptions &options,
+                             bool record_match_indexes,
+                             ResultHeap &&heap) {
+  vector<MatchResult> vec;
+  while (heap.size()) {
+    const MatchResult &result = heap.top();
+    if (record_match_indexes) {
+      result.matchIndexes.reset(new vector<int>(query.size()));
+      string lower = str_to_lower(*result.value);
+      score_match(
+        result.value->c_str(),
+        lower.c_str(),
+        query.c_str(),
+        query_case.c_str(),
+        options,
+        0.0,
+        result.matchIndexes.get()
+      );
+    }
+    vec.push_back(result);
+    heap.pop();
+  }
+  reverse(vec.begin(), vec.end());
+  return vec;
+}
+
+void thread_worker(
+  const string &query,
+  const string &query_case,
+  const MatchOptions &options,
+  bool use_last_match,
+  std::atomic<float>* min_score,
+  size_t max_results,
+  vector<MatcherBase::CandidateData> &candidates,
+  size_t start,
+  size_t end,
+  ResultHeap &result
+) {
+  uint64_t bitmask = letter_bitmask(query_case.c_str());
+  for (size_t i = start; i < end; i++) {
+    auto &candidate = candidates[i];
+    if (use_last_match && !candidate.last_match) {
+      continue;
+    }
+    if ((bitmask & candidate.bitmask) == bitmask) {
+      float score;
+      if(query == "") {
+        score = 1;
+      } else if(options.fuzzaldrin) {
+        score = fuzzaldrin_score(candidate.value, query);
+        score = fuzzaldrin_basename_score(candidate.value, query, score);
+      } else {
+        score = score_match(
+          candidate.value.c_str(),
+          candidate.lowercase.c_str(),
+          query.c_str(),
+          query_case.c_str(),
+          options,
+          min_score->load()
+        );
+      }
+      if (score > 0) {
+        push_heap(
+          result,
+          score,
+          score_based_root_path(options, candidate),
+          candidate.id,
+          &candidate.value,
+          max_results
+        );
+        if (result.size() == max_results) {
+          float current_max = result.top().score;
+          float min_score_value = min_score->load();
+          // Unfortunately there's no thread-safe "max"...
+          // When running compare_exchange_weak it's possible that another
+          // thread wrote to it in the meantime, in which case we have to check
+          // again. Since it's always increasing this is guaranteed to converge.
+          while (current_max > min_score_value) {
+            min_score->compare_exchange_weak(min_score_value, current_max);
+          }
+        }
+        candidate.last_match = true;
+      } else {
+        candidate.last_match = false;
+      }
+    }
+  }
+}
+
+vector<MatchResult> MatcherBase::findMatches(const std::string &query,
+                                             const MatcherOptions &options) {
+  size_t max_results = options.max_results;
+  size_t num_threads = options.num_threads;
+  if (max_results == 0) {
+    max_results = numeric_limits<size_t>::max();
+  }
+  MatchOptions matchOptions;
+  matchOptions.case_sensitive = options.case_sensitive;
+  matchOptions.smart_case = false;
+  matchOptions.max_gap = options.max_gap;
+  matchOptions.root_path = options.root_path;
+  matchOptions.fuzzaldrin = options.fuzzaldrin;
+
+  string new_query;
+  // Ignore all whitespace in the query.
+  for (auto c : query) {
+    if (!isspace(c)) {
+      new_query += c;
+    }
+    if (options.smart_case && isupper(c) && !matchOptions.case_sensitive) {
+      matchOptions.smart_case = true;
+    }
+  }
+
+  string query_case;
+  if (!options.case_sensitive) {
+    query_case = str_to_lower(new_query);
+  } else {
+    query_case = query;
+  }
+
+  // If our current query is just an extension of the last query,
+  // quickly ignore all previous non-matches as an optimization.
+  bool use_last_match = query_case.substr(0, lastQuery_.size()) == lastQuery_;
+  lastQuery_ = query_case;
+
+  ResultHeap combined;
+  std::atomic<float> min_score(0);
+  if (num_threads == 0 || candidates_.size() < 10000) {
+    thread_worker(new_query, query_case, matchOptions, use_last_match, &min_score,
+                  max_results, candidates_, 0, candidates_.size(), combined);
+  } else {
+    vector<ResultHeap> thread_results(num_threads);
+    vector<thread> threads;
+    size_t cur_start = 0;
+    for (size_t i = 0; i < num_threads; i++) {
+      size_t chunk_size = candidates_.size() / num_threads;
+      // Distribute remainder among the chunks.
+      if (i < candidates_.size() % num_threads) {
+        chunk_size++;
+      }
+      threads.emplace_back(
+        thread_worker,
+        ref(new_query),
+        ref(query_case),
+        ref(matchOptions),
+        use_last_match,
+        &min_score,
+        max_results,
+        ref(candidates_),
+        cur_start,
+        cur_start + chunk_size,
+        ref(thread_results[i])
+      );
+      cur_start += chunk_size;
+    }
+
+    for (size_t i = 0; i < num_threads; i++) {
+      threads[i].join();
+      while (thread_results[i].size()) {
+        auto &top = thread_results[i].top();
+        push_heap(
+          combined,
+          top.score,
+          top.score_based_root_path,
+          top.id,
+          top.value,
+          max_results
+        );
+        thread_results[i].pop();
+      }
+    }
+  }
+
+  return finalize(
+    new_query,
+    query_case,
+    matchOptions,
+    options.record_match_indexes,
+    move(combined)
+  );
+}
+
+void MatcherBase::addCandidate(uint32_t id, const string &candidate) {
+  auto it = lookup_.find(id);
+  if (it == lookup_.end()) {
+    string lowercase = str_to_lower(candidate);
+    lookup_[id] = candidates_.size();
+    CandidateData data;
+    data.id = id;
+    data.value = candidate;
+    data.bitmask = letter_bitmask(lowercase.c_str());
+    data.lowercase = move(lowercase);
+    data.last_match = true;
+    data.num_dirs = num_dirs(candidate);
+    candidates_.emplace_back(move(data));
+  }
+}
+
+void MatcherBase::removeCandidate(uint32_t id) {
+  auto it = lookup_.find(id);
+  if (it != lookup_.end()) {
+    if (it->second + 1 != candidates_.size()) {
+      swap(candidates_[it->second], candidates_.back());
+      lookup_[candidates_[it->second].id] = it->second;
+    }
+    candidates_.pop_back();
+    lookup_.erase(id);
+  }
+}
+
+void MatcherBase::clear() {
+  candidates_.clear();
+  lookup_.clear();
+}
+
+void MatcherBase::reserve(size_t n) {
+  candidates_.reserve(n);
+  lookup_.reserve(n);
+}
+
+size_t MatcherBase::size() const {
+  return candidates_.size();
+}

package/src/MatcherBase.h

@@ -0,0 +1,93 @@
+#pragma once
+
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+struct MatcherOptions {
+  bool case_sensitive = false;
+  bool smart_case = false;
+  bool fuzzaldrin = false;
+  size_t num_threads = 0;
+  size_t max_results = 0;
+  size_t max_gap = 0;
+  bool record_match_indexes = false;
+  std::string root_path;
+};
+
+struct MatchResult {
+  float score;
+  uint32_t id;
+  // We can't afford to copy strings around while we're ranking them.
+  // These are not guaranteed to last very long and should be copied out ASAP.
+  const std::string *value;
+  // Only computed if `record_match_indexes` was set to true.
+  mutable std::shared_ptr<std::vector<int>> matchIndexes = nullptr;
+  int score_based_root_path;
+
+  MatchResult(float score,
+              int score_based_root_path,
+              uint32_t id,
+              const std::string *value)
+    : score(score),
+      id(id),
+      value(value),
+      score_based_root_path(score_based_root_path) {}
+
+  // Order small scores to the top of any priority queue.
+  // We need a min-heap to maintain the top-N results.
+  bool operator<(const MatchResult& other) const {
+    if (score == other.score) {
+      // In case of a tie, favour shorter strings.
+      if (score_based_root_path == other.score_based_root_path) {
+        return value->length() < other.value->length();
+      }
+      return score_based_root_path > other.score_based_root_path;
+    }
+    return score > other.score;
+  }
+};
+
+class MatcherBase {
+public:
+  struct CandidateData {
+    uint32_t id;
+    std::string value;
+    std::string lowercase;
+    int num_dirs;
+    /**
+     * A bitmask representing the counts of letters a-z contained in the string.
+     * Bits i*2 and i*2 + 1 represent a count of the i-th letter:
+     * 00 = 0
+     * 01 = 1
+     * 11 = 2
+     * With this scheme, if (bitmask(X) & bitmask(Y)) == bitmask(X) then we
+     * instantly know that Y contains at least all the letters in X.
+     */
+    uint64_t bitmask;
+    /**
+     * True if this was a match against lastQuery_.
+     * Since the most common use case for this library is for typeaheads,
+     * we can often avoid a ton of work by skiping past negatives.
+     * We'll use this only if the new query strictly extends lastQuery_.
+     */
+    bool last_match;
+  };
+
+  std::vector<MatchResult> findMatches(const std::string &query,
+                                       const MatcherOptions &options);
+  void addCandidate(uint32_t id, const std::string &candidate);
+  void removeCandidate(uint32_t id);
+  void clear();
+  void reserve(size_t n);
+  size_t size() const;
+
+private:
+  // Storing candidate data in an array makes table scans significantly faster.
+  // This makes add/remove slightly more expensive, but in our case queries
+  // are significantly more frequent.
+  std::vector<CandidateData> candidates_;
+  std::unordered_map<uint32_t, size_t> lookup_;
+  std::string lastQuery_;
+};