@gmod/trix 1.0.0 → 2.0.2

package/CHANGELOG.md

@@ -1,3 +1,17 @@
+# v2.0.2
+
+- Publish src directory for better source maps
+
+# v2.0.1
+
+- Redeploy to npm with preversion script added
+
+# v2.0.0
+
+- Fix issue with infinite loop
+- Add abortsignal support
+- Only query first word when string with multiple words is entered
+
 # v1.0.0
 
 - Change result format from just the "result" string returned to be "term,result"

package/README.md

@@ -1,4 +1,7 @@
+[![Build Status](https://img.shields.io/github/workflow/status/GMOD/trix-js/Push/main?logo=github&style=for-the-badge)](https://github.com/GMOD/trix-js/actions?query=branch%3Amain+workflow%3APush+)
+
 # trix-js
+
 Read UCSC Trix indexes in pure JavaScript
 
 ## Usage
@@ -9,37 +12,43 @@ import { RemoteFile } from 'generic-filehandle'
 
 // any filehandle object that supports the Nodejs FileHandle API will work.
 // We use generic-filehandle here to demonstrate searching files on remote servers.
-const ixxFile = new RemoteFile('https://hgdownload.soe.ucsc.edu/gbdb/hg38/knownGene.ixx');
-const ixFile = new RemoteFile('https://hgdownload.soe.ucsc.edu/gbdb/hg38/knownGene.ix');
+const ixxFile = new RemoteFile(
+  'https://hgdownload.soe.ucsc.edu/gbdb/hg38/knownGene.ixx',
+)
+const ixFile = new RemoteFile(
+  'https://hgdownload.soe.ucsc.edu/gbdb/hg38/knownGene.ix',
+)
 
-const trix = new Trix(ixxFile, ixFile);
+const trix = new Trix(ixxFile, ixFile)
 
 async function doStuff() {
-  const results = await trix.search('oca');
-  console.log(results);
+  const results = await trix.search('oca')
+  console.log(results)
 }
-doStuff();
-
+doStuff()
 ```
 
 ## Documentation
+
 ### Trix constructor
+
 The Trix class constructor accepts arguments:
+
 - `ixxFile` - a filehandle object for the trix .ixx file
 - `ixFile` - a filehandle object for the trix .ix file
 - `maxResults = 20` - an optional number specifying the maximum number of results to return on `trix.search()`
 
-
 ### Trix search
+
 **Search the index files for a term and find its keys.**<br>
 **In the case of searching with multiple words, `trix.search()` finds the intersection of the result sets.**<br>
 The Trix search function accepts argument:
+
 - `searchString` - a string of space-separated words for what to search the index file and find keys for<br>
-  
+
 The Trix search function returns: <br>
-- `Promise<string[]>` - a promised array of strings where each string is an itemId result
-  
 
+- `Promise<[term,result][] as [string,string][]>` - an array of [term, result] pairs where each term is the left column in the trix and the right column is the trix match
 
 ## Examples
 
@@ -47,49 +56,37 @@ The Trix search function returns: <br>
 import { LocalFile } from 'generic-filehandle'
 import Trix from '@gmod/trix'
 
-const ixxFile = new LocalFile('out.ixx');
-const ixFile = new LocalFile('out.ix');
+const ixxFile = new LocalFile('out.ixx')
+const ixFile = new LocalFile('out.ix')
 
 // limit maxResults to 5
-const trix = new Trix(ixxFile, ixFile, 5);
+const trix = new Trix(ixxFile, ixFile, 5)
 
 async function doStuff() {
-const results1 = await trix.search('herc');
-  console.log(results1);
+  const results1 = await trix.search('herc')
+  console.log(results1)
 
   // increase maxResults to 30
-  trix.maxResults = 30;
+  trix.maxResults = 30
 
-  const results2 = await trix.search('linc');
-  console.log(results2);
+  const results2 = await trix.search('linc')
+  console.log(results2)
 }
 
-doStuff();
+doStuff()
 ```
-<br><br>
-
-
-
-
-
-
-
-
-
-
-
-
-
 
+<br><br>
 
 ## Development
 
-
 ### Test trix-js
+
 First, clone this repo and install npm packages. <br>
 Then, run `npm test`. <br>
 
 ### Test the USCS TrixSearch - Requires Linux
+
 First, clone this repo.
 To run test searches on a track hub using the USCS `TrixSearch`, navigate to `tests/testdata/test#` and run `bash test#script.sh` where # is the test number.
 To change search terms, edit `searchterms.txt`.
@@ -97,7 +94,9 @@ To change search terms, edit `searchterms.txt`.
 **Wondering what to search for?**<br>
 Open up `tests/testdata/test#/input.txt`.
 
-
 **How to test my own .gff.gz data?**<br>
 Navigate to `/test/rawGenomes` and create a directory with your .gff.gz file in it. From within that directory, run `bash ../../programs/gff3ToInput.sh <.gff3.gz FILE> <OUTPUT NAME>`.
 
+## Reference
+
+See https://genome.ucsc.edu/goldenPath/help/trix.html for basic concepts of trix and https://github.com/GMOD/ixixx-js for a javascript implementation of the ixIxx command

package/dist/index.d.ts

@@ -1,59 +1,12 @@
 import type { GenericFilehandle } from 'generic-filehandle';
 export default class Trix {
-    private index;
     private ixFile;
+    private ixxFile;
     maxResults: number;
-    /**
-     * @param ixxFile [anyFile] the second-level trix index file produced by ixIxx.
-     * @param ixFile [anyFile] the first-level trix index file produced by ixIxx.
-     * @param maxResults [number] the maximum number of results to return. Default is set to 20.
-     */
     constructor(ixxFile: GenericFilehandle, ixFile: GenericFilehandle, maxResults?: number);
-    /**
-     * Search trix for the given searchWord(s). Return up to {this.maxResults} results.
-     * This method matches each index prefix against each searchWord. It does not do fuzzy matching.
-     *
-     * @param searchString [string] term(s) separated by spaces to search for id(s).
-     * @returns results [Array<string>] where each string is a corresponding itemId.
-     */
-    search(searchString: string): Promise<string[]>;
-    /**
-     * Seek ahead to the correct position in the .ix file,
-     * then load that chunk of .ix into a buffer.
-     *
-     * @param searchWord [string]
-     * @returns a Buffer holding the sections we want to search.
-     */
+    search(searchString: string, opts?: {
+        signal?: AbortSignal;
+    }): Promise<[string, string][]>;
+    private getIndex;
     private _getBuffer;
-    /**
-     * Given the end position of the last buffer,
-     * load the next chunk  of .ix data into a buffer and return it.
-     *
-     * @param seekPosStart [number] where to start loading data into the new buffer.
-     * @returns a Buffer holding the chunk we want to search.
-     */
-    private _getNextChunk;
-    /**
-     * Create and return a buffer given the start and end position
-     * of what to load from the .ix file.
-     *
-     * @param seekPosStart [number] byte the buffer should start reading from file.
-     * @param seekPosEnd [number] byte the buffer should stop reading from file.
-     * @returns a Buffer holding the chunk of data.
-     */
-    private _createBuffer;
-    /**
-     * Takes in a hit string and returns an array of result terms.
-     *
-     * @param line [string] The line of .ix that is a hit.
-     * @returns results [Array<hit>]. Each hit contains the itemId [string], and wordPos [number].
-     */
-    private _parseHitString;
-    /**
-     * Parses ixx file and constructs a map of {word: ixFileLocation}
-     *
-     * @param ixxFile [anyFile] second level index that is produced by ixIxx.
-     * @returns a ParsedIxx map.
-     */
-    private _parseIxx;
 }

package/dist/index.js

@@ -1,4 +1,15 @@
 "use strict";
+var __assign = (this && this.__assign) || function () {
+    __assign = Object.assign || function(t) {
+        for (var s, i = 1, n = arguments.length; i < n; i++) {
+            s = arguments[i];
+            for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
+                t[p] = s[p];
+        }
+        return t;
+    };
+    return __assign.apply(this, arguments);
+};
 var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
     function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
     return new (P || (P = Promise))(function (resolve, reject) {
@@ -35,17 +46,6 @@ var __generator = (this && this.__generator) || function (thisArg, body) {
         if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
     }
 };
-var __values = (this && this.__values) || function(o) {
-    var s = typeof Symbol === "function" && Symbol.iterator, m = s && o[s], i = 0;
-    if (m) return m.call(o);
-    if (o && typeof o.length === "number") return {
-        next: function () {
-            if (o && i >= o.length) o = void 0;
-            return { value: o && o[i++], done: !o };
-        }
-    };
-    throw new TypeError(s ? "Object is not iterable." : "Symbol.iterator is not defined.");
-};
 var __read = (this && this.__read) || function (o, n) {
     var m = typeof Symbol === "function" && o[Symbol.iterator];
     if (!m) return o;
@@ -63,364 +63,160 @@ var __read = (this && this.__read) || function (o, n) {
     return ar;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-var trixPrefixSize = 5;
-// Define this object with .ixx and .ix files.
-// Then use the trixSearch() function to search for a word.
+var TRIX_PREFIX_SIZE = 5;
+var CHUNK_SIZE = 65536;
+// https://stackoverflow.com/a/9229821/2129219
+function uniqBy(a, key) {
+    var seen = new Set();
+    return a.filter(function (item) {
+        var k = key(item);
+        return seen.has(k) ? false : seen.add(k);
+    });
+}
 var Trix = /** @class */ (function () {
-    /**
-     * @param ixxFile [anyFile] the second-level trix index file produced by ixIxx.
-     * @param ixFile [anyFile] the first-level trix index file produced by ixIxx.
-     * @param maxResults [number] the maximum number of results to return. Default is set to 20.
-     */
     function Trix(ixxFile, ixFile, maxResults) {
         if (maxResults === void 0) { maxResults = 20; }
-        this.index = this._parseIxx(ixxFile);
         this.ixFile = ixFile;
+        this.ixxFile = ixxFile;
         this.maxResults = maxResults;
     }
-    /**
-     * Search trix for the given searchWord(s). Return up to {this.maxResults} results.
-     * This method matches each index prefix against each searchWord. It does not do fuzzy matching.
-     *
-     * @param searchString [string] term(s) separated by spaces to search for id(s).
-     * @returns results [Array<string>] where each string is a corresponding itemId.
-     */
-    Trix.prototype.search = function (searchString) {
+    Trix.prototype.search = function (searchString, opts) {
         return __awaiter(this, void 0, void 0, function () {
-            var resultArr, firstWord, initialSet, searchWords, w, searchWord, bufData, buf, bufPos, resultSet, linePtr, numValues, startsWith, done, i, tempBufData, cur, line, arr, arr_1, arr_1_1, hit;
-            var e_1, _a;
-            return __generator(this, function (_b) {
-                switch (_b.label) {
+            var resultArr, searchWords, searchWord, res, seekPosEnd, buffer, done, _loop_1, this_1, state_1;
+            return __generator(this, function (_a) {
+                switch (_a.label) {
                     case 0:
                         resultArr = [];
-                        firstWord = true;
-                        initialSet = new Set();
                         searchWords = searchString.split(' ');
-                        w = 0;
-                        _b.label = 1;
+                        searchWord = searchWords[0].toLowerCase();
+                        return [4 /*yield*/, this._getBuffer(searchWord, opts)];
                     case 1:
-                        if (!(w < searchWords.length)) return [3 /*break*/, 10];
-                        searchWord = searchWords[w];
-                        searchWord = searchWord.toLowerCase();
-                        return [4 /*yield*/, this._getBuffer(searchWord)];
-                    case 2:
-                        bufData = _b.sent();
-                        buf = bufData.buf;
-                        bufPos = bufData.bufEndPos;
-                        resultSet = new Set();
-                        linePtr = 0;
-                        numValues = 0;
-                        _b.label = 3;
-                    case 3:
-                        if (!(linePtr < buf.byteLength)) return [3 /*break*/, 8];
-                        startsWith = true;
-                        done = false;
-                        i = linePtr;
-                        _b.label = 4;
-                    case 4:
-                        if (!(buf[i] != 10)) return [3 /*break*/, 7];
-                        if (!(i >= buf.byteLength)) return [3 /*break*/, 6];
-                        return [4 /*yield*/, this._getNextChunk(bufPos)];
-                    case 5:
-                        tempBufData = _b.sent();
-                        if (tempBufData) {
-                            buf = tempBufData.buf;
-                            bufPos = tempBufData.bufEndPos;
-                            i = 0;
-                            linePtr = 0;
-                        }
-                        else {
-                            // If tempBufData is null, we reached the end of the file, so we are done.
-                            done = true;
-                            return [3 /*break*/, 7];
-                        }
-                        _b.label = 6;
-                    case 6:
-                        if (startsWith) {
-                            cur = String.fromCharCode(buf[i]);
-                            if (i < linePtr + searchWord.length &&
-                                searchWord[i - linePtr] > cur) {
-                                // searchWord[i] > cur, so keep looping.
-                                startsWith = false;
-                            }
-                            else if (i < linePtr + searchWord.length &&
-                                searchWord[i - linePtr] < cur) {
-                                // searchWord[i] < cur, so we lexicographically will not find any more results.
-                                startsWith = false;
-                                done = true;
-                                return [3 /*break*/, 7];
-                            }
-                            else {
-                                // This condition indicates we found a match.
-                                if (buf[i] === 44) {
-                                    // We found a ',' so increment numValues by one.
-                                    numValues++;
-                                    // If we're searching for one word and we have enough results, break out at the next space.
-                                    if (numValues >= this.maxResults && searchWords.length === 1) {
-                                        while (buf[i] != 32)
-                                            i++;
-                                        return [3 /*break*/, 7];
-                                    }
-                                }
-                            }
+                        res = _a.sent();
+                        if (!res) {
+                            return [2 /*return*/, []];
                         }
-                        i++;
-                        return [3 /*break*/, 4];
-                    case 7:
-                        if (done)
-                            return [3 /*break*/, 8];
-                        // If the line starts with the searchWord, we have a hit!
-                        if (startsWith) {
-                            line = buf.slice(linePtr, i).toString();
-                            arr = this._parseHitString(line);
-                            if (searchWords.length === 1) {
-                                // Only a single search word so add results to array.
-                                resultArr = resultArr.concat(arr);
-                                // Once we have enough results, stop searching.
-                                if (resultArr.length >= this.maxResults)
-                                    return [3 /*break*/, 8];
-                            }
-                            else {
-                                try {
-                                    // Handle multiple words using sets.
-                                    for (arr_1 = (e_1 = void 0, __values(arr)), arr_1_1 = arr_1.next(); !arr_1_1.done; arr_1_1 = arr_1.next()) {
-                                        hit = arr_1_1.value;
-                                        hit = hit.toLowerCase();
-                                        if (firstWord) {
-                                            resultSet.add(hit);
-                                        }
-                                        else {
-                                            if (initialSet.has(hit)) {
-                                                resultSet.add(hit);
-                                                // If it is on the last iteration of words, break after we reach maxResults
-                                                if (w === searchWords.length - 1 &&
-                                                    resultSet.size >= this.maxResults)
-                                                    break;
+                        seekPosEnd = res.seekPosEnd, buffer = res.buffer;
+                        done = false;
+                        _loop_1 = function () {
+                            var foundSomething, str, lines, hits, res2;
+                            return __generator(this, function (_b) {
+                                switch (_b.label) {
+                                    case 0:
+                                        foundSomething = false;
+                                        str = buffer.toString();
+                                        lines = str
+                                            .slice(0, str.lastIndexOf('\n'))
+                                            .split('\n')
+                                            .filter(function (f) { return !!f; });
+                                        hits = lines
+                                            // eslint-disable-next-line @typescript-eslint/no-loop-func
+                                            .filter(function (line) {
+                                            var word = line.split(' ')[0];
+                                            var match = word.startsWith(searchWord);
+                                            if (!foundSomething && match) {
+                                                foundSomething = true;
+                                            }
+                                            // we are done scanning if we are lexicographically greater than the
+                                            // search string
+                                            if (word > searchWord) {
+                                                done = true;
                                             }
+                                            return match;
+                                        })
+                                            .map(function (line) {
+                                            var _a = __read(line.split(' ')), term = _a[0], parts = _a.slice(1);
+                                            return parts.map(function (elt) { return [term, elt.split(',')[0]]; });
+                                        })
+                                            .flat();
+                                        if (!(resultArr.length + hits.length < this_1.maxResults && !done)) return [3 /*break*/, 2];
+                                        return [4 /*yield*/, this_1.ixFile.read(Buffer.alloc(CHUNK_SIZE), 0, CHUNK_SIZE, seekPosEnd, opts)
+                                            // early break if empty response
+                                        ];
+                                    case 1:
+                                        res2 = _b.sent();
+                                        // early break if empty response
+                                        if (!res2.bytesRead) {
+                                            resultArr = resultArr.concat(hits);
+                                            return [2 /*return*/, "break"];
                                         }
-                                    }
-                                }
-                                catch (e_1_1) { e_1 = { error: e_1_1 }; }
-                                finally {
-                                    try {
-                                        if (arr_1_1 && !arr_1_1.done && (_a = arr_1.return)) _a.call(arr_1);
-                                    }
-                                    finally { if (e_1) throw e_1.error; }
+                                        buffer = Buffer.concat([buffer, res2.buffer]);
+                                        seekPosEnd += CHUNK_SIZE;
+                                        return [3 /*break*/, 3];
+                                    case 2:
+                                        if (resultArr.length + hits.length >= this_1.maxResults || done) {
+                                            resultArr = resultArr.concat(hits);
+                                            return [2 /*return*/, "break"];
+                                        }
+                                        _b.label = 3;
+                                    case 3: return [2 /*return*/];
                                 }
-                            }
-                        }
-                        linePtr = i + 1;
-                        return [3 /*break*/, 3];
-                    case 8:
-                        initialSet = resultSet;
-                        firstWord = false;
-                        // If there aren't any results, stop looping, because an intersection with an empty set is an empty set.
-                        if (resultArr.length === 0 && initialSet.size === 0)
-                            return [2 /*return*/, []];
-                        _b.label = 9;
-                    case 9:
-                        w++;
-                        return [3 /*break*/, 1];
-                    case 10:
-                        // 4. Return the hitList [list of string]
-                        if (searchWords.length === 1) {
-                            return [2 /*return*/, resultArr];
-                        }
-                        // Else we need to return our set converted to an array
-                        resultArr = Array.from(initialSet);
-                        if (resultArr.length > this.maxResults)
-                            return [2 /*return*/, resultArr.slice(0, this.maxResults)];
-                        return [2 /*return*/, resultArr];
+                            });
+                        };
+                        this_1 = this;
+                        _a.label = 2;
+                    case 2:
+                        if (!!done) return [3 /*break*/, 4];
+                        return [5 /*yield**/, _loop_1()];
+                    case 3:
+                        state_1 = _a.sent();
+                        if (state_1 === "break")
+                            return [3 /*break*/, 4];
+                        return [3 /*break*/, 2];
+                    case 4: 
+                    // deduplicate results based on the detail column (resultArr[1])
+                    return [2 /*return*/, uniqBy(resultArr, function (elt) { return elt[1]; }).slice(0, this.maxResults)];
                 }
             });
         });
     };
-    // Private Methods:
-    /**
-     * Seek ahead to the correct position in the .ix file,
-     * then load that chunk of .ix into a buffer.
-     *
-     * @param searchWord [string]
-     * @returns a Buffer holding the sections we want to search.
-     */
-    Trix.prototype._getBuffer = function (searchWord) {
+    Trix.prototype.getIndex = function (opts) {
         return __awaiter(this, void 0, void 0, function () {
-            var seekPosStart, seekPosEnd, indexes, indexes_1, indexes_1_1, _a, key, value, trimmedKey;
-            var e_2, _b;
-            return __generator(this, function (_c) {
-                switch (_c.label) {
-                    case 0:
-                        seekPosStart = 0;
-                        seekPosEnd = -1;
-                        return [4 /*yield*/, this.index];
+            var file;
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0: return [4 /*yield*/, this.ixxFile.readFile(__assign({ encoding: 'utf8' }, opts))];
                     case 1:
-                        indexes = _c.sent();
-                        try {
-                            for (indexes_1 = __values(indexes), indexes_1_1 = indexes_1.next(); !indexes_1_1.done; indexes_1_1 = indexes_1.next()) {
-                                _a = __read(indexes_1_1.value, 2), key = _a[0], value = _a[1];
-                                trimmedKey = key.slice(0, searchWord.length);
-                                if (seekPosEnd === -1) {
-                                    if (trimmedKey >= searchWord) {
-                                        // We reached the end pos in the file.
-                                        seekPosEnd = value - 1;
-                                        break;
-                                    }
-                                    else {
-                                        seekPosStart = value;
-                                    }
-                                }
-                            }
-                        }
-                        catch (e_2_1) { e_2 = { error: e_2_1 }; }
-                        finally {
-                            try {
-                                if (indexes_1_1 && !indexes_1_1.done && (_b = indexes_1.return)) _b.call(indexes_1);
-                            }
-                            finally { if (e_2) throw e_2.error; }
-                        }
-                        // Return the buffer and its end position in the file.
-                        return [2 /*return*/, this._createBuffer(seekPosStart, seekPosEnd)];
+                        file = _a.sent();
+                        return [2 /*return*/, file
+                                .split('\n')
+                                .filter(function (f) { return !!f; })
+                                .map(function (line) {
+                                var prefix = line.slice(0, TRIX_PREFIX_SIZE);
+                                var posStr = line.slice(TRIX_PREFIX_SIZE);
+                                var pos = Number.parseInt(posStr, 16);
+                                return [prefix, pos];
+                            })];
                 }
             });
         });
     };
-    /**
-     * Given the end position of the last buffer,
-     * load the next chunk  of .ix data into a buffer and return it.
-     *
-     * @param seekPosStart [number] where to start loading data into the new buffer.
-     * @returns a Buffer holding the chunk we want to search.
-     */
-    Trix.prototype._getNextChunk = function (seekPosStart) {
+    Trix.prototype._getBuffer = function (searchWord, opts) {
         return __awaiter(this, void 0, void 0, function () {
-            var seekPosEnd, indexes, indexes_2, indexes_2_1, _a, key, value;
-            var e_3, _b;
-            return __generator(this, function (_c) {
-                switch (_c.label) {
+            var seekPosStart, seekPosEnd, indexes, len, res;
+            return __generator(this, function (_a) {
+                switch (_a.label) {
                     case 0:
-                        if (seekPosStart == -1)
-                            return [2 /*return*/, null];
+                        seekPosStart = 0;
                         seekPosEnd = -1;
-                        return [4 /*yield*/, this.index];
+                        return [4 /*yield*/, this.getIndex(opts)];
                     case 1:
-                        indexes = _c.sent();
-                        try {
-                            for (indexes_2 = __values(indexes), indexes_2_1 = indexes_2.next(); !indexes_2_1.done; indexes_2_1 = indexes_2.next()) {
-                                _a = __read(indexes_2_1.value, 2), key = _a[0], value = _a[1];
-                                if (value <= seekPosStart + 1)
-                                    continue;
-                                seekPosEnd = value;
-                                break;
-                            }
-                        }
-                        catch (e_3_1) { e_3 = { error: e_3_1 }; }
-                        finally {
-                            try {
-                                if (indexes_2_1 && !indexes_2_1.done && (_b = indexes_2.return)) _b.call(indexes_2);
+                        indexes = _a.sent();
+                        indexes.forEach(function (_a) {
+                            var _b = __read(_a, 2), key = _b[0], value = _b[1];
+                            var trimmedKey = key.slice(0, searchWord.length);
+                            if (trimmedKey < searchWord) {
+                                seekPosStart = value;
+                                seekPosEnd = value + 65536;
                             }
-                            finally { if (e_3) throw e_3.error; }
+                        });
+                        len = seekPosEnd - seekPosStart;
+                        if (len < 0) {
+                            return [2 /*return*/, undefined];
                         }
-                        seekPosStart--;
-                        // Return the buffer and its end position in the file.
-                        return [2 /*return*/, this._createBuffer(seekPosStart, seekPosEnd)];
-                }
-            });
-        });
-    };
-    /**
-     * Create and return a buffer given the start and end position
-     * of what to load from the .ix file.
-     *
-     * @param seekPosStart [number] byte the buffer should start reading from file.
-     * @param seekPosEnd [number] byte the buffer should stop reading from file.
-     * @returns a Buffer holding the chunk of data.
-     */
-    Trix.prototype._createBuffer = function (seekPosStart, seekPosEnd) {
-        return __awaiter(this, void 0, void 0, function () {
-            var bufLength, stat, buf;
-            return __generator(this, function (_a) {
-                switch (_a.label) {
-                    case 0:
-                        if (!(seekPosEnd < 0)) return [3 /*break*/, 2];
-                        return [4 /*yield*/, this.ixFile.stat()];
-                    case 1:
-                        stat = _a.sent();
-                        bufLength = stat.size - seekPosStart;
-                        return [3 /*break*/, 3];
+                        return [4 /*yield*/, this.ixFile.read(Buffer.alloc(len), 0, len, seekPosStart, opts)];
                     case 2:
-                        bufLength = seekPosEnd - seekPosStart;
-                        _a.label = 3;
-                    case 3:
-                        buf = Buffer.alloc(bufLength);
-                        return [4 /*yield*/, this.ixFile.read(buf, 0, bufLength, seekPosStart)];
-                    case 4:
-                        _a.sent();
-                        // Return the buffer and its end position in the file.
-                        return [2 /*return*/, { buf: buf, bufEndPos: seekPosEnd }];
-                }
-            });
-        });
-    };
-    /**
-     * Takes in a hit string and returns an array of result terms.
-     *
-     * @param line [string] The line of .ix that is a hit.
-     * @returns results [Array<hit>]. Each hit contains the itemId [string], and wordPos [number].
-     */
-    Trix.prototype._parseHitString = function (line) {
-        var e_4, _a;
-        var arr = [];
-        var _b = __read(line.split(' ')), term = _b[0], parts = _b.slice(1); // skip term
-        try {
-            // Each result is of format: "{itemId},{wordPos}"
-            // Parse the entire line of these and return
-            for (var parts_1 = __values(parts), parts_1_1 = parts_1.next(); !parts_1_1.done; parts_1_1 = parts_1.next()) {
-                var part = parts_1_1.value;
-                var pair = part.split(',');
-                if (pair.length === 2) {
-                    var itemId = pair[0];
-                    var wordPos = Number.parseInt(pair[1]);
-                    if (typeof wordPos !== 'number' || isNaN(wordPos))
-                        throw new Error("Error in ix index format at term " + itemId + " for word " + parts[0]);
-                    arr.push(term + "," + itemId);
-                }
-                else if (pair.length > 1) {
-                    throw new Error("Error in ix index format at word " + parts[0]);
-                }
-            }
-        }
-        catch (e_4_1) { e_4 = { error: e_4_1 }; }
-        finally {
-            try {
-                if (parts_1_1 && !parts_1_1.done && (_a = parts_1.return)) _a.call(parts_1);
-            }
-            finally { if (e_4) throw e_4.error; }
-        }
-        return arr;
-    };
-    /**
-     * Parses ixx file and constructs a map of {word: ixFileLocation}
-     *
-     * @param ixxFile [anyFile] second level index that is produced by ixIxx.
-     * @returns a ParsedIxx map.
-     */
-    Trix.prototype._parseIxx = function (ixxFile) {
-        return __awaiter(this, void 0, void 0, function () {
-            var file, lines;
-            return __generator(this, function (_a) {
-                switch (_a.label) {
-                    case 0: return [4 /*yield*/, ixxFile.readFile('utf8')];
-                    case 1:
-                        file = (_a.sent());
-                        lines = file.split('\n');
-                        return [2 /*return*/, new Map(lines
-                                .filter(function (f) { return !!f; })
-                                .map(function (line) {
-                                var prefix = line.substr(0, trixPrefixSize);
-                                var posStr = line.substr(trixPrefixSize);
-                                var pos = Number.parseInt(posStr, 16);
-                                return [prefix, pos];
-                            }))];
+                        res = _a.sent();
+                        return [2 /*return*/, __assign(__assign({}, res), { seekPosEnd: seekPosEnd })];
                 }
             });
         });
@@ -428,3 +224,4 @@ var Trix = /** @class */ (function () {
     return Trix;
 }());
 exports.default = Trix;
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEA,IAAM,gBAAgB,GAAG,CAAC,CAAA;AAE1B,IAAM,UAAU,GAAG,KAAK,CAAA;AAExB,8CAA8C;AAC9C,SAAS,MAAM,CAAC,CAAqB,EAAE,GAAsC;IAC3E,IAAM,IAAI,GAAG,IAAI,GAAG,EAAE,CAAA;IACtB,OAAO,CAAC,CAAC,MAAM,CAAC,UAAA,IAAI;QAClB,IAAM,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,CAAA;QACnB,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;IAC1C,CAAC,CAAC,CAAA;AACJ,CAAC;AAED;IAOE,cACE,OAA0B,EAC1B,MAAyB,EACzB,UAAe;QAAf,2BAAA,EAAA,eAAe;QAEf,IAAI,CAAC,MAAM,GAAG,MAAM,CAAA;QACpB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAA;QACtB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAA;IAC9B,CAAC;IAEK,qBAAM,GAAZ,UAAa,YAAoB,EAAE,IAA+B;;;;;;wBAC5D,SAAS,GAAG,EAAwB,CAAA;wBAClC,WAAW,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;wBAGrC,UAAU,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAA;wBACnC,qBAAM,IAAI,CAAC,UAAU,CAAC,UAAU,EAAE,IAAI,CAAC,EAAA;;wBAA7C,GAAG,GAAG,SAAuC;wBACnD,IAAI,CAAC,GAAG,EAAE;4BACR,sBAAO,EAAE,EAAA;yBACV;wBAEK,UAAU,GAAa,GAAG,WAAhB,EAAE,MAAM,GAAK,GAAG,OAAR,CAAQ;wBAC5B,IAAI,GAAG,KAAK,CAAA;;;;;;wCAEV,cAAc,GAAG,KAAK,CAAA;wCACpB,GAAG,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAA;wCAIvB,KAAK,GAAG,GAAG;6CACd,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;6CAC/B,KAAK,CAAC,IAAI,CAAC;6CACX,MAAM,CAAC,UAAA,CAAC,IAAI,OAAA,CAAC,CAAC,CAAC,EAAH,CAAG,CAAC,CAAA;wCAEb,IAAI,GAAG,KAAK;4CAChB,2DAA2D;6CAC1D,MAAM,CAAC,UAAA,IAAI;4CACV,IAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAA;4CAC/B,IAAM,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,CAAA;4CACzC,IAAI,CAAC,cAAc,IAAI,KAAK,EAAE;gDAC5B,cAAc,GAAG,IAAI,CAAA;6CACtB;4CAED,oEAAoE;4CACpE,gBAAgB;4CAChB,IAAI,IAAI,GAAG,UAAU,EAAE;gDACrB,IAAI,GAAG,IAAI,CAAA;6CACZ;4CACD,OAAO,KAAK,CAAA;wCACd,CAAC,CAAC;6CACD,GAAG,CAAC,UAAA,IAAI;4CACD,IAAA,KAAA,OAAmB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA,EAAjC,IAAI,QAAA,EAAK,KAAK,cAAmB,CAAA;4CACxC,OAAO,KAAK,CAAC,GAAG,CAAC,UAAA,GAAG,IAAI,OAAA,CAAC,IAAI,EAAE,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAzB,CAAyB,CAAC,CAAA;wCACpD,CAAC,CAAC;6CACD,IAAI,EAAwB,CAAA;6CAI3B,CAAA,SAAS,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,GAAG,OAAK,UAAU,IAAI,CAAC,IAAI,CAAA,EAAzD,wBAAyD;wCAE9C,qBAAM,OAAK,MAAM,CAAC,IAAI,CACjC,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,EACxB,CAAC,EACD,UAAU,EACV,UAAU,EACV,IAAI,CACL;4CAED,gCAAgC;0CAF/B;;wCANK,IAAI,GAAG,SAMZ;wCAED,gCAAgC;wCAChC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE;4CACnB,SAAS,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;;yCAEnC;wCACD,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAA;wCAC7C,UAAU,IAAI,UAAU,CAAA;;;wCAKrB,IAAI,SAAS,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,OAAK,UAAU,IAAI,IAAI,EAAE;4CAClE,SAAS,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;;yCAEnC;;;;;;;;;6BA3DI,CAAC,IAAI;;;;;;;;oBA8DZ,gEAAgE;oBAChE,sBAAO,MAAM,CAAC,SAAS,EAAE,UAAA,GAAG,IAAI,OAAA,GAAG,CAAC,CAAC,CAAC,EAAN,CAAM,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC,EAAA;;;;KAClE;IAEa,uBAAQ,GAAtB,UAAuB,IAA+B;;;;;4BACvC,qBAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,YACtC,QAAQ,EAAE,MAAM,IACb,IAAI,EACP,EAAA;;wBAHI,IAAI,GAAG,SAGX;wBACF,sBAAO,IAAI;iCACR,KAAK,CAAC,IAAI,CAAC;iCACX,MAAM,CAAC,UAAA,CAAC,IAAI,OAAA,CAAC,CAAC,CAAC,EAAH,CAAG,CAAC;iCAChB,GAAG,CAAC,UAAA,IAAI;gCACP,IAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,gBAAgB,CAAC,CAAA;gCAC9C,IAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAA;gCAC3C,IAAM,GAAG,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;gCACvC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAqB,CAAA;4BAC1C,CAAC,CAAC,EAAA;;;;KACL;IAEa,yBAAU,GAAxB,UACE,UAAkB,EAClB,IAA+B;;;;;;wBAE3B,YAAY,GAAG,CAAC,CAAA;wBAChB,UAAU,GAAG,CAAC,CAAC,CAAA;wBACH,qBAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAA;;wBAAnC,OAAO,GAAG,SAAyB;wBACzC,OAAO,CAAC,OAAO,CAAC,UAAC,EAAY;gCAAZ,KAAA,aAAY,EAAX,GAAG,QAAA,EAAE,KAAK,QAAA;4BAC1B,IAAM,UAAU,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,MAAM,CAAC,CAAA;4BAClD,IAAI,UAAU,GAAG,UAAU,EAAE;gCAC3B,YAAY,GAAG,KAAK,CAAA;gCACpB,UAAU,GAAG,KAAK,GAAG,KAAK,CAAA;6BAC3B;wBACH,CAAC,CAAC,CAAA;wBAGI,GAAG,GAAG,UAAU,GAAG,YAAY,CAAA;wBACrC,IAAI,GAAG,GAAG,CAAC,EAAE;4BACX,sBAAO,SAAS,EAAA;yBACjB;wBACW,qBAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAChC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,EACjB,CAAC,EACD,GAAG,EACH,YAAY,EACZ,IAAI,CACL,EAAA;;wBANK,GAAG,GAAG,SAMX;wBACD,4CACK,GAAG,KACN,UAAU,YAAA,KACX;;;;KACF;IACH,WAAC;AAAD,CAAC,AAhJD,IAgJC"}

package/esm/index.d.ts

@@ -0,0 +1,12 @@
+import type { GenericFilehandle } from 'generic-filehandle';
+export default class Trix {
+    private ixFile;
+    private ixxFile;
+    maxResults: number;
+    constructor(ixxFile: GenericFilehandle, ixFile: GenericFilehandle, maxResults?: number);
+    search(searchString: string, opts?: {
+        signal?: AbortSignal;
+    }): Promise<[string, string][]>;
+    private getIndex;
+    private _getBuffer;
+}

package/esm/index.js

@@ -0,0 +1,121 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const TRIX_PREFIX_SIZE = 5;
+const CHUNK_SIZE = 65536;
+// https://stackoverflow.com/a/9229821/2129219
+function uniqBy(a, key) {
+    const seen = new Set();
+    return a.filter(item => {
+        const k = key(item);
+        return seen.has(k) ? false : seen.add(k);
+    });
+}
+class Trix {
+    constructor(ixxFile, ixFile, maxResults = 20) {
+        this.ixFile = ixFile;
+        this.ixxFile = ixxFile;
+        this.maxResults = maxResults;
+    }
+    async search(searchString, opts) {
+        let resultArr = [];
+        const searchWords = searchString.split(' ');
+        // we only search one word at a time
+        const searchWord = searchWords[0].toLowerCase();
+        const res = await this._getBuffer(searchWord, opts);
+        if (!res) {
+            return [];
+        }
+        let { seekPosEnd, buffer } = res;
+        let done = false;
+        while (!done) {
+            let foundSomething = false;
+            const str = buffer.toString();
+            // slice to lastIndexOf('\n') to make sure we get complete records
+            // since the buffer fetch could get halfway into a record
+            const lines = str
+                .slice(0, str.lastIndexOf('\n'))
+                .split('\n')
+                .filter(f => !!f);
+            const hits = lines
+                // eslint-disable-next-line @typescript-eslint/no-loop-func
+                .filter(line => {
+                const word = line.split(' ')[0];
+                const match = word.startsWith(searchWord);
+                if (!foundSomething && match) {
+                    foundSomething = true;
+                }
+                // we are done scanning if we are lexicographically greater than the
+                // search string
+                if (word > searchWord) {
+                    done = true;
+                }
+                return match;
+            })
+                .map(line => {
+                const [term, ...parts] = line.split(' ');
+                return parts.map(elt => [term, elt.split(',')[0]]);
+            })
+                .flat();
+            // if we are not done, and we haven't filled up maxResults with hits yet,
+            // then refetch
+            if (resultArr.length + hits.length < this.maxResults && !done) {
+                // eslint-disable-next-line no-await-in-loop
+                const res2 = await this.ixFile.read(Buffer.alloc(CHUNK_SIZE), 0, CHUNK_SIZE, seekPosEnd, opts);
+                // early break if empty response
+                if (!res2.bytesRead) {
+                    resultArr = resultArr.concat(hits);
+                    break;
+                }
+                buffer = Buffer.concat([buffer, res2.buffer]);
+                seekPosEnd += CHUNK_SIZE;
+            }
+            // if we have filled up the hits, or we are detected to be done via the
+            // filtering, then return
+            else if (resultArr.length + hits.length >= this.maxResults || done) {
+                resultArr = resultArr.concat(hits);
+                break;
+            }
+        }
+        // deduplicate results based on the detail column (resultArr[1])
+        return uniqBy(resultArr, elt => elt[1]).slice(0, this.maxResults);
+    }
+    async getIndex(opts) {
+        const file = await this.ixxFile.readFile({
+            encoding: 'utf8',
+            ...opts,
+        });
+        return file
+            .split('\n')
+            .filter(f => !!f)
+            .map(line => {
+            const prefix = line.slice(0, TRIX_PREFIX_SIZE);
+            const posStr = line.slice(TRIX_PREFIX_SIZE);
+            const pos = Number.parseInt(posStr, 16);
+            return [prefix, pos];
+        });
+    }
+    async _getBuffer(searchWord, opts) {
+        let seekPosStart = 0;
+        let seekPosEnd = -1;
+        const indexes = await this.getIndex(opts);
+        indexes.forEach(([key, value]) => {
+            const trimmedKey = key.slice(0, searchWord.length);
+            if (trimmedKey < searchWord) {
+                seekPosStart = value;
+                seekPosEnd = value + 65536;
+            }
+        });
+        // Return the buffer and its end position in the file.
+        const len = seekPosEnd - seekPosStart;
+        if (len < 0) {
+            return undefined;
+        }
+        const res = await this.ixFile.read(Buffer.alloc(len), 0, len, seekPosStart, opts);
+        return {
+            ...res,
+            seekPosEnd,
+        };
+    }
+}
+exports.default = Trix;
+//# sourceMappingURL=index.js.map

package/esm/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;AAEA,MAAM,gBAAgB,GAAG,CAAC,CAAA;AAE1B,MAAM,UAAU,GAAG,KAAK,CAAA;AAExB,8CAA8C;AAC9C,SAAS,MAAM,CAAC,CAAqB,EAAE,GAAsC;IAC3E,MAAM,IAAI,GAAG,IAAI,GAAG,EAAE,CAAA;IACtB,OAAO,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;QACrB,MAAM,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,CAAA;QACnB,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;IAC1C,CAAC,CAAC,CAAA;AACJ,CAAC;AAED,MAAqB,IAAI;IAOvB,YACE,OAA0B,EAC1B,MAAyB,EACzB,UAAU,GAAG,EAAE;QAEf,IAAI,CAAC,MAAM,GAAG,MAAM,CAAA;QACpB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAA;QACtB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAA;IAC9B,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,YAAoB,EAAE,IAA+B;QAChE,IAAI,SAAS,GAAG,EAAwB,CAAA;QACxC,MAAM,WAAW,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;QAE3C,oCAAoC;QACpC,MAAM,UAAU,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAA;QAC/C,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,EAAE,IAAI,CAAC,CAAA;QACnD,IAAI,CAAC,GAAG,EAAE;YACR,OAAO,EAAE,CAAA;SACV;QAED,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,GAAG,GAAG,CAAA;QAChC,IAAI,IAAI,GAAG,KAAK,CAAA;QAChB,OAAO,CAAC,IAAI,EAAE;YACZ,IAAI,cAAc,GAAG,KAAK,CAAA;YAC1B,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAA;YAE7B,kEAAkE;YAClE,yDAAyD;YACzD,MAAM,KAAK,GAAG,GAAG;iBACd,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;iBAC/B,KAAK,CAAC,IAAI,CAAC;iBACX,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;YAEnB,MAAM,IAAI,GAAG,KAAK;gBAChB,2DAA2D;iBAC1D,MAAM,CAAC,IAAI,CAAC,EAAE;gBACb,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAA;gBAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,CAAA;gBACzC,IAAI,CAAC,cAAc,IAAI,KAAK,EAAE;oBAC5B,cAAc,GAAG,IAAI,CAAA;iBACtB;gBAED,oEAAoE;gBACpE,gBAAgB;gBAChB,IAAI,IAAI,GAAG,UAAU,EAAE;oBACrB,IAAI,GAAG,IAAI,CAAA;iBACZ;gBACD,OAAO,KAAK,CAAA;YACd,CAAC,CAAC;iBACD,GAAG,CAAC,IAAI,CAAC,EAAE;gBACV,MAAM,CAAC,IAAI,EAAE,GAAG,KAAK,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;gBACxC,OAAO,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;YACpD,CAAC,CAAC;iBACD,IAAI,EAAwB,CAAA;YAE/B,yEAAyE;YACzE,eAAe;YACf,IAAI,SAAS,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,UAAU,IAAI,CAAC,IAAI,EAAE;gBAC7D,4CAA4C;gBAC5C,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CACjC,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,EACxB,CAAC,EACD,UAAU,EACV,UAAU,EACV,IAAI,CACL,CAAA;gBAED,gCAAgC;gBAChC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE;oBACnB,SAAS,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;oBAClC,MAAK;iBACN;gBACD,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAA;gBAC7C,UAAU,IAAI,UAAU,CAAA;aACzB;YAED,uEAAuE;YACvE,yBAAyB;iBACpB,IAAI,SAAS,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,IAAI,IAAI,EAAE;gBAClE,SAAS,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;gBAClC,MAAK;aACN;SACF;QAED,gEAAgE;QAChE,OAAO,MAAM,CAAC,SAAS,EAAE,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC,CAAA;IACnE,CAAC;IAEO,KAAK,CAAC,QAAQ,CAAC,IAA+B;QACpD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC;YACvC,QAAQ,EAAE,MAAM;YAChB,GAAG,IAAI;SACR,CAAC,CAAA;QACF,OAAO,IAAI;aACR,KAAK,CAAC,IAAI,CAAC;aACX,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;aAChB,GAAG,CAAC,IAAI,CAAC,EAAE;YACV,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,gBAAgB,CAAC,CAAA;YAC9C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAA;YAC3C,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;YACvC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAqB,CAAA;QAC1C,CAAC,CAAC,CAAA;IACN,CAAC;IAEO,KAAK,CAAC,UAAU,CACtB,UAAkB,EAClB,IAA+B;QAE/B,IAAI,YAAY,GAAG,CAAC,CAAA;QACpB,IAAI,UAAU,GAAG,CAAC,CAAC,CAAA;QACnB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;QACzC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;YAC/B,MAAM,UAAU,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,MAAM,CAAC,CAAA;YAClD,IAAI,UAAU,GAAG,UAAU,EAAE;gBAC3B,YAAY,GAAG,KAAK,CAAA;gBACpB,UAAU,GAAG,KAAK,GAAG,KAAK,CAAA;aAC3B;QACH,CAAC,CAAC,CAAA;QAEF,sDAAsD;QACtD,MAAM,GAAG,GAAG,UAAU,GAAG,YAAY,CAAA;QACrC,IAAI,GAAG,GAAG,CAAC,EAAE;YACX,OAAO,SAAS,CAAA;SACjB;QACD,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAChC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,EACjB,CAAC,EACD,GAAG,EACH,YAAY,EACZ,IAAI,CACL,CAAA;QACD,OAAO;YACL,GAAG,GAAG;YACN,UAAU;SACX,CAAA;IACH,CAAC;CACF;AAhJD,uBAgJC"}

package/package.json

@@ -1,33 +1,43 @@
 {
-  "version": "1.0.0",
+  "version": "2.0.2",
   "license": "Apache-2.0",
   "main": "dist/index.js",
+  "module": "esm/index.js",
   "files": [
-    "dist"
+    "dist",
+    "esm",
+    "src"
   ],
   "engines": {
     "node": ">=10"
   },
   "scripts": {
-    "prebuild": "rimraf dist",
-    "build": "tsc",
+    "lint": "eslint --report-unused-disable-directives --max-warnings 0 --ext .js,.ts src ",
+    "prebuild": "rimraf dist esm",
+    "build:esm": "tsc --target es2018 --outDir esm",
+    "build:es5": "tsc --target es5 --outDir dist",
+    "build": "npm run build:esm && npm run build:es5",
+    "preversion": "npm run lint && npm test && npm run build",
     "postversion": "git push --follow-tags",
     "test": "jest"
   },
-  "prettier": {
-    "printWidth": 80,
-    "semi": true,
-    "singleQuote": true,
-    "trailingComma": "es5"
-  },
   "name": "@gmod/trix",
   "author": "Matt Morgan",
   "repository": "GMOD/trix-js",
   "devDependencies": {
-    "@types/jest": "^26.0.24",
-    "@types/node": "^14.14.37",
+    "@types/jest": "^27.0.3",
+    "@types/node": "^16.11.13",
+    "@typescript-eslint/eslint-plugin": "^5.7.0",
+    "@typescript-eslint/parser": "^5.7.0",
+    "eslint": "^7.0.0",
+    "eslint-config-airbnb-base": "^15.0.0",
+    "eslint-config-airbnb-typescript": "^16.1.0",
+    "eslint-config-prettier": "^8.3.0",
+    "eslint-plugin-import": "^2.25.3",
+    "eslint-plugin-prettier": "^4.0.0",
     "generic-filehandle": "^2.1.0",
     "jest": "^27.0.6",
+    "prettier": "^2.5.1",
     "rimraf": "^3.0.2",
     "ts-jest": "^27.0.4",
     "typescript": "^4.3.5"

package/src/index.ts

@@ -0,0 +1,160 @@
+import type { GenericFilehandle } from 'generic-filehandle'
+
+const TRIX_PREFIX_SIZE = 5
+
+const CHUNK_SIZE = 65536
+
+// https://stackoverflow.com/a/9229821/2129219
+function uniqBy(a: [string, string][], key: (elt: [string, string]) => string) {
+  const seen = new Set()
+  return a.filter(item => {
+    const k = key(item)
+    return seen.has(k) ? false : seen.add(k)
+  })
+}
+
+export default class Trix {
+  private ixFile: GenericFilehandle
+
+  private ixxFile: GenericFilehandle
+
+  maxResults: number
+
+  constructor(
+    ixxFile: GenericFilehandle,
+    ixFile: GenericFilehandle,
+    maxResults = 20,
+  ) {
+    this.ixFile = ixFile
+    this.ixxFile = ixxFile
+    this.maxResults = maxResults
+  }
+
+  async search(searchString: string, opts?: { signal?: AbortSignal }) {
+    let resultArr = [] as [string, string][]
+    const searchWords = searchString.split(' ')
+
+    // we only search one word at a time
+    const searchWord = searchWords[0].toLowerCase()
+    const res = await this._getBuffer(searchWord, opts)
+    if (!res) {
+      return []
+    }
+
+    let { seekPosEnd, buffer } = res
+    let done = false
+    while (!done) {
+      let foundSomething = false
+      const str = buffer.toString()
+
+      // slice to lastIndexOf('\n') to make sure we get complete records
+      // since the buffer fetch could get halfway into a record
+      const lines = str
+        .slice(0, str.lastIndexOf('\n'))
+        .split('\n')
+        .filter(f => !!f)
+
+      const hits = lines
+        // eslint-disable-next-line @typescript-eslint/no-loop-func
+        .filter(line => {
+          const word = line.split(' ')[0]
+          const match = word.startsWith(searchWord)
+          if (!foundSomething && match) {
+            foundSomething = true
+          }
+
+          // we are done scanning if we are lexicographically greater than the
+          // search string
+          if (word > searchWord) {
+            done = true
+          }
+          return match
+        })
+        .map(line => {
+          const [term, ...parts] = line.split(' ')
+          return parts.map(elt => [term, elt.split(',')[0]])
+        })
+        .flat() as [string, string][]
+
+      // if we are not done, and we haven't filled up maxResults with hits yet,
+      // then refetch
+      if (resultArr.length + hits.length < this.maxResults && !done) {
+        // eslint-disable-next-line no-await-in-loop
+        const res2 = await this.ixFile.read(
+          Buffer.alloc(CHUNK_SIZE),
+          0,
+          CHUNK_SIZE,
+          seekPosEnd,
+          opts,
+        )
+
+        // early break if empty response
+        if (!res2.bytesRead) {
+          resultArr = resultArr.concat(hits)
+          break
+        }
+        buffer = Buffer.concat([buffer, res2.buffer])
+        seekPosEnd += CHUNK_SIZE
+      }
+
+      // if we have filled up the hits, or we are detected to be done via the
+      // filtering, then return
+      else if (resultArr.length + hits.length >= this.maxResults || done) {
+        resultArr = resultArr.concat(hits)
+        break
+      }
+    }
+
+    // deduplicate results based on the detail column (resultArr[1])
+    return uniqBy(resultArr, elt => elt[1]).slice(0, this.maxResults)
+  }
+
+  private async getIndex(opts?: { signal?: AbortSignal }) {
+    const file = await this.ixxFile.readFile({
+      encoding: 'utf8',
+      ...opts,
+    })
+    return file
+      .split('\n')
+      .filter(f => !!f)
+      .map(line => {
+        const prefix = line.slice(0, TRIX_PREFIX_SIZE)
+        const posStr = line.slice(TRIX_PREFIX_SIZE)
+        const pos = Number.parseInt(posStr, 16)
+        return [prefix, pos] as [string, number]
+      })
+  }
+
+  private async _getBuffer(
+    searchWord: string,
+    opts?: { signal?: AbortSignal },
+  ) {
+    let seekPosStart = 0
+    let seekPosEnd = -1
+    const indexes = await this.getIndex(opts)
+    indexes.forEach(([key, value]) => {
+      const trimmedKey = key.slice(0, searchWord.length)
+      if (trimmedKey < searchWord) {
+        seekPosStart = value
+        seekPosEnd = value + 65536
+      }
+    })
+
+    // Return the buffer and its end position in the file.
+    const len = seekPosEnd - seekPosStart
+    if (len < 0) {
+      return undefined
+    }
+    const res = await this.ixFile.read(
+      Buffer.alloc(len),
+      0,
+      len,
+      seekPosStart,
+      opts,
+    )
+    return {
+      ...res,
+      seekPosEnd,
+    }
+  }
+}