jexidb 1.0.8 → 1.1.0

package/README.md

@@ -26,7 +26,8 @@ To create a new instance of the database, you need to provide a file path where
 import { Database } from 'jexidb'; // ESM
 
 const db = new Database('path/to/database.jdb', { // file will be created if it does not already exist
-  v8: false, // false by default, set to true to use V8 serialization instead of JSON.
+  v8: false, //
+  create: true, // create the file if it does not exist (default is true)
   compress: false, // set to true to compress each entry
   compressIndex: false, // set to true to compress the index only
   indexes: { // keys to use in queries, only those key values ​​are kept in memory, so fewer specified keys lead to improved performance

package/dist/Database.cjs

@@ -53,10 +53,12 @@ var Database = exports.Database = /*#__PURE__*/function (_EventEmitter) {
     _this2 = _callSuper(this, Database);
     _this2.opts = Object.assign({
       v8: false,
+      create: true,
+      indexes: {},
       index: {
         data: {}
       },
-      indexes: {},
+      includeLinePosition: true,
       compress: false,
       compressIndex: false,
       maxMemoryUsage: 64 * 1024 // 64KB
@@ -122,66 +124,86 @@ var Database = exports.Database = /*#__PURE__*/function (_EventEmitter) {
               throw new Error('Cleared, empty file');
             case 14:
               _context.next = 16;
-              return this.fileHandler.readLastLine();
+              return this.fileHandler.readLastLine()["catch"](function () {
+                return 0;
+              });
             case 16:
               lastLine = _context.sent;
               if (!(!lastLine || !lastLine.length)) {
-                _context.next = 19;
+                _context.next = 23;
+                break;
+              }
+              if (!this.opts.create) {
+                _context.next = 22;
                 break;
               }
               throw new Error('File does not exists or is a empty file');
-            case 19:
-              _context.next = 21;
+            case 22:
+              throw new Error('File is not a valid database file, expected offsets at the end of the file');
+            case 23:
+              _context.next = 25;
               return this.serializer.deserialize(lastLine, {
                 compress: this.opts.compressIndex
               });
-            case 21:
+            case 25:
               offsets = _context.sent;
               if (Array.isArray(offsets)) {
-                _context.next = 24;
+                _context.next = 30;
                 break;
               }
-              throw new Error('File to parse offsets, expected an array');
-            case 24:
+              if (!this.opts.create) {
+                _context.next = 29;
+                break;
+              }
+              throw new Error('File does not exists or is a empty file');
+            case 29:
+              throw new Error('File is not a valid database file, expected offsets at the end of the file to be an array');
+            case 30:
               this.indexOffset = offsets[offsets.length - 2];
               this.offsets = offsets;
               ptr = this.locate(offsets.length - 2);
               this.offsets = this.offsets.slice(0, -2);
               this.shouldTruncate = true;
-              _context.next = 31;
+              _context.next = 37;
               return (_this$fileHandler = this.fileHandler).readRange.apply(_this$fileHandler, _toConsumableArray(ptr));
-            case 31:
+            case 37:
               indexLine = _context.sent;
-              _context.next = 34;
+              _context.next = 40;
               return this.serializer.deserialize(indexLine, {
                 compress: this.opts.compressIndex
               });
-            case 34:
+            case 40:
               index = _context.sent;
               index && this.indexManager.load(index);
-              _context.next = 43;
+              _context.next = 53;
               break;
-            case 38:
-              _context.prev = 38;
+            case 44:
+              _context.prev = 44;
               _context.t0 = _context["catch"](9);
               if (Array.isArray(this.offsets)) {
                 this.offsets = [];
               }
               this.indexOffset = 0;
+              if (!(!this.opts.create && !this.opts.clear)) {
+                _context.next = 52;
+                break;
+              }
+              throw _context.t0;
+            case 52:
               if (!String(_context.t0).includes('empty file')) {
                 console.error('Error loading database:', _context.t0);
               }
-            case 43:
-              _context.prev = 43;
+            case 53:
+              _context.prev = 53;
               this.initializing = false;
               this.initialized = true;
               this.emit('init');
-              return _context.finish(43);
-            case 48:
+              return _context.finish(53);
+            case 58:
             case "end":
               return _context.stop();
           }
-        }, _callee, this, [[9, 38, 43, 48]]);
+        }, _callee, this, [[9, 44, 53, 58]]);
       }));
       function init() {
         return _init.apply(this, arguments);
@@ -336,84 +358,38 @@ var Database = exports.Database = /*#__PURE__*/function (_EventEmitter) {
         return n !== undefined;
       });
     }
-  }, {
-    key: "readLines",
-    value: function () {
-      var _readLines = _asyncToGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee4(map, ranges) {
-        var results, i, start;
-        return _regeneratorRuntime().wrap(function _callee4$(_context4) {
-          while (1) switch (_context4.prev = _context4.next) {
-            case 0:
-              if (!ranges) ranges = this.getRanges(map);
-              _context4.next = 3;
-              return this.fileHandler.readRanges(ranges, this.serializer.deserialize.bind(this.serializer));
-            case 3:
-              results = _context4.sent;
-              i = 0;
-              _context4.t0 = _regeneratorRuntime().keys(results);
-            case 6:
-              if ((_context4.t1 = _context4.t0()).done) {
-                _context4.next = 14;
-                break;
-              }
-              start = _context4.t1.value;
-              if (!(!results[start] || results[start]._ !== undefined)) {
-                _context4.next = 10;
-                break;
-              }
-              return _context4.abrupt("continue", 6);
-            case 10:
-              while (this.offsets[i] != start && i < map.length) i++; // weak comparison as 'start' is a string
-              results[start]._ = map[i++];
-              _context4.next = 6;
-              break;
-            case 14:
-              return _context4.abrupt("return", Object.values(results).filter(function (r) {
-                return r !== undefined;
-              }));
-            case 15:
-            case "end":
-              return _context4.stop();
-          }
-        }, _callee4, this);
-      }));
-      function readLines(_x, _x2) {
-        return _readLines.apply(this, arguments);
-      }
-      return readLines;
-    }()
   }, {
     key: "insert",
     value: function () {
-      var _insert = _asyncToGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee5(data) {
+      var _insert = _asyncToGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee4(data) {
         var line, position;
-        return _regeneratorRuntime().wrap(function _callee5$(_context5) {
-          while (1) switch (_context5.prev = _context5.next) {
+        return _regeneratorRuntime().wrap(function _callee4$(_context4) {
+          while (1) switch (_context4.prev = _context4.next) {
             case 0:
               if (!this.destroyed) {
-                _context5.next = 2;
+                _context4.next = 2;
                 break;
               }
               throw new Error('Database is destroyed');
             case 2:
               if (this.initialized) {
-                _context5.next = 5;
+                _context4.next = 5;
                 break;
               }
-              _context5.next = 5;
+              _context4.next = 5;
               return this.init();
             case 5:
               if (this.shouldTruncate) {
                 this.writeBuffer.push(this.indexOffset);
                 this.shouldTruncate = false;
               }
-              _context5.next = 8;
+              _context4.next = 8;
               return this.serializer.serialize(data, {
                 compress: this.opts.compress,
                 v8: this.opts.v8
               });
             case 8:
-              line = _context5.sent;
+              line = _context4.sent;
               // using Buffer for offsets accuracy
               position = this.offsets.length;
               this.offsets.push(this.indexOffset);
@@ -421,21 +397,21 @@ var Database = exports.Database = /*#__PURE__*/function (_EventEmitter) {
               this.emit('insert', data, position);
               this.writeBuffer.push(line);
               if (!(!this.flushing && this.currentWriteBufferSize() > this.opts.maxMemoryUsage)) {
-                _context5.next = 17;
+                _context4.next = 17;
                 break;
               }
-              _context5.next = 17;
+              _context4.next = 17;
               return this.flush();
             case 17:
               this.indexManager.add(data, position);
               this.shouldSave = true;
             case 19:
             case "end":
-              return _context5.stop();
+              return _context4.stop();
           }
-        }, _callee5, this);
+        }, _callee4, this);
       }));
-      function insert(_x3) {
+      function insert(_x) {
         return _insert.apply(this, arguments);
       }
       return insert;
@@ -474,23 +450,23 @@ var Database = exports.Database = /*#__PURE__*/function (_EventEmitter) {
   }, {
     key: "_flush",
     value: function () {
-      var _flush2 = _asyncToGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee6() {
+      var _flush2 = _asyncToGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee5() {
         var release, fd, data, pos, err;
-        return _regeneratorRuntime().wrap(function _callee6$(_context6) {
-          while (1) switch (_context6.prev = _context6.next) {
+        return _regeneratorRuntime().wrap(function _callee5$(_context5) {
+          while (1) switch (_context5.prev = _context5.next) {
             case 0:
-              _context6.next = 2;
+              _context5.next = 2;
               return this.mutex.acquire();
             case 2:
-              release = _context6.sent;
-              _context6.next = 5;
+              release = _context5.sent;
+              _context5.next = 5;
               return _fs["default"].promises.open(this.fileHandler.file, 'a');
             case 5:
-              fd = _context6.sent;
-              _context6.prev = 6;
+              fd = _context5.sent;
+              _context5.prev = 6;
             case 7:
               if (!this.writeBuffer.length) {
-                _context6.next = 26;
+                _context5.next = 26;
                 break;
               }
               data = void 0;
@@ -498,20 +474,20 @@ var Database = exports.Database = /*#__PURE__*/function (_EventEmitter) {
                 return typeof b === 'number';
               });
               if (!(pos === 0)) {
-                _context6.next = 21;
+                _context5.next = 21;
                 break;
               }
-              _context6.next = 13;
+              _context5.next = 13;
               return fd.close();
             case 13:
-              _context6.next = 15;
+              _context5.next = 15;
               return this.fileHandler.truncate(this.writeBuffer.shift());
             case 15:
-              _context6.next = 17;
+              _context5.next = 17;
               return _fs["default"].promises.open(this.fileHandler.file, 'a');
             case 17:
-              fd = _context6.sent;
-              return _context6.abrupt("continue", 7);
+              fd = _context5.sent;
+              return _context5.abrupt("continue", 7);
             case 21:
               if (pos === -1) {
                 data = Buffer.concat(this.writeBuffer);
@@ -521,34 +497,34 @@ var Database = exports.Database = /*#__PURE__*/function (_EventEmitter) {
                 this.writeBuffer.splice(0, pos);
               }
             case 22:
-              _context6.next = 24;
+              _context5.next = 24;
               return fd.write(data);
             case 24:
-              _context6.next = 7;
+              _context5.next = 7;
               break;
             case 26:
               this.shouldSave = true;
-              _context6.next = 32;
+              _context5.next = 32;
               break;
             case 29:
-              _context6.prev = 29;
-              _context6.t0 = _context6["catch"](6);
-              console.error('Error flushing:', _context6.t0);
+              _context5.prev = 29;
+              _context5.t0 = _context5["catch"](6);
+              console.error('Error flushing:', _context5.t0);
             case 32:
-              _context6.prev = 32;
-              _context6.next = 35;
+              _context5.prev = 32;
+              _context5.next = 35;
               return fd.close()["catch"](function (e) {
                 return err = e;
               });
             case 35:
               release();
               err && console.error('Error closing file:', err);
-              return _context6.finish(32);
+              return _context5.finish(32);
             case 38:
             case "end":
-              return _context6.stop();
+              return _context5.stop();
           }
-        }, _callee6, this, [[6, 29, 32, 38]]);
+        }, _callee5, this, [[6, 29, 32, 38]]);
       }));
       function _flush() {
         return _flush2.apply(this, arguments);
@@ -560,9 +536,9 @@ var Database = exports.Database = /*#__PURE__*/function (_EventEmitter) {
     value: function walk(map) {
       var _this = this;
       var options = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : {};
-      return _wrapAsyncGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee7() {
-        var ranges, groupedRanges, fd, _iterator2, _step2, groupedRange, _iteratorAbruptCompletion, _didIteratorError, _iteratorError, _iterator, _step, row, entry;
-        return _regeneratorRuntime().wrap(function _callee7$(_context7) {
+      return _wrapAsyncGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee6() {
+        var ranges, groupedRanges, fd, count, _iterator4, _step4, groupedRange, _iteratorAbruptCompletion, _didIteratorError, _iteratorError, _loop, _ret, _iterator, _step;
+        return _regeneratorRuntime().wrap(function _callee6$(_context7) {
           while (1) switch (_context7.prev = _context7.next) {
             case 0:
               if (!_this.destroyed) {
@@ -611,129 +587,181 @@ var Database = exports.Database = /*#__PURE__*/function (_EventEmitter) {
             case 18:
               fd = _context7.sent;
               _context7.prev = 19;
-              _iterator2 = _createForOfIteratorHelper(groupedRanges);
-              _context7.prev = 21;
-              _iterator2.s();
-            case 23:
-              if ((_step2 = _iterator2.n()).done) {
-                _context7.next = 66;
+              count = 0;
+              _iterator4 = _createForOfIteratorHelper(groupedRanges);
+              _context7.prev = 22;
+              _iterator4.s();
+            case 24:
+              if ((_step4 = _iterator4.n()).done) {
+                _context7.next = 63;
                 break;
               }
-              groupedRange = _step2.value;
+              groupedRange = _step4.value;
+              if (!(options.limit && count >= options.limit)) {
+                _context7.next = 28;
+                break;
+              }
+              return _context7.abrupt("break", 63);
+            case 28:
               _iteratorAbruptCompletion = false;
               _didIteratorError = false;
-              _context7.prev = 27;
+              _context7.prev = 30;
+              _loop = /*#__PURE__*/_regeneratorRuntime().mark(function _loop() {
+                var row, entry;
+                return _regeneratorRuntime().wrap(function _loop$(_context6) {
+                  while (1) switch (_context6.prev = _context6.next) {
+                    case 0:
+                      row = _step.value;
+                      if (!(options.limit && count >= options.limit)) {
+                        _context6.next = 3;
+                        break;
+                      }
+                      return _context6.abrupt("return", 0);
+                    case 3:
+                      _context6.next = 5;
+                      return _awaitAsyncGenerator(_this.serializer.deserialize(row.line, {
+                        compress: _this.opts.compress,
+                        v8: _this.opts.v8
+                      }));
+                    case 5:
+                      entry = _context6.sent;
+                      if (!(entry === null)) {
+                        _context6.next = 8;
+                        break;
+                      }
+                      return _context6.abrupt("return", 1);
+                    case 8:
+                      count++;
+                      if (!options.includeOffsets) {
+                        _context6.next = 14;
+                        break;
+                      }
+                      _context6.next = 12;
+                      return {
+                        entry: entry,
+                        start: row.start,
+                        _: row._ || _this.offsets.findIndex(function (n) {
+                          return n === row.start;
+                        })
+                      };
+                    case 12:
+                      _context6.next = 17;
+                      break;
+                    case 14:
+                      if (_this.opts.includeLinePosition) {
+                        entry._ = row._ || _this.offsets.findIndex(function (n) {
+                          return n === row.start;
+                        });
+                      }
+                      _context6.next = 17;
+                      return entry;
+                    case 17:
+                    case "end":
+                      return _context6.stop();
+                  }
+                }, _loop);
+              });
               _iterator = _asyncIterator(_this.fileHandler.readGroupedRange(groupedRange, fd));
-            case 29:
-              _context7.next = 31;
+            case 33:
+              _context7.next = 35;
               return _awaitAsyncGenerator(_iterator.next());
-            case 31:
+            case 35:
               if (!(_iteratorAbruptCompletion = !(_step = _context7.sent).done)) {
-                _context7.next = 48;
+                _context7.next = 45;
                 break;
               }
-              row = _step.value;
-              _context7.next = 35;
-              return _awaitAsyncGenerator(_this.serializer.deserialize(row.line, {
-                compress: _this.opts.compress,
-                v8: _this.opts.v8
-              }));
-            case 35:
-              entry = _context7.sent;
-              if (!(entry === null)) {
-                _context7.next = 38;
+              return _context7.delegateYield(_loop(), "t1", 37);
+            case 37:
+              _ret = _context7.t1;
+              if (!(_ret === 0)) {
+                _context7.next = 40;
                 break;
               }
-              return _context7.abrupt("continue", 45);
-            case 38:
-              if (!options.includeOffsets) {
-                _context7.next = 43;
+              return _context7.abrupt("break", 45);
+            case 40:
+              if (!(_ret === 1)) {
+                _context7.next = 42;
                 break;
               }
-              _context7.next = 41;
-              return {
-                entry: entry,
-                start: row.start
-              };
-            case 41:
-              _context7.next = 45;
-              break;
-            case 43:
-              _context7.next = 45;
-              return entry;
-            case 45:
+              return _context7.abrupt("continue", 42);
+            case 42:
               _iteratorAbruptCompletion = false;
-              _context7.next = 29;
+              _context7.next = 33;
               break;
-            case 48:
-              _context7.next = 54;
+            case 45:
+              _context7.next = 51;
               break;
-            case 50:
-              _context7.prev = 50;
-              _context7.t1 = _context7["catch"](27);
+            case 47:
+              _context7.prev = 47;
+              _context7.t2 = _context7["catch"](30);
               _didIteratorError = true;
-              _iteratorError = _context7.t1;
-            case 54:
-              _context7.prev = 54;
-              _context7.prev = 55;
+              _iteratorError = _context7.t2;
+            case 51:
+              _context7.prev = 51;
+              _context7.prev = 52;
               if (!(_iteratorAbruptCompletion && _iterator["return"] != null)) {
-                _context7.next = 59;
+                _context7.next = 56;
                 break;
               }
-              _context7.next = 59;
+              _context7.next = 56;
               return _awaitAsyncGenerator(_iterator["return"]());
-            case 59:
-              _context7.prev = 59;
+            case 56:
+              _context7.prev = 56;
               if (!_didIteratorError) {
-                _context7.next = 62;
+                _context7.next = 59;
                 break;
               }
               throw _iteratorError;
-            case 62:
-              return _context7.finish(59);
-            case 63:
-              return _context7.finish(54);
-            case 64:
-              _context7.next = 23;
+            case 59:
+              return _context7.finish(56);
+            case 60:
+              return _context7.finish(51);
+            case 61:
+              _context7.next = 24;
               break;
-            case 66:
-              _context7.next = 71;
+            case 63:
+              _context7.next = 68;
               break;
+            case 65:
+              _context7.prev = 65;
+              _context7.t3 = _context7["catch"](22);
+              _iterator4.e(_context7.t3);
             case 68:
               _context7.prev = 68;
-              _context7.t2 = _context7["catch"](21);
-              _iterator2.e(_context7.t2);
+              _iterator4.f();
+              return _context7.finish(68);
             case 71:
               _context7.prev = 71;
-              _iterator2.f();
-              return _context7.finish(71);
-            case 74:
-              _context7.prev = 74;
-              _context7.next = 77;
+              _context7.next = 74;
               return _awaitAsyncGenerator(fd.close());
-            case 77:
-              return _context7.finish(74);
-            case 78:
+            case 74:
+              return _context7.finish(71);
+            case 75:
             case "end":
               return _context7.stop();
           }
-        }, _callee7, null, [[19,, 74, 78], [21, 68, 71, 74], [27, 50, 54, 64], [55,, 59, 63]]);
+        }, _callee6, null, [[19,, 71, 75], [22, 65, 68, 71], [30, 47, 51, 61], [52,, 56, 60]]);
       }))();
     }
   }, {
     key: "query",
     value: function () {
-      var _query = _asyncToGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee8(criteria) {
+      var _query = _asyncToGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee7(criteria) {
         var options,
           results,
+          _iteratorAbruptCompletion2,
+          _didIteratorError2,
+          _iteratorError2,
+          _iterator2,
+          _step2,
+          entry,
           _options$orderBy$spli,
           _options$orderBy$spli2,
           field,
           _options$orderBy$spli3,
           direction,
-          matchingLines,
           _args8 = arguments;
-        return _regeneratorRuntime().wrap(function _callee8$(_context8) {
+        return _regeneratorRuntime().wrap(function _callee7$(_context8) {
           while (1) switch (_context8.prev = _context8.next) {
             case 0:
               options = _args8.length > 1 && _args8[1] !== undefined ? _args8[1] : {};
@@ -758,14 +786,54 @@ var Database = exports.Database = /*#__PURE__*/function (_EventEmitter) {
               _context8.next = 10;
               return this.save()["catch"](console.error);
             case 10:
-              if (!Array.isArray(criteria)) {
-                _context8.next = 19;
+              results = [];
+              _iteratorAbruptCompletion2 = false;
+              _didIteratorError2 = false;
+              _context8.prev = 13;
+              _iterator2 = _asyncIterator(this.walk(criteria, options));
+            case 15:
+              _context8.next = 17;
+              return _iterator2.next();
+            case 17:
+              if (!(_iteratorAbruptCompletion2 = !(_step2 = _context8.sent).done)) {
+                _context8.next = 23;
                 break;
               }
-              _context8.next = 13;
-              return this.readLines(criteria);
-            case 13:
-              results = _context8.sent;
+              entry = _step2.value;
+              results.push(entry);
+            case 20:
+              _iteratorAbruptCompletion2 = false;
+              _context8.next = 15;
+              break;
+            case 23:
+              _context8.next = 29;
+              break;
+            case 25:
+              _context8.prev = 25;
+              _context8.t1 = _context8["catch"](13);
+              _didIteratorError2 = true;
+              _iteratorError2 = _context8.t1;
+            case 29:
+              _context8.prev = 29;
+              _context8.prev = 30;
+              if (!(_iteratorAbruptCompletion2 && _iterator2["return"] != null)) {
+                _context8.next = 34;
+                break;
+              }
+              _context8.next = 34;
+              return _iterator2["return"]();
+            case 34:
+              _context8.prev = 34;
+              if (!_didIteratorError2) {
+                _context8.next = 37;
+                break;
+              }
+              throw _iteratorError2;
+            case 37:
+              return _context8.finish(34);
+            case 38:
+              return _context8.finish(29);
+            case 39:
               if (options.orderBy) {
                 _options$orderBy$spli = options.orderBy.split(' '), _options$orderBy$spli2 = _slicedToArray(_options$orderBy$spli, 2), field = _options$orderBy$spli2[0], _options$orderBy$spli3 = _options$orderBy$spli2[1], direction = _options$orderBy$spli3 === void 0 ? 'asc' : _options$orderBy$spli3;
                 results.sort(function (a, b) {
@@ -774,32 +842,14 @@ var Database = exports.Database = /*#__PURE__*/function (_EventEmitter) {
                   return 0;
                 });
               }
-              if (options.limit) {
-                results = results.slice(0, options.limit);
-              }
               return _context8.abrupt("return", results);
-            case 19:
-              _context8.next = 21;
-              return this.indexManager.query(criteria, options);
-            case 21:
-              matchingLines = _context8.sent;
-              if (!(!matchingLines || !matchingLines.size)) {
-                _context8.next = 24;
-                break;
-              }
-              return _context8.abrupt("return", []);
-            case 24:
-              _context8.next = 26;
-              return this.query(_toConsumableArray(matchingLines), options);
-            case 26:
-              return _context8.abrupt("return", _context8.sent);
-            case 27:
+            case 41:
             case "end":
               return _context8.stop();
           }
-        }, _callee8, this);
+        }, _callee7, this, [[13, 25, 29, 39], [30,, 34, 38]]);
       }));
-      function query(_x4) {
+      function query(_x2) {
         return _query.apply(this, arguments);
       }
       return query;
@@ -807,119 +857,145 @@ var Database = exports.Database = /*#__PURE__*/function (_EventEmitter) {
   }, {
     key: "update",
     value: function () {
-      var _update = _asyncToGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee9(criteria, data) {
+      var _update = _asyncToGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee8(criteria, data) {
         var _this8 = this;
         var options,
           matchingLines,
           ranges,
           validMatchingLines,
           entries,
-          lines,
+          _iteratorAbruptCompletion3,
+          _didIteratorError3,
+          _iteratorError3,
           _iterator3,
           _step3,
-          _loop,
+          entry,
+          lines,
+          _i,
+          _entries,
+          _entry,
+          updated,
+          ret,
           offsets,
           byteOffset,
           k,
-          _args10 = arguments;
-        return _regeneratorRuntime().wrap(function _callee9$(_context10) {
-          while (1) switch (_context10.prev = _context10.next) {
+          _args9 = arguments;
+        return _regeneratorRuntime().wrap(function _callee8$(_context9) {
+          while (1) switch (_context9.prev = _context9.next) {
             case 0:
-              options = _args10.length > 2 && _args10[2] !== undefined ? _args10[2] : {};
+              options = _args9.length > 2 && _args9[2] !== undefined ? _args9[2] : {};
               if (this.shouldTruncate) {
                 this.writeBuffer.push(this.indexOffset);
                 this.shouldTruncate = false;
               }
               if (!this.destroyed) {
-                _context10.next = 4;
+                _context9.next = 4;
                 break;
               }
               throw new Error('Database is destroyed');
             case 4:
               if (this.initialized) {
-                _context10.next = 7;
+                _context9.next = 7;
                 break;
               }
-              _context10.next = 7;
+              _context9.next = 7;
               return this.init();
             case 7:
-              _context10.t0 = this.shouldSave;
-              if (!_context10.t0) {
-                _context10.next = 11;
+              _context9.t0 = this.shouldSave;
+              if (!_context9.t0) {
+                _context9.next = 11;
                 break;
               }
-              _context10.next = 11;
+              _context9.next = 11;
               return this.save()["catch"](console.error);
             case 11:
-              _context10.next = 13;
+              _context9.next = 13;
               return this.indexManager.query(criteria, options);
             case 13:
-              matchingLines = _context10.sent;
+              matchingLines = _context9.sent;
               if (!(!matchingLines || !matchingLines.size)) {
-                _context10.next = 16;
+                _context9.next = 16;
                 break;
               }
-              return _context10.abrupt("return", []);
+              return _context9.abrupt("return", []);
             case 16:
               ranges = this.getRanges(_toConsumableArray(matchingLines));
               validMatchingLines = new Set(ranges.map(function (r) {
                 return r.index;
               }));
               if (validMatchingLines.size) {
-                _context10.next = 20;
+                _context9.next = 20;
                 break;
               }
-              return _context10.abrupt("return", []);
+              return _context9.abrupt("return", []);
             case 20:
-              _context10.next = 22;
-              return this.readLines(_toConsumableArray(validMatchingLines), ranges);
-            case 22:
-              entries = _context10.sent;
-              lines = [];
-              _iterator3 = _createForOfIteratorHelper(entries);
-              _context10.prev = 25;
-              _loop = /*#__PURE__*/_regeneratorRuntime().mark(function _loop() {
-                var entry, err, updated, ret;
-                return _regeneratorRuntime().wrap(function _loop$(_context9) {
-                  while (1) switch (_context9.prev = _context9.next) {
-                    case 0:
-                      entry = _step3.value;
-                      updated = Object.assign(entry, data);
-                      _context9.next = 4;
-                      return _this8.serializer.serialize(updated)["catch"](function (e) {
-                        return err = e;
-                      });
-                    case 4:
-                      ret = _context9.sent;
-                      err || lines.push(ret);
-                    case 6:
-                    case "end":
-                      return _context9.stop();
-                  }
-                }, _loop);
-              });
-              _iterator3.s();
-            case 28:
-              if ((_step3 = _iterator3.n()).done) {
-                _context10.next = 32;
+              entries = [];
+              _iteratorAbruptCompletion3 = false;
+              _didIteratorError3 = false;
+              _context9.prev = 23;
+              _iterator3 = _asyncIterator(this.walk(criteria, options));
+            case 25:
+              _context9.next = 27;
+              return _iterator3.next();
+            case 27:
+              if (!(_iteratorAbruptCompletion3 = !(_step3 = _context9.sent).done)) {
+                _context9.next = 33;
                 break;
               }
-              return _context10.delegateYield(_loop(), "t1", 30);
+              entry = _step3.value;
+              entries.push(entry);
             case 30:
-              _context10.next = 28;
+              _iteratorAbruptCompletion3 = false;
+              _context9.next = 25;
               break;
-            case 32:
-              _context10.next = 37;
+            case 33:
+              _context9.next = 39;
               break;
-            case 34:
-              _context10.prev = 34;
-              _context10.t2 = _context10["catch"](25);
-              _iterator3.e(_context10.t2);
-            case 37:
-              _context10.prev = 37;
-              _iterator3.f();
-              return _context10.finish(37);
-            case 40:
+            case 35:
+              _context9.prev = 35;
+              _context9.t1 = _context9["catch"](23);
+              _didIteratorError3 = true;
+              _iteratorError3 = _context9.t1;
+            case 39:
+              _context9.prev = 39;
+              _context9.prev = 40;
+              if (!(_iteratorAbruptCompletion3 && _iterator3["return"] != null)) {
+                _context9.next = 44;
+                break;
+              }
+              _context9.next = 44;
+              return _iterator3["return"]();
+            case 44:
+              _context9.prev = 44;
+              if (!_didIteratorError3) {
+                _context9.next = 47;
+                break;
+              }
+              throw _iteratorError3;
+            case 47:
+              return _context9.finish(44);
+            case 48:
+              return _context9.finish(39);
+            case 49:
+              lines = [];
+              _i = 0, _entries = entries;
+            case 51:
+              if (!(_i < _entries.length)) {
+                _context9.next = 61;
+                break;
+              }
+              _entry = _entries[_i];
+              updated = Object.assign(_entry, data);
+              _context9.next = 56;
+              return this.serializer.serialize(updated);
+            case 56:
+              ret = _context9.sent;
+              lines.push(ret);
+            case 58:
+              _i++;
+              _context9.next = 51;
+              break;
+            case 61:
               offsets = [];
               byteOffset = 0, k = 0;
               this.offsets.forEach(function (n, i) {
@@ -933,22 +1009,22 @@ var Database = exports.Database = /*#__PURE__*/function (_EventEmitter) {
               });
               this.offsets = offsets;
               this.indexOffset += byteOffset;
-              _context10.next = 47;
+              _context9.next = 68;
               return this.fileHandler.replaceLines(ranges, lines);
-            case 47:
+            case 68:
               _toConsumableArray(validMatchingLines).forEach(function (lineNumber, i) {
                 _this8.indexManager.dryRemove(lineNumber);
                 _this8.indexManager.add(entries[i], lineNumber);
               });
               this.shouldSave = true;
-              return _context10.abrupt("return", entries);
-            case 50:
+              return _context9.abrupt("return", entries);
+            case 71:
             case "end":
-              return _context10.stop();
+              return _context9.stop();
           }
-        }, _callee9, this, [[25, 34, 37, 40]]);
+        }, _callee8, this, [[23, 35, 39, 49], [40,, 44, 48]]);
       }));
-      function update(_x5, _x6) {
+      function update(_x3, _x4) {
         return _update.apply(this, arguments);
       }
       return update;
@@ -956,7 +1032,7 @@ var Database = exports.Database = /*#__PURE__*/function (_EventEmitter) {
   }, {
     key: "delete",
     value: function () {
-      var _delete2 = _asyncToGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee10(criteria) {
+      var _delete2 = _asyncToGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee9(criteria) {
         var options,
           matchingLines,
           ranges,
@@ -964,51 +1040,51 @@ var Database = exports.Database = /*#__PURE__*/function (_EventEmitter) {
           offsets,
           byteOffset,
           k,
-          _args11 = arguments;
-        return _regeneratorRuntime().wrap(function _callee10$(_context11) {
-          while (1) switch (_context11.prev = _context11.next) {
+          _args10 = arguments;
+        return _regeneratorRuntime().wrap(function _callee9$(_context10) {
+          while (1) switch (_context10.prev = _context10.next) {
             case 0:
-              options = _args11.length > 1 && _args11[1] !== undefined ? _args11[1] : {};
+              options = _args10.length > 1 && _args10[1] !== undefined ? _args10[1] : {};
               if (this.shouldTruncate) {
                 this.writeBuffer.push(this.indexOffset);
                 this.shouldTruncate = false;
               }
               if (!this.destroyed) {
-                _context11.next = 4;
+                _context10.next = 4;
                 break;
               }
               throw new Error('Database is destroyed');
             case 4:
               if (this.initialized) {
-                _context11.next = 7;
+                _context10.next = 7;
                 break;
               }
-              _context11.next = 7;
+              _context10.next = 7;
               return this.init();
             case 7:
-              _context11.t0 = this.shouldSave;
-              if (!_context11.t0) {
-                _context11.next = 11;
+              _context10.t0 = this.shouldSave;
+              if (!_context10.t0) {
+                _context10.next = 11;
                 break;
               }
-              _context11.next = 11;
+              _context10.next = 11;
               return this.save()["catch"](console.error);
             case 11:
-              _context11.next = 13;
+              _context10.next = 13;
               return this.indexManager.query(criteria, options);
             case 13:
-              matchingLines = _context11.sent;
+              matchingLines = _context10.sent;
               if (!(!matchingLines || !matchingLines.size)) {
-                _context11.next = 16;
+                _context10.next = 16;
                 break;
               }
-              return _context11.abrupt("return", 0);
+              return _context10.abrupt("return", 0);
             case 16:
               ranges = this.getRanges(_toConsumableArray(matchingLines));
               validMatchingLines = new Set(ranges.map(function (r) {
                 return r.index;
               }));
-              _context11.next = 20;
+              _context10.next = 20;
               return this.fileHandler.replaceLines(ranges, []);
             case 20:
               offsets = [];
@@ -1026,14 +1102,14 @@ var Database = exports.Database = /*#__PURE__*/function (_EventEmitter) {
               this.indexOffset += byteOffset;
               this.indexManager.remove(_toConsumableArray(validMatchingLines));
               this.shouldSave = true;
-              return _context11.abrupt("return", ranges.length);
+              return _context10.abrupt("return", ranges.length);
             case 28:
             case "end":
-              return _context11.stop();
+              return _context10.stop();
           }
-        }, _callee10, this);
+        }, _callee9, this);
       }));
-      function _delete(_x7) {
+      function _delete(_x5) {
         return _delete2.apply(this, arguments);
       }
       return _delete;
@@ -1041,16 +1117,16 @@ var Database = exports.Database = /*#__PURE__*/function (_EventEmitter) {
   }, {
     key: "destroy",
     value: function () {
-      var _destroy = _asyncToGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee11() {
-        return _regeneratorRuntime().wrap(function _callee11$(_context12) {
-          while (1) switch (_context12.prev = _context12.next) {
+      var _destroy = _asyncToGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee10() {
+        return _regeneratorRuntime().wrap(function _callee10$(_context11) {
+          while (1) switch (_context11.prev = _context11.next) {
             case 0:
-              _context12.t0 = this.shouldSave;
-              if (!_context12.t0) {
-                _context12.next = 4;
+              _context11.t0 = this.shouldSave;
+              if (!_context11.t0) {
+                _context11.next = 4;
                 break;
               }
-              _context12.next = 4;
+              _context11.next = 4;
               return this.save()["catch"](console.error);
             case 4:
               this.destroyed = true;
@@ -1061,9 +1137,9 @@ var Database = exports.Database = /*#__PURE__*/function (_EventEmitter) {
               this.fileHandler.destroy();
             case 10:
             case "end":
-              return _context12.stop();
+              return _context11.stop();
           }
-        }, _callee11, this);
+        }, _callee10, this);
       }));
       function destroy() {
         return _destroy.apply(this, arguments);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "jexidb",
-  "version": "1.0.8",
+  "version": "1.1.0",
   "description": "JexiDB is a pure JS NPM library for managing data on disk using JSONL efficiently, without the need for a server.",
   "main": "./dist/Database.cjs",
   "module": "./src/Database.mjs",

package/src/Database.mjs

@@ -10,8 +10,10 @@ export class Database extends EventEmitter {
     super()
     this.opts = Object.assign({
       v8: false,
-      index: { data: {} },
+      create: true,
       indexes: {},
+      index: { data: {} },
+      includeLinePosition: true,
       compress: false,
       compressIndex: false,
       maxMemoryUsage: 64 * 1024 // 64KB
@@ -41,13 +43,20 @@ export class Database extends EventEmitter {
         await this.fileHandler.truncate(0).catch(console.error)
         throw new Error('Cleared, empty file')
       }
-      const lastLine = await this.fileHandler.readLastLine()
+      const lastLine = await this.fileHandler.readLastLine().catch(() => 0)
       if (!lastLine || !lastLine.length) {
-        throw new Error('File does not exists or is a empty file')
+        if (this.opts.create) {
+          throw new Error('File does not exists or is a empty file')
+        } else {
+          throw new Error('File is not a valid database file, expected offsets at the end of the file')
+        }
       }
       const offsets = await this.serializer.deserialize(lastLine, { compress: this.opts.compressIndex })
       if (!Array.isArray(offsets)) {
-        throw new Error('File to parse offsets, expected an array')
+        if(this.opts.create) {
+          throw new Error('File does not exists or is a empty file')
+        }
+        throw new Error('File is not a valid database file, expected offsets at the end of the file to be an array')
       }
       this.indexOffset = offsets[offsets.length - 2]
       this.offsets = offsets
@@ -62,7 +71,9 @@ export class Database extends EventEmitter {
         this.offsets = []
       }
       this.indexOffset = 0
-      if (!String(e).includes('empty file')) {
+      if(!this.opts.create && !this.opts.clear) {
+        throw e
+      } else if (!String(e).includes('empty file')) {
         console.error('Error loading database:', e)
       }
     } finally {
@@ -130,18 +141,6 @@ export class Database extends EventEmitter {
     }).filter(n => n !== undefined)
   }
 
-  async readLines(map, ranges) {
-    if (!ranges) ranges = this.getRanges(map)
-    const results = await this.fileHandler.readRanges(ranges, this.serializer.deserialize.bind(this.serializer))
-    let i = 0
-    for (const start in results) {
-      if (!results[start] || results[start]._ !== undefined) continue
-      while (this.offsets[i] != start && i < map.length) i++ // weak comparison as 'start' is a string
-      results[start]._ = map[i++]
-    }
-    return Object.values(results).filter(r => r !== undefined)
-  }
-
   async insert(data) {
     if (this.destroyed) throw new Error('Database is destroyed')
     if (!this.initialized) await this.init()
@@ -232,13 +231,24 @@ export class Database extends EventEmitter {
     const groupedRanges = await this.fileHandler.groupedRanges(ranges)
     const fd = await fs.promises.open(this.fileHandler.file, 'r')
     try {
+      let count = 0
       for (const groupedRange of groupedRanges) {
+        if (options.limit && count >= options.limit) {
+          break
+        }
         for await (const row of this.fileHandler.readGroupedRange(groupedRange, fd)) {
+          if (options.limit && count >= options.limit) {
+            break
+          }
           const entry = await this.serializer.deserialize(row.line, { compress: this.opts.compress, v8: this.opts.v8 })
           if (entry === null) continue
+          count++
           if (options.includeOffsets) {
-            yield { entry, start: row.start }
+            yield { entry, start: row.start, _: row._ || this.offsets.findIndex(n => n === row.start) }
           } else {
+            if (this.opts.includeLinePosition) {
+              entry._ = row._ || this.offsets.findIndex(n => n === row.start)
+            }
             yield entry
           }
         }
@@ -252,27 +262,17 @@ export class Database extends EventEmitter {
     if (this.destroyed) throw new Error('Database is destroyed')
     if (!this.initialized) await this.init()
     this.shouldSave && await this.save().catch(console.error)
-    if (Array.isArray(criteria)) {
-      let results = await this.readLines(criteria)
-      if (options.orderBy) {
-        const [field, direction = 'asc'] = options.orderBy.split(' ')
-        results.sort((a, b) => {
-          if (a[field] > b[field]) return direction === 'asc' ? 1 : -1
-          if (a[field] < b[field]) return direction === 'asc' ? -1 : 1
-          return 0;
-        })
-      }
-      if (options.limit) {
-        results = results.slice(0, options.limit);
-      }
-      return results
-    } else {
-      const matchingLines = await this.indexManager.query(criteria, options)
-      if (!matchingLines || !matchingLines.size) {
-        return []
-      }
-      return await this.query([...matchingLines], options)
+    let results = []
+    for await (const entry of this.walk(criteria, options)) results.push(entry)
+    if (options.orderBy) {
+      const [field, direction = 'asc'] = options.orderBy.split(' ')
+      results.sort((a, b) => {
+        if (a[field] > b[field]) return direction === 'asc' ? 1 : -1
+        if (a[field] < b[field]) return direction === 'asc' ? -1 : 1
+        return 0;
+      })
     }
+    return results
   }
 
   async update(criteria, data, options={}) {
@@ -292,13 +292,13 @@ export class Database extends EventEmitter {
     if (!validMatchingLines.size) {
       return []
     }
-    const entries = await this.readLines([...validMatchingLines], ranges)
+    let entries = []
+    for await (const entry of this.walk(criteria, options)) entries.push(entry)
     const lines = []
     for(const entry of entries) {
-      let err
       const updated = Object.assign(entry, data)
-      const ret = await this.serializer.serialize(updated).catch(e => err = e)
-      err || lines.push(ret)
+      const ret = await this.serializer.serialize(updated)
+      lines.push(ret)
     }
     const offsets = []
     let byteOffset = 0, k = 0

package/test/test.mjs

@@ -83,14 +83,19 @@ const runTests = async (id, name, format, opts) => {
     console.assert(pass2, `Round 2 - READ: Test failed: ${character.name} seems to have been teleported out of the database!`);
     if(pass1 && pass2) console.log(`Round 2 - READ: Flawless Victory! All characters inserted successfully, led by ${character.name}.`);
 
-    // 4. Test data update
+    // 4. Test indexes
+    const pass3 = await db.indexManager.readColumnIndex('name').has(character.name)
+    console.assert(pass3, `Round 3 - INDEX: Test failed: ${character.name} is not in the index.`);
+    if(pass3) console.log(`Round 3 - INDEX: Flawless Victory! ${character.name} is in the index.`);
+
+    // 5. Test data update
     await db.update({ id: 1 }, { name: character.name + ' Updated' });
     results = await db.query({ id: 1 });
     const pass4 = results.length === 1 && results[0].name === character.name + ' Updated';
     console.assert(pass4, `Round 3 - UPDATE: Test failed: ${character.updateMessage}`);
     if(pass4) console.log(`Round 3 - UPDATE: Flawless Victory! ${character.name} has been updated successfully.`);
 
-    // 5. Test data deletion
+    // 6. Test data deletion
     await db.delete({ name: character.name + ' Updated' });
     results = await db.query({ id: { '<=': 2 } });
     const pass5 = results.length === 1;
@@ -119,7 +124,7 @@ const runTests = async (id, name, format, opts) => {
 }
 
 async function runAllTests() {
-    const depth = 100
+    const depth = 10
     let err, i = 1
     let tests = [
         ['json', 'JSON', { indexes: { id: 'number', name: 'string' }, v8: false, compress: false, compressIndex: false }],