datafolder 1.0.22 → 1.0.24

package/ChangeLog.md

@@ -0,0 +1,33 @@
+1.0.24 Add HNSW test cases (test/testQyHNSW.ts).
+
+1.0.23 Add ChangeLog.md
+
+1.0.21 Add ArangoDB Query Language (AQL) test case (test/testQyAQL.js)
+
+1.0.20 Add HNSW support.
+
+1.0.19 Add DocumentDB test case (test/testQyDocumentDB.js)
+
+1.0.18 Add Embedding Storage Support. Related APIs: dataFolder.insertEmbedding() and dataFolder.delEmbedding(). TODO: Add HNSW support and vector query.
+
+1.0.17 Add Northwind sample data test (test/testQyNorthwind.js)
+
+1.0.16 Add GraphQL sample data test (test/testQyGraphQL.js) to show how to write complex queries.
+
+1.0.15 Use [Workers](https://nodejs.org/docs/latest/api/worker_threads.html) to implement snapshot compactor, expire service, outdated data cleaner etc.
+
+1.0.14 Expire supports specifying milliseconds.
+
+1.0.13 Added expire support to delete a dir in a specific future time.
+
+1.0.12 Added expire support to expire a file or some properties of the JSON data in future.
+
+1.0.11 Added [SKILL.md](https://github.com/datafolderdev/datafolder/blob/main/SKILL.md) for feeding to AI
+
+1.0.10 Added document [datafolder for high speed SSDs](https://github.com/datafolderdev/datafolder/wiki/datafolder-for-high-speed-SSDs)
+
+1.0.9 Added Typescript type declarations.
+
+1.0.7 Added trigger editor and [Internals of datafolder](https://github.com/datafolderdev/datafolder/wiki/Internals-of-datafolder)
+
+1.0.6 Added code mirror as the query editor.

package/README.md

@@ -4,7 +4,7 @@ datafolder is a lightning fluid JSON database for high speed SSDs, featuring an
 
 ### Changelog
 
-[ChangeLog] (ChangeLog.md)
+[ChangeLog](./ChangeLog.md)
 
 ## Installation
 

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "datafolder",
-	"version": "1.0.22",
+	"version": "1.0.24",
 	"author": {
 		"name": "lysh",
 		"email": "datafolderdev@outlook.com"

package/static/codemirror.bundle/editor.bundle.js

@@ -62,7 +62,7 @@
   }
 
   function prevClusterBreak(str, pos, includeExtending) {
-    while (pos > 0) {
+    while (pos > 1) {
       let found = nextClusterBreak(str, pos - 2, includeExtending);
       if (found < pos) return found
       pos--;
@@ -1750,6 +1750,7 @@
               }
           };
       }
+      get extension() { return this; }
   }
   function compareArray(a, b, compare) {
       if (a.length != b.length)
@@ -1947,6 +1948,7 @@
           this.inner = inner;
           this.prec = prec;
       }
+      get extension() { return this; }
   }
   /**
   Extension compartments can be used to make a configuration
@@ -1981,6 +1983,7 @@
           this.compartment = compartment;
           this.inner = inner;
       }
+      get extension() { return this; }
   }
   class Configuration {
       constructor(base, compartments, dynamicSlots, address, staticValues, facets) {
@@ -2090,6 +2093,8 @@
           else {
               let content = ext.extension;
               if (!content)
+                  throw new Error(`Unrecognized extension value in extension set (${ext}).`);
+              if (content == ext)
                   throw new Error(`Unrecognized extension value in extension set (${ext}). This sometimes happens because multiple instances of @codemirror/state are loaded, breaking instanceof checks.`);
               inner(content, prec);
           }
@@ -4815,8 +4820,11 @@
   function maxOffset(node) {
       return node.nodeType == 3 ? node.nodeValue.length : node.childNodes.length;
   }
-  function flattenRect(rect, left) {
-      let x = left ? rect.left : rect.right;
+  function flattenRect(rect, toLeft) {
+      let { left, right } = rect;
+      if (left == right)
+          return rect;
+      let x = toLeft ? left : right;
       return { left: x, right: x, top: rect.top, bottom: rect.bottom };
   }
   function windowRect(win) {
@@ -6080,7 +6088,7 @@
           return this.posBefore(tile) + tile.length;
       }
       covers(side) { return true; }
-      coordsIn(pos, side) { return null; }
+      coordsIn(pos, side, rtl) { return null; }
       domPosFor(off, side) {
           let index = domIndex(this.dom);
           let after = this.length ? off > 0 : side > 0;
@@ -6267,6 +6275,9 @@
       }
       get domAttrs() { return this.attrs; }
       // Find the tile associated with a given position in this line.
+      // Side -2/2 is handled specially, in that it allows the position
+      // returned to be before (-2) or after (2) widgets that would always
+      // be after/before a cursor position.
       resolveInline(pos, side, forCoords) {
           let before = null, beforeOff = -1, after = null, afterOff = -1;
           function scan(tile, pos) {
@@ -6276,12 +6287,12 @@
                       if (child.isComposite()) {
                           scan(child, pos - off);
                       }
-                      else if ((!after || after.isHidden && (side > 0 || forCoords && onSameLine(after, child))) &&
-                          (end > pos || (child.flags & 32 /* TileFlag.After */))) {
+                      else if ((!after || after.isHidden && (side > 0 && !(after.flags & 32 /* TileFlag.After */) || forCoords && onSameLine(after, child))) &&
+                          (end > pos || (child.flags & 32 /* TileFlag.After */) && side <= 1)) {
                           after = child;
                           afterOff = pos - off;
                       }
-                      else if (off < pos || (child.flags & 16 /* TileFlag.Before */) && !child.isHidden) {
+                      else if (off < pos || (child.flags & 16 /* TileFlag.Before */) && !child.isHidden && side >= -1) {
                           before = child;
                           beforeOff = pos - off;
                       }
@@ -6293,11 +6304,11 @@
           let target = ((side < 0 ? before : after) || before || after);
           return target ? { tile: target, offset: target == before ? beforeOff : afterOff } : null;
       }
-      coordsIn(pos, side) {
+      coordsIn(pos, side, rtl) {
           let found = this.resolveInline(pos, side, true);
           if (!found)
               return fallbackRect(this);
-          return found.tile.coordsIn(Math.max(0, found.offset), side);
+          return found.tile.coordsIn(Math.max(0, found.offset), side, rtl);
       }
       domIn(pos, side) {
           let found = this.resolveInline(pos, side);
@@ -6361,7 +6372,7 @@
       }
       isText() { return true; }
       toString() { return JSON.stringify(this.text); }
-      coordsIn(pos, side) {
+      coordsIn(pos, side, rtl) {
           let length = this.dom.nodeValue.length;
           if (pos > length)
               pos = length;
@@ -6371,7 +6382,7 @@
                   if (pos) {
                       from--;
                       flatten = 1;
-                  } // FIXME this is wrong in RTL text
+                  }
                   else if (to < length) {
                       to++;
                       flatten = -1;
@@ -6390,7 +6401,7 @@
           let rect = rects[(flatten ? flatten < 0 : side >= 0) ? 0 : rects.length - 1];
           if (browser.safari && !flatten && rect.width == 0)
               rect = Array.prototype.find.call(rects, r => r.width) || rect;
-          return flatten ? flattenRect(rect, flatten < 0) : rect || null;
+          return rtl == null ? rect : flattenRect(rect, (flatten ? flatten > 0 : side < 0) == rtl);
       }
       static of(text, dom) {
           let tile = new TextTile(dom || document.createTextNode(text), text);
@@ -6466,7 +6477,10 @@
       }
       get isHidden() { return true; }
       get overrideDOMText() { return Text.empty; }
-      coordsIn(pos) { return this.dom.getBoundingClientRect(); }
+      coordsIn(pos, side, rtl) {
+          let rect = this.dom.getBoundingClientRect();
+          return rtl == null ? rect : flattenRect(rect, (side > 0) == rtl);
+      }
   }
   // Represents a position in the tile tree.
   class TilePointer {
@@ -6484,7 +6498,15 @@
           let { tile, index, beforeBreak, parents } = this;
           while (dist || side > 0) {
               if (!tile.isComposite()) {
-                  if (index == tile.length) {
+                  let len = tile.length;
+                  if (index < len && dist) {
+                      let take = Math.min(dist, len - index);
+                      if (walker)
+                          walker.skip(tile, index, index + take);
+                      dist -= take;
+                      index += take;
+                  }
+                  if (index == len) {
                       beforeBreak = !!tile.breakAfter;
                       ({ tile, index } = parents.pop());
                       index++;
@@ -6492,13 +6514,6 @@
                   else if (!dist) {
                       break;
                   }
-                  else {
-                      let take = Math.min(dist, tile.length - index);
-                      if (walker)
-                          walker.skip(tile, index, index + take);
-                      dist -= take;
-                      index += take;
-                  }
               }
               else if (beforeBreak) {
                   if (!dist)
@@ -6828,8 +6843,8 @@
       find(cls, test, type = 2 /* Reused.DOM */) {
           let i = cls.bucket;
           let bucket = this.buckets[i], off = this.index[i];
-          for (let j = bucket.length - 1; j >= 0; j--) {
-              // Look at the most recently added items first (last-in, first-out)
+          for (let j = 0; j < bucket.length; j++) {
+              // Look at the most oldest items first (first-in, first-out)
               let index = (j + off) % bucket.length, tile = bucket[index];
               if ((!test || test(tile)) && !this.reused.has(tile)) {
                   bucket.splice(index, 1);
@@ -7032,7 +7047,7 @@
       }
       emit(from, to) {
           let pendingLineAttrs = null;
-          let b = this.builder, markCount = 0;
+          let b = this.builder, markCount = -1;
           let openEnd = RangeSet.spans(this.decorations, from, to, {
               point: (from, to, deco, active, openStart, index) => {
                   if (deco instanceof PointDecoration) {
@@ -7083,9 +7098,11 @@
                       }
                       pendingLineAttrs = null;
                   }
+                  markCount = active.length;
               }
           });
-          this.openWidget = openEnd > markCount;
+          if (markCount > -1)
+              this.openWidget = openEnd > markCount;
           if (!this.openWidget)
               b.addLineStartIfNotCovered(pendingLineAttrs);
           this.openMarks = openEnd;
@@ -7494,7 +7511,7 @@
       domAtPos(pos, side) {
           let { tile, offset } = this.tile.resolveBlock(pos, side);
           if (tile.isWidget())
-              return tile.domPosFor(pos, side);
+              return tile.domPosFor(offset, side);
           return tile.domIn(offset, side);
       }
       inlineDOMNearPos(pos, side) {
@@ -7531,14 +7548,16 @@
               after = null;
           return before && side < 0 || !after ? before.domIn(beforeOff, side) : after.domIn(afterOff, side);
       }
-      coordsAt(pos, side) {
+      // Get the coord of the element at the given side of the given
+      // position. If rtl is given, flatten it using that text direction.
+      coordsAt(pos, side, rtl) {
           let { tile, offset } = this.tile.resolveBlock(pos, side);
           if (tile.isWidget()) {
               if (tile.widget instanceof BlockGapWidget)
                   return null;
               return tile.coordsInWidget(offset, side, true);
           }
-          return tile.coordsIn(offset, side);
+          return tile.coordsIn(offset, side, rtl);
       }
       lineAt(pos, side) {
           let { tile } = this.tile.resolveBlock(pos, side);
@@ -7712,7 +7731,6 @@
           this.blockWrappers = this.view.state.facet(blockWrappers).map(v => typeof v == "function" ? v(this.view) : v);
       }
       scrollIntoView(target) {
-          var _a;
           if (target.isSnapshot) {
               let ref = this.view.viewState.lineBlockAt(target.range.head);
               this.view.scrollDOM.scrollTop = ref.top - target.yMargin;
@@ -7729,7 +7747,7 @@
               }
           }
           let { range } = target;
-          let rect = this.coordsAt(range.head, (_a = range.assoc) !== null && _a !== void 0 ? _a : (range.empty ? 0 : range.head > range.anchor ? -1 : 1)), other;
+          let rect = this.coordsAt(range.head, range.assoc || (range.head > range.anchor ? -1 : 1)), other;
           if (!rect)
               return;
           if (!range.empty && (other = this.coordsAt(range.anchor, range.anchor > range.head ? -1 : 1)))
@@ -8771,6 +8789,7 @@
           this.view = view;
           this.lastKeyCode = 0;
           this.lastKeyTime = 0;
+          this.touchActive = false;
           this.lastTouchTime = 0;
           this.lastTouchX = 0;
           this.lastTouchY = 0;
@@ -8782,6 +8801,10 @@
           // (after which we retroactively handle them and reset the DOM) to
           // avoid messing up the virtual keyboard state.
           this.pendingIOSKey = undefined;
+          // Set to a time stap by scroll events when touch isn't active on
+          // iOS, to work around an issue where Safari will abort the scroll
+          // momentum if we set scrollTop
+          this.lastIOSMomentumScroll = 0;
           // When enabled (>-1), tab presses are not given to key handlers,
           // leaving the browser's default behavior. If >0, the mode expires
           // at that timestamp, and any other keypress clears it.
@@ -9212,8 +9235,11 @@
       });
   }
   observers.scroll = view => {
-      view.inputState.lastScrollTop = view.scrollDOM.scrollTop;
-      view.inputState.lastScrollLeft = view.scrollDOM.scrollLeft;
+      let iState = view.inputState;
+      iState.lastScrollTop = view.scrollDOM.scrollTop;
+      iState.lastScrollLeft = view.scrollDOM.scrollLeft;
+      if (browser.ios && !iState.touchActive)
+          iState.lastIOSMomentumScroll = Date.now();
   };
   observers.wheel = observers.mousewheel = view => {
       view.inputState.lastWheelEvent = Date.now();
@@ -9226,6 +9252,7 @@
   };
   observers.touchstart = (view, e) => {
       let iState = view.inputState, touch = e.targetTouches[0];
+      iState.touchActive = true;
       iState.lastTouchTime = Date.now();
       if (touch) {
           iState.lastTouchX = touch.clientX;
@@ -9236,6 +9263,9 @@
   observers.touchmove = view => {
       view.inputState.setSelectionOrigin("select.pointer");
   };
+  observers.touchend = (view, e) => {
+      view.inputState.touchActive = false;
+  };
   handlers.mousedown = (view, event) => {
       view.observer.flush();
       if (view.inputState.lastTouchTime > Date.now() - 2000)
@@ -11261,6 +11291,8 @@
           backgroundColor: "#f5f5f5",
           color: "black"
       },
+      ".cm-panels-top": { top: "0" },
+      ".cm-panels-bottom": { bottom: "0" },
       "&light .cm-panels-top": {
           borderBottom: "1px solid #ddd"
       },
@@ -12497,6 +12529,7 @@
                                   this.viewState.lineBlockAt(scrollAnchorPos).top;
                               let diff = (newAnchorHeight - scrollAnchorHeight) / this.scaleY;
                               if ((diff > 1 || diff < -1) &&
+                                  !(browser.ios && this.inputState.lastIOSMomentumScroll > Date.now() - 100) &&
                                   (scroll == this.scrollDOM || this.hasFocus ||
                                       Math.max(this.inputState.lastWheelEvent, this.inputState.lastTouchTime) > Date.now() - 100)) {
                                   scrollOffset = scrollOffset + diff;
@@ -12788,12 +12821,9 @@
       */
       coordsAtPos(pos, side = 1) {
           this.readMeasured();
-          let rect = this.docView.coordsAt(pos, side);
-          if (!rect || rect.left == rect.right)
-              return rect;
           let line = this.state.doc.lineAt(pos), order = this.bidiSpans(line);
           let span = order[BidiSpan.find(order, pos - line.from, -1, side)];
-          return flattenRect(rect, (span.dir == Direction.LTR) == (side > 0));
+          return this.docView.coordsAt(pos, side, span.dir == Direction.RTL);
       }
       /**
       Return the rectangle around a given character. If `pos` does not
@@ -13863,9 +13893,10 @@
       },
       class: "cm-selectionLayer"
   });
+  const selectionBg = browser.gecko && browser.gecko_version >= 153 ? "#ffffff01" : "transparent";
   const hideNativeSelection = /*@__PURE__*/Prec.highest(/*@__PURE__*/EditorView.theme({
       ".cm-line": {
-          "& ::selection, &::selection": { backgroundColor: "transparent !important" },
+          "& ::selection, &::selection": { backgroundColor: `${selectionBg} !important` },
           caretColor: "transparent !important"
       },
       ".cm-content": {
@@ -15262,7 +15293,6 @@
           if (!this.dom) {
               this.dom = document.createElement("div");
               this.dom.className = this.top ? "cm-panels cm-panels-top" : "cm-panels cm-panels-bottom";
-              this.dom.style[this.top ? "top" : "bottom"] = "0";
               let parent = this.container || this.view.dom;
               parent.insertBefore(this.dom, this.top ? parent.firstChild : null);
           }

package/test/testQyHNSW.js

@@ -0,0 +1,34 @@
+import test from "node:test";
+import { ok, equal, deepEqual } from "node:assert/strict";
+import { QyHNSW, cosineSimilarity, euclideanSimilarity } from "../src/QyHNSW.js";
+import { logger } from "../src/QyLogger.js";
+logger.level = "warn";
+test("test QyHNSW", async ({ mock }) => {
+    let qyHNSW = new QyHNSW(8);
+    deepEqual(qyHNSW.probs, [
+        0.875, 0.984375, 0.998046875, 0.999755859375, 0.999969482421875, 0.9999961853027344, 0.9999995231628418,
+        0.9999999403953552, 0.9999999925494194, 0.9999999990686774, 1,
+    ]);
+    equal(qyHNSW._determineLayer(0.395), 0);
+    equal(qyHNSW._determineLayer(0.9), 1);
+    equal(qyHNSW._determineLayer(0.99981), 4);
+    equal(qyHNSW._determineLayer(0.9999999990686774), 9);
+    equal(qyHNSW._determineLayer(0.9999999991), 10);
+    qyHNSW = new QyHNSW(16);
+    deepEqual(qyHNSW.probs, [
+        0.9375, 0.99609375, 0.999755859375, 0.9999847412109375, 0.9999990463256836, 0.9999999403953552,
+        0.9999999962747097, 0.9999999997671694, 1,
+    ]);
+    qyHNSW = new QyHNSW(100);
+    deepEqual(qyHNSW.probs, [0.99, 0.9999, 0.999999, 0.99999999, 0.9999999999, 1]);
+    equal(qyHNSW._determineLayer(0.395), 0);
+    equal(qyHNSW._determineLayer(0.9), 0);
+    equal(qyHNSW._determineLayer(0.99981), 1);
+    equal(qyHNSW._determineLayer(0.9999999990686774), 4);
+    equal(qyHNSW._determineLayer(0.9999999991), 4);
+    equal(qyHNSW._determineLayer(0.9999999999001), 5);
+    equal(cosineSimilarity([1, 1, 1], [2, 2, 2]), 1.0000000000000002);
+    equal(euclideanSimilarity([1, 1, 1], [2, 2, 2]), 0.36602540378443865);
+    equal(cosineSimilarity([5, 2, 9], [4, 6, 1]), 0.5369694517472824);
+    equal(euclideanSimilarity([5, 2, 9], [4, 6, 1]), 0.1);
+});