hdt 3.1.1 → 3.2.1

package/deps/libhdt/src/util/StopWatch.cpp

@@ -163,7 +163,7 @@ std::ostream &operator<<(std::ostream &stream, StopWatch &sw) {
 }
 
 std::string StopWatch::toHuman(unsigned long long time) {
-    uint64_t tot_secs = time/1000000;
+    unsigned long long tot_secs = time/1000000;
 
 	unsigned int hours = tot_secs/3600;
 	unsigned int mins = (tot_secs/60) % 60;

package/lib/HdtDocument.cc

@@ -55,6 +55,7 @@ const Nan::Persistent<Function>& HdtDocument::GetConstructor() {
     constructorTemplate->InstanceTemplate()->SetInternalFieldCount(1);
     // Create prototype
     Nan::SetPrototypeMethod(constructorTemplate, "_searchTriples", SearchTriples);
+    Nan::SetPrototypeMethod(constructorTemplate, "_searchBindings", SearchBindings);
     Nan::SetPrototypeMethod(constructorTemplate, "_searchLiterals", SearchLiterals);
     Nan::SetPrototypeMethod(constructorTemplate, "_searchTerms",  SearchTerms);
     Nan::SetPrototypeMethod(constructorTemplate, "_fetchDistinctTerms", FetchDistinctTerms);
@@ -234,6 +235,141 @@ NAN_METHOD(HdtDocument::SearchTriples) {
     new Nan::Callback(info[5].As<Function>()), info.This()));
 }
 
+/******** HdtDocument#_searchBindings ********/
+
+class SearchBindingsWorker : public Nan::AsyncWorker {
+  HdtDocument* document;
+  // JavaScript function arguments
+  string subject, predicate, object;
+  uint32_t offset, limit;
+  // Callback return values
+  vector<TripleID> triples;
+  map<unsigned int, string> subjects, predicates, objects;
+  uint32_t totalCount;
+  bool hasExactCount;
+  bool varS, varP, varO;
+
+public:
+  SearchBindingsWorker(HdtDocument* document, char* subject, char* predicate, char* object,
+                       uint32_t offset, uint32_t limit, Nan::Callback* callback, Local<Object> self)
+    : Nan::AsyncWorker(callback),
+      document(document), subject(subject), predicate(predicate), object(object),
+      offset(offset), limit(limit), totalCount(0) {
+    SaveToPersistent(SELF, self);
+  };
+
+  void Execute() {
+    IteratorTripleID* it = NULL;
+    try {
+      // Determine which terms are variables
+      varS = isVariable(subject);
+      varP = isVariable(predicate);
+      varO = isVariable(object);
+
+      // Prepare the triple pattern
+      Dictionary* dict = document->GetHDT()->getDictionary();
+      TripleString triple(varS ? "" : subject, varP ? "" : predicate, varO ? "" : toHdtLiteral(object));
+      TripleID tripleId;
+      dict->tripleStringtoTripleID(triple, tripleId);
+      // If any of the components does not exist, there are no matches
+      if ((!varS && subject[0]   && !tripleId.getSubject())   ||
+          (!varP && predicate[0] && !tripleId.getPredicate()) ||
+          (!varO && object[0]    && !tripleId.getObject())) {
+        hasExactCount = true;
+        return;
+      }
+
+      // Estimate the total number of triples
+      it = document->GetHDT()->getTriples()->search(tripleId);
+      totalCount = it->estimatedNumResults();
+      hasExactCount = it->numResultEstimation() == EXACT;
+
+      // Go to the right offset
+      if (it->canGoTo())
+        try { it->skip(offset), offset = 0; }
+        catch (const runtime_error error) { /* invalid offset */ }
+      else
+        while (offset && it->hasNext()) it->next(), offset--;
+
+      // Add matching triples to the result vector
+      if (!offset) {
+        while (it->hasNext() && triples.size() < limit) {
+          TripleID& triple = *it->next();
+          triples.push_back(triple);
+          if (varS && !subjects.count(triple.getSubject())) {
+            subjects[triple.getSubject()] = dict->idToString(triple.getSubject(), SUBJECT);
+          }
+          if (varP && !predicates.count(triple.getPredicate())) {
+            predicates[triple.getPredicate()] = dict->idToString(triple.getPredicate(), PREDICATE);
+          }
+          if (varO && !objects.count(triple.getObject())) {
+            string object(dict->idToString(triple.getObject(), OBJECT));
+            objects[triple.getObject()] = fromHdtLiteral(object);
+          }
+        }
+      }
+    }
+    catch (const runtime_error error) { SetErrorMessage(error.what()); }
+    if (it)
+      delete it;
+  }
+
+  void HandleOKCallback() {
+    Nan::HandleScope scope;
+    // Convert the triple components into strings
+    map<unsigned int, string>::const_iterator it;
+    map<unsigned int, Local<String> > subjectStrings, predicateStrings, objectStrings;
+    for (it = subjects.begin(); it != subjects.end(); it++)
+      subjectStrings[it->first] = Nan::New(it->second.c_str()).ToLocalChecked();
+    for (it = predicates.begin(); it != predicates.end(); it++)
+      predicateStrings[it->first] = Nan::New(it->second.c_str()).ToLocalChecked();
+    for (it = objects.begin(); it != objects.end(); it++)
+      objectStrings[it->first] = Nan::New(it->second.c_str()).ToLocalChecked();
+
+    // Convert the triples into a double JavaScript array
+    uint32_t count = 0;
+    Local<Array> bindingsArray = Nan::New<Array>(triples.size());
+    uint32_t variables = (varS ? 1 : 0) + (varP ? 1 : 0) + (varO ? 1 : 0);
+    for (vector<TripleID>::const_iterator it = triples.begin(); it != triples.end(); it++) {
+      uint32_t countInner = 0;
+      Local<Object> bindingsArrayInner = Nan::New<Array>(variables);
+      if (varS) {
+        Nan::Set(bindingsArrayInner, countInner++, subjectStrings[it->getSubject()]);
+      }
+      if (varP) {
+        Nan::Set(bindingsArrayInner, countInner++, predicateStrings[it->getPredicate()]);
+      }
+      if (varO) {
+        Nan::Set(bindingsArrayInner, countInner++, objectStrings[it->getObject()]);
+      }
+      Nan::Set(bindingsArray, count++, bindingsArrayInner);
+    }
+
+    // Send the JavaScript array through the callback
+    const unsigned argc = 4;
+    Local<Value> argv[argc] = { Nan::Null(), bindingsArray,
+                                Nan::New<Integer>((uint32_t)totalCount),
+                                Nan::New<Boolean>((bool)hasExactCount) };
+    callback->Call(Nan::To<v8::Object>(GetFromPersistent(SELF)).ToLocalChecked(), argc, argv, async_resource);
+  }
+
+  void HandleErrorCallback() {
+    Nan::HandleScope scope;
+    Local<Value> argv[] = { Exception::Error(Nan::New(ErrorMessage()).ToLocalChecked()) };
+    callback->Call(Nan::To<v8::Object>(GetFromPersistent(SELF)).ToLocalChecked(), 1, argv, async_resource);
+  }
+};
+
+// Searches for a triple pattern in the document and return bindings.
+// JavaScript signature: HdtDocument#_searchBindings(subject, predicate, object, offset, limit, callback)
+NAN_METHOD(HdtDocument::SearchBindings) {
+  assert(info.Length() == 6);
+  Nan::AsyncQueueWorker(new SearchBindingsWorker(Unwrap<HdtDocument>(info.This()),
+    *Nan::Utf8String(info[0]), *Nan::Utf8String(info[1]), *Nan::Utf8String(info[2]),
+    Nan::To<uint32_t>(info[3]).FromJust(), Nan::To<uint32_t>(info[4]).FromJust(),
+    new Nan::Callback(info[5].As<Function>()), info.This()));
+}
+
 
 
 /******** HdtDocument#_searchLiterals ********/
@@ -612,6 +748,10 @@ NAN_PROPERTY_GETTER(HdtDocument::Closed) {
 //   "literal"^^<http://example.org/datatype>
 // The functions below convert when needed.
 
+// Check if a term is a variable
+bool isVariable(string& term) {
+  return term[0] == '?';
+}
 
 // Converts a JavaScript literal to an HDT literal
 string& toHdtLiteral(string& literal) {

package/lib/HdtDocument.h

@@ -32,6 +32,8 @@ class HdtDocument : public node::ObjectWrap {
 
   // HdtDocument#_searchTriples(subject, predicate, object, offset, limit, callback, self)
   static NAN_METHOD(SearchTriples);
+  // HdtDocument#_searchBindings(subject, predicate, object, offset, limit, callback, self)
+  static NAN_METHOD(SearchBindings);
   // HdtDocument#_searchLiterals(substring, offset, limit, callback, self)
   static NAN_METHOD(SearchLiterals);
   // HdtDocument#_searchTerms(prefix, limit, position, callback)
@@ -50,6 +52,8 @@ class HdtDocument : public node::ObjectWrap {
   static NAN_PROPERTY_GETTER(Closed);
 };
 
+// Check if a term is a variable
+bool isVariable(string& term);
 // Converts a JavaScript literal to an HDT literal
 std::string& toHdtLiteral(std::string& literal);
 // Converts an HDT literal to a JavaScript literal

package/lib/hdt.js

@@ -13,6 +13,10 @@ function isValidHdtTerm(term) {
   return term && term.termType && (term.termType === 'Literal' || term.termType === 'BlankNode' || term.termType === 'NamedNode');
 }
 
+function isValidHdtTermForSearchBindings(term) {
+  return isValidHdtTerm(term) || (term && term.termType === 'Variable');
+}
+
 // Searches the document for triples with the given subject, predicate, and object.
 HdtDocumentPrototype.searchTriples = function (subject, predicate, object, options) {
   if (this.closed) return closedError;
@@ -29,6 +33,80 @@ HdtDocumentPrototype.searchTriples = function (subject, predicate, object, optio
   });
 };
 
+// Searches the document for triples with the given subject, predicate, and object, and return them as bindings
+HdtDocumentPrototype.searchBindings = function (bindingsFactory, subject, predicate, object, options) {
+  if (this.closed) return closedError;
+  if (!isValidHdtTermForSearchBindings(subject) || !isValidHdtTermForSearchBindings(predicate) || !isValidHdtTermForSearchBindings(object)) throw new Error('Passed invalid subject term');
+  options = options || {};
+  return new Promise((resolve, reject) => {
+    // Collect variable terms
+    const variables = [];
+    if (subject.termType === 'Variable')   variables.push(subject);
+    if (predicate.termType === 'Variable') variables.push(predicate);
+    if (object.termType === 'Variable')    variables.push(object);
+
+    // Check if we need to do post-filtering for overlapping variables
+    let shouldFilterIndexes = false;
+    const filterIndexes = variables.map((variable, i) => {
+      const equalVariables = [];
+      for (let j = i + 1; j < variables.length; j++) {
+        if (variable.equals(variables[j])) {
+          equalVariables.push(j);
+          shouldFilterIndexes = true;
+        }
+      }
+      return equalVariables;
+    });
+
+    // If we have offset and limit with overlapping variables, they must be handled in JS-land.
+    const offset = parseOffset(options);
+    const limit = parseLimit(options);
+
+    this._searchBindings(termToString(subject), termToString(predicate), termToString(object),
+      shouldFilterIndexes ? 0 : offset, shouldFilterIndexes ? MAX : limit,
+      (err, bindingsRaw, totalCount, hasExactCount) => {
+        if (err)
+          return reject(err);
+
+        // If we had overlapping variables, potentially filter bindings
+        if (shouldFilterIndexes) {
+          // Filter bindings
+          bindingsRaw = bindingsRaw.filter(binding => {
+            for (let i = 0; i < binding.length; i++) {
+              const bindingEntry = binding[i];
+              const filterI = filterIndexes[i];
+              if (filterI) {
+                for (const j of filterI) {
+                  if (j !== undefined && bindingEntry !== binding[j]) {
+                    totalCount--;
+                    return false;
+                  }
+                }
+              }
+            }
+            return true;
+          });
+
+          // Apply offsets and limits in JS-land
+          if (offset > 0 || limit < MAX) {
+            hasExactCount = true;
+            totalCount = bindingsRaw.length;
+            bindingsRaw = bindingsRaw.slice(offset, offset + limit);
+          }
+        }
+
+        // Create bindings objects
+        const bindings = bindingsRaw.map(b => bindingsFactory.bindings(b.map((value, i) => [variables[i], stringToTerm(value)])));
+
+        return resolve({
+          bindings,
+          totalCount,
+          hasExactCount,
+        });
+      });
+  });
+};
+
 // Gives an approximate number of matches of triples with the given subject, predicate, and object.
 HdtDocumentPrototype.countTriples = function (subject, predicate, object) {
   return this.search(subject, predicate, object, { offset: 0, limit: 0 });
@@ -152,6 +230,7 @@ module.exports = {
         // Document the features of the HDT file
         document.features = Object.freeze({
           searchTriples:  true, // supported by default
+          searchBindings: true, // supported by default
           countTriples:   true, // supported by default
           searchLiterals: !!(document._features & 1),
           readHeader:     true, // supported by default

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hdt",
-  "version": "3.1.1",
+  "version": "3.2.1",
   "description": "Native bindings to access HDT compressed triple files.",
   "author": "Ruben Verborgh <ruben.verborgh@gmail.com>",
   "keywords": [
@@ -15,6 +15,12 @@
   "main": "./lib/hdt",
   "bin": "./bin/hdt",
   "types": "./lib/hdt.d.ts",
+  "files": [
+    "bin",
+    "binding.gyp",
+    "deps",
+    "lib"
+  ],
   "repository": {
     "type": "git",
     "url": "https://github.com/RubenVerborgh/HDT-Node.git"
@@ -28,7 +34,8 @@
   "scripts": {
     "test": "rm test/*.hdt.index.* 2> /dev/null; mocha",
     "lint": "eslint --fix lib/*.js test/*.js bin/*",
-    "validate": "npm ls"
+    "validate": "npm ls",
+    "preinstall": "if [ -e replace-in-file.json ]; then replace-in-file --configFile=replace-in-file.json; fi && node-gyp rebuild"
   },
   "dependencies": {
     "minimist": "^1.1.0",
@@ -40,7 +47,9 @@
     "eslint": "^5.3.0",
     "mocha": "^6.2.2",
     "precommit-hook": "^3.0.0",
-    "should": "^13.1.0"
+    "replace-in-file": "^8.3.0",
+    "should": "^13.1.0",
+    "@comunica/utils-bindings-factory": "^4.0.2"
   },
   "pre-commit": [
     "lint",

package/.eslintrc

@@ -1,178 +0,0 @@
-{
-  env: {
-    node: true,
-  },
-  parserOptions: {
-    ecmaVersion: 6
-  },
-  globals: {
-    Promise: true
-  },
-  rules: {
-    // Possible Errors
-    comma-dangle: [2, "always-multiline"],
-    no-cond-assign: 0,
-    no-console: 2,
-    no-constant-condition: 0,
-    no-debugger: 2,
-    no-dupe-args: 2,
-    no-dupe-keys: 2,
-    no-duplicate-case: 2,
-    no-empty: 2,
-    no-empty-character-class: 2,
-    no-ex-assign: 0,
-    no-extra-boolean-cast: 2,
-    no-extra-parens: 0,
-    no-extra-semi: 2,
-    no-func-assign: 2,
-    no-inner-declarations: 0,
-    no-invalid-regexp: 2,
-    no-irregular-whitespace: 2,
-    no-negated-in-lhs: 2,
-    no-obj-calls: 2,
-    no-regex-spaces: 2,
-    no-sparse-arrays: 2,
-    no-unreachable: 2,
-    use-isnan: 2,
-    valid-jsdoc: 0,
-    valid-typeof: 2,
-    no-unexpected-multiline: 2,
-
-    // Best Practices
-    accessor-pairs: 2,
-    block-scoped-var: 2,
-    complexity: 0,
-    consistent-return: 0,
-    curly: [2, "multi-or-nest"],
-    default-case: 0,
-    dot-notation: 2,
-    dot-location: [2, "property"],
-    eqeqeq: 2,
-    guard-for-in: 0,
-    no-alert: 2,
-    no-caller: 2,
-    no-div-regex: 2,
-    no-else-return: 0,
-    no-labels: 2,
-    no-eq-null: 2,
-    no-eval: 2,
-    no-extend-native: 2,
-    no-extra-bind: 2,
-    no-fallthrough: 2,
-    no-floating-decimal: 2,
-    no-implicit-coercion: 0,
-    no-implied-eval: 2,
-    no-invalid-this: 2,
-    no-iterator: 2,
-    no-lone-blocks: 2,
-    no-loop-func: 0,
-    no-multi-spaces: 0,
-    no-multi-str: 2,
-    no-native-reassign: 2,
-    no-new-func: 2,
-    no-new-wrappers: 2,
-    no-new: 2,
-    no-octal-escape: 2,
-    no-octal: 2,
-    no-param-reassign: 0,
-    no-process-env: 2,
-    no-proto: 2,
-    no-redeclare: 2,
-    no-return-assign: 0,
-    no-script-url: 2,
-    no-self-compare: 2,
-    no-sequences: 0, // allow the comma operator
-    no-throw-literal: 2,
-    no-unused-expressions: 0,
-    no-useless-call: 2,
-    no-void: 2,
-    no-warning-comments: 0,
-    no-with: 2,
-    radix: 2,
-    vars-on-top: 0,
-    wrap-iife: [2, "inside"],
-    yoda: 2,
-
-    // Strict Mode
-    strict: [2, "never"],
-
-    // Variables
-    init-declarations: 0,
-    no-catch-shadow: 0,
-    no-delete-var: 2,
-    no-label-var: 2,
-    no-shadow-restricted-names: 2,
-    no-shadow: 0,
-    no-undef-init: 2,
-    no-undef: 2,
-    no-undefined: 0,
-    no-unused-vars: [ 2, { args: "none" }],
-    no-use-before-define: [2, "nofunc"],
-
-    // Node.js
-    callback-return: 0,
-    handle-callback-err: 2,
-    no-mixed-requires: 0,
-    no-new-require: 2,
-    no-path-concat: 2,
-    no-process-exit: 0,
-    no-restricted-modules: 2,
-    no-sync: 0,
-
-    // Stylistic Issues
-    array-bracket-spacing: 2,
-    block-spacing: 2,
-    brace-style: [2, "stroustrup", { allowSingleLine: true }],
-    camelcase: 2,
-    comma-spacing: 2,
-    comma-style: 2,
-    computed-property-spacing: 2,
-    consistent-this: 0,
-    eol-last: 2,
-    func-names: 0,
-    func-style: [2, "declaration"],
-    id-length: 0,
-    id-match: 2,
-    indent-legacy: [2, 2, { VariableDeclarator: 2 }],
-    key-spacing: 0,
-    lines-around-comment: 2,
-    linebreak-style: 2,
-    max-nested-callbacks: [2, 2],
-    new-cap: 2,
-    new-parens: 2,
-    newline-after-var: 0,
-    no-array-constructor: 2,
-    no-continue: 2,
-    no-inline-comments: 0,
-    no-lonely-if: 2,
-    no-mixed-spaces-and-tabs: 2,
-    no-multiple-empty-lines: 0,
-    no-nested-ternary: 0,
-    no-new-object: 2,
-    no-spaced-func: 2,
-    no-ternary: 0,
-    no-trailing-spaces: 2,
-    no-underscore-dangle: 0,
-    no-unneeded-ternary: 2,
-    object-curly-spacing: [2, "always"],
-    object-curly-newline: 0,
-    object-property-newline: 0,
-    one-var: 0,
-    operator-assignment: 2,
-    operator-linebreak: [2, "after", { overrides: { ":": "ignore" } }],
-    padded-blocks: [2, "never"],
-    quote-props: [2, "consistent-as-needed"],
-    quotes: [2, "single", "avoid-escape"],
-    semi-spacing: 2,
-    semi: 2,
-    sort-vars: 0,
-    keyword-spacing: 2,
-    space-before-blocks: 2,
-    space-before-function-paren: [2, {"anonymous": "always", "named": "never"}],
-    space-in-parens: 2,
-    space-infix-ops: 2,
-    space-unary-ops: 2,
-    spaced-comment: [2, "always", { block: { markers: ["!"] } }],
-    wrap-regex: 0,
-  },
-}

package/.github/workflows/ci.yml

@@ -1,72 +0,0 @@
-name: CI
-on: [push, pull_request]
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-
-  test:
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        os:
-          - ubuntu-20.04
-          - ubuntu-22.04
-          - ubuntu-latest
-          - macos-12
-          - macos-13
-          - macos-14
-          - macos-latest
-        node-version:
-          - 18.x
-          - 20.x
-          - 22.x
-    steps:
-      - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v3
-        with:
-          node-version: ${{ matrix.node-version }}
-      - name: Ensure line endings are consistent
-        run: git config --global core.autocrlf input
-      - name: Check out repository
-        uses: actions/checkout@v3
-        with:
-          submodules: 'recursive'
-      - name: Load cache
-        uses: actions/cache@v3
-        with:
-          path: |
-            **/node_modules
-            .rdf-test-suite-cache
-            .rdf-test-suite-ldf-cache
-          key: ${{ runner.os }}-${{ runner.node-version }}-test-modules-${{ hashFiles('**/package-lock.json') }}
-      - name: Install dependencies
-        run: npm install
-      - name: Run tests
-        run: npm run test
-      - name: Run bin
-        run: bin/hdt test/test.hdt --format turtle --query 'http://example.org/s1 ?p ?o' | grep 'http://example.org/p1'
-
-  lint:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Use Node.js
-        uses: actions/setup-node@v3
-        with:
-          node-version: 18.x
-      - name: Check out repository
-        uses: actions/checkout@v3
-        with:
-          submodules: 'recursive'
-      - name: Load cache
-        uses: actions/cache@v3
-        with:
-          path: '**/node_modules'
-          key: ${{ runner.os }}-lint-modules-${{ hashFiles('**/package-lock.json') }}
-      - name: Install dependencies
-        run: npm install
-      - name: Run linter
-        run: npm run lint

package/.gitmodules

@@ -1,4 +0,0 @@
-[submodule "deps"]
-	path = deps
-	url = https://github.com/rdfhdt/hdt-cpp.git
-	branch = master

package/.jshintignore

@@ -1 +0,0 @@
-node_modules

package/.jshintrc

@@ -1,10 +0,0 @@
-{
-  "node": true,
-
-  "curly": true,
-  "latedef": true,
-  "quotmark": true,
-  "undef": true,
-  "unused": true,
-  "trailing": true
-}

package/deps/libcds/README

@@ -1,12 +0,0 @@
-libcds implements low-level succinct data structures such as bitmaps,
-sequences, permutations, etc. The main goal is to provide a set of
-structures that form the building block of most compressed/succinct
-solutions. In the near future we are planning to add compression
-algorithms and support for succinct trees.
-
-You can find more information at http://libcds.recoded.cl/
-
-Right now we are just fixing bugs and testing some new things in here,
-but most of the effort is being put into libcds2, this rewrite aims at
-solving many problems with 64bits systems. This is necessary in order
-to index bigger inputs.