@builder6/query-mongodb 0.6.1

package/dist/utils.js

@@ -0,0 +1,45 @@
+"use strict";
+
+var _extends = Object.assign || function (target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i]; for (var key in source) { if (Object.prototype.hasOwnProperty.call(source, key)) { target[key] = source[key]; } } } return target; };
+
+function _toConsumableArray(arr) { if (Array.isArray(arr)) { for (var i = 0, arr2 = Array(arr.length); i < arr.length; i++) { arr2[i] = arr[i]; } return arr2; } else { return Array.from(arr); } }
+
+var replaceId = function replaceId(item) {
+  return item._id ? _extends({}, item, { _id: item._id.toHexString() }) : item;
+};
+
+var createSummaryQueryExecutor = function createSummaryQueryExecutor(limit) {
+  var queriesExecuted = 0;
+
+  return function (fn) {
+    return !limit || ++queriesExecuted <= limit ? fn() : Promise.resolve();
+  };
+};
+
+var merge = function merge(os) {
+  return Object.assign.apply(Object, [{}].concat(_toConsumableArray(os)));
+};
+
+var debug = function debug(id, f) {
+  var options = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : {};
+
+  var output = options.output || console.log;
+  var processResult = options.processResult || function (result) {
+    return result;
+  };
+  var processArgs = options.processArgs || function (args) {
+    return args;
+  };
+  return function () {
+    for (var _len = arguments.length, args = Array(_len), _key = 0; _key < _len; _key++) {
+      args[_key] = arguments[_key];
+    }
+
+    output("DEBUG(" + id + "): ", processArgs(args));
+    var result = f.apply(undefined, args);
+    output("DEBUG(" + id + "/result): ", processResult(result));
+    return result;
+  };
+};
+
+module.exports = { replaceId: replaceId, createSummaryQueryExecutor: createSummaryQueryExecutor, merge: merge, debug: debug };

package/dist/utils.test.js

@@ -0,0 +1,36 @@
+'use strict';
+
+var chai = require('chai');
+var assert = chai.assert;
+var sinon = require('sinon');
+
+var _require = require('./utils'),
+    replaceId = _require.replaceId,
+    createSummaryQueryExecutor = _require.createSummaryQueryExecutor;
+
+suite('utils', function () {
+  suite('replaceId', function () {
+    test('works', function () {
+      assert.deepEqual(replaceId({ _id: { toHexString: function toHexString() {
+            return '42';
+          } } }), {
+        _id: '42'
+      });
+    });
+  });
+
+  suite('createSummaryQueryExecutor', function () {
+    test('works', function (done) {
+      var exec = createSummaryQueryExecutor(3);
+      var func = sinon.stub().resolves();
+      var promises = [exec(func), exec(func), exec(func), exec(func), exec(func)];
+
+      Promise.all(promises).then(function (rs) {
+        assert.equal(rs.length, 5);
+
+        assert.equal(func.callCount, 3);
+        done();
+      });
+    });
+  });
+});

package/index.js

@@ -0,0 +1 @@
+module.exports = require('./dist/index');

package/options.js

@@ -0,0 +1 @@
+module.exports = require('./dist/options');

package/package.json

@@ -0,0 +1,60 @@
+{
+  "name": "@builder6/query-mongodb",
+  "version": "0.6.1",
+  "description": "Querying a MongoDB collection using DevExtreme data store load parameters",
+  "main": "index.js",
+  "files": [
+    "index.js",
+    "options.js",
+    "dist",
+    "src",
+    "LICENSE",
+    "README.md"
+  ],
+  "scripts": {
+    "prepare": "babel src --out-dir ./dist",
+    "build": "babel src --out-dir ./dist"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/builder6/builder6-server.git"
+  },
+  "keywords": [
+    "DevExtreme",
+    "MongoDB"
+  ],
+  "author": "Oliver Sturm <oliver@oliversturm.com>",
+  "license": "SEE LICENSE IN LICENSE",
+  "bugs": {
+    "url": "https://github.com/builder6/builder6-server/issues"
+  },
+  "homepage": "https://github.com/builder6/builder6-server#readme",
+  "engines": {
+    "node": ">=7.3"
+  },
+  "dependencies": {
+    "yup": "^1.1.0"
+  },
+  "devDependencies": {
+    "babel-cli": "^6.26.0",
+    "babel-core": "^6.26.3",
+    "babel-eslint": "^10.1.0",
+    "babel-plugin-remove-comments": "^2.0.0",
+    "babel-plugin-transform-es2015-modules-umd": "^6.24.1",
+    "babel-plugin-transform-object-rest-spread": "^6.26.0",
+    "babel-polyfill": "^6.26.0",
+    "babel-preset-env": "^1.7.0",
+    "chai": "^4.3.7",
+    "eslint": "^8.38.0",
+    "eslint-plugin-promise": "^6.1.1",
+    "mocha": "^10.2.0",
+    "mongodb": "^5.2.0",
+    "nyc": "^15.1.0",
+    "qs": "^6.11.1",
+    "sinon": "^15.0.3"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "gitHead": "2f816481fc81096a973e75ed27ff9b8bb0638e0c"
+}

package/src/index.js

@@ -0,0 +1,497 @@
+const {
+  createGroupFieldName,
+  createGroupKeyPipeline,
+  createGroupingPipeline,
+  createSkipTakePipeline,
+  createCountPipeline,
+  createMatchPipeline,
+  createSortPipeline,
+  createSummaryPipeline,
+  createSelectProjectExpression,
+  createSelectPipeline,
+  createCompleteFilterPipeline,
+  createRemoveNestedFieldsPipeline,
+} = require('./pipelines');
+const {
+  replaceId,
+  createSummaryQueryExecutor,
+  merge,
+  debug,
+} = require('./utils');
+const { getOptions } = require('./options');
+
+function createContext(contextOptions, loadOptions) {
+  const aggregateCall = (collection, pipeline, identifier) =>
+    ((aggregateOptions) => collection.aggregate(pipeline, aggregateOptions))(
+      contextOptions.dynamicAggregateOptions
+        ? filterAggregateOptions(
+            contextOptions.dynamicAggregateOptions(
+              identifier,
+              pipeline,
+              collection
+            )
+          )
+        : contextOptions.aggregateOptions
+    );
+
+  const getCount = (collection, pipeline) =>
+    aggregateCall(collection, pipeline, 'getCount')
+      .toArray()
+      // Strangely, the pipeline returns an empty array when the "match" part
+      // filters out all rows - I would expect to still see the "count" stage
+      // working, but it doesn't. Ask mongo.
+      .then((r) => (r.length > 0 ? r[0].count : 0));
+
+  const populateSummaryResults = (target, summary, summaryResults) => {
+    if (summary) {
+      target.summary = [];
+
+      for (const s of summary) {
+        switch (s.summaryType) {
+          case 'sum':
+            target.summary.push(summaryResults['___sum' + s.selector]);
+            break;
+          case 'avg':
+            target.summary.push(summaryResults['___avg' + s.selector]);
+            break;
+          case 'min':
+            target.summary.push(summaryResults['___min' + s.selector]);
+            break;
+          case 'max':
+            target.summary.push(summaryResults['___max' + s.selector]);
+            break;
+          case 'count':
+            target.summary.push(summaryResults.___count);
+            break;
+          default:
+            console.error(`Invalid summaryType ${s.summaryType}, ignoring`);
+        }
+      }
+    }
+    return target;
+  };
+
+  const queryGroupData = (
+    collection,
+    desc,
+    includeDataItems,
+    countSeparately,
+    itemProjection,
+    groupKeyPipeline,
+    sortPipeline,
+    filterPipelineDetails,
+    skipTakePipeline,
+    matchPipeline
+  ) =>
+    aggregateCall(
+      collection,
+      [
+        ...contextOptions.preProcessingPipeline,
+        // sort pipeline first, apparently that enables it to use indexes
+        ...sortPipeline,
+        ...filterPipelineDetails.pipeline,
+        ...matchPipeline,
+        ...createRemoveNestedFieldsPipeline(
+          filterPipelineDetails.nestedFields,
+          contextOptions
+        ),
+        ...createGroupingPipeline(
+          desc,
+          includeDataItems,
+          countSeparately,
+          groupKeyPipeline,
+          itemProjection,
+          contextOptions
+        ),
+        ...skipTakePipeline,
+      ],
+      'queryGroupData'
+    )
+      .toArray()
+      .then((r) =>
+        includeDataItems
+          ? r.map((i) => ({
+              ...i,
+              items: contextOptions.replaceIds
+                ? i.items.map(replaceId)
+                : i.items,
+            }))
+          : r
+      );
+
+  const queryGroup = (
+    collection,
+    groupIndex,
+    runSummaryQuery,
+    filterPipelineDetails,
+    skipTakePipeline = [],
+    summaryPipeline = [],
+    matchPipeline = []
+  ) => {
+    const group = loadOptions.group[groupIndex];
+    const lastGroup = groupIndex === loadOptions.group.length - 1;
+    const itemDataRequired = lastGroup && group.isExpanded;
+    const separateCountRequired = !lastGroup;
+
+    // The current implementation of the dxDataGrid, at least, assumes that sub-group details are
+    // always included in the result, whether or not the group is marked isExpanded.
+    const subGroupsRequired = !lastGroup; // && group.isExpanded;
+    const summariesRequired =
+      loadOptions.groupSummary && loadOptions.groupSummary.length > 0;
+
+    const groupKeyPipeline = createGroupKeyPipeline(
+      group.selector,
+      group.groupInterval,
+      groupIndex,
+      contextOptions
+    );
+
+    const augmentWithSubGroups = (groupData) =>
+      subGroupsRequired
+        ? groupData.map((item) =>
+            queryGroup(
+              collection,
+              groupIndex + 1,
+              runSummaryQuery,
+              filterPipelineDetails, // used unchanged in lower levels
+              [], // skip/take doesn't apply on lower levels - correct?
+              summaryPipeline, // unmodified
+              // matchPipeline modified to filter down into group level
+              [
+                ...matchPipeline,
+                // not completely clean to include this in the match pipeline, but the field
+                // added in the groupKeyPipeline is required specifically for the following match
+                ...groupKeyPipeline,
+                ...createMatchPipeline(
+                  createGroupFieldName(groupIndex),
+                  item.key,
+                  contextOptions
+                ),
+              ]
+            ).then((r) => {
+              item.items = r;
+              item.count = r.length;
+              return r;
+            })
+          )
+        : [];
+
+    const augmentWithSeparateCount = (groupData) => {
+      if (separateCountRequired) {
+        // We need to count separately because this is not the lowest level group,
+        // but since we didn't query details about our nested group, we can't just go
+        // for the length of the result array. An extra query is required in this case.
+        // Even though the count is a type of summary for the group, it is special - different
+        // from other group level summaries. The difference is that for each group, a summary
+        // is usually calculated with its data, even if that data isn't actually visible in the
+        // UI at the time. The count on the other hand is meant to represent the number of
+        // elements in the group, and in case these elements are sub-groups instead of data
+        // items, count represents a value that must not be calculated using the data items.
+
+        const nextGroup = loadOptions.group[groupIndex + 1];
+        const nextGroupKeyPipeline = createGroupKeyPipeline(
+          nextGroup.selector,
+          nextGroup.groupInterval,
+          groupIndex + 1,
+          contextOptions
+        );
+        return groupData.map((item) =>
+          getCount(collection, [
+            ...contextOptions.preProcessingPipeline,
+            ...filterPipelineDetails.pipeline,
+            ...groupKeyPipeline,
+
+            ...matchPipeline,
+            ...createMatchPipeline(
+              createGroupFieldName(groupIndex),
+              item.key,
+              contextOptions
+            ),
+            ...createGroupingPipeline(
+              nextGroup.desc,
+              false,
+              true,
+              nextGroupKeyPipeline,
+              contextOptions
+            ),
+            ...createCountPipeline(contextOptions),
+          ]).then((r) => {
+            item.count = r;
+            return r;
+          })
+        );
+      } else return [];
+    };
+
+    const augmentWithSummaries = (groupData) =>
+      summariesRequired
+        ? groupData.map((item) =>
+            runSummaryQuery(() =>
+              aggregateCall(
+                collection,
+                [
+                  ...contextOptions.preProcessingPipeline,
+                  ...filterPipelineDetails.pipeline,
+                  ...groupKeyPipeline,
+                  ...matchPipeline,
+                  ...createMatchPipeline(
+                    createGroupFieldName(groupIndex),
+                    item.key,
+                    contextOptions
+                  ),
+                  ...summaryPipeline,
+                ],
+                'augmentWithSummaries'
+              ).toArray()
+            ).then((r) =>
+              populateSummaryResults(item, loadOptions.groupSummary, r[0])
+            )
+          )
+        : [];
+
+    return queryGroupData(
+      collection,
+      group.desc,
+      itemDataRequired,
+      separateCountRequired,
+      createSelectProjectExpression(loadOptions.select, true),
+      groupKeyPipeline,
+      itemDataRequired
+        ? createSortPipeline(loadOptions.sort, contextOptions)
+        : [],
+      filterPipelineDetails,
+      skipTakePipeline,
+      matchPipeline
+    ).then(
+      (groupData) =>
+        /* eslint-disable promise/no-nesting */
+        Promise.all([
+          ...augmentWithSubGroups(groupData),
+          ...augmentWithSeparateCount(groupData),
+          ...augmentWithSummaries(groupData),
+        ]).then(() => groupData)
+      /* eslint-enable promise/no-nesting */
+    );
+  };
+
+  const totalCount = (collection, completeFilterPipelineDetails) =>
+    loadOptions.requireTotalCount || loadOptions.totalSummary
+      ? contextOptions.preferMetadataCount &&
+        contextOptions.preProcessingPipeline.length === 0 &&
+        completeFilterPipelineDetails.pipeline.length <= 1
+        ? [
+            collection
+              .count(
+                completeFilterPipelineDetails.pipeline.length === 1
+                  ? completeFilterPipelineDetails.pipeline[0]['$match']
+                  : undefined
+              )
+              .then((r) => ({ totalCount: r })),
+          ]
+        : [
+            getCount(collection, [
+              ...contextOptions.preProcessingPipeline,
+              ...completeFilterPipelineDetails.pipeline,
+              ...createCountPipeline(contextOptions),
+            ]).then((r) => ({ totalCount: r })),
+          ]
+      : [];
+
+  const summary =
+    (collection, completeFilterPipelineDetails) => (resultObject) =>
+      resultObject.totalCount > 0 && loadOptions.totalSummary
+        ? aggregateCall(
+            collection,
+            [
+              ...contextOptions.preProcessingPipeline,
+              ...completeFilterPipelineDetails.pipeline,
+              ...createSummaryPipeline(
+                loadOptions.totalSummary,
+                contextOptions
+              ),
+            ],
+            'summary'
+          )
+            .toArray()
+            .then((r) =>
+              populateSummaryResults(
+                resultObject,
+                loadOptions.totalSummary,
+                r[0]
+              )
+            )
+        : Promise.resolve(resultObject);
+
+  const queryGroups = (collection) => {
+    const completeFilterPipelineDetails = createCompleteFilterPipeline(
+      loadOptions.searchExpr,
+      loadOptions.searchOperation,
+      loadOptions.searchValue,
+      loadOptions.filter,
+      contextOptions
+    );
+    const summaryPipeline = createSummaryPipeline(
+      loadOptions.groupSummary,
+      contextOptions
+    );
+    const skipTakePipeline = createSkipTakePipeline(
+      loadOptions.skip,
+      loadOptions.take,
+      contextOptions
+    );
+
+    const mainQueryResult = () =>
+      queryGroup(
+        collection,
+        0,
+        createSummaryQueryExecutor(undefined),
+        completeFilterPipelineDetails,
+        skipTakePipeline,
+        summaryPipeline
+      ).then((r) => ({ data: r }));
+
+    const groupCount = () => {
+      if (loadOptions.requireGroupCount) {
+        const group = loadOptions.group[0];
+
+        return [
+          getCount(collection, [
+            ...contextOptions.preProcessingPipeline,
+            ...completeFilterPipelineDetails.pipeline,
+            ...createGroupingPipeline(
+              group.desc,
+              false,
+              true,
+              createGroupKeyPipeline(
+                group.selector,
+                group.groupInterval,
+                0,
+                contextOptions
+              ),
+              contextOptions
+            ),
+            ...createCountPipeline(contextOptions),
+          ]).then((r) => ({ groupCount: r })),
+        ];
+      } else return [];
+    };
+
+    return Promise.all([
+      mainQueryResult(),
+      ...groupCount(),
+      ...totalCount(collection, completeFilterPipelineDetails),
+    ])
+      .then(merge)
+      .then(summary(collection, completeFilterPipelineDetails));
+  };
+
+  const querySimple = (collection) => {
+    const completeFilterPipelineDetails = createCompleteFilterPipeline(
+      loadOptions.searchExpr,
+      loadOptions.searchOperation,
+      loadOptions.searchValue,
+      loadOptions.filter,
+      contextOptions
+    );
+    const sortPipeline = createSortPipeline(loadOptions.sort, contextOptions);
+    const skipTakePipeline = createSkipTakePipeline(
+      loadOptions.skip,
+      loadOptions.take,
+      contextOptions
+    );
+    const selectPipeline = createSelectPipeline(
+      loadOptions.select,
+      contextOptions
+    );
+    const removeNestedFieldsPipeline = createRemoveNestedFieldsPipeline(
+      completeFilterPipelineDetails.nestedFields,
+      contextOptions
+    );
+
+    const mainQueryResult = () =>
+      aggregateCall(
+        collection,
+        [
+          ...contextOptions.preProcessingPipeline,
+          ...completeFilterPipelineDetails.pipeline,
+          ...sortPipeline,
+          ...skipTakePipeline,
+          ...selectPipeline,
+          ...removeNestedFieldsPipeline,
+        ],
+        'mainQueryResult'
+      )
+        .toArray()
+        .then((r) => (contextOptions.replaceIds ? r.map(replaceId) : r))
+        .then((r) => ({ data: r }));
+
+    return Promise.all([
+      mainQueryResult(),
+      ...totalCount(collection, completeFilterPipelineDetails),
+    ])
+      .then(merge)
+      .then(summary(collection, completeFilterPipelineDetails));
+  };
+
+  return { queryGroups, querySimple };
+}
+
+function filterAggregateOptions(proposedOptions) {
+  const acceptableAggregateOptionNames = [
+    'allowDiskUse',
+    'maxTimeMS',
+    'readConcern',
+    'collation',
+    'hint',
+    'comment',
+  ];
+  return Object.keys(proposedOptions).reduce(
+    (r, v) =>
+      acceptableAggregateOptionNames.includes(v)
+        ? { ...r, [v]: proposedOptions[v] }
+        : r,
+    {}
+  );
+}
+
+function query(collection, loadOptions = {}, options = {}) {
+  const proposedAggregateOptions = options.aggregateOptions;
+  delete options.aggregateOptions;
+
+  const standardContextOptions = {
+    replaceIds: true,
+    summaryQueryLimit: 100,
+    // timezone offset for the query, in the form returned by
+    // Date.getTimezoneOffset
+    timezoneOffset: 0,
+    preProcessingPipeline: [],
+    caseInsensitiveRegex: true,
+  };
+  const contextOptions = Object.assign(standardContextOptions, options);
+
+  if (!options.dynamicAggregateOptions && proposedAggregateOptions)
+    contextOptions.aggregateOptions = filterAggregateOptions(
+      proposedAggregateOptions
+    );
+
+  const context = createContext(contextOptions, loadOptions);
+
+  return loadOptions.group && loadOptions.group.length > 0
+    ? context.queryGroups(collection)
+    : context.querySimple(collection);
+}
+
+function querySimple(collection, loadOptions = {}, options = {}) {
+  delete loadOptions.group;
+  return query(collection, loadOptions, options);
+}
+
+function queryGroups(collection, loadOptions = {}, options = {}) {
+  return query(collection, loadOptions, options);
+}
+
+module.exports = {
+  queryGroups,
+  querySimple,
+  getOptions
+};