electrodb 3.5.3 → 3.6.1

package/index.js

@@ -18,7 +18,9 @@ const {
 const { createConversions } = require("./src/conversions");
 
 const {
-  ComparisonTypes
+  ComparisonTypes,
+  EntityIdentifiers,
+  EntityIdentifierFields,
 } = require('./src/types');
 
 module.exports = {
@@ -35,4 +37,6 @@ module.exports = {
   ElectroUserValidationError,
   ElectroAttributeValidationError,
   createConversions,
+  EntityIdentifiers,
+  EntityIdentifierFields,
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "electrodb",
-  "version": "3.5.3",
+  "version": "3.6.1",
   "description": "A library to more easily create and interact with multiple entities and heretical relationships in dynamodb",
   "main": "index.js",
   "scripts": {

package/src/clauses.js

@@ -1267,6 +1267,7 @@ let clauses = {
             operation: options._isTransaction
               ? MethodTypes.transactWrite
               : undefined,
+            state,
           },
         });
 

package/src/client.js

@@ -2,6 +2,7 @@ const lib = require("@aws-sdk/lib-dynamodb");
 const util = require("@aws-sdk/util-dynamodb");
 const { isFunction } = require("./validations");
 const { ElectroError, ErrorCodes } = require("./errors");
+const { EntityIdentifiers } = require("./types");
 const DocumentClientVersions = {
   v2: "v2",
   v3: "v3",
@@ -289,9 +290,14 @@ function normalizeClient(client) {
 }
 
 function normalizeConfig(config = {}) {
+  const identifiers = config.identifiers || {};
   return {
     ...config,
     client: normalizeClient(config.client),
+    identifiers: {
+      entity: identifiers.entity || EntityIdentifiers.entity,
+      version: identifiers.version || EntityIdentifiers.version
+    }
   };
 }
 

package/src/entity.js

@@ -30,9 +30,10 @@ const {
   CastKeyOptions,
   ComparisonTypes,
   DataOptions,
+  IndexProjectionOptions,
 } = require("./types");
 const { FilterFactory } = require("./filters");
-const { FilterOperations } = require("./operations");
+const { FilterOperations, formatExpressionName } = require("./operations");
 const { WhereFactory } = require("./where");
 const { clauses, ChainState } = require("./clauses");
 const { EventManager } = require("./events");
@@ -49,16 +50,16 @@ const ImpactedIndexTypeSource = {
 
 class Entity {
   constructor(model, config = {}) {
-    config = c.normalizeConfig(config);
+    this.config = c.normalizeConfig(config);
+    this.identifiers = this.config.identifiers;
+    this.client = this.config.client;
     this.eventManager = new EventManager({
-      listeners: config.listeners,
+      listeners: this.config.listeners,
     });
-    this.eventManager.add(config.logger);
+    this.eventManager.add(this.config.logger);
     this._validateModel(model);
     this.version = EntityVersions.v1;
-    this.config = config;
-    this.client = config.client;
-    this.model = this._parseModel(model, config);
+    this.model = this._parseModel(model, this.config);
     /** start beta/v1 condition **/
     this.config.table = config.table || model.table;
     /** end beta/v1 condition **/
@@ -94,24 +95,31 @@ class Entity {
         ).query(...values);
       };
     }
-
-    this.config.identifiers = config.identifiers || {};
-    this.identifiers = {
-      entity: this.config.identifiers.entity || "__edb_e__",
-      version: this.config.identifiers.version || "__edb_v__",
-    };
     this._instance = ElectroInstance.entity;
     this._instanceType = ElectroInstanceTypes.entity;
     this.schema = model;
   }
 
   get scan() {
-    return this._makeChain(
+    const result = this._makeChain(
       TableIndex,
       this._clausesWithFilters,
       clauses.index,
       { _isPagination: true },
     ).scan();
+
+    for (const accessPattern in this.model.indexes) {
+      const index = this.model.indexes[accessPattern].index;
+
+      result[accessPattern] = this._makeChain(
+        index,
+        this._clausesWithFilters,
+        clauses.index,
+        { _isPagination: true },
+      ).scan();
+    }
+
+    return result;
   }
 
   setIdentifier(type = "", identifier = "") {
@@ -145,40 +153,38 @@ class Entity {
     );
   }
 
-  _attributesIncludeKeys(attributes = []) {
+  _itemIncludesKeys(item) {
     let { pk, sk } = this.model.prefixes[TableIndex];
-    let pkFound = false;
-    let skFound = false;
-    for (let i = 0; i < attributes.length; i++) {
-      const attribute = attributes[i];
-      if (attribute === sk.field) {
-        skFound = true;
-      }
-      if (attribute === pk.field) {
-        skFound = true;
-      }
-      if (pkFound && skFound) {
-        return true;
-      }
-    }
-    return false;
+    return item[pk.field] !== undefined && (
+      sk.field === undefined || item[sk.field] !== undefined
+    );
   }
 
   ownsKeys(key = {}) {
+    const accessPattern =
+      this.model.translations.indexes.fromIndexToAccessPattern[TableIndex];
     let { pk, sk } = this.model.prefixes[TableIndex];
     let hasSK = this.model.lookup.indexHasSortKeys[TableIndex];
     const typeofPkProvided = typeof key[pk.field];
     const pkPrefixMatch =
-      typeofPkProvided === "string" && key[pk.field].startsWith(pk.prefix);
+      typeofPkProvided === "string" &&
+      key[pk.field].startsWith(pk.prefix) &&
+      (!pk.postfix || key[pk.field].endsWith(pk.postfix));
     const isNumericPk = typeofPkProvided === "number" && pk.cast === "number";
     let pkMatch = pkPrefixMatch || isNumericPk;
     let skMatch = pkMatch && !hasSK;
     if (pkMatch && hasSK) {
       const typeofSkProvided = typeof key[sk.field];
-      const skPrefixMatch =
-        typeofSkProvided === "string" && key[sk.field].startsWith(sk.prefix);
-      const isNumericSk = typeofSkProvided === "number" && sk.cast === "number";
-      skMatch = skPrefixMatch || isNumericSk;
+      if (typeofSkProvided === "number") {
+        skMatch = sk.cast === "number";
+      } else if (typeofSkProvided === "string") {
+        // if sk is a string, check prefix and postfix match
+        const hasNoPrefixOrStartsWithPrefix = !sk.prefix || key[sk.field].startsWith(sk.prefix);
+        const hasNoPostfixOrEndsWithPostfix = !sk.postfix || key[sk.field].endsWith(sk.postfix);
+        skMatch =
+          hasNoPrefixOrStartsWithPrefix &&
+          hasNoPostfixOrEndsWithPostfix;
+      }
     }
 
     return (
@@ -652,6 +658,7 @@ class Entity {
       _isCollectionQuery: false,
       preserveBatchOrder: true,
       ignoreOwnership: config._providedIgnoreOwnership,
+      attributes: config._providedAttributes,
     });
 
     const unprocessed = [];
@@ -689,7 +696,9 @@ class Entity {
     let iterations = 0;
     let count = 0;
     let hydratedUnprocessed = [];
-    const shouldHydrate = config.hydrate && method === MethodTypes.query;
+    const shouldHydrate =
+      config.hydrate &&
+      (method === MethodTypes.query || method === MethodTypes.scan);
     do {
       let response = await this._exec(
         method,
@@ -908,6 +917,18 @@ class Entity {
     }
   }
 
+  is(item, config) {
+    return (
+      config.ignoreOwnership &&
+      !this._itemIncludesKeys(item)
+    ) || (
+      (config.ignoreOwnership || config.hydrate) &&
+      this.ownsKeys(item)
+    ) || (
+      this.ownsItem(item)
+    );
+  }
+
   formatResponse(response, index, config = {}) {
     let stackTrace;
     if (!config.originalErr) {
@@ -930,15 +951,7 @@ class Entity {
         results = response;
       } else {
         if (response.Item) {
-          if (
-            (config.ignoreOwnership &&
-              config.attributes &&
-              config.attributes.length > 0 &&
-              !this._attributesIncludeKeys(config.attributes)) ||
-            ((config.ignoreOwnership || config.hydrate) &&
-              this.ownsKeys(response.Item)) ||
-            this.ownsItem(response.Item)
-          ) {
+          if (this.is(response.Item, config)) {
             results = this.model.schema.formatItemForRetrieval(
               response.Item,
               config,
@@ -949,15 +962,7 @@ class Entity {
         } else if (response.Items) {
           results = [];
           for (let item of response.Items) {
-            if (
-              (config.ignoreOwnership &&
-                config.attributes &&
-                config.attributes.length > 0 &&
-                !this._attributesIncludeKeys(config.attributes)) ||
-              ((config.ignoreOwnership || config.hydrate) &&
-                this.ownsKeys(item)) ||
-              this.ownsItem(item)
-            ) {
+            if (this.is(item, config)) {
               let record = this.model.schema.formatItemForRetrieval(
                 item,
                 config,
@@ -1640,6 +1645,7 @@ class Entity {
       listeners: [],
       preserveBatchOrder: false,
       attributes: [],
+      _providedAttributes: [],
       terminalOperation: undefined,
       formatCursor: u.cursorFormatter,
       order: undefined,
@@ -1649,6 +1655,19 @@ class Entity {
       _includeOnResponseItem: {},
     };
 
+    // Auto-set ignoreOwnership: true for INCLUDE or KEYS_ONLY indexes
+    if (context.state && context.state.query && context.state.query.index) {
+      const indexName = context.state.query.index;
+      const accessPattern =
+        this.model.translations.indexes.fromIndexToAccessPattern[indexName];
+      if (accessPattern) {
+        const indexDefinition = this.model.indexes[accessPattern];
+        config.ignoreOwnership =
+          indexDefinition.projection === IndexProjectionOptions.keys_only ||
+          (Array.isArray(indexDefinition.projection) && !this.model.indexes[accessPattern].identifiersAreProjected);
+      }
+    }
+
     return provided.filter(Boolean).reduce((config, option) => {
       if (typeof option.order === "string") {
         switch (option.order.toLowerCase()) {
@@ -1727,6 +1746,7 @@ class Entity {
 
       if (Array.isArray(option.attributes)) {
         config.attributes = config.attributes.concat(option.attributes);
+        config._providedAttributes = option.attributes;
       }
 
       if (option.preserveBatchOrder === true) {
@@ -1850,7 +1870,7 @@ class Entity {
         }
       }
 
-      if (option.ignoreOwnership) {
+      if (option.ignoreOwnership !== undefined) {
         config.ignoreOwnership = option.ignoreOwnership;
         config._providedIgnoreOwnership = option.ignoreOwnership;
       }
@@ -1875,6 +1895,10 @@ class Entity {
       if (option.hydrate) {
         config.hydrate = true;
         config.ignoreOwnership = true;
+        // if we will hydrate later, we don't want to provide a ProjectionExpression since the attributes
+        // may contain non-projected attributes that the user expects to receive from the main table
+        // after hydration
+        config.attributes = [];
       }
 
       if (validations.isFunction(option.hydrator)) {
@@ -2015,6 +2039,7 @@ class Entity {
         params = this._makeScanParam(
           filter[ExpressionTypes.FilterExpression],
           config,
+          state,
         );
         break;
       /* istanbul ignore next */
@@ -2058,27 +2083,41 @@ class Entity {
       TerminalOperation[config.terminalOperation] === TerminalOperation.go ||
       TerminalOperation[config.terminalOperation] === TerminalOperation.page;
 
-    // 1. Take stock of invalid attributes, if there are any this should be considered
-    //    unintentional and should throw to prevent unintended results
-    // 2. Convert all attribute names to their respective "field" names
-    const unknownAttributes = [];
-    let attributeFields = new Set();
-    for (const attributeName of attributes) {
-      const fieldName = this.model.schema.getFieldName(attributeName);
-      if (typeof fieldName !== "string") {
-        unknownAttributes.push(attributeName);
-      } else {
-        attributeFields.add(fieldName);
+    // Convert all attribute names to their respective "field" names
+    let hasTableIndexPk = false;
+    let hasTableIndexSk = !this.model.translations.keys[TableIndex].sk;
+    const attributeFields = new Map();
+    for (let [key, name] of Object.entries(parameters.ExpressionAttributeNames || {})) {
+      if (key.startsWith("#")) {
+        key = key.slice(1);
+      }
+      attributeFields.set(key, name);
+      if (name === this.model.translations.keys[TableIndex].pk) {
+        hasTableIndexPk = true;
       }
+      if (name === this.model.translations.keys[TableIndex].sk) {
+        hasTableIndexSk = true;
+      }
+    }
+
+    if (!hasTableIndexPk) {
+      const field = this.model.translations.keys[TableIndex].pk;
+      const name = formatExpressionName(field, attributeFields);
+      attributeFields.set(name, field);
     }
 
-    // Stop doing work, prepare error message and throw
-    if (attributeFields.size === 0 || unknownAttributes.length > 0) {
-      let message = "Unknown attributes provided in query options";
-      if (unknownAttributes.length) {
-        message += `: ${u.commaSeparatedString(unknownAttributes)}`;
+    if (!hasTableIndexSk) {
+      const field = this.model.translations.keys[TableIndex].sk;
+      const name = formatExpressionName(field, attributeFields);
+      attributeFields.set(name, field);
+    }
+
+    for (const attributeName of attributes) {
+      const fieldName = this.model.schema.getFieldName(attributeName) || attributeName;
+      if (fieldName) {
+        const formatted = formatExpressionName(fieldName, attributeFields);
+        attributeFields.set(formatted, fieldName);
       }
-      throw new e.ElectroError(e.ErrorCodes.InvalidOptions, message);
     }
 
     // add ExpressionAttributeNames if it doesn't exist already
@@ -2095,8 +2134,8 @@ class Entity {
       enforcesOwnership
     ) {
       // add entity identifiers to so items can be identified
-      attributeFields.add(this.identifiers.entity);
-      attributeFields.add(this.identifiers.version);
+      attributeFields.set(this.identifiers.entity, this.identifiers.entity);
+      attributeFields.set(this.identifiers.version, this.identifiers.version);
 
       // if pagination is required you may enter into a scenario where
       // the LastEvaluatedKey doesn't belong to entity and one must be formed.
@@ -2111,22 +2150,23 @@ class Entity {
 
         for (const attribute of [...tableIndexFacets.all, ...indexFacets.all]) {
           const fieldName = this.model.schema.getFieldName(attribute.name);
-          attributeFields.add(fieldName);
+          const formatted = formatExpressionName(attribute.name, attributeFields);
+          attributeFields.set(formatted, fieldName);
         }
       }
     }
 
-    for (const attributeField of attributeFields) {
+    for (const [attributeField, attributeName] of attributeFields.entries()) {
       // prefix the ExpressionAttributeNames because some prefixes are not allowed
       parameters.ExpressionAttributeNames["#" + attributeField] =
-        attributeField;
+        attributeName;
     }
 
     // if there is already a ProjectionExpression (e.g. config "params"), merge it
     if (typeof parameters.ProjectionExpression === "string") {
       parameters.ProjectionExpression = [
         parameters.ProjectionExpression,
-        ...Object.keys([parameters.ExpressionAttributeNames]),
+        ...Object.keys(parameters.ExpressionAttributeNames),
       ].join(", ");
     } else {
       parameters.ProjectionExpression = Object.keys(
@@ -2234,17 +2274,17 @@ class Entity {
   }
 
   /* istanbul ignore next */
-  _makeScanParam(filter = {}, options = {}) {
-    let indexBase = TableIndex;
-    let hasSortKey = this.model.lookup.indexHasSortKeys[indexBase];
+  _makeScanParam(filter = {}, options = {}, state = {}) {
+    let index = state.query.index || TableIndex;
+    let hasSortKey = this.model.lookup.indexHasSortKeys[index];
     let accessPattern =
-      this.model.translations.indexes.fromIndexToAccessPattern[indexBase];
+      this.model.translations.indexes.fromIndexToAccessPattern[index];
     let pkField = this.model.indexes[accessPattern].pk.field;
     let { pk, sk } = this._makeIndexKeys({
-      index: indexBase,
+      index,
     });
 
-    let keys = this._makeParameterKey(indexBase, pk, ...sk);
+    let keys = this._makeParameterKey(index, pk, ...sk);
     // trim empty key values (this can occur when keys are defined by users)
     for (let key in keys) {
       if (keys[key] === undefined || keys[key] === "") {
@@ -2297,6 +2337,10 @@ class Entity {
       params.FilterExpression = filterExpressions.join(" AND ");
     }
 
+    if (index) {
+      params.IndexName = index;
+    }
+
     return this._applyProjectionExpressions({
       parameters: params,
       config: options,
@@ -4227,7 +4271,7 @@ class Entity {
     return model;
   }
 
-  _normalizeIndexes(indexes) {
+  _normalizeIndexes(indexes, config) {
     let normalized = {};
     let indexFieldTranslation = {};
     let indexHasSortKeys = {};
@@ -4253,6 +4297,7 @@ class Entity {
       fields: [],
       attributes: [],
       labels: {},
+      projections: [],
     };
     let seenIndexes = {};
     let seenIndexFields = {};
@@ -4370,7 +4415,7 @@ class Entity {
               e.ErrorCodes.DuplicateIndexCompositeAttributes,
               `The Access Pattern '${accessPattern}' contains duplicate references the composite attribute(s): ${u.commaSeparatedString(
                 duplicates,
-              )}. Composite attributes can only be used more than once in an index if your sort key is limitted to a single attribute. This is to prevent unexpected runtime errors related to the inability to generate keys.`,
+              )}. Composite attributes can only be used more than once in an index if your sort key is limited to a single attribute. This is to prevent unexpected runtime errors related to the inability to generate keys.`,
             );
           }
         }
@@ -4387,8 +4432,39 @@ class Entity {
         scope: indexScope,
         condition: indexCondition,
         conditionDefined: conditionDefined,
+        projection: index.projection,
+        identifiersAreProjected: false,
       };
 
+      let projections = [];
+
+      if (index.projection !== undefined) {
+        if (typeof index.projection === "string" && (
+            index.projection.toLowerCase() === IndexProjectionOptions.keys_only ||
+            index.projection.toLowerCase() === IndexProjectionOptions.all
+        )) {
+          definition.projection = index.projection.toLowerCase();
+        } else if (Array.isArray(index.projection) && index.projection.length > 0 && index.projection.every((attr) => typeof attr === "string")) {
+          definition.projection = index.projection;
+          let nameIdentifierPresent = false;
+          let versionIdentiferPresent = false;
+          for (const name of definition.projection) {
+            projections.push({ name, accessPattern });
+            if (name === config.identifiers.entity) {
+              nameIdentifierPresent = true;
+            } else if (name === config.identifiers.version) {
+              versionIdentiferPresent = true;
+            }
+          }
+          definition.identifiersAreProjected = nameIdentifierPresent && versionIdentiferPresent;
+        } else {
+          throw new e.ElectroError(
+            e.ErrorCodes.InvalidProjectionDefinition,
+            `The Access Pattern '${accessPattern}' contains an invalid "projection" value: ${u.toDisplayString(index.projection)}. Valid projection values include ${u.commaSeparatedString(Object.values(IndexProjectionOptions))}, or an array of attribute names with a length greater than one.`
+          )
+        }
+      }
+
       indexHasSubCollections[indexName] =
         inCollection && Array.isArray(collection);
 
@@ -4444,6 +4520,7 @@ class Entity {
       };
 
       facets.attributes = [...facets.attributes, ...attributes];
+      facets.projections = [...facets.projections, ...projections];
 
       facets.fields.push(pk.field);
 
@@ -4844,10 +4921,14 @@ class Entity {
       indexHasSortKeys,
       indexAccessPattern,
       indexHasSubCollections,
-    } = this._normalizeIndexes(model.indexes);
+    } = this._normalizeIndexes(model.indexes, config);
     let schema = new Schema(model.attributes, facets, {
-      getClient: () => this.client,
       isRoot: true,
+      getClient: () => this.client,
+      identifiers: {
+        [config.identifiers.entity]: config.identifiers.entity,
+        [config.identifiers.version]: config.identifiers.version,
+      },
     });
 
     let filters = this._normalizeFilters(model.filters);
@@ -4951,6 +5032,7 @@ function matchToEntityAlias({
   record,
   entities = {},
   allowMatchOnKeys = false,
+  config = {},
 } = {}) {
   let entity;
   if (paramItem && v.isFunction(paramItem[TransactionCommitSymbol])) {
@@ -4984,7 +5066,7 @@ function matchToEntityAlias({
       // 		entityAlias = alias;
       // 	}
       // }
-    } else if (entities[alias] && entities[alias].ownsKeys(record)) {
+    } else if (entities[alias] && entities[alias].is(record, config)) {
       entityAlias = alias;
       break;
     }

package/src/errors.js

@@ -133,18 +133,6 @@ const ErrorCodes = {
     name: "IncompatibleKeyCasing",
     sym: ErrorCode,
   },
-  InvalidListenerProvided: {
-    code: 1020,
-    section: "invalid-listener-provided",
-    name: "InvalidListenerProvided",
-    sym: ErrorCode,
-  },
-  InvalidLoggerProvided: {
-    code: 1020,
-    section: "invalid-listener-provided",
-    name: "InvalidListenerProvided",
-    sym: ErrorCode,
-  },
   InvalidClientProvided: {
     code: 1021,
     section: "invalid-client-provided",
@@ -157,6 +145,24 @@ const ErrorCodes = {
     name: "Inconsistent Index Definition",
     sym: ErrorCode,
   },
+  InvalidListenerProvided: {
+    code: 1023,
+    section: "invalid-listener-provided",
+    name: "InvalidListenerProvided",
+    sym: ErrorCode,
+  },
+  InvalidLoggerProvided: {
+    code: 1024,
+    section: "invalid-listener-provided",
+    name: "InvalidListenerProvided",
+    sym: ErrorCode,
+  },
+  InvalidProjectionDefinition: {
+    code: 1025,
+    section: "invalid-projection-definition",
+    name: "InvalidProjectionDefinition",
+    sym: ErrorCode,
+  },
   MissingAttribute: {
     code: 2001,
     section: "missing-attribute",
@@ -292,6 +298,7 @@ class ElectroError extends Error {
     if (code && code.sym === ErrorCode) {
       detail = code;
     }
+    this.cause = cause;
     this._message = message;
     // this.message = `${message} - For more detail on this error reference: ${getHelpLink(detail.section)}`;
     this.message = makeMessage(message, detail.section);

package/src/operations.js

@@ -9,6 +9,37 @@ const { FilterOperations } = require("./filterOperations");
 const e = require("./errors");
 const u = require("./util");
 
+
+/**
+ * formatExpressionName: formats a field name for expression attribute names parameters.
+ * @param {string} name
+ * @param {Map<string, string>} seen
+ */
+function formatExpressionName(name, seen) {
+  const nameWasNotANumber = isNaN(name);
+  const originalName = `${name}`;
+  let formattedName = originalName.replaceAll(/[^\w]/g, "");
+
+  if (formattedName.length === 0) {
+    formattedName = "p";
+  } else if (nameWasNotANumber !== isNaN(formattedName)) {
+    // name became number due to replace
+    formattedName = `p${formattedName}`;
+  }
+
+  const originalFormattedName = formattedName;
+  let nameSuffix = 1;
+  while (
+    seen.has(formattedName) &&
+    seen.get(formattedName) !== originalName
+    ) {
+    formattedName = `${originalFormattedName}_${++nameSuffix}`;
+  }
+
+  seen.set(formattedName, originalName);
+  return formattedName;
+}
+
 class ExpressionState {
   constructor({ prefix } = {}) {
     this.names = {};
@@ -30,29 +61,7 @@ class ExpressionState {
   }
 
   formatName(name = "") {
-    const nameWasNotANumber = isNaN(name);
-    const originalName = `${name}`;
-    let formattedName = originalName.replaceAll(/[^\w]/g, "");
-
-    if (formattedName.length === 0) {
-      formattedName = "p";
-    } else if (nameWasNotANumber !== isNaN(formattedName)) {
-      // name became number due to replace
-      formattedName = `p${formattedName}`;
-    }
-
-    const originalFormattedName = formattedName;
-    let nameSuffix = 1;
-
-    while (
-      this.formattedNameToOriginalNameMap.has(formattedName) &&
-      this.formattedNameToOriginalNameMap.get(formattedName) !== originalName
-    ) {
-      formattedName = `${originalFormattedName}_${++nameSuffix}`;
-    }
-
-    this.formattedNameToOriginalNameMap.set(formattedName, originalName);
-    return formattedName;
+    return formatExpressionName(name, this.formattedNameToOriginalNameMap);
   }
 
   // todo: make the structure: name, value, paths
@@ -403,4 +412,5 @@ module.exports = {
   FilterOperationNames,
   ExpressionState,
   AttributeOperationProxy,
+  formatExpressionName,
 };