electrodb 2.0.0 → 2.1.0

package/README.md

@@ -12,6 +12,7 @@
 ------------
 
 <h1 align="center">ElectroDB has now reached 2.0.0!</h1>
+
 For existing users, checkout the [CHANGELOG](./CHANGELOG.md) and/or the section [Version 2 Migration](#version-2-migration) to learn more about the recent move to 2.0.0 and the changes neccessary to move to the newest version.
 
 ------------
@@ -344,7 +345,7 @@ import { Entity } from "electrodb";
 > When using TypeScript, for strong type checking, be sure to either add your model as an object literal to the Entity constructor or create your model using const assertions with the `as const` syntax.
 
 # Services
-In ***ElectroDB*** a `Service` represents a collection of related Entities. Services allow you to build queries span across Entities. Similar to Entities, Services can coexist on a single table without collision. You can use Entities independent of Services, you do not need to import models into a Service to use them individually. However, you do you need to use a Service if you intend make queries that `join` multiple Entities.
+In ***ElectroDB*** a `Service` represents a collection of related Entities. Services allow you to build queries that span across Entities. Similar to Entities, Services can coexist on a single table without collision. You can use Entities independent of Services, you do not need to import models into a Service to use them individually. However, you do you need to use a Service if you intend make queries that `join` multiple Entities.
 
 Require:
 ```javascript  
@@ -628,7 +629,11 @@ attributes: {
         cast?: "number"|"string"|"boolean";
 		get?: (attribute: <type>, schema: any) => <type> | void | undefined;
 		set?: (attribute?: <type>, schema?: any) => <type> | void | undefined; 
-		watch: "*" | string[]
+		watch?: "*" | string[];
+		padding?: {
+		    length: number;
+		    char: string;
+		}
 	}
 }
 ```
@@ -637,22 +642,23 @@ attributes: {
 
 #### Attribute Definition
 
-Property      | Type                                                       | Required | Types     | Description
- ------------ | :--------------------------------------------------------: | :------: | :-------: | -----------
-`type`        | `string`, `ReadonlyArray<string>`, `string[]`              | yes      | all       | Accepts the values: `"string"`, `"number"` `"boolean"`, `"map"`, `"list"`, `"set"`, an array of strings representing a finite list of acceptable values: `["option1", "option2", "option3"]`, or `"any"` which disables value type checking on that attribute.
-`required`    | `boolean`                                                  | no       | all       | Flag an attribute as required to be present when creating a record. This attribute also acts as a type of `NOT NULL` flag, preventing it from being removed directly. When applied to nested properties, be mindful that default map values can cause required child attributes to fail validation.
-`hidden`      | `boolean`                                                  | no       | all       | Flag an attribute as hidden to remove the property from results before they are returned. 
-`default`     | `value`, `() => value`                                     | no       | all       | Either the default value itself or a synchronous function that returns the desired value. Applied before `set` and before `required` check. In the case of nested attributes, default values will apply defaults to children attributes until an undefined value is reached
-`validate`    | `RegExp`, `(value: any) => void`, `(value: any) => string` | no       | all       | Either regex or a synchronous callback to return an error string (will result in exception using the string as the error's message), or thrown exception in the event of an error.   
-`field`       | `string`                                                   | no       | all       | The name of the attribute as it exists in DynamoDB, if named differently in the schema attributes. Defaults to the `AttributeName` as defined in the schema.
-`readOnly`    | `boolean`                                                  | no       | all       | Prevents an attribute from being updated after the record has been created. Attributes used in the composition of the table's primary Partition Key and Sort Key are read-only by default. The one exception to `readOnly` is for properties that also use the `watch` property, read [attribute watching](#attribute-watching) for more detail. 
-`label`       | `string`                                                   | no       | all       | Used in index composition to prefix key composite attributes. By default, the `AttributeName` is used as the label.
-`set`         | `(attribute, schema) => value`                             | no       | all       | A synchronous callback allowing you to apply changes to a value before it is set in params or applied to the database. First value represents the value passed to ElectroDB, second value are the attributes passed on that update/put
-`get`         | `(attribute, schema) => value`                             | no       | all       | A synchronous callback allowing you to apply changes to a value after it is retrieved from the database. First value represents the value passed to ElectroDB, second value are the attributes retrieved from the database.
-`watch`       | `Attribute[], "*"`                                         | no       | root-only | Define other attributes that will always trigger your attribute's getter and setter callback after their getter/setter callbacks are executed. Only available on root level attributes.
-`properties`  | `{[key: string]: Attribute}`                               | yes*     | map       | Define the properties available on a `"map"` attribute, required if your attribute is a map. Syntax for map properties is the same as root level attributes.
-`items`       | `Attribute`                                                | yes*     | list      | Define the attribute type your list attribute will contain, required if your attribute is a list. Syntax for list items is the same as a single attribute.
-`items`       | "string" | "number"                                        | yes*     | set       | Define the primitive type your set attribute will contain, required if your attribute is a set. Unlike lists, a set defines it's items with a string of either "string" or "number".
+Property      | Type                                                       | Required | Types          | Description
+ ------------ | :--------------------------------------------------------: | :------: | :------------: | -----------
+`type`        | `string`, `ReadonlyArray<string>`, `string[]`              | yes      | all            | Accepts the values: `"string"`, `"number"` `"boolean"`, `"map"`, `"list"`, `"set"`, an array of strings representing a finite list of acceptable values: `["option1", "option2", "option3"]`, or `"any"` which disables value type checking on that attribute.
+`required`    | `boolean`                                                  | no       | all            | Flag an attribute as required to be present when creating a record. This attribute also acts as a type of `NOT NULL` flag, preventing it from being removed directly. When applied to nested properties, be mindful that default map values can cause required child attributes to fail validation.
+`hidden`      | `boolean`                                                  | no       | all            | Flag an attribute as hidden to remove the property from results before they are returned.
+`default`     | `value`, `() => value`                                     | no       | all            | Either the default value itself or a synchronous function that returns the desired value. Applied before `set` and before `required` check. In the case of nested attributes, default values will apply defaults to children attributes until an undefined value is reached
+`validate`    | `RegExp`, `(value: any) => void`, `(value: any) => string` | no       | all            | Either regex or a synchronous callback to return an error string (will result in exception using the string as the error's message), or thrown exception in the event of an error.
+`field`       | `string`                                                   | no       | all            | The name of the attribute as it exists in DynamoDB, if named differently in the schema attributes. Defaults to the `AttributeName` as defined in the schema.
+`readOnly`    | `boolean`                                                  | no       | all            | Prevents an attribute from being updated after the record has been created. Attributes used in the composition of the table's primary Partition Key and Sort Key are read-only by default. The one exception to `readOnly` is for properties that also use the `watch` property, read [attribute watching](#attribute-watching) for more detail.
+`label`       | `string`                                                   | no       | all            | Used in index key composition to prefix key composite attributes. By default, the `AttributeName` is used as the label.
+`padding`     | `{ length: number; char: string; }`                        | no       | string, number | Similar to `label`, this property only impacts the attribute's value during index key composition. Padding allows you to define a string pattern to left pad your attribute when ElectroDB builds your partition or sort key. This can be helpful to implementing zero-padding patterns with numbers and strings in sort keys. Note, this will _not_ impact your attribute's stored value, if you want to transform the attribute's field value, use the `set` callback described below.
+`set`         | `(attribute, schema) => value`                             | no       | all            | A synchronous callback allowing you to apply changes to a value before it is set in params or applied to the database. First value represents the value passed to ElectroDB, second value are the attributes passed on that update/put
+`get`         | `(attribute, schema) => value`                             | no       | all            | A synchronous callback allowing you to apply changes to a value after it is retrieved from the database. First value represents the value passed to ElectroDB, second value are the attributes retrieved from the database.
+`watch`       | `Attribute[], "*"`                                         | no       | root-only      | Define other attributes that will always trigger your attribute's getter and setter callback after their getter/setter callbacks are executed. Only available on root level attributes.
+`properties`  | `{[key: string]: Attribute}`                               | yes*     | map            | Define the properties available on a `"map"` attribute, required if your attribute is a map. Syntax for map properties is the same as root level attributes.
+`items`       | `Attribute`                                                | yes*     | list           | Define the attribute type your list attribute will contain, required if your attribute is a list. Syntax for list items is the same as a single attribute.
+`items`       | "string" | "number"                                        | yes*     | set            | Define the primitive type your set attribute will contain, required if your attribute is a set. Unlike lists, a set defines it's items with a string of either "string" or "number".
 
 #### Enum Attributes
 
@@ -2643,8 +2649,7 @@ Equivalent DocClient Parameters:
 ### Batch Get
 Provide all Table Index composite attributes in an array of objects to the `get` method to perform a BatchGet query.
 
-> _NOTE: Performing a BatchGet will return a response structure unique to BatchGet: a two-dimensional array with the results of the query and any unprocessed records. See the example below._
-> Additionally, when performing a BatchGet the `.params()` method will return an _array_ of parameters, rather than just the parameters for one docClient query. This is because ElectroDB BatchGet queries larger than the docClient's limit of 100 records.
+> _NOTE: When performing a BatchGet the `.params()` method will return an _array_ of parameters, rather than just the parameters for one docClient query. This is because ElectroDB BatchGet allows queries larger than the docClient's limit of 100 records.
 
 If the number of records you are requesting is above the BatchGet threshold of 100 records, ElectroDB will make multiple requests to DynamoDB and return the results in a single array. By default, ElectroDB will make these requests in series, one after another. If you are confident your table can handle the throughput, you can use the [Query Option](#query-options) `concurrent`. This value can be set to any number greater than zero, and will execute that number of requests simultaneously.
 
@@ -4258,6 +4263,7 @@ By default, **ElectroDB** enables you to work with records as the names and prop
   listeners Array<(event) => void>;
   preserveBatchOrder?: boolean;
   attributes?: string[];
+  order?: 'asc' | 'desc';
 };
 ```
 
@@ -4275,7 +4281,7 @@ response           | `"default"`          | Used as a convenience for applying t
 ignoreOwnership    | `false`              | By default, **ElectroDB** interrogates items returned from a query for the presence of matching entity "identifiers". This helps to ensure other entities, or other versions of an entity, are filtered from your results. If you are using ElectroDB with an existing table/dataset you can turn off this feature by setting this property to `true`.
 limit              | _none_               | A target for the number of items to return from DynamoDB. If this option is passed, Queries on entities and through collections will paginate DynamoDB until this limit is reached or all items for that query have been returned.
 pages              | 1                    | How many DynamoDB pages should a query iterate through before stopping. To have ElectroDB automatically paginate through all results, pass the string value `'all'`.
-sort               | 'asc'                | Convenience option for `ScanIndexForward`, to the change the order of queries based on your index's Sort Key -- valid options include 'asc' and 'desc'. [[read more](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_Query.html)]
+order              | 'asc'                | Convenience option for `ScanIndexForward`, to the change the order of queries based on your index's Sort Key -- valid options include 'asc' and 'desc'. [[read more](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_Query.html)]
 listeners          | `[]`                 | An array of callbacks that are invoked when [internal ElectroDB events](#events) occur.
 logger             | _none_               | A convenience option for a single event listener that semantically can be used for logging.
 preserveBatchOrder | `false`              | When used with a [batchGet](#batch-get) operation, ElectroDB will ensure the order returned by a batchGet will be the same as the order provided. When enabled, if a record is returned from DynamoDB as "unprocessed" ([read more here](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_BatchGetItem.html)), ElectroDB will return a null value at that index.

package/index.d.ts

@@ -781,6 +781,10 @@ export interface BooleanAttribute {
     readonly field?: string;
     readonly label?: string;
     readonly watch?: ReadonlyArray<string> | "*";
+    readonly padding?: {
+        length: number;
+        char: string;
+    }
 }
 
 export interface NestedNumberAttribute {
@@ -807,6 +811,10 @@ export interface NumberAttribute {
     readonly field?: string;
     readonly label?: string;
     readonly watch?: ReadonlyArray<string> | "*";
+    readonly padding?: {
+        length: number;
+        char: string;
+    }
 }
 
 export interface NestedStringAttribute {
@@ -833,6 +841,10 @@ export interface StringAttribute {
     readonly field?: string;
     readonly label?: string;
     readonly watch?: ReadonlyArray<string> | "*";
+    readonly padding?: {
+        length: number;
+        char: string;
+    }
 }
 
 export interface NestedEnumAttribute {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "electrodb",
-  "version": "2.0.0",
+  "version": "2.1.0",
   "description": "A library to more easily create and interact with multiple entities and heretical relationships in dynamodb",
   "main": "index.js",
   "scripts": {

package/src/entity.js

@@ -2178,12 +2178,17 @@ class Entity {
 		}
 		let key = prefix;
 		for (let i = 0; i < labels.length; i++) {
-			let {name, label} = labels[i];
-
+			const { name, label } = labels[i];
+			const attribute = this.model.schema.getAttribute(name);
+			let value = supplied[name];
 			if (supplied[name] === undefined && excludeLabelTail) {
 				break;
 			}
 
+			if (attribute && validations.isFunction(attribute.format)) {
+				value = attribute.format(`${value}`);
+			}
+
 			if (isCustom) {
 				key = `${key}${label}`;
 			} else {
@@ -2194,7 +2199,7 @@ class Entity {
 				break;
 			}
 
-			key = `${key}${supplied[name]}`;
+			key = `${key}${value}`;
 		}
 
 		return u.formatKeyCasing(key, casing);

package/src/operations.js

@@ -388,7 +388,7 @@ class AttributeOperationProxy {
                             const attributeValues = [];
                             let hasNestedValue = false;
                             for (let value of values) {
-                                value = target.format(value);
+                                value = target.applyFixings(value);
                                 // template.length is to see if function takes value argument
                                 if (template.length > 3) {
                                     if (seen.has(value)) {

package/src/schema.js

@@ -1,4 +1,4 @@
-const { CastTypes, ValueTypes, KeyCasing, AttributeTypes, AttributeMutationMethods, AttributeWildCard, PathTypes } = require("./types");
+const { CastTypes, ValueTypes, KeyCasing, AttributeTypes, AttributeMutationMethods, AttributeWildCard, PathTypes, TableIndex } = require("./types");
 const AttributeTypeNames = Object.keys(AttributeTypes);
 const ValidFacetTypes = [AttributeTypes.string, AttributeTypes.number, AttributeTypes.boolean, AttributeTypes.enum];
 const e = require("./errors");
@@ -100,6 +100,9 @@ class Attribute {
 		this.isKeyField = !!definition.isKeyField;
 		this.unformat = this._makeDestructureKey(definition);
 		this.format = this._makeStructureKey(definition);
+		this.padding = definition.padding;
+		this.applyFixings = this._makeApplyFixings(definition);
+		this.applyPadding = this._makePadding(definition);
 		this.indexes = [...(definition.indexes || [])];
 		let {isWatched, isWatcher, watchedBy, watching, watchAll} = Attribute._destructureWatcher(definition);
 		this._isWatched = isWatched
@@ -233,29 +236,72 @@ class Attribute {
 		return set || ((attr) => attr);
 	}
 
-	_makeStructureKey({prefix = "", postfix = "", casing= KeyCasing.none} = {}) {
-		return (key) => {
-			let value = key;
-			if (this.type === AttributeTypes.string && v.isStringHasLength(key)) {
-				value = `${prefix}${key}${postfix}`;
+	_makeApplyFixings({ prefix = "", postfix = "", casing= KeyCasing.none } = {}) {
+		return (value) => {
+			if ([AttributeTypes.string, AttributeTypes.enum].includes(this.type)) {
+				value = `${prefix}${value}${postfix}`;
 			}
+
 			return u.formatAttributeCasing(value, casing);
 		}
 	}
 
-	_makeDestructureKey({prefix = "", postfix = "", casing= KeyCasing.none} = {}) {
+	_makeStructureKey() {
+		return (key) => {
+			return this.applyPadding(key);
+		}
+	}
+
+	_isPaddingEligible(padding = {} ) {
+		return !!padding && padding.length && v.isStringHasLength(padding.char);
+	}
+
+	_makePadding({ padding = {} }) {
+		return (value) => {
+			if (typeof value !== 'string') {
+				return value;
+			} else if (this._isPaddingEligible(padding)) {
+				return u.addPadding({padding, value});
+			} else {
+				return value;
+			}
+		}
+	}
+
+	_makeRemoveFixings({prefix = "", postfix = "", casing= KeyCasing.none} = {}) {
 		return (key) => {
 			let value = "";
 			if (![AttributeTypes.string, AttributeTypes.enum].includes(this.type) || typeof key !== "string") {
-				return key;
-			} else if (key.length > prefix.length) {
+				value = key;
+			} else if (prefix.length > 0 && key.length > prefix.length) {
 				for (let i = prefix.length; i < key.length - postfix.length; i++) {
 					value += key[i];
 				}
 			} else {
 				value = key;
 			}
-			return u.formatAttributeCasing(value, casing);
+
+			return value;
+		}
+	}
+
+	_makeDestructureKey({prefix = "", postfix = "", casing= KeyCasing.none, padding = {}} = {}) {
+		return (key) => {
+			let value = "";
+			if (![AttributeTypes.string, AttributeTypes.enum].includes(this.type) || typeof key !== "string") {
+				return key;
+			} else if (key.length > prefix.length) {
+				value = u.removeFixings({prefix, postfix, value: key});
+			} else {
+				value = key;
+			}
+
+			// todo: if an attribute is also used as a pk or sk directly in one index, but a composite in another, then padding is going to be broken
+			// if (padding && padding.length) {
+			// 	value = u.removePadding({padding, value});
+			// }
+
+			return value;
 		};
 	}
 
@@ -958,7 +1004,7 @@ class Schema {
 					let definition = facets.byField[field][indexName];
 					if (definition.facets.length > 1) {
 						throw new e.ElectroError(
-							e.ErrorCodes.InvalidIndexCompositeWithAttributeName,
+							e.ErrorCodes.InvalidIndexWithAttributeName,
 							`Invalid definition for "${definition.type}" field on index "${u.formatIndexNameForDisplay(indexName)}". The ${definition.type} field "${definition.field}" shares a field name with an attribute defined on the Entity, and therefore is not allowed to contain composite references to other attributes. Please either change the field name of the attribute, or redefine the index to use only the single attribute "${definition.field}".`
 						)
 					}
@@ -1005,10 +1051,19 @@ class Schema {
 							`Invalid use of a collection on index "${u.formatIndexNameForDisplay(indexName)}". The ${definition.type} field "${definition.field}" shares a field name with an attribute defined on the Entity, and therefore the index is not allowed to participate in a Collection. Please either change the field name of the attribute, or remove all collection(s) from the index.`
 						)
 					}
+
+					if (definition.field === field) {
+						if (attribute.padding !== undefined) {
+							throw new e.ElectroError(
+								e.ErrorCodes.InvalidAttributeDefinition,
+								`Invalid padding definition for the attribute "${name}". Padding is not currently supported for attributes that are also defined as table indexes.`
+							);
+						}
+					}
 				}
 			}
 
-			let isKey = !!facets.byIndex && facets.byIndex[""].all.find((facet) => facet.name === name);
+			let isKey = !!facets.byIndex && facets.byIndex[TableIndex].all.find((facet) => facet.name === name);
 			let definition = {
 				name,
 				field,
@@ -1033,6 +1088,7 @@ class Schema {
 				properties: attribute.properties,
 				parentPath: attribute.parentPath,
 				parentType: attribute.parentType,
+				padding: attribute.padding,
 			};
 
 			if (definition.type === AttributeTypes.custom) {

package/src/service.js

@@ -427,7 +427,7 @@ class Service {
 		return [!!collectionDifferences.length, collectionDifferences];
 	}
 
-	_compareEntityAttributes(definition = {}, providedAttributes = {}) {
+	_compareEntityAttributes(entityName, definition = {}, providedAttributes = {}, keys) {
 		let results = {
 			additions: {},
 			invalid: [],
@@ -438,17 +438,29 @@ class Service {
 				results.additions[name] = detail;
 			} else if (defined.field !== detail.field) {
 				results.invalid.push(
-					`Attribute provided "${name}" with Table Field "${detail.field}" does not match established Table Field "${defined.field}"`,
+					`The attribute "${name}" with Table Field "${detail.field}" does not match established Table Field "${defined.field}"`,
 				);
 			}
+			if (defined && detail && (defined.padding || detail.padding)) {
+				const definedPadding = defined.padding || {};
+				const detailPadding = detail.padding || {};
+				if (keys.pk.facets.includes(name) &&
+					(definedPadding.length !== detailPadding.length ||
+					definedPadding.char !== detailPadding.char)
+				) {
+					results.invalid.push(
+						`The attribute "${name}" contains inconsistent padding definitions that impact how keys are formed`,
+					);
+				}
+			}
 		}
 		return [!!results.invalid.length, results];
 	}
 
-	_processEntityAttributes(definition = {}, providedAttributes = {}) {
-		let [attributesAreIncompatible, attributeResults] = this._compareEntityAttributes(definition, providedAttributes);
+	_processEntityAttributes(entityName, definition = {}, providedAttributes = {}, keys) {
+		let [attributesAreIncompatible, attributeResults] = this._compareEntityAttributes(entityName, definition, providedAttributes, keys);
 		if (attributesAreIncompatible) {
-			throw new e.ElectroError(e.ErrorCodes.InvalidJoin, `Invalid entity attributes. The following attributes have already been defined on this model but with incompatible or conflicting properties: ${attributeResults.invalid.join(", ")}`);
+			throw new e.ElectroError(e.ErrorCodes.InvalidJoin, `Inconsistent attribute(s) on the entity "${entityName}". The following attribute(s) are defined with incompatible or conflicting definitions across participating entities: ${attributeResults.invalid.join(", ")}. These attribute definitions must match among all members of the collection.`);
 		} else {
 			return {
 				...definition,
@@ -566,7 +578,7 @@ class Service {
 			this.collectionSchema[collection].table = entity._getTableName();
 		}
 		this.collectionSchema[collection].keys = this._processEntityKeys(name, this.collectionSchema[collection].keys, providedIndex);
-		this.collectionSchema[collection].attributes = this._processEntityAttributes(this.collectionSchema[collection].attributes, entity.model.schema.attributes);
+		this.collectionSchema[collection].attributes = this._processEntityAttributes(name, this.collectionSchema[collection].attributes, entity.model.schema.attributes, this.collectionSchema[collection].keys);
 		this.collectionSchema[collection].entities[name] = entity;
 		this.collectionSchema[collection].identifiers = this._processEntityIdentifiers(this.collectionSchema[collection].identifiers, entity.getIdentifierExpressions(name));
 		this.collectionSchema[collection].index = this._processEntityCollectionIndex(this.collectionSchema[collection].index, providedIndex.index, name, collection);

package/src/util.js

@@ -184,9 +184,48 @@ const cursorFormatter = {
   }
 }
 
+function removeFixings({prefix = '', postfix = '', value = ''} = {}) {
+  const start = value.toLowerCase().startsWith(prefix.toLowerCase()) ? prefix.length : 0;
+  const end = value.length - (value.toLowerCase().endsWith(postfix.toLowerCase()) ? postfix.length : 0);
+
+  let formatted = '';
+  for (let i = start; i < end; i++) {
+    formatted += value[i];
+  }
+
+  return formatted;
+}
+
+function addPadding({padding = {}, value = ''} = {}) {
+  return value.padStart(padding.length, padding.char);
+}
+
+function removePadding({padding = {}, value = ''} = {}) {
+  if (!padding.length || value.length >= padding.length) {
+    return value;
+  }
+
+  let formatted = '';
+  let useRemaining = false;
+  for (let i = 0; i < value.length; i++) {
+    const char = value[i];
+    if (useRemaining || i >= padding.length) {
+      formatted += char;
+    } else if (char !== padding.char) {
+      formatted += char;
+      useRemaining = true;
+    }
+  }
+
+  return formatted;
+}
+
 module.exports = {
   getUnique,
   batchItems,
+  addPadding,
+  removePadding,
+  removeFixings,
   parseJSONPath,
   getInstanceType,
   getModelVersion,

package/src/validations.js

@@ -64,6 +64,18 @@ const Attribute = {
 			type: "any",
 			format: "isFunction",
 		},
+		padding: {
+			type: "object",
+			required: ['length', 'char'],
+			properties: {
+				length: {
+					type: 'number'
+				},
+				char: {
+					type: 'string',
+				}
+			}
+		}
 	},
 };