electrodb 1.12.1 → 2.1.0

package/src/schema.js

@@ -1,4 +1,4 @@
-const { CastTypes, ValueTypes, KeyCasing, AttributeTypes, AttributeMutationMethods, AttributeWildCard, PathTypes } = require("./types");
+const { CastTypes, ValueTypes, KeyCasing, AttributeTypes, AttributeMutationMethods, AttributeWildCard, PathTypes, TableIndex } = require("./types");
 const AttributeTypeNames = Object.keys(AttributeTypes);
 const ValidFacetTypes = [AttributeTypes.string, AttributeTypes.number, AttributeTypes.boolean, AttributeTypes.enum];
 const e = require("./errors");
@@ -100,6 +100,9 @@ class Attribute {
 		this.isKeyField = !!definition.isKeyField;
 		this.unformat = this._makeDestructureKey(definition);
 		this.format = this._makeStructureKey(definition);
+		this.padding = definition.padding;
+		this.applyFixings = this._makeApplyFixings(definition);
+		this.applyPadding = this._makePadding(definition);
 		this.indexes = [...(definition.indexes || [])];
 		let {isWatched, isWatcher, watchedBy, watching, watchAll} = Attribute._destructureWatcher(definition);
 		this._isWatched = isWatched
@@ -233,29 +236,72 @@ class Attribute {
 		return set || ((attr) => attr);
 	}
 
-	_makeStructureKey({prefix = "", postfix = "", casing= KeyCasing.none} = {}) {
-		return (key) => {
-			let value = key;
-			if (this.type === AttributeTypes.string && v.isStringHasLength(key)) {
-				value = `${prefix}${key}${postfix}`;
+	_makeApplyFixings({ prefix = "", postfix = "", casing= KeyCasing.none } = {}) {
+		return (value) => {
+			if ([AttributeTypes.string, AttributeTypes.enum].includes(this.type)) {
+				value = `${prefix}${value}${postfix}`;
 			}
+
 			return u.formatAttributeCasing(value, casing);
 		}
 	}
 
-	_makeDestructureKey({prefix = "", postfix = "", casing= KeyCasing.none} = {}) {
+	_makeStructureKey() {
+		return (key) => {
+			return this.applyPadding(key);
+		}
+	}
+
+	_isPaddingEligible(padding = {} ) {
+		return !!padding && padding.length && v.isStringHasLength(padding.char);
+	}
+
+	_makePadding({ padding = {} }) {
+		return (value) => {
+			if (typeof value !== 'string') {
+				return value;
+			} else if (this._isPaddingEligible(padding)) {
+				return u.addPadding({padding, value});
+			} else {
+				return value;
+			}
+		}
+	}
+
+	_makeRemoveFixings({prefix = "", postfix = "", casing= KeyCasing.none} = {}) {
 		return (key) => {
 			let value = "";
 			if (![AttributeTypes.string, AttributeTypes.enum].includes(this.type) || typeof key !== "string") {
-				return key;
-			} else if (key.length > prefix.length) {
+				value = key;
+			} else if (prefix.length > 0 && key.length > prefix.length) {
 				for (let i = prefix.length; i < key.length - postfix.length; i++) {
 					value += key[i];
 				}
 			} else {
 				value = key;
 			}
-			return u.formatAttributeCasing(value, casing);
+
+			return value;
+		}
+	}
+
+	_makeDestructureKey({prefix = "", postfix = "", casing= KeyCasing.none, padding = {}} = {}) {
+		return (key) => {
+			let value = "";
+			if (![AttributeTypes.string, AttributeTypes.enum].includes(this.type) || typeof key !== "string") {
+				return key;
+			} else if (key.length > prefix.length) {
+				value = u.removeFixings({prefix, postfix, value: key});
+			} else {
+				value = key;
+			}
+
+			// todo: if an attribute is also used as a pk or sk directly in one index, but a composite in another, then padding is going to be broken
+			// if (padding && padding.length) {
+			// 	value = u.removePadding({padding, value});
+			// }
+
+			return value;
 		};
 	}
 
@@ -958,7 +1004,7 @@ class Schema {
 					let definition = facets.byField[field][indexName];
 					if (definition.facets.length > 1) {
 						throw new e.ElectroError(
-							e.ErrorCodes.InvalidIndexCompositeWithAttributeName,
+							e.ErrorCodes.InvalidIndexWithAttributeName,
 							`Invalid definition for "${definition.type}" field on index "${u.formatIndexNameForDisplay(indexName)}". The ${definition.type} field "${definition.field}" shares a field name with an attribute defined on the Entity, and therefore is not allowed to contain composite references to other attributes. Please either change the field name of the attribute, or redefine the index to use only the single attribute "${definition.field}".`
 						)
 					}
@@ -1005,10 +1051,19 @@ class Schema {
 							`Invalid use of a collection on index "${u.formatIndexNameForDisplay(indexName)}". The ${definition.type} field "${definition.field}" shares a field name with an attribute defined on the Entity, and therefore the index is not allowed to participate in a Collection. Please either change the field name of the attribute, or remove all collection(s) from the index.`
 						)
 					}
+
+					if (definition.field === field) {
+						if (attribute.padding !== undefined) {
+							throw new e.ElectroError(
+								e.ErrorCodes.InvalidAttributeDefinition,
+								`Invalid padding definition for the attribute "${name}". Padding is not currently supported for attributes that are also defined as table indexes.`
+							);
+						}
+					}
 				}
 			}
 
-			let isKey = !!facets.byIndex && facets.byIndex[""].all.find((facet) => facet.name === name);
+			let isKey = !!facets.byIndex && facets.byIndex[TableIndex].all.find((facet) => facet.name === name);
 			let definition = {
 				name,
 				field,
@@ -1033,6 +1088,7 @@ class Schema {
 				properties: attribute.properties,
 				parentPath: attribute.parentPath,
 				parentType: attribute.parentType,
+				padding: attribute.padding,
 			};
 
 			if (definition.type === AttributeTypes.custom) {

package/src/service.js

@@ -243,13 +243,8 @@ class Service {
 
 	cleanseRetrievedData(collection = "", entities, data = {}, config = {}) {
 		if (config.raw) {
-			if (config._isPagination) {
-				return [data.LastEvaluatedKey, data];
-			} else {
-				return data;
-			}
+			return data;
 		}
-
 		data.Items = data.Items || [];
 		let index = this.collectionSchema[collection].index;
 		let results = {};
@@ -268,18 +263,15 @@ class Service {
 			if (!entityAlias) {
 				continue;
 			}
-
 			// pager=false because we don't want the entity trying to parse the lastEvaluatedKey
-			let items = this.collectionSchema[collection].entities[entityAlias].formatResponse({Item: record}, index, {...config, pager: false});
-			results[entityAlias].push(items);
-		}
-
-		if (config._isPagination) {
-			let page = this._formatReturnPager(config, index, data.LastEvaluatedKey, data.Items[data.Items.length - 1]);
-			return [page, results];
-		} else {
-			return results;
+			let items = this.collectionSchema[collection].entities[entityAlias].formatResponse({Item: record}, index, {
+				...config,
+				pager: false,
+				parse: undefined
+			});
+			results[entityAlias].push(items.data);
 		}
+		return results;
 	}
 
 	findKeyOwner(lastEvaluatedKey) {
@@ -287,92 +279,25 @@ class Service {
 			.find((entity) => entity.ownsLastEvaluatedKey(lastEvaluatedKey));
 	}
 
-	findItemPagerOwner(collection, pager = {}) {
-		if (this.collectionSchema[collection] === undefined) {
-			throw new Error("Invalid collection");
-		}
-		let matchingEntities = [];
-		if (pager === null) {
-			return matchingEntities;
-		}
-		for (let entity of Object.values(this.collectionSchema[collection].entities)) {
-			if (entity.ownsPager(pager, this.collectionSchema[collection].index)) {
-				matchingEntities.push(entity);
-			}
-		}
-		return matchingEntities;
-	}
-
-	findNamedPagerOwner(pager = {}) {
-		let identifiers = this._getEntityIdentifiers(this.entities);
-		for (let identifier of identifiers) {
-			let hasCorrectFieldProperties = typeof pager[identifier.nameField] === "string" && typeof pager[identifier.versionField] === "string";
-			let hasMatchingFieldValues = pager[identifier.nameField] === identifier.name && pager[identifier.versionField] === identifier.version;
-			if (hasCorrectFieldProperties && hasMatchingFieldValues) {
-				return identifier.entity;
-			}
+	expectKeyOwner(lastEvaluatedKey) {
+		const owner = this.findKeyOwner(lastEvaluatedKey);
+		if (owner === undefined) {
+			throw new e.ElectroError(e.ErrorCodes.NoOwnerForCursor, `Supplied cursor does not resolve to Entity within the Service ${this.service.name}`);
 		}
+		return owner;
 	}
 
-	expectPagerOwner(type, collection, pager) {
-		if (type === Pager.raw) {
-			return Object.values(this.collectionSchema[collection].entities)[0];
-		} else if (type === Pager.named || type === undefined) {
-			let owner = this.findNamedPagerOwner(pager);
-			if (owner === undefined) {
-				throw new e.ElectroError(e.ErrorCodes.NoOwnerForPager, "Supplied Pager does not resolve to Entity within Service");
-			}
-			return owner;
-		} else if (type === Pager.item) {
-			let owners = this.findItemPagerOwner(collection, pager);
-			if (owners.length === 1) {
-				return owners[0];
-			} else {
-				throw new e.ElectroError(e.ErrorCodes.PagerNotUnique, "Supplied Pager did not resolve to single Entity");
-			}
-		} else {
-			throw new e.ElectroError(e.ErrorCodes.InvalidOptions, `Invalid value for option "pager" provider: "${pager}". Allowed values include ${u.commaSeparatedString(Object.keys(Pager))}.`)
-		}
-	}
-
-	findPagerIdentifiers(collection, pager = {}) {
-		if (this.collectionSchema[collection] === undefined) {
-			throw new Error("Invalid collection");
-		}
-		let matchingIdentifiers = [];
-		if (pager === null) {
-			return matchingIdentifiers;
-		}
-		for (let entity of Object.values(this.collectionSchema[collection].entities)) {
-			if (entity.ownsPager(pager, this.collectionSchema[collection].index)) {
-				matchingIdentifiers.push({
-					[entity.identifiers.entity]: entity.getName(),
-					[entity.identifiers.version]: entity.getVersion(),
-				});
-			}
-		}
-		return matchingIdentifiers;
+	findCursorOwner(cursor) {
+		return Object.values(this.entities)
+			.find(entity => entity.ownsCursor(cursor));
 	}
 
-	_formatReturnPager(config, index, lastEvaluatedKey, lastReturned) {
-		if (config.pager === "raw") {
-			return lastEvaluatedKey;
-		} else if (!lastEvaluatedKey) {
-			return null;
-		} else {
-			let entity = this.findKeyOwner(lastEvaluatedKey);
-			if (!entity) {
-				return lastReturned !== undefined
-					? this._formatReturnPager(config, index, lastReturned)
-					: null
-			}
-			let page = entity._formatKeysToItem(index, lastEvaluatedKey);
-			if (config.pager === "named") {
-				page[entity.identifiers.entity] = entity.getName();
-				page[entity.identifiers.version] = entity.getVersion();
-			}
-			return page;
+	expectCursorOwner(cursor) {
+		const owner = this.findCursorOwner(cursor);
+		if (owner === undefined) {
+			throw new e.ElectroError(e.ErrorCodes.NoOwnerForCursor, `Supplied cursor does not resolve to Entity within the Service ${this.service.name}`);
 		}
+		return owner;
 	}
 
 	_getTableName() {
@@ -389,21 +314,7 @@ class Service {
 	_makeCollectionChain(name = "", attributes = {}, initialClauses = {}, expressions = {}, entities = {}, entity = {}, facets = {}) {
 		let filterBuilder = new FilterFactory(attributes, FilterOperations);
 		let whereBuilder = new WhereFactory(attributes, FilterOperations);
-
-		let pageClause = {...initialClauses.page};
-		let pageAction = initialClauses.page.action;
-		pageClause.action = (entity, state, page = null, options = {}) => {
-			try {
-				if (page === null) {
-					return pageAction(entity, state, page, options);
-				}
-				let owner = this.expectPagerOwner(options.pager, name, page);
-				return pageAction(owner, state, page, options);
-			} catch(err) {
-				return Promise.reject(err);
-			}
-		}
-		let clauses = {...initialClauses, page: pageClause};
+		let clauses = {...initialClauses};
 
 		clauses = filterBuilder.injectFilterClauses(clauses);
 		clauses = whereBuilder.injectWhereClauses(clauses);
@@ -412,6 +323,14 @@ class Service {
 			// expressions, // DynamoDB doesnt return what I expect it would when provided with these entity filters
 			parse: (options, data) => {
 				return this.cleanseRetrievedData(name, entities, data, options);
+			},
+			formatCursor: {
+				serialize: (key) => {
+					return this.expectKeyOwner(key).serializeCursor(key);
+				},
+				deserialize: (cursor) => {
+					return this.expectCursorOwner(cursor).deserilizeCursor(cursor);
+				}
 			}
 		};
 
@@ -508,7 +427,7 @@ class Service {
 		return [!!collectionDifferences.length, collectionDifferences];
 	}
 
-	_compareEntityAttributes(definition = {}, providedAttributes = {}) {
+	_compareEntityAttributes(entityName, definition = {}, providedAttributes = {}, keys) {
 		let results = {
 			additions: {},
 			invalid: [],
@@ -519,17 +438,29 @@ class Service {
 				results.additions[name] = detail;
 			} else if (defined.field !== detail.field) {
 				results.invalid.push(
-					`Attribute provided "${name}" with Table Field "${detail.field}" does not match established Table Field "${defined.field}"`,
+					`The attribute "${name}" with Table Field "${detail.field}" does not match established Table Field "${defined.field}"`,
 				);
 			}
+			if (defined && detail && (defined.padding || detail.padding)) {
+				const definedPadding = defined.padding || {};
+				const detailPadding = detail.padding || {};
+				if (keys.pk.facets.includes(name) &&
+					(definedPadding.length !== detailPadding.length ||
+					definedPadding.char !== detailPadding.char)
+				) {
+					results.invalid.push(
+						`The attribute "${name}" contains inconsistent padding definitions that impact how keys are formed`,
+					);
+				}
+			}
 		}
 		return [!!results.invalid.length, results];
 	}
 
-	_processEntityAttributes(definition = {}, providedAttributes = {}) {
-		let [attributesAreIncompatible, attributeResults] = this._compareEntityAttributes(definition, providedAttributes);
+	_processEntityAttributes(entityName, definition = {}, providedAttributes = {}, keys) {
+		let [attributesAreIncompatible, attributeResults] = this._compareEntityAttributes(entityName, definition, providedAttributes, keys);
 		if (attributesAreIncompatible) {
-			throw new e.ElectroError(e.ErrorCodes.InvalidJoin, `Invalid entity attributes. The following attributes have already been defined on this model but with incompatible or conflicting properties: ${attributeResults.invalid.join(", ")}`);
+			throw new e.ElectroError(e.ErrorCodes.InvalidJoin, `Inconsistent attribute(s) on the entity "${entityName}". The following attribute(s) are defined with incompatible or conflicting definitions across participating entities: ${attributeResults.invalid.join(", ")}. These attribute definitions must match among all members of the collection.`);
 		} else {
 			return {
 				...definition,
@@ -647,7 +578,7 @@ class Service {
 			this.collectionSchema[collection].table = entity._getTableName();
 		}
 		this.collectionSchema[collection].keys = this._processEntityKeys(name, this.collectionSchema[collection].keys, providedIndex);
-		this.collectionSchema[collection].attributes = this._processEntityAttributes(this.collectionSchema[collection].attributes, entity.model.schema.attributes);
+		this.collectionSchema[collection].attributes = this._processEntityAttributes(name, this.collectionSchema[collection].attributes, entity.model.schema.attributes, this.collectionSchema[collection].keys);
 		this.collectionSchema[collection].entities[name] = entity;
 		this.collectionSchema[collection].identifiers = this._processEntityIdentifiers(this.collectionSchema[collection].identifiers, entity.getIdentifierExpressions(name));
 		this.collectionSchema[collection].index = this._processEntityCollectionIndex(this.collectionSchema[collection].index, providedIndex.index, name, collection);

package/src/types.js

@@ -110,7 +110,8 @@ const AttributeMutationMethods = {
 const Pager = {
 	raw: "raw",
 	named: "named",
-	item: "item"
+	item: "item",
+	cursor: "cursor"
 }
 
 const UnprocessedTypes = {
@@ -194,6 +195,15 @@ const TerminalOperation = {
 	page: 'page',
 }
 
+const AllPages = 'all';
+
+const ResultOrderOption = {
+	'asc': true,
+	'desc': false
+};
+
+const ResultOrderParam = 'ScanIndexForward';
+
 module.exports = {
 	Pager,
 	KeyTypes,
@@ -223,5 +233,8 @@ module.exports = {
 	AttributeProxySymbol,
 	ElectroInstanceTypes,
 	EventSubscriptionTypes,
-	AttributeMutationMethods
+	AttributeMutationMethods,
+	AllPages,
+	ResultOrderOption,
+	ResultOrderParam,
 };

package/src/util.js

@@ -161,9 +161,71 @@ function getUnique(arr1, arr2) {
   ]));
 }
 
+const cursorFormatter = {
+  serialize: (key) => {
+    if (!key) {
+      return null;
+    } else if (typeof val !== 'string') {
+      key = JSON.stringify(key);
+    }
+    return Buffer.from(key).toString('base64url');
+  },
+  deserialize: (cursor) => {
+    if (!cursor) {
+      return undefined;
+    } else if (typeof cursor !== 'string') {
+      throw new Error(`Invalid cursor provided, expected type 'string' recieved: ${JSON.stringify(cursor)}`);
+    }
+    try {
+      return JSON.parse(Buffer.from(cursor, 'base64url').toString('utf8'));
+    } catch(err) {
+      throw new Error('Unable to parse cursor');
+    }
+  }
+}
+
+function removeFixings({prefix = '', postfix = '', value = ''} = {}) {
+  const start = value.toLowerCase().startsWith(prefix.toLowerCase()) ? prefix.length : 0;
+  const end = value.length - (value.toLowerCase().endsWith(postfix.toLowerCase()) ? postfix.length : 0);
+
+  let formatted = '';
+  for (let i = start; i < end; i++) {
+    formatted += value[i];
+  }
+
+  return formatted;
+}
+
+function addPadding({padding = {}, value = ''} = {}) {
+  return value.padStart(padding.length, padding.char);
+}
+
+function removePadding({padding = {}, value = ''} = {}) {
+  if (!padding.length || value.length >= padding.length) {
+    return value;
+  }
+
+  let formatted = '';
+  let useRemaining = false;
+  for (let i = 0; i < value.length; i++) {
+    const char = value[i];
+    if (useRemaining || i >= padding.length) {
+      formatted += char;
+    } else if (char !== padding.char) {
+      formatted += char;
+      useRemaining = true;
+    }
+  }
+
+  return formatted;
+}
+
 module.exports = {
   getUnique,
   batchItems,
+  addPadding,
+  removePadding,
+  removeFixings,
   parseJSONPath,
   getInstanceType,
   getModelVersion,
@@ -171,6 +233,7 @@ module.exports = {
   genericizeJSONPath,
   commaSeparatedString,
   formatAttributeCasing,
+  cursorFormatter,
   applyBetaModelOverrides,
   formatIndexNameForDisplay,
   BatchGetOrderMaintainer,

package/src/validations.js

@@ -64,6 +64,18 @@ const Attribute = {
 			type: "any",
 			format: "isFunction",
 		},
+		padding: {
+			type: "object",
+			required: ['length', 'char'],
+			properties: {
+				length: {
+					type: 'number'
+				},
+				char: {
+					type: 'string',
+				}
+			}
+		}
 	},
 };
 

package/src/where.js

@@ -120,7 +120,7 @@ class WhereFactory {
 			injected[name] = {
 				name,
 				action: this.buildClause(filter),
-				children: ["params", "go", "page", "where", ...modelFilters],
+				children: ["params", "go", "where", ...modelFilters],
 			};
 		}
 		filterChildren.push("where");
@@ -129,7 +129,7 @@ class WhereFactory {
 			action: (entity, state, fn) => {
 				return this.buildClause(fn)(entity, state);
 			},
-			children: ["params", "go", "page", "where", ...modelFilters],
+			children: ["params", "go", "where", ...modelFilters],
 		};
 		for (let parent of filterParents) {
 			injected[parent] = { ...injected[parent] };

package/notes

@@ -1,45 +0,0 @@
-/*
-
-todo: better param typing
-done: get typing
-todo: get implementation
-done: get[] typing
-todo: get[] implementation
-done: delete typing
-todo: delete implementation
-done: delete[] typing
-todo: delete[] implementation
-done: put[] typing
-todo: put[] implementation
-done: put typing
-todo: put implementation
-todo: update typing
-done: update implementation
-    done: SetRecord
-    done: RemoveRecord
-    done: DataUpdateMethodRecord
-
-done: scan typing
-todo: scan implementation
-done: query typing
-todo: query implementation
-done: find typing
-todo: find implementation
-done: match typing
-todo: match implementation
-done: parse typing
-todo: parse implementation (requires cursor?)
-todo: collection query
-todo: collection impl
-
-todo: add where to go?
-todo: rethink option names
-
-todo: add cursor to query options
-todo: remove old doc notes on `join` and slim down `service`
-
-
-
-find and test any add on set and number
-try number enum?
-integration test enum set ts

package/output

@@ -1,106 +0,0 @@
-page {
-    "data": [
-        {
-            "prop2": "e3b142b1-e5d7-4b17-8a66-38fcdf6ef1ef",
-            "prop1": "value1",
-            "prop4": "value4"
-        },
-        {
-            "prop2": "e8ed19db-3ac4-45d6-8fb8-281ada117e96",
-            "prop1": "value1",
-            "prop4": "value4"
-        },
-        {
-            "prop2": "dcca03f5-7478-4afb-9297-20a68ff56559",
-            "prop1": "value1",
-            "prop4": "value4"
-        },
-        {
-            "prop2": "03ccd79e-c5e5-4179-8213-196eafcbd4b9",
-            "prop1": "value1",
-            "prop4": "value4"
-        },
-        {
-            "prop2": "2900735e-aaa4-4661-a52d-7aecb752e694",
-            "prop1": "value1",
-            "prop4": "value4"
-        },
-        {
-            "prop2": "0f1da461-5257-4f6c-8cb3-af7b56819100",
-            "prop1": "value1",
-            "prop4": "value4"
-        },
-        {
-            "prop2": "7562c43b-8ac9-483c-b950-4f0e1366c11a",
-            "prop1": "value1",
-            "prop4": "value4"
-        },
-        {
-            "prop2": "ee844e97-dc1c-4242-bd2b-45685e39d7fd",
-            "prop1": "value1",
-            "prop4": "value4"
-        },
-        {
-            "prop2": "4b55e242-9875-412e-b483-24e51dabf6c8",
-            "prop1": "value1",
-            "prop4": "value4"
-        },
-        {
-            "prop2": "77a5b66e-c658-4ec6-9630-4d2469653bd4",
-            "prop1": "value1",
-            "prop4": "value4"
-        },
-        {
-            "prop2": "194efbfe-7be6-4662-9982-a1c8eb6ab36d",
-            "prop1": "value1",
-            "prop4": "value4"
-        },
-        {
-            "prop2": "value2",
-            "prop1": "value1",
-            "prop4": "value4"
-        },
-        {
-            "prop2": "516d8256-3b3e-41e1-a44a-0720fe748e23",
-            "prop1": "value1",
-            "prop4": "value4"
-        },
-        {
-            "prop2": "35bf3a7e-9dff-4817-a157-2be09c5fc8a8",
-            "prop1": "value1",
-            "prop4": "value4"
-        },
-        {
-            "prop2": "3836452b-6c00-415e-85ce-a9596644b6eb",
-            "prop1": "value1",
-            "prop4": "value4"
-        },
-        {
-            "prop2": "value2abc",
-            "prop1": "value1",
-            "prop4": "value4"
-        },
-        {
-            "prop2": "887e5301-113c-4ca6-b6d0-ebaf27e217d5",
-            "prop1": "value1",
-            "prop4": "value4"
-        },
-        {
-            "prop2": "6c6d9363-3821-40c8-b3d0-7ec2c6d3f1eb",
-            "prop1": "value1",
-            "prop4": "value4"
-        },
-        {
-            "prop2": "fcff14a3-ffa9-498e-a0ae-596c747b027b",
-            "prop1": "value1",
-            "prop4": "value4"
-        },
-        {
-            "prop2": "1d199eeb-ccc1-4c25-a1b6-1685126d44b3",
-            "prop1": "value1",
-            "prop4": "value4"
-        }
-    ],
-    "count": 2,
-    "next": null
-}