electrodb 2.5.1 → 2.6.1

package/src/entity.js

@@ -1,6 +1,7 @@
 "use strict";
 const { Schema } = require("./schema");
-const { AllPages, 
+const {
+	AllPages,
 	KeyCasing, 
 	TableIndex, 
 	FormatToReturnValues, 
@@ -35,7 +36,6 @@ const validations = require("./validations");
 const c = require('./client');
 const u = require("./util");
 const e = require("./errors");
-const { validate } = require("jsonschema");
 const v = require('./validations');
 
 class Entity {
@@ -59,6 +59,21 @@ class Entity {
 		this._clausesWithFilters = this._whereBuilder.injectWhereClauses(this._clausesWithFilters);
 
 		this.query = {};
+		this.conversions = {
+			fromComposite: {
+				toKeys: (composite, options = {}) => this._fromCompositeToKeys({ provided: composite }, options),
+				toCursor: (composite) => this._fromCompositeToCursor({ provided: composite }, { strict: 'all' }),
+			},
+			fromKeys: {
+				toCursor: (keys) => this._fromKeysToCursor({provided: keys}, {}),
+				toComposite: (keys) => this._fromKeysToComposite({ provided: keys }),
+			},
+			fromCursor: {
+				toKeys: (cursor) => this._fromCursorToKeys({provided: cursor}),
+				toComposite: (cursor) => this._fromCursorToComposite({provided: cursor}),
+			},
+			byAccessPattern: {}
+		};
 		for (let accessPattern in this.model.indexes) {
 			let index = this.model.indexes[accessPattern].index;
 			this.query[accessPattern] = (...values) => {
@@ -67,7 +82,24 @@ class Entity {
 				}
 				return this._makeChain(index, this._clausesWithFilters, clauses.index, options).query(...values);
 			};
+
+			this.conversions.byAccessPattern[accessPattern] = {
+				fromKeys: {
+					toCursor: (keys) => this._fromKeysToCursorByIndex({indexName: index, provided: keys}),
+					toComposite: (keys) => this._fromKeysToCompositeByIndex({indexName: index, provided: keys}),
+				},
+				fromCursor: {
+					toKeys: (cursor) => this._fromCursorToKeysByIndex({indexName: index, provided: cursor}),
+					toComposite: (cursor) => this._fromCursorToCompositeByIndex({indexName: index, provided: cursor}),
+				},
+				fromComposite: {
+					toCursor: (composite) => this._fromCompositeToCursorByIndex({indexName: index, provided: composite}, { strict: 'all' }),
+					toKeys: (composite, options = {}) => this._fromCompositeToKeysByIndex({indexName: index, provided: composite}, options),
+				},
+			};
+
 		}
+
 		this.config.identifiers = config.identifiers || {};
 		this.identifiers = {
 			entity: this.config.identifiers.entity || "__edb_e__",
@@ -98,6 +130,36 @@ class Entity {
 		return this.model.version;
 	}
 
+	// ownsItem(item) {
+	// 	return (
+	// 		item &&
+	// 		this.getName() === item[this.identifiers.entity] &&
+	// 		this.getVersion() === item[this.identifiers.version] &&
+	// 		validations.isStringHasLength(item[this.identifiers.entity]) &&
+	// 		validations.isStringHasLength(item[this.identifiers.version])
+	// 	) || !!this.ownsKeys(item)
+	// }
+
+	// ownsKeys({keys = {}}) {
+	// 	let {pk, sk} = this.model.prefixes[TableIndex];
+	// 	let hasSK = this.model.lookup.indexHasSortKeys[TableIndex];
+	// 	let pkMatch = typeof keys[pk.field] === "string" && keys[pk.field].startsWith(pk.prefix);
+	// 	let skMatch = pkMatch && !hasSK;
+	// 	if (pkMatch && hasSK) {
+	// 		skMatch = typeof keys[sk.field] === "string" && keys[sk.field].startsWith(sk.prefix);
+	// 	}
+	//
+	// 	return (pkMatch && skMatch &&
+	// 		this._formatKeysToItem(TableIndex, key) !== null);
+	// }
+
+	// ownsCursor({ cursor }) {
+	// 	if (typeof cursor === 'string') {
+	// 		cursor = u.cursorFormatter.deserialize(cursor);
+	// 	}
+	// 	return this.ownsKeys({ keys: cursor });
+	// }
+
 	ownsItem(item) {
 		return (
 			item &&
@@ -108,21 +170,43 @@ class Entity {
 		)
 	}
 
-	ownsLastEvaluatedKey(key = {}) {
+	_attributesIncludeKeys(attributes = []) {
+		let {pk, sk} = this.model.prefixes[TableIndex];
+		let pkFound = false;
+		let skFound = false;
+		for (let i = 0; i < attributes.length; i++) {
+			const attribute = attributes[i];
+			if (attribute === sk.field) {
+				skFound = true;
+			}
+			if (attribute === pk.field) {
+				skFound = true;
+			}
+			if (pkFound && skFound) {
+				return true;
+			}
+		}
+		return false;
+	}
+
+	ownsKeys(key = {}) {
 		let {pk, sk} = this.model.prefixes[TableIndex];
 		let hasSK = this.model.lookup.indexHasSortKeys[TableIndex];
 		let pkMatch = typeof key[pk.field] === "string" && key[pk.field].startsWith(pk.prefix);
+		let skMatch = pkMatch && !hasSK;
 		if (pkMatch && hasSK) {
-			return typeof key[sk.field] === "string" && key[sk.field].startsWith(sk.prefix);
+			skMatch = typeof key[sk.field] === "string" && key[sk.field].startsWith(sk.prefix);
 		}
-		return pkMatch;
+
+		return (pkMatch && skMatch &&
+			this._formatKeysToItem(TableIndex, key) !== null);
 	}
 
 	ownsCursor(cursor) {
 		if (typeof cursor === 'string') {
 			cursor = u.cursorFormatter.deserialize(cursor);
 		}
-		return this.ownsLastEvaluatedKey(cursor);
+		return this.ownsKeys(cursor);
 	}
 
 	serializeCursor(key) {
@@ -445,11 +529,54 @@ class Entity {
 		return { data: resultsAll, unprocessed: unprocessedAll };
 	}
 
+	async hydrate(index, keys = [], config) {
+		const items = [];
+		const validKeys = [];
+		for (let i = 0; i < keys.length; i++) {
+			const key = keys[i];
+			const item = this._formatKeysToItem(index, key);
+			if (item !== null) {
+				items.push(item);
+				validKeys.push(key);
+			}
+		}
+
+		const results = await this.get(items).go({
+			...config,
+			hydrate: false,
+			parse: undefined,
+			hydrator: undefined,
+			_isCollectionQuery: false,
+			preserveBatchOrder: true,
+			ignoreOwnership: config._providedIgnoreOwnership
+		});
+
+		const unprocessed = [];
+		const data = [];
+
+		for (let i = 0; i < results.data.length; i++) {
+			const key = validKeys[i];
+			const item = results.data[i];
+			if (!item) {
+				if (key) {
+					unprocessed.push(key);
+				}
+			} else {
+				data.push(item);
+			}
+		}
+
+		return {
+			unprocessed,
+			data,
+		};
+	}
+
 	async executeQuery(method, parameters, config = {}) {
 		let results = config._isCollectionQuery
 			? {}
 			: [];
-		let ExclusiveStartKey = this._formatExclusiveStartKey(config);
+		let ExclusiveStartKey = this._formatExclusiveStartKey({config, indexName: parameters.IndexName });
 		if (ExclusiveStartKey === null) {
 			ExclusiveStartKey = undefined;
 		}
@@ -457,13 +584,20 @@ class Entity {
 		let max = this._normalizeLimitValue(config.limit);
 		let iterations = 0;
 		let count = 0;
+		let hydratedUnprocessed = [];
+		const shouldHydrate = config.hydrate && method === MethodTypes.query;
 		do {
 			let limit = max === undefined
 				? parameters.Limit
 				: max - count;
-			let response = await this._exec(method, {ExclusiveStartKey, ...parameters, Limit: limit}, config);
+			let response = await this._exec(method, { ExclusiveStartKey, ...parameters, Limit: limit }, config);
 			ExclusiveStartKey = response.LastEvaluatedKey;
-			response = this.formatResponse(response, parameters.IndexName, config);
+			response = this.formatResponse(response, parameters.IndexName, {
+				...config,
+				includeKeys: shouldHydrate || config.includeKeys,
+				ignoreOwnership: shouldHydrate || config.ignoreOwnership,
+			});
+
 			if (config.raw) {
 				return response;
 			} else if (config._isCollectionQuery) {
@@ -471,14 +605,26 @@ class Entity {
 					if (max) {
 						count += response.data[entity].length;
 					}
+					let items = response.data[entity];
+					if (shouldHydrate && items.length) {
+						const hydrated = await config.hydrator(entity, parameters.IndexName, items, config);
+						items = hydrated.data;
+						hydratedUnprocessed = hydratedUnprocessed.concat(hydrated.unprocessed);
+					}
 					results[entity] = results[entity] || [];
-					results[entity] = [...results[entity], ...response.data[entity]];
+					results[entity] = [...results[entity], ...items];
 				}
 			} else if (Array.isArray(response.data)) {
 				if (max) {
 					count += response.data.length;
 				}
-				results = [...results, ...response.data];
+				let items = response.data;
+				if (shouldHydrate) {
+					const hydrated = await this.hydrate(parameters.IndexName, items, config);
+					items = hydrated.data;
+					hydratedUnprocessed = hydratedUnprocessed.concat(hydrated.unprocessed);
+				}
+				results = [...results, ...items];
 			} else {
 				return response;
 			}
@@ -490,6 +636,14 @@ class Entity {
 		);
 
 		const cursor = this._formatReturnPager(config, ExclusiveStartKey);
+
+		if (shouldHydrate) {
+			return {
+				cursor,
+				data: results,
+				unprocessed: hydratedUnprocessed,
+			};
+		}
 		return { data: results, cursor };
 	}
 
@@ -628,7 +782,11 @@ class Entity {
 				results = response;
 			} else {
 				if (response.Item) {
-					if (config.ignoreOwnership || this.ownsItem(response.Item)) {
+					if (
+						(config.ignoreOwnership && config.attributes && config.attributes.length > 0 && !this._attributesIncludeKeys(config.attributes)) ||
+						((config.ignoreOwnership || config.hydrate) && this.ownsKeys(response.Item)) ||
+						this.ownsItem(response.Item)
+					) {
 						results = this.model.schema.formatItemForRetrieval(response.Item, config);
 						if (Object.keys(results).length === 0) {
 							results = null;
@@ -639,7 +797,11 @@ class Entity {
 				} else if (response.Items) {
 					results = [];
 					for (let item of response.Items) {
-						if (config.ignoreOwnership || this.ownsItem(item)) {
+						if (
+							(config.ignoreOwnership && config.attributes && config.attributes.length > 0 && !this._attributesIncludeKeys(config.attributes)) ||
+							((config.ignoreOwnership || config.hydrate) && this.ownsKeys(item)) ||
+							this.ownsItem(item)
+						) {
 							let record = this.model.schema.formatItemForRetrieval(item, config);
 							if (Object.keys(record).length > 0) {
 								results.push(record);
@@ -675,7 +837,6 @@ class Entity {
 		}
 	}
 
-
 	parse(item, options = {}) {
 		if (item === undefined || item === null) {
 			return null;
@@ -687,6 +848,216 @@ class Entity {
 		return this.formatResponse(item, TableIndex, config);
 	}
 
+	_fromCompositeToKeys({provided}, options = {}) {
+		if (!provided || Object.keys(provided).length === 0) {
+			throw new e.ElectroError(e.ErrorCodes.InvalidConversionCompositeProvided, 'Invalid conversion composite provided');
+		}
+
+		let keys = {};
+		const secondaryIndexStrictMode = options.strict === 'all' || options.strict === 'pk' ? 'pk' : 'none';
+		for (const { index } of Object.values(this.model.indexes)) {
+			const indexKeys = this._fromCompositeToKeysByIndex({ indexName: index, provided }, {
+				strict: index === TableIndex ? options.strict : secondaryIndexStrictMode,
+			});
+			if (indexKeys) {
+				keys = {
+					...keys,
+					...indexKeys,
+				}
+			}
+		}
+
+
+		if (Object.keys(keys).length === 0) {
+			throw new e.ElectroError(e.ErrorCodes.InvalidConversionCompositeProvided, 'Invalid conversion composite provided');
+		}
+
+		return keys;
+	}
+
+	_fromCompositeToCursor({provided}, options = {}) {
+		const keys = this._fromCompositeToKeys({provided}, options);
+		if (!keys || Object.keys(keys).length === 0) {
+			throw new e.ElectroError(e.ErrorCodes.InvalidConversionCompositeProvided, 'Invalid conversion composite provided');
+		}
+		return u.cursorFormatter.serialize(keys);
+	}
+
+	_fromKeysToCursor({provided}, options = {}) {
+		if (!provided || Object.keys(provided).length === 0) {
+			throw new e.ElectroError(e.ErrorCodes.InvalidConversionKeysProvided, 'Invalid keys provided');
+		}
+		return u.cursorFormatter.serialize(provided);
+	}
+
+	_fromKeysToComposite({provided}, options = {}) {
+		if (!provided || Object.keys(provided).length === 0) {
+			throw new e.ElectroError(e.ErrorCodes.InvalidConversionKeysProvided, 'Provided keys could not be used to form composite attributes');
+		}
+
+		let keys = {};
+		for (const { index } of Object.values(this.model.indexes)) {
+			const composite = this._fromKeysToCompositeByIndex({indexName: index, provided}, options);
+			if (composite) {
+				for (const attribute in composite) {
+					if (keys[attribute] === undefined) {
+						keys[attribute] = composite[attribute];
+					}
+				}
+			}
+		}
+
+		if (Object.keys(keys).length === 0) {
+			throw new e.ElectroError(e.ErrorCodes.InvalidConversionKeysProvided, 'Provided keys could not be used to form composite attributes');
+		}
+
+		return keys;
+	}
+
+	_fromCursorToKeys({provided}, options = {}) {
+		if (typeof provided !== 'string') {
+			throw new e.ElectroError(e.ErrorCodes.InvalidConversionCursorProvided, 'Invalid conversion cursor provided');
+		}
+
+		return u.cursorFormatter.deserialize(provided);
+	}
+
+	_fromCursorToComposite({provided}, options = {}) {
+		if (typeof provided !== 'string') {
+			throw new e.ElectroError(e.ErrorCodes.InvalidConversionCursorProvided, 'Invalid conversion cursor provided');
+		}
+
+		const keys = this._fromCursorToKeys({provided}, options);
+		if (!keys){
+			throw new e.ElectroError(e.ErrorCodes.InvalidConversionCursorProvided, 'Invalid conversion cursor provided');
+		}
+
+		return this._fromKeysToComposite({provided: keys}, options);
+	}
+
+	_fromCompositeToCursorByIndex({indexName = TableIndex, provided}, options = {}) {
+		if (!provided || Object.keys(provided).length === 0) {
+			throw new e.ElectroError(e.ErrorCodes.InvalidConversionCompositeProvided, 'Invalid conversion composite provided');
+		}
+
+		const keys = this._formatSuppliedPager(indexName, provided, {
+			relaxedPk: false,
+			relaxedSk: false,
+		});
+
+		return this._fromKeysToCursorByIndex({indexName, provided: keys}, options);
+	}
+
+	_fromCompositeToKeysByIndex({indexName = TableIndex, provided}, options = {}) {
+		return this._formatSuppliedPager(indexName, provided, {
+			relaxedPk: options.strict !== 'pk' && options.strict !== 'all',
+			relaxedSk: options.strict !== 'all',
+		});
+	}
+
+	_fromCursorToKeysByIndex({ provided }, options = {}) {
+		if (typeof provided !== 'string' || provided.length < 1) {
+			throw new e.ElectroError(e.ErrorCodes.InvalidConversionCursorProvided, 'Invalid conversion cursor provided');
+		}
+
+		return u.cursorFormatter.deserialize(provided);
+	}
+
+	_fromKeysToCursorByIndex({indexName = TableIndex, provided}, options = {}) {
+		const isValidTableIndex = this._verifyKeys({indexName: TableIndex, provided});
+		const isValidIndex = this._verifyKeys({indexName, provided});
+		if (!isValidTableIndex) {
+			throw new e.ElectroError(e.ErrorCodes.InvalidConversionKeysProvided, 'Provided keys did not include valid properties for the primary index');
+		} else if (!isValidIndex) {
+			throw new e.ElectroError(e.ErrorCodes.InvalidConversionKeysProvided, `Provided keys did not include valid properties for the index "${indexName}"`);
+		}
+
+		const keys = this._trimKeysToIndex({ indexName, provided });
+
+		if (!keys || Object.keys(keys).length === 0) {
+			throw new e.ElectroError(e.ErrorCodes.InvalidConversionKeysProvided, `Provided keys not defined`);
+		}
+
+		return u.cursorFormatter.serialize(provided);
+	}
+
+	_fromKeysToCompositeByIndex({indexName = TableIndex, provided}, options = {}) {
+		let allKeys = {};
+
+		const indexKeys = this._deconstructIndex({ index: indexName, keys: provided });
+		if (!indexKeys) {
+			throw new e.ElectroError(e.ErrorCodes.InvalidConversionKeysProvided, `Provided keys did not include valid properties for the index "${indexName}"`);
+		}
+
+		allKeys = {
+			...indexKeys,
+		}
+
+		let tableKeys;
+		if (indexName !== TableIndex) {
+			tableKeys = this._deconstructIndex({index: TableIndex, keys: provided});
+		}
+
+		if (tableKeys === null) {
+			return allKeys;
+		}
+
+		allKeys = {
+			...allKeys,
+			...tableKeys,
+		};
+
+		if (Object.keys(allKeys).length === 0) {
+			throw new e.ElectroError(e.ErrorCodes.InvalidConversionKeysProvided, 'Provided keys could not be used to form composite attributes');
+		}
+
+		return allKeys;
+	}
+
+	_fromCursorToCompositeByIndex({indexName = TableIndex, provided}, options = {}) {
+		const keys = this._fromCursorToKeysByIndex({indexName, provided}, options);
+		if (!keys || Object.keys(keys).length === 0) {
+			throw new e.ElectroError(e.ErrorCodes.InvalidConversionCursorProvided, 'Invalid conversion cursor provided');
+		}
+		return this._fromKeysToCompositeByIndex({indexName, provided: keys}, options);
+	}
+
+	_trimKeysToIndex({indexName = TableIndex, provided}) {
+		if (!provided) {
+			return null;
+		}
+
+		const pkName = this.model.translations.keys[indexName].pk;
+		const skName = this.model.translations.keys[indexName].sk;
+		const tablePKName = this.model.translations.keys[TableIndex].pk;
+		const tableSKName = this.model.translations.keys[TableIndex].sk;
+
+		const keys = {
+			[pkName]: provided[pkName],
+			[skName]: provided[skName],
+			[tablePKName]: provided[tablePKName],
+			[tableSKName]: provided[tableSKName],
+		};
+
+		if (!keys || Object.keys(keys).length === 0) {
+			return null;
+		}
+
+		return keys;
+	}
+
+	_verifyKeys({indexName, provided}) {
+		if (!provided) {
+			throw new e.ElectroError(e.ErrorCodes.InvalidConversionKeysProvided, `Provided keys not defined`);
+		}
+
+		const pkName = this.model.translations.keys[indexName].pk;
+		const skName = this.model.translations.keys[indexName].sk;
+
+		return provided[pkName] !== undefined &&
+			(!skName || provided[skName] !== undefined);
+	}
+
 	_formatReturnPager(config, lastEvaluatedKey) {
 		let page = lastEvaluatedKey || null;
 		if (config.raw || config.pager === Pager.raw) {
@@ -695,12 +1066,22 @@ class Entity {
 		return config.formatCursor.serialize(page) || null;
 	}
 
-	_formatExclusiveStartKey(config) {
+	_formatExclusiveStartKey({config, indexName = TableIndex}) {
 		let exclusiveStartKey = config.cursor;
 		if (config.raw || config.pager === Pager.raw) {
-			return exclusiveStartKey || null;
+			return this._trimKeysToIndex({ provided: exclusiveStartKey, indexName }) || null;
+		}
+		let keys;
+		if (config.pager === Pager.item) {
+			keys = this._fromCompositeToKeysByIndex({indexName, provided: exclusiveStartKey});
+		} else {
+			keys = config.formatCursor.deserialize(exclusiveStartKey);
+		}
+		if (!keys) {
+			return null;
 		}
-		return config.formatCursor.deserialize(exclusiveStartKey) || null;
+
+		return this._trimKeysToIndex({provided: keys, indexName}) || null;
 	}
 
 	setClient(client) {
@@ -782,121 +1163,154 @@ class Entity {
 		return value;
 	}
 
-	_deconstructKeys(index, keyType, key, backupFacets = {}) {
-		if (typeof key !== "string" || key.length === 0) {
-			return null;
-		}
-		
-		let accessPattern = this.model.translations.indexes.fromIndexToAccessPattern[index];
-		let {prefix, isCustom} = this.model.prefixes[index][keyType];
-		let {facets} = this.model.indexes[accessPattern][keyType];
+	_createKeyDeconstructor(prefixes = {}, labels = [], attributes = {}) {
+		let {prefix, isCustom, postfix} = prefixes;
 		let names = [];
 		let types = [];
-		let pattern = `^${this._regexpEscape(prefix)}`;
-		let labels = this.model.facets.labels[index][keyType] || [];
+		let pattern = `^${this._regexpEscape(prefix || '')}`;
 		for (let {name, label} of labels) {
-			let attr = this.model.schema.attributes[name];
+			let attr = attributes[name];
+			if (isCustom && !name && label) {
+				// this case is for when someone uses a direct attribute reference but with a postfix (zoinks ;P)
+				pattern += `${this._regexpEscape(label)}`;
+			} else if (isCustom) {
+				pattern += `${this._regexpEscape(label === undefined ? "" : label)}(.+)`;
+			} else {
+				pattern += `#${this._regexpEscape(label === undefined ? name : label)}_(.+)`;
+			}
+			names.push(name);
 			if (attr) {
-				if (isCustom) {
-					pattern += `${this._regexpEscape(label === undefined ? "" : label)}(.+)`;
-				} else {
-					pattern += `#${this._regexpEscape(label === undefined ? name : label)}_(.+)`;
-				}
-				names.push(name);
 				types.push(attr.type);
 			}
 		}
+		if (typeof postfix === 'string') {
+			pattern += this._regexpEscape(postfix);
+		}
 		pattern += "$";
-		let regex = RegExp(pattern);
-		let match = key.match(regex);
-		let results = {};
-		if (match) {
-			for (let i = 0; i < names.length; i++) {
-				let key = names[i];
-				let value = match[i+1];
-				let type = types[i];
-				switch (type) {
-					case "number":
-						value = parseFloat(value);
-						break;
-					case "boolean":
-						value = value === "true";
-						break;
-				}
-				results[key] = value;
-			}
-		} else {
-			if (Object.keys(backupFacets || {}).length === 0) {
-				// this can occur when a scan is performed but returns no results given the current filters or record timing
-				return {};
+
+		let regex = new RegExp(pattern, "i");
+
+		return ({ key } = {}) => {
+			if (!['string', 'number'].includes(typeof key)) {
+				return null;
 			}
-			for (let facet of facets) {
-				if (backupFacets[facet] === undefined) {
-					throw new e.ElectroError(e.ErrorCodes.LastEvaluatedKey, 'LastEvaluatedKey contains entity that does not match the entity used to query. Use {pager: "raw"} query option.');
-				} else {
-					results[facet] = backupFacets[facet];
+			key = `${key}`;
+			let match = key.match(regex);
+			let results = {};
+			if (match) {
+				for (let i = 0; i < names.length; i++) {
+					let key = names[i];
+					let value = match[i + 1];
+					let type = types[i];
+					switch (type) {
+						case "number": {
+							value = parseFloat(value);
+							break;
+						}
+						case "boolean": {
+							value = value === "true";
+							break;
+						}
+					}
+					if (key && value !== undefined) {
+						results[key] = value;
+					}
 				}
+			} else {
+				results = null;
 			}
+
+			return results;
 		}
-		return results;
 	}
 
-
-
-	_deconstructIndex(index = TableIndex, lastEvaluated, lastReturned) {
+	_deconstructIndex({index = TableIndex, keys = {}} = {}) {
+		const hasIndex = !!this.model.translations.keys[index];
+		if (!hasIndex) {
+			return null;
+		}
 		let pkName = this.model.translations.keys[index].pk;
 		let skName = this.model.translations.keys[index].sk;
-		let pkFacets = this._deconstructKeys(index, KeyTypes.pk, lastEvaluated[pkName], lastReturned);
-		let skFacets = this._deconstructKeys(index, KeyTypes.sk, lastEvaluated[skName], lastReturned);
-		let facets = {...pkFacets};
-		if (skFacets && Object.keys(skFacets).length) {
-			facets = {...skFacets, ...pkFacets};
+		const indexHasSortKey = this.model.lookup.indexHasSortKeys[index];
+		const deconstructors = this.model.keys.deconstructors[index];
+		const pk = keys[pkName];
+		if (pk === undefined) {
+			return null;
+		}
+		const pkComposites = deconstructors.pk({key: pk});
+		if (pkComposites === null) {
+			return null;
+		}
+		let skComposites = {};
+		if (indexHasSortKey) {
+			const sk = keys[skName];
+			if (!sk) {
+				return null;
+			}
+			skComposites = deconstructors.sk({key: sk});
+			if (skComposites === null) {
+				return null;
+			}
+		}
+		return {
+			...pkComposites,
+			...skComposites,
 		}
-		return facets;
 	}
 
-	_formatKeysToItem(index = TableIndex, lastEvaluated, lastReturned) {
-		if (lastEvaluated === null || typeof lastEvaluated !== "object" || Object.keys(lastEvaluated).length === 0) {
-			return lastEvaluated;
+	_formatKeysToItem(index = TableIndex, keys) {
+		if (keys === null || typeof keys !== "object" || Object.keys(keys).length === 0) {
+			return keys;
 		}
 		let tableIndex = TableIndex;
-		let pager = this._deconstructIndex(index, lastEvaluated, lastReturned);
+		let indexParts = this._deconstructIndex({index, keys});
+		if (indexParts === null) {
+			return null;
+		}
 		// lastEvaluatedKeys from query calls include the index pk/sk as well as the table index's pk/sk
 		if (index !== tableIndex) {
-			pager = {...pager, ...this._deconstructIndex(tableIndex, lastEvaluated, lastReturned)};
+			const tableIndexParts = this._deconstructIndex({index: tableIndex, keys});
+			if (tableIndexParts === null) {
+				return null;
+			}
+			indexParts = { ...indexParts, ...tableIndexParts };
 		}
-		let pagerIsEmpty = Object.keys(pager).length === 0;
-		let pagerIsIncomplete = this.model.facets.byIndex[tableIndex].all.find(facet => pager[facet.name] === undefined);
-		if (pagerIsEmpty || pagerIsIncomplete) {
+		let noPartsFound = Object.keys(indexParts).length === 0;
+		let partsAreIncomplete = this.model.facets.byIndex[tableIndex].all.find(facet => indexParts[facet.name] === undefined);
+		if (noPartsFound || partsAreIncomplete) {
 			// In this case no suitable record could be found be the deconstructed pager.
 			// This can be valid in cases where a scan is performed but returns no results.
 			return null;
 		}
 
-		return pager;
+		return indexParts;
 	}
 
-	_constructPagerIndex(index = TableIndex, item) {
-		let pkAttributes = this._expectFacets(item, this.model.facets.byIndex[index].pk);
-		let skAttributes = this._expectFacets(item, this.model.facets.byIndex[index].sk);
+	_constructPagerIndex(index = TableIndex, item, options = {}) {
+		let pkAttributes = options.relaxedPk ? item : this._expectFacets(item, this.model.facets.byIndex[index].pk);
+		let skAttributes = options.relaxedSk ? item : this._expectFacets(item, this.model.facets.byIndex[index].sk);
+
 		let keys = this._makeIndexKeys({
 			index,
 			pkAttributes,
 			skAttributes: [skAttributes],
 		});
+
 		return this._makeParameterKey(index, keys.pk, ...keys.sk);
 	}
 
-	_formatSuppliedPager(index = TableIndex, item) {
+	_formatSuppliedPager(index = TableIndex, item, options = {}) {
 		if (typeof item !== "object" || Object.keys(item).length === 0) {
 			return item;
 		}
+
 		let tableIndex = TableIndex;
-		let pager = this._constructPagerIndex(index, item);
+		let pager = this._constructPagerIndex(index, item, options);
 		if (index !== tableIndex) {
-			pager = {...pager, ...this._constructPagerIndex(tableIndex, item)}
+			pager = {...pager, ...this._constructPagerIndex(tableIndex, item, options)};
 		}
-		return pager
+
+		return pager;
 	}
 
 	_normalizeExecutionOptions({ provided = [], context = {} } = {}) {
@@ -916,6 +1330,7 @@ class Entity {
 			cursor: null,
 			data: 'attributes',
 			ignoreOwnership: false,
+			_providedIgnoreOwnership: false,
 			_isPagination: false,
 			_isCollectionQuery: false,
 			pages: 1,
@@ -925,6 +1340,8 @@ class Entity {
 			terminalOperation: undefined,
 			formatCursor: u.cursorFormatter,
 			order: undefined,
+			hydrate: false,
+			hydrator: (_entity, _indexName, items) => items,
 		};
 
 		return provided.filter(Boolean).reduce((config, option) => {
@@ -1060,6 +1477,7 @@ class Entity {
 
 			if (option.ignoreOwnership) {
 				config.ignoreOwnership = option.ignoreOwnership;
+				config._providedIgnoreOwnership = option.ignoreOwnership;
 			}
 
 			if (option.listeners) {
@@ -1076,6 +1494,15 @@ class Entity {
 				}
 			}
 
+			if (option.hydrate) {
+				config.hydrate = true;
+				config.ignoreOwnership = true;
+			}
+
+			if (validations.isFunction(option.hydrator)) {
+				config.hydrator = option.hydrator;
+			}
+
 			config.page = Object.assign({}, config.page, option.page);
 			config.params = Object.assign({}, config.params, option.params);
 			return config;
@@ -1520,12 +1947,12 @@ class Entity {
 		};
 	}
 
-	_makeUpsertParams({update, upsert} = {}, pk, sk) {
+	_makeUpsertParams({ update, upsert } = {}, pk, sk) {
 		const { updatedKeys, setAttributes, indexKey } = this._getPutKeys(pk, sk && sk.facets, upsert.data);
 		const upsertAttributes = this.model.schema.translateToFields(setAttributes);
 		const keyNames = Object.keys(indexKey);
-		update.set(this.identifiers.entity, this.getName());
-		update.set(this.identifiers.version, this.getVersion());
+		// update.set(this.identifiers.entity, this.getName());
+		// update.set(this.identifiers.version, this.getVersion());
 		for (const field of [...Object.keys(upsertAttributes), ...Object.keys(updatedKeys)]) {
 			const value = u.getFirstDefined(upsertAttributes[field], updatedKeys[field]);
 			if (!keyNames.includes(field)) {
@@ -2238,7 +2665,11 @@ class Entity {
 		const subCollections = this.model.subCollections[collection];
 		const index = this.model.translations.collections.fromCollectionToIndex[collection];
 		const accessPattern = this.model.translations.indexes.fromIndexToAccessPattern[index];
+		const prefixes = this.model.prefixes[index];
 		const prefix = this._makeCollectionPrefix(subCollections);
+		if (prefixes.sk && prefixes.sk.isCustom) {
+			return '';
+		}
 		return this._formatKeyCasing(accessPattern, prefix);
 	}
 
@@ -2293,7 +2724,8 @@ class Entity {
 		if (!prefixes) {
 			throw new Error(`Invalid index: ${index}`);
 		}
-		let partitionKey = this._makeKey(prefixes.pk, facets.pk, pkFacets, this.model.facets.labels[index].pk, {excludeLabelTail: true});
+		// let partitionKey = this._makeKey(prefixes.pk, facets.pk, pkFacets, this.model.facets.labels[index].pk, { excludeLabelTail: true });
+		let partitionKey = this._makeKey(prefixes.pk, facets.pk, pkFacets, this.model.facets.labels[index].pk);
 		let pk = partitionKey.key;
 		let sk = [];
 		let fulfilled = false;
@@ -2388,6 +2820,7 @@ class Entity {
 				key: supplied[facets[0]],
 			};
 		}
+
 		let key = prefix;
 		let foundCount = 0;
 		for (let i = 0; i < labels.length; i++) {
@@ -2425,9 +2858,11 @@ class Entity {
 			key += postfix;
 		}
 
+		const transformedKey = transform(u.formatKeyCasing(key, casing));
+
 		return {
 			fulfilled,
-			key: transform(u.formatKeyCasing(key, casing))
+			key: transformedKey,
 		};
 	}
 
@@ -3125,6 +3560,17 @@ class Entity {
 			indexes[indexAccessPattern.fromIndexToAccessPattern[indexName]].pk.labels = facets.labels[indexName].pk;
 			indexes[indexAccessPattern.fromIndexToAccessPattern[indexName]].sk.labels = facets.labels[indexName].sk;
 		}
+		const deconstructors = {};
+		for (const indexName of Object.keys(facets.labels)) {
+			const keyTypes = prefixes[indexName] || {};
+			deconstructors[indexName] = {};
+			for (const keyType in keyTypes) {
+				const prefixes = keyTypes[keyType];
+				const labels = facets.labels[indexName][keyType] || [];
+				const attributes = schema.attributes;
+				deconstructors[indexName][keyType] = this._createKeyDeconstructor(prefixes, labels, attributes);
+			}
+		}
 
 		return {
 			name,
@@ -3150,6 +3596,9 @@ class Entity {
 				indexes: indexAccessPattern,
 				collections: indexCollection,
 			},
+			keys: {
+				deconstructors,
+			},
 			original: model,
 		};
 	}
@@ -3173,20 +3622,31 @@ function getEntityIdentifiers(entities) {
 	return identifiers;
 }
 
-function matchToEntityAlias({ paramItem, identifiers, record } = {}) {
+function matchToEntityAlias({ paramItem, identifiers, record, entities = {}, allowMatchOnKeys = false } = {}) {
 	let entity;
-	let entityAlias;
-
 	if (paramItem && v.isFunction(paramItem[TransactionCommitSymbol])) {
 		const committed = paramItem[TransactionCommitSymbol]();
 		entity = committed.entity;
 	}
 
+	let entityAlias;
 	for (let {name, version, nameField, versionField, alias} of identifiers) {
 		if (entity && entity.model.entity === name && entity.model.version === version) {
 			entityAlias = alias;
 			break;
-		} else if (record[nameField] !== undefined && record[nameField] === name && record[versionField] !== undefined && record[versionField] === version) {
+		} else if (record[nameField] !== undefined && record[versionField] !== undefined && record[nameField] === name && record[versionField] === version) {
+			entityAlias = alias;
+			break;
+		// } else if (allowMatchOnKeys && entities[alias] && entities[alias].ownsKeys({keys: record})) {
+		// 	if (entityAlias) {
+		// 		if (alias !== entityAlias) {
+		// 			throw new Error('Key ownership found to be not distinct');
+		// 		}
+		// 	} else {
+		// 		entityAlias = alias;
+		// 	}
+		// }
+		} else if (entities[alias] && entities[alias].ownsKeys(record)) {
 			entityAlias = alias;
 			break;
 		}