electrodb 1.12.1 → 2.1.0

package/src/clauses.js

@@ -50,11 +50,11 @@ let clauses = {
 				return state;
 			}
 		},
-		children: ["params", "go", "page"],
+		children: ["params", "go"],
 	},
 	scan: {
 		name: "scan",
-		action(entity, state) {
+		action(entity, state, config) {
 			if (state.getError() !== null) {
 				return state;
 			}
@@ -65,7 +65,7 @@ let clauses = {
 				return state;
 			}
 		},
-		children: ["params", "go", "page"],
+		children: ["params", "go"],
 	},
 	get: {
 		name: "get",
@@ -397,6 +397,7 @@ let clauses = {
 				return state;
 			}
 			try {
+				state.addOption('_isPagination', true);
 				const attributes = state.getCompositeAttributes();
 				return state
 					.setMethod(MethodTypes.query)
@@ -410,7 +411,7 @@ let clauses = {
 				return state;
 			}
 		},
-		children: ["between", "gte", "gt", "lte", "lt", "begins", "params", "go", "page"],
+		children: ["between", "gte", "gt", "lte", "lt", "begins", "params", "go"],
 	},
 	between: {
 		name: "between",
@@ -430,7 +431,7 @@ let clauses = {
 				return state;
 			}
 		},
-		children: ["go", "params", "page"],
+		children: ["go", "params"],
 	},
 	begins: {
 		name: "begins",
@@ -450,7 +451,7 @@ let clauses = {
 				return state;
 			}
 		},
-		children: ["go", "params", "page"],
+		children: ["go", "params"],
 	},
 	gt: {
 		name: "gt",
@@ -470,7 +471,7 @@ let clauses = {
 				return state;
 			}
 		},
-		children: ["go", "params", "page"],
+		children: ["go", "params"],
 	},
 	gte: {
 		name: "gte",
@@ -490,7 +491,7 @@ let clauses = {
 				return state;
 			}
 		},
-		children: ["go", "params", "page"],
+		children: ["go", "params"],
 	},
 	lt: {
 		name: "lt",
@@ -509,7 +510,7 @@ let clauses = {
 				return state;
 			}
 		},
-		children: ["go", "params", "page"],
+		children: ["go", "params"],
 	},
 	lte: {
 		name: "lte",
@@ -528,7 +529,7 @@ let clauses = {
 				return state;
 			}
 		},
-		children: ["go", "params", "page"],
+		children: ["go", "params"],
 	},
 	params: {
 		name: "params",
@@ -589,28 +590,6 @@ let clauses = {
 		},
 		children: [],
 	},
-	page: {
-		name: "page",
-		action(entity, state, page = null, options = {}) {
-			if (state.getError() !== null) {
-				return Promise.reject(state.error);
-			}
-			try {
-				if (entity.client === undefined) {
-					throw new e.ElectroError(e.ErrorCodes.NoClientDefined, "No client defined on model");
-				}
-				options.page = page;
-				options._isPagination = true;
-				options.terminalOperation = TerminalOperation.page;
-				let params = clauses.params.action(entity, state, options);
-				let {config} = entity._applyParameterOptions({}, state.getOptions(), options);
-				return entity.go(state.getMethod(), params, config);
-			} catch(err) {
-				return Promise.reject(err);
-			}
-		},
-		children: []
-	},
 };
 
 class ChainState {
@@ -677,6 +656,10 @@ class ChainState {
 		return this.query.options;
 	}
 
+	addOption(key, value) {
+		this.query.options[key] = value;
+	}
+
 	_appendProvided(type, attributes) {
 		const newAttributes = Object.keys(attributes).map(attribute => {
 			return {

package/src/entity.js

@@ -1,6 +1,27 @@
 "use strict";
 const { Schema } = require("./schema");
-const { KeyCasing, TableIndex, FormatToReturnValues, ReturnValues, EntityVersions, ItemOperations, UnprocessedTypes, Pager, ElectroInstance, KeyTypes, QueryTypes, MethodTypes, Comparisons, ExpressionTypes, ModelVersions, ElectroInstanceTypes, MaxBatchItems, TerminalOperation } = require("./types");
+const { AllPages, 
+	KeyCasing, 
+	TableIndex, 
+	FormatToReturnValues, 
+	ReturnValues, 
+	EntityVersions, 
+	ItemOperations, 
+	UnprocessedTypes, 
+	Pager, 
+	ElectroInstance, 
+	KeyTypes, 
+	QueryTypes, 
+	MethodTypes, 
+	Comparisons, 
+	ExpressionTypes, 
+	ModelVersions, 
+	ElectroInstanceTypes, 
+	MaxBatchItems, 
+	TerminalOperation, 
+	ResultOrderOption,
+	ResultOrderParam
+} = require("./types");
 const { FilterFactory } = require("./filters");
 const { FilterOperations } = require("./operations");
 const { WhereFactory } = require("./where");
@@ -10,6 +31,7 @@ const validations = require("./validations");
 const c = require('./client');
 const u = require("./util");
 const e = require("./errors");
+const { validate } = require("jsonschema");
 
 class Entity {
 	constructor(model, config = {}) {
@@ -30,7 +52,7 @@ class Entity {
 		this._whereBuilder = new WhereFactory(this.model.schema.attributes, FilterOperations);
 		this._clausesWithFilters = this._filterBuilder.injectFilterClauses(clauses, this.model.filters);
 		this._clausesWithFilters = this._whereBuilder.injectWhereClauses(this._clausesWithFilters);
-		this.scan = this._makeChain(TableIndex, this._clausesWithFilters, clauses.index).scan();
+		this.scan = this._makeChain(TableIndex, this._clausesWithFilters, clauses.index, {_isPagination: true}).scan();
 		this.query = {};
 		for (let accessPattern in this.model.indexes) {
 			let index = this.model.indexes[accessPattern].index;
@@ -84,14 +106,30 @@ class Entity {
 		return pkMatch;
 	}
 
+	ownsCursor(cursor) {
+		if (typeof cursor === 'string') {
+			cursor = u.cursorFormatter.deserialize(cursor);
+		}
+		return this.ownsLastEvaluatedKey(cursor);
+	}
+
+	serializeCursor(key) {
+		return u.cursorFormatter.serialize(key);
+	}
+
+	deserializeCursor(cursor) {
+		return u.cursorFormatter.deserialize(cursor);
+	}
+
+	/** @depricated pagers no longer exist, use the new cursor api */
 	ownsPager(pager, index = TableIndex) {
 		if (pager === null) {
 			return false;
 		}
-		let tableIndexFacets = this.model.facets.byIndex[TableIndex];
+		let tableIndexFacets = this.model.facets.byIndex[index];
 		// todo: is the fact it doesn't use the provided index a bug?
 		// feels like collections may have played a roll into why this is this way
-		let indexFacets = this.model.facets.byIndex[TableIndex];
+		let indexFacets = this.model.facets.byIndex[index];
 
 		// Unknown index
 		if (tableIndexFacets === undefined || indexFacets === undefined) {
@@ -114,9 +152,10 @@ class Entity {
 	}
 
 	match(facets = {}) {
-		let match = this._findBestIndexKeyMatch(facets);
+		const options = { _isPagination: true };
+		const match = this._findBestIndexKeyMatch(facets);
 		if (match.shouldScan) {
-			return this._makeChain(TableIndex, this._clausesWithFilters, clauses.index)
+			return this._makeChain(TableIndex, this._clausesWithFilters, clauses.index, options)
 				.scan()
 				.filter(attr => {
 					let eqFilters = [];
@@ -128,7 +167,7 @@ class Entity {
 				return eqFilters.join(" AND ");
 			});
 		} else {
-			return this._makeChain(match.index, this._clausesWithFilters, clauses.index)
+			return this._makeChain(match.index, this._clausesWithFilters, clauses.index, options)
 				.query(facets)
 				.filter(attr => {
 					let eqFilters = [];
@@ -143,16 +182,17 @@ class Entity {
 	}
 
 	find(facets = {}) {
-		let match = this._findBestIndexKeyMatch(facets);
+		const options = { _isPagination: true };
+		const match = this._findBestIndexKeyMatch(facets);
 		if (match.shouldScan) {
-			return this._makeChain(TableIndex, this._clausesWithFilters, clauses.index).scan();
+			return this._makeChain(TableIndex, this._clausesWithFilters, clauses.index, options).scan();
 		} else {
-			return this._makeChain(match.index, this._clausesWithFilters, clauses.index).query(facets);
+			return this._makeChain(match.index, this._clausesWithFilters, clauses.index, options).query(facets);
 		}
 	}
 
 	collection(collection = "", clauses = {}, facets = {}, {expressions = {}, parse} = {}) {
-		let options = {
+		const options = {
 			parse,
 			expressions: {
 				names: expressions.names || {},
@@ -239,7 +279,8 @@ class Entity {
 				case MethodTypes.batchGet:
 					return await this.executeBulkGet(parameters, config);
 				case MethodTypes.query:
-					return await this.executeQuery(parameters, config)
+				case MethodTypes.scan:
+					return await this.executeQuery(method, parameters, config);
 				default:
 					return await this.executeOperation(method, parameters, config);
 			}
@@ -305,20 +346,20 @@ class Entity {
 			await Promise.all(operation.map(async params => {
 				let response = await this._exec(MethodTypes.batchWrite, params, config);
 				if (validations.isFunction(config.parse)) {
-					let parsed = await config.parse(config, response);
+					let parsed = config.parse(config, response);
 					if (parsed) {
 						results.push(parsed);
 					}
-					return;
-				}
-				let unprocessed = this.formatBulkWriteResponse(response, config);
-				for (let u of unprocessed) {
-					results.push(u);
+				} else {
+					let {unprocessed} = this.formatBulkWriteResponse(response, config);
+					for (let u of unprocessed) {
+						results.push(u);
+					}
 				}
 			}));
 		}
 
-		return results;
+		return { unprocessed: results };
 	}
 
 	_createNewBatchGetOrderMaintainer(config = {}) {
@@ -356,26 +397,29 @@ class Entity {
 			await Promise.all(operation.map(async params => {
 				let response = await this._exec(MethodTypes.batchGet, params, config);
 				if (validations.isFunction(config.parse)) {
-					resultsAll.push(await config.parse(config, response));
-					return;
+					resultsAll.push(config.parse(config, response));
+				} else {
+					this.applyBulkGetResponseFormatting({
+						orderMaintainer,
+						resultsAll,
+						unprocessedAll,
+						response,
+						config
+					});
 				}
-				this.applyBulkGetResponseFormatting({
-					orderMaintainer,
-					resultsAll,
-					unprocessedAll,
-					response,
-					config
-				});
 			}));
 		}
-		return [resultsAll, unprocessedAll];
+		return { data: resultsAll, unprocessed: unprocessedAll };
 	}
 
-	async executeQuery(parameters, config = {}) {
+	async executeQuery(method, parameters, config = {}) {
 		let results = config._isCollectionQuery
 			? {}
 			: [];
-		let ExclusiveStartKey;
+		let ExclusiveStartKey = this._formatExclusiveStartKey(config);
+		if (ExclusiveStartKey === null) {
+			ExclusiveStartKey = undefined;
+		}
 		let pages = this._normalizePagesValue(config.pages);
 		let max = this._normalizeLimitValue(config.limit);
 		let iterations = 0;
@@ -384,53 +428,48 @@ class Entity {
 			let limit = max === undefined
 				? parameters.Limit
 				: max - count;
-			let response = await this._exec("query", {ExclusiveStartKey, ...parameters, Limit: limit}, config);
-
+			let response = await this._exec(method, {ExclusiveStartKey, ...parameters, Limit: limit}, config);
 			ExclusiveStartKey = response.LastEvaluatedKey;
-
-			if (validations.isFunction(config.parse)) {
-				response = config.parse(config, response);
-			} else {
-				response = this.formatResponse(response, parameters.IndexName, config);
-			}
-
-			if (config.raw || config._isPagination) {
+			response = this.formatResponse(response, parameters.IndexName, config);
+			if (config.raw) {
 				return response;
 			} else if (config._isCollectionQuery) {
-				for (const entity in response) {
+				for (const entity in response.data) {
 					if (max) {
-						count += response[entity].length;
+						count += response.data[entity].length;
 					}
 					results[entity] = results[entity] || [];
-					results[entity] = [...results[entity], ...response[entity]];
+					results[entity] = [...results[entity], ...response.data[entity]];
 				}
-			} else if (Array.isArray(response)) {
+			} else if (Array.isArray(response.data)) {
 				if (max) {
-					count += response.length;
+					count += response.data.length;
 				}
-				results = [...results, ...response];
+				results = [...results, ...response.data];
 			} else {
 				return response;
 			}
-
 			iterations++;
-		} while(ExclusiveStartKey && iterations < pages && (max === undefined || count < max));
-		return results;
+		} while(
+			ExclusiveStartKey &&
+			(pages === AllPages || iterations < pages) &&
+			(max === undefined || count < max)
+		);
+
+		const cursor = this._formatReturnPager(config, ExclusiveStartKey);
+		return { data: results, cursor };
 	}
 
 	async executeOperation(method, parameters, config) {
 		let response = await this._exec(method, parameters, config);
-		if (validations.isFunction(config.parse)) {
-			return config.parse(config, response);
-		}
 		switch (parameters.ReturnValues) {
 			case FormatToReturnValues.none:
-				return null;
+				return { data: null };
 			case FormatToReturnValues.all_new:
 			case FormatToReturnValues.all_old:
 			case FormatToReturnValues.updated_new:
 			case FormatToReturnValues.updated_old:
-				return this.formatResponse(response, parameters.IndexName, config);
+				return this.formatResponse(response, config);
 			case FormatToReturnValues.default:
 			default:
 				return this._formatDefaultResponse(method, parameters.IndexName, parameters, config, response);
@@ -475,9 +514,9 @@ class Entity {
 		const index = TableIndex;
 		let unprocessed = response.UnprocessedItems[table];
 		if (Array.isArray(unprocessed) && unprocessed.length) {
-			return unprocessed.map(request => {
+			unprocessed = unprocessed.map(request => {
 				if (request.PutRequest) {
-					return this.formatResponse(request.PutRequest, index, config);
+					return this.formatResponse(request.PutRequest, index, config).data;
 				} else if (request.DeleteRequest) {
 					if (config.unprocessed === UnprocessedTypes.raw) {
 						return request.DeleteRequest.Key;
@@ -487,10 +526,12 @@ class Entity {
 				} else {
 					throw new Error("Unknown response format");
 				}
-			})
+			});
 		} else {
-			return []
+			unprocessed = [];
 		}
+		
+		return { unprocessed };
 	}
 
 	applyBulkGetResponseFormatting({
@@ -526,9 +567,9 @@ class Entity {
 				const slot = orderMaintainer.getOrder(item);
 				const formatted = this.formatResponse({Item: item}, index, config);
 				if (slot !== -1) {
-					resultsAll[slot] = formatted;
+					resultsAll[slot] = formatted.data;
 				} else {
-					resultsAll.push(formatted);
+					resultsAll.push(formatted.data);
 				}
 			}
 		}
@@ -541,14 +582,16 @@ class Entity {
 		}
 		try {
 			let results = {};
-			if (config.raw && !config._isPagination) {
+			if (validations.isFunction(config.parse)) {
+				results = config.parse(config, response);
+			} else if (config.raw && !config._isPagination) {
 				if (response.TableName) {
 					results = {};
 				} else {
 					results = response;
 				}
 			} else if (config.raw && (config._isPagination || config.lastEvaluatedKeyRaw)) {
-				return [response.LastEvaluatedKey || null, response];
+				results = response;
 			} else {
 				if (response.Item) {
 					if (config.ignoreOwnership || this.ownsItem(response.Item)) {
@@ -573,18 +616,18 @@ class Entity {
 						results = null;
 					}
 				} else if (config._objectOnEmpty) {
-					return {};
+					return { data: {} };
 				} else {
 					results = null;
 				}
 			}
 
-			if (config._isPagination) {
-				let nextPage = this._formatReturnPager(config, index, response.LastEvaluatedKey, results[results.length - 1]);
-				results = [nextPage, results];
+			if (config._isPagination || response.LastEvaluatedKey) {
+				const nextPage = this._formatReturnPager(config, response.LastEvaluatedKey);
+				return { cursor: nextPage ?? null, data: results };
 			}
 
-			return results;
+			return { data: results };
 
 		} catch (err) {
 			if (config.originalErr || stackTrace === undefined) {
@@ -608,16 +651,20 @@ class Entity {
 		return this.formatResponse(item, TableIndex, config);
 	}
 
-	_formatReturnPager(config, index, lastEvaluatedKey, lastReturned) {
-		let page = lastEvaluatedKey || null;
-		if (config.pager !== Pager.raw) {
-			page = this._formatKeysToItem(index, page, lastReturned);
-			if (page && config.pager === Pager.named) {
-				page[this.identifiers.entity] = this.getName();
-				page[this.identifiers.version] = this.getVersion();
-			}
+	_formatReturnPager(config, lastEvaluatedKey) {
+		let page = lastEvaluatedKey ?? null;
+		if (config.raw || config.pager === Pager.raw) {
+			return page;
+		}
+		return config.formatCursor.serialize(page) ?? null;
+	}
+
+	_formatExclusiveStartKey(config) {
+		let exclusiveStartKey = config.cursor;
+		if (config.raw || config.pager === Pager.raw) {
+			return exclusiveStartKey ?? null;
 		}
-		return page;
+		return config.formatCursor.deserialize(exclusiveStartKey) ?? null;
 	}
 
 	_getTableName() {
@@ -674,10 +721,13 @@ class Entity {
 		return value;
 	}
 
-	_normalizePagesValue(value = Number.MAX_SAFE_INTEGER) {
+	_normalizePagesValue(value) {
+		if (value === AllPages) {
+			return value;
+		}
 		value = parseInt(value);
 		if (isNaN(value) || value < 1) {
-			throw new e.ElectroError(e.ErrorCodes.InvalidPagesOption, "Query option 'pages' must be of type 'number' and greater than zero.");
+			throw new e.ElectroError(e.ErrorCodes.InvalidPagesOption, `Query option 'pages' must be of type 'number' and greater than zero or the string value '${AllPages}'`);
 		}
 		return value;
 	}
@@ -819,17 +869,34 @@ class Entity {
 			pager: Pager.named,
 			unprocessed: UnprocessedTypes.item,
 			response: 'default',
+			cursor: null,
+			data: 'attributes',
 			ignoreOwnership: false,
 			_isPagination: false,
 			_isCollectionQuery: false,
-			pages: undefined,
+			pages: 1,
 			listeners: [],
 			preserveBatchOrder: false,
 			attributes: [],
 			terminalOperation: undefined,
+			formatCursor: u.cursorFormatter,
+			order: undefined,
 		};
 
 		config = options.reduce((config, option) => {
+			if (typeof option.order === 'string') {
+				switch (option.order.toLowerCase()) {
+					case 'asc':
+						config.params[ResultOrderParam] = ResultOrderOption.asc;
+						break;
+					case 'desc':
+						config.params[ResultOrderParam] = ResultOrderOption.desc;
+						break;
+					default:
+						throw new e.ElectroError(e.ErrorCodes.InvalidOptions, `Invalid value for query option "order" provided. Valid options include 'asc' and 'desc, received: "${option.order}"`);
+				}
+			}
+
 			if (typeof option.response === 'string' && option.response.length) {
 				const format = 	ReturnValues[option.response];
 				if (format === undefined) {
@@ -839,6 +906,18 @@ class Entity {
 				config.params.ReturnValues = FormatToReturnValues[format];
 			}
 
+			if (option.formatCursor) {
+				const isValid = ['serialize', 'deserialize'].every(method => 
+					method in option.formatCursor && 
+						validate.isFunction(option.formatCursor[method])
+				);
+				if (isValid) {
+					config.formatCursor = option.formatCursor;
+				} else {
+					throw new e.ElectroError(e.ErrorCodes.InvalidOptions, `Invalid value for query option "formatCursor" provided. Formatter interface must have serialize and deserialize functions`);
+				}
+			}
+
 			if (option.terminalOperation in TerminalOperation) {
 				config.terminalOperation = TerminalOperation[option.terminalOperation];
 			}
@@ -881,6 +960,22 @@ class Entity {
 				config.unprocessed = UnprocessedTypes.raw;
 			}
 
+			if (option.cursor) {
+				config.cursor = option.cursor;
+			}
+
+			if (option.data) {
+				config.data = option.data;
+				switch(option.data) {
+					case 'raw':
+						config.raw = true;
+						break;
+					case 'includeKeys':
+						config.includeKeys = true;
+						break;
+				}
+			}
+
 			if (option.limit !== undefined) {
 				config.limit = option.limit;
 				config.params.Limit = option.limit;
@@ -945,15 +1040,7 @@ class Entity {
 			}
 		}
 
-		if (Object.keys(config.page || {}).length) {
-			if (config.raw || config.pager === Pager.raw) {
-				parameters.ExclusiveStartKey = config.page;
-			} else {
-				parameters.ExclusiveStartKey = this._formatSuppliedPager(params.IndexName, config.page);
-			}
-		}
-
-		return {parameters, config};
+		return { parameters, config };
 	}
 
 	addListeners(logger) {
@@ -2091,12 +2178,17 @@ class Entity {
 		}
 		let key = prefix;
 		for (let i = 0; i < labels.length; i++) {
-			let {name, label} = labels[i];
-
+			const { name, label } = labels[i];
+			const attribute = this.model.schema.getAttribute(name);
+			let value = supplied[name];
 			if (supplied[name] === undefined && excludeLabelTail) {
 				break;
 			}
 
+			if (attribute && validations.isFunction(attribute.format)) {
+				value = attribute.format(`${value}`);
+			}
+
 			if (isCustom) {
 				key = `${key}${label}`;
 			} else {
@@ -2107,7 +2199,7 @@ class Entity {
 				break;
 			}
 
-			key = `${key}${supplied[name]}`;
+			key = `${key}${value}`;
 		}
 
 		return u.formatKeyCasing(key, casing);

package/src/errors.js

@@ -217,6 +217,12 @@ const ErrorCodes = {
     name: "NoOwnerForPager",
     sym: ErrorCode,
   },
+  NoOwnerForCursor: {
+    code: 5004,
+    section: "no-owner-for-pager",
+    name: "NoOwnerForCursor",
+    sym: ErrorCode,
+  },
   PagerNotUnique: {
     code: 5005,
     section: "pager-not-unique",

package/src/filters.js

@@ -84,7 +84,7 @@ class FilterFactory {
 			injected[name] = {
 				name: name,
 				action: this.buildClause(filter),
-				children: ["params", "go", "page", "filter", ...modelFilters],
+				children: ["params", "go", "filter", ...modelFilters],
 			};
 		}
 		filterChildren.push("filter");
@@ -93,7 +93,7 @@ class FilterFactory {
 			action: (entity, state, fn) => {
 				return this.buildClause(fn)(entity, state);
 			},
-			children: ["params", "go", "page", "filter", ...modelFilters],
+			children: ["params", "go", "filter", ...modelFilters],
 		};
 		for (let parent of filterParents) {
 			injected[parent] = { ...injected[parent] };

package/src/operations.js

@@ -388,7 +388,7 @@ class AttributeOperationProxy {
                             const attributeValues = [];
                             let hasNestedValue = false;
                             for (let value of values) {
-                                value = target.format(value);
+                                value = target.applyFixings(value);
                                 // template.length is to see if function takes value argument
                                 if (template.length > 3) {
                                     if (seen.has(value)) {