@peerbit/indexer-sqlite3 1.1.4 → 1.2.0

package/src/schema.ts

@@ -14,8 +14,9 @@ import {
 	serialize,
 	variant,
 } from "@dao-xyz/borsh";
-import { toHexString } from "@peerbit/crypto";
+import { fromHexString, toHexString } from "@peerbit/crypto";
 import * as types from "@peerbit/indexer-interface";
+import { type PlanningSession, flattenQuery } from "./query-planner.js";
 
 const SQLConversionMap: any = {
 	u8: "INTEGER",
@@ -54,6 +55,15 @@ export type BindableValue =
 	| ArrayBuffer
 	| null;
 
+let JSON_GROUP_ARRAY = "json_group_array";
+let JSON_OBJECT = "distinct json_object";
+
+export const u64ToI64 = (u64: bigint | number) => {
+	return (typeof u64 === "number" ? BigInt(u64) : u64) - 9223372036854775808n;
+};
+export const i64ToU64 = (i64: number | bigint) =>
+	(typeof i64 === "number" ? BigInt(i64) : i64) + 9223372036854775808n;
+
 export const convertToSQLType = (
 	value: boolean | bigint | string | number | Uint8Array,
 	type?: FieldType,
@@ -64,12 +74,18 @@ export const convertToSQLType = (
 		if (type === "bool") {
 			return value ? 1 : 0;
 		}
+		if (type === "u64") {
+			// shift to fit in i64
+
+			return u64ToI64(value as number | bigint);
+		}
 	}
 	return value as BindableValue;
 };
 
 const nullAsUndefined = (value: any) => (value === null ? undefined : value);
-export const escapeColumnName = (name: string) => `"${name}"`;
+export const escapeColumnName = (name: string, char = '"') =>
+	`${char}${name}${char}`;
 
 export class MissingFieldError extends Error {
 	constructor(message: string) {
@@ -100,9 +116,15 @@ export const convertFromSQLType = (
 			: nullAsUndefined(value);
 	}
 	if (type === "u64") {
-		return typeof value === "number" || typeof value === "string"
-			? BigInt(value)
-			: nullAsUndefined(value);
+		if (typeof value === "number" || typeof value === "bigint") {
+			return i64ToU64(value as number | bigint); // TODO is not always value type bigint?
+		}
+		if (value == null) {
+			return nullAsUndefined(value);
+		}
+		throw new Error(
+			`Unexpected value type for value ${value} expected number or bigint for u64 field`,
+		);
 	}
 	return nullAsUndefined(value);
 };
@@ -135,6 +157,7 @@ type SQLField = {
 	type: string;
 	isPrimary: boolean;
 	from: Field | undefined;
+	unwrappedType: FieldType | undefined;
 	path: string[];
 	describesExistenceOfAnother?: string;
 };
@@ -155,6 +178,7 @@ export interface Table {
 	parent: Table | undefined;
 	referencedInArray: boolean;
 	isSimpleValue: boolean;
+	indices: Set<string>;
 }
 
 export const getSQLTable = (
@@ -203,6 +227,7 @@ export const getSQLTable = (
 			referencedInArray: false,
 			isSimpleValue: false,
 			inline,
+			indices: new Set<string>(),
 		};
 		ret.push(table);
 		for (const dep of dependencies) {
@@ -242,6 +267,14 @@ export const getTableName = (
 	path: string[] = [],
 	clazz: string | Constructor<any>,
 ) => {
+	let pathKey = path.length > 0 ? path.join("__") + "__" : "";
+	if (typeof clazz !== "string") {
+		const tableName = (clazz as any)["__table_" + pathKey];
+		if (tableName) {
+			return tableName;
+		}
+	}
+
 	let name: string = typeof clazz === "string" ? clazz : getNameOfClass(clazz);
 
 	// prefix the generated table name so that the name is a valid SQL identifier (table name)
@@ -249,9 +282,11 @@ export const getTableName = (
 
 	// leading _ to allow path to have numbers
 
-	const ret =
-		(path.length > 0 ? path.join("__") + "__" : "") +
-		name.replace(/[^a-zA-Z0-9_]/g, "_");
+	const ret = pathKey + name.replace(/[^a-zA-Z0-9_]/g, "_");
+
+	if (typeof clazz !== "string") {
+		(clazz as any)["__table_" + pathKey] = ret;
+	}
 	return ret;
 };
 
@@ -301,13 +336,14 @@ export const getSQLFields = (
 			? addJoinFieldFromParent
 			: (fields: SQLField[], contstraints: SQLConstraint[]) => {
 					// we resolve primary field here since it might be unknown until this point
-					const primaryField =
+					const parentPrimaryField =
 						primary != null
 							? sqlFields.find((field) => field.name === primary)
 							: undefined;
-					const parentPrimaryFieldName = primaryField?.key || CHILD_TABLE_ID;
-					const parentPrimaryFieldType = primaryField
-						? primaryField.type
+					const parentPrimaryFieldName =
+						parentPrimaryField?.key || CHILD_TABLE_ID;
+					const parentPrimaryFieldType = parentPrimaryField
+						? parentPrimaryField.type
 						: "INTEGER";
 
 					fields.unshift(
@@ -318,6 +354,7 @@ export const getSQLFields = (
 							type: "INTEGER",
 							isPrimary: true,
 							from: undefined,
+							unwrappedType: undefined,
 							path: [CHILD_TABLE_ID],
 						},
 
@@ -327,8 +364,9 @@ export const getSQLFields = (
 							key: PARENT_TABLE_ID,
 							definition: `${PARENT_TABLE_ID} ${parentPrimaryFieldType}`,
 							type: parentPrimaryFieldType,
+							from: parentPrimaryField?.from,
+							unwrappedType: parentPrimaryField?.unwrappedType,
 							isPrimary: false,
-							from: undefined,
 							path: [PARENT_TABLE_ID],
 						},
 					);
@@ -395,6 +433,7 @@ export const getSQLFields = (
 							type: "INTEGER",
 							isPrimary: false,
 							from: undefined,
+							unwrappedType: undefined,
 							path: [ARRAY_INDEX_COLUMN],
 						},
 						...table.fields.slice(2),
@@ -425,6 +464,7 @@ export const getSQLFields = (
 			type: fieldType,
 			isPrimary,
 			from: field,
+			unwrappedType: unwrapNestedType(field.type),
 			path: [...path.slice(1), key],
 		});
 	};
@@ -512,6 +552,7 @@ export const getSQLFields = (
 				type: "bool",
 				isPrimary: false,
 				from: undefined,
+				unwrappedType: undefined,
 				path: [...path.slice(1), key],
 				describesExistenceOfAnother: path[path.length - 1],
 			});
@@ -613,7 +654,7 @@ const getTableFromValue = (
 	field: Field,
 	value?: any,
 ): Table => {
-	let clazzName: string | undefined = undefined;
+	let clazzName: string | Constructor<any> | undefined = undefined;
 	if (!isNestedType(field.type)) {
 		clazzName = WRAPPED_SIMPLE_VALUE_VARIANT;
 	} else {
@@ -632,7 +673,7 @@ const getTableFromValue = (
 				continue;
 			}
 			if (ctor) {
-				clazzName = getNameOfClass(ctor);
+				clazzName = ctor;
 				break;
 			}
 		}
@@ -764,7 +805,7 @@ export const insert = async (
 					for (const _field of subTable.fields) {
 						bindableValues.push(null);
 					}
-					bindableValues[bindableValues.length - 1] = false; // assign the value "false" to the exist field column
+					bindableValues[bindableValues.length - 1] = 0; // assign the value "false" to the exist field column
 					continue;
 				}
 
@@ -773,7 +814,7 @@ export const insert = async (
 						if (table.inline) {
 							bindableValues.push(...values); // insert the bindable values into the parent bindable array
 							if (field.type instanceof OptionKind) {
-								bindableValues.push(true); // assign the value "true" to the exist field column
+								bindableValues.push(1); // assign the value "true" to the exist field column
 							}
 							return undefined;
 						} else {
@@ -889,7 +930,7 @@ export const generateSelectQuery = (
 	table: Table,
 	selects: { from: string; as: string }[],
 ) => {
-	return `SELECT ${selects.map((x) => `${x.from} as ${x.as}`).join(", ")} FROM ${table.name}`;
+	return `select ${selects.map((x) => `${x.from} as ${x.as}`).join(", ")} FROM ${table.name}`;
 };
 
 export const selectAllFieldsFromTables = (
@@ -901,24 +942,26 @@ export const selectAllFieldsFromTables = (
 			from: string;
 			as: string;
 		}[];
-		joins: Map<string, JoinTable>;
+		joins: Map<string, JoinOrRootTable>;
+		groupBy: string | undefined;
 	}[] = [];
 
 	for (const table of tables) {
-		const { selects, join: joinFromSelect } = selectAllFieldsFromTable(
-			table,
-			shape,
-		);
-		selectsPerTable.push({ selects, joins: joinFromSelect });
+		const {
+			selects,
+			join: joinFromSelect,
+			groupBy,
+		} = selectAllFieldsFromTable(table, shape);
+
+		selectsPerTable.push({ selects, joins: joinFromSelect, groupBy });
 	}
 
 	// pad with empty selects to make sure all selects have the same length
-	/* const maxSelects = Math.max(...selectsPerTable.map(x => x.selects.length)); */
-
 	let newSelects: {
 		from: string;
 		as: string;
 	}[][] = [];
+
 	for (const [i, selects] of selectsPerTable.entries()) {
 		const newSelect = [];
 		for (const [j, selectsOther] of selectsPerTable.entries()) {
@@ -931,11 +974,6 @@ export const selectAllFieldsFromTables = (
 			}
 		}
 		newSelects.push(newSelect);
-
-		/* let pad = 0;
-		while (select.selects.length < maxSelects) {
-			select.selects.push({ from: "NULL", as: `'pad#${++pad}'` });
-		} */
 	}
 	// also return table name
 	for (const [i, selects] of selectsPerTable.entries()) {
@@ -952,8 +990,67 @@ export const selectAllFieldsFromTable = (
 	let stack: { table: Table; shape?: types.Shape }[] = [{ table, shape }];
 	let join: Map<string, JoinTable> = new Map();
 	const fieldResolvers: { from: string; as: string }[] = [];
+	let groupByParentId = false;
 	for (const tableAndShape of stack) {
-		if (!tableAndShape.table.inline) {
+		if (tableAndShape.table.referencedInArray) {
+			let selectBuilder = `${JSON_GROUP_ARRAY}(${JSON_OBJECT}(`;
+
+			groupByParentId = true; // we need to group by the parent id as else we will not be returned with more than 1 result
+
+			let first = false;
+			const as = createReconstructReferenceName(tableAndShape.table);
+
+			for (const field of tableAndShape.table.fields) {
+				if (
+					(field.isPrimary ||
+						!tableAndShape.shape ||
+						matchFieldInShape(tableAndShape.shape, [], field) ||
+						// also always include the index field
+						field.name === ARRAY_INDEX_COLUMN) &&
+					field.name !== PARENT_TABLE_ID
+				) {
+					let resolveField = `${as}.${escapeColumnName(field.name)}`;
+					// if field is bigint we need to convert it to string, so that later in a JSON.parse scenario it is not converted to a number, but remains a string until we can convert it back to a bigint manually
+					if (field.unwrappedType === "u64") {
+						resolveField = `CAST(${resolveField} AS TEXT)`;
+					}
+
+					// if field is blob we need to convert it to hex string
+					if (field.type === "BLOB") {
+						resolveField = `HEX(${resolveField})`;
+					}
+
+					if (first) {
+						selectBuilder += `, `;
+					}
+					first = true;
+					selectBuilder += `${escapeColumnName(field.name, "'")}, ${resolveField}`;
+				}
+			}
+			selectBuilder += `))  `; // FILTER (WHERE ${tableAndShape.table.name}.${tableAndShape.table.primary} IS NOT NULL)
+
+			fieldResolvers.push({
+				from: selectBuilder,
+				as,
+			});
+
+			join.set(createReconstructReferenceName(tableAndShape.table), {
+				as,
+				table: tableAndShape.table,
+				type: "left" as const,
+				columns: [],
+			});
+		} else if (!tableAndShape.table.inline) {
+			// we end up here when we have simple joins we want to make that are not arrays, and not inlined
+			if (tableAndShape.table.parent != null) {
+				join.set(createReconstructReferenceName(tableAndShape.table), {
+					as: tableAndShape.table.name,
+					table: tableAndShape.table,
+					type: "left" as const,
+					columns: [],
+				});
+			}
+
 			for (const field of tableAndShape.table.fields) {
 				if (
 					field.isPrimary ||
@@ -969,10 +1066,6 @@ export const selectAllFieldsFromTable = (
 		}
 
 		for (const child of tableAndShape.table.children) {
-			if (child.referencedInArray) {
-				continue;
-			}
-
 			let childShape: types.Shape | undefined = undefined;
 			if (tableAndShape.shape) {
 				const parentPath = child.parentPath?.slice(1);
@@ -991,11 +1084,7 @@ export const selectAllFieldsFromTable = (
 							? maybeShape[0]
 							: maybeShape;
 			}
-
 			stack.push({ table: child, shape: childShape });
-			if (!child.inline) {
-				join.set(child.name, { as: child.name, table: child });
-			}
 		}
 	}
 
@@ -1004,6 +1093,10 @@ export const selectAllFieldsFromTable = (
 	}
 
 	return {
+		groupBy: groupByParentId
+			? `${table.name}.${escapeColumnName(table.primary as string)}` ||
+				undefined
+			: undefined,
 		selects: fieldResolvers, //  `SELECT ${fieldResolvers.join(", ")} FROM ${table.name}`,
 		join,
 	};
@@ -1062,24 +1155,55 @@ export const resolveInstanceFromValue = async <
 					: maybeShape;
 
 		if (isArray) {
-			let once = false;
+			/* let once = false; */
 			let resolvedArr = [];
 
 			for (const subtable of subTables) {
-				// TODO types
-				let rootTable = getNonInlinedTable(table);
-				const arr = await resolveChildren(
-					fromTablePrefixedValues[
-						getTablePrefixedField(
-							rootTable,
-							rootTable.primary as string,
-							!tablePrefixed,
-						)
-					],
-					subtable,
-				);
-				if (arr) {
-					once = true;
+				// check if the array already in the provided row
+				let arr: any[] | undefined = undefined;
+				const tableName = createReconstructReferenceName(subtable);
+				if (fromTablePrefixedValues[tableName]) {
+					arr = JSON.parse(fromTablePrefixedValues[tableName]) as Array<any>;
+					arr = arr.filter((x) => x[subtable.primary as string] != null);
+
+					// we need to go over all fields that are to be bigints and convert
+					// them back to bigints
+					// for blob fields we need to convert them back to Uint8Array
+					for (const field of subtable.fields) {
+						if (field.name === PARENT_TABLE_ID) {
+							continue;
+						}
+						if (field.unwrappedType === "u64") {
+							for (const item of arr!) {
+								item[field.name] = BigInt(item[field.name]);
+							}
+						} else if (field.type === "BLOB") {
+							for (const item of arr!) {
+								item[field.name] = fromHexString(item[field.name]);
+							}
+						}
+					}
+				} else {
+					if (subtable.children) {
+						// TODO we only end up where when we resolve nested arrays,
+						// which shoulld instead be resolved in a nested select (with json_group_array and json_object)
+						let rootTable = getNonInlinedTable(table);
+						const parentId =
+							fromTablePrefixedValues[
+								getTablePrefixedField(
+									rootTable,
+									rootTable.primary as string,
+									!tablePrefixed,
+								)
+							];
+
+						arr = await resolveChildren(parentId, subtable);
+					} else {
+						arr = [];
+					}
+				}
+				if (arr && arr.length > 0) {
+					/* once = true; */
 					for (const element of arr) {
 						const resolved: SimpleNested | any = await resolveInstanceFromValue(
 							element,
@@ -1097,11 +1221,7 @@ export const resolveInstanceFromValue = async <
 				}
 			}
 
-			if (!once) {
-				obj[field.key] = undefined;
-			} else {
-				obj[field.key] = resolvedArr;
-			}
+			obj[field.key] = resolvedArr; // we can not do option(vec('T')) since we dont store the option type for Arrays (TODO)
 		} else {
 			// resolve nested object from row directly
 			/* let extracted: any = {} */
@@ -1234,7 +1354,7 @@ export const convertDeleteRequestToQuery = (
 ): { sql: string; bindable: any[] } => {
 	const { query, bindable } = convertRequestToQuery(
 		"delete",
-		request,
+		{ query: types.toQuery(request.query) },
 		tables,
 		table,
 	);
@@ -1251,7 +1371,7 @@ export const convertSumRequestToQuery = (
 ): { sql: string; bindable: any[] } => {
 	const { query, bindable } = convertRequestToQuery(
 		"sum",
-		request,
+		{ query: types.toQuery(request.query), key: request.key },
 		tables,
 		table,
 	);
@@ -1276,7 +1396,7 @@ export const convertCountRequestToQuery = (
 ): { sql: string; bindable: any[] } => {
 	const { query, bindable } = convertRequestToQuery(
 		"count",
-		request,
+		{ query: request?.query ? types.toQuery(request.query) : undefined },
 		tables,
 		table,
 	);
@@ -1286,13 +1406,76 @@ export const convertCountRequestToQuery = (
 	};
 };
 
+const buildOrderBy = (
+	sort: types.Sort[] | types.Sort | undefined,
+	tables: Map<string, Table>,
+	table: Table,
+	joinBuilder: Map<string, JoinOrRootTable>,
+	resolverBuilder: { from: string; as: string }[],
+	path: string[] = [],
+	options?: {
+		fetchAll?: boolean;
+		planner?: PlanningSession;
+	},
+) => {
+	let orderByBuilder: string | undefined = undefined;
+
+	if (
+		(!sort || (Array.isArray(sort) && sort.length === 0)) &&
+		!options?.fetchAll
+	) {
+		sort =
+			table.primary && path.length === 0
+				? [{ key: [table.primary], direction: types.SortDirection.ASC }]
+				: undefined;
+	}
+
+	if (sort) {
+		let sortArr = Array.isArray(sort) ? sort : [sort];
+		if (sortArr.length > 0) {
+			orderByBuilder = "";
+			let once = false;
+			for (const sort of sortArr) {
+				const { foreignTables, queryKey } = resolveTableToQuery(
+					table,
+					tables,
+					joinBuilder,
+					[...path, ...sort.key],
+					undefined,
+					true,
+				);
+
+				for (const foreignTable of foreignTables) {
+					if (once) {
+						orderByBuilder += ", ";
+					}
+					once = true;
+
+					foreignTable.columns.push(queryKey); // add the sort key to the list of columns that will be used for this query
+					orderByBuilder += `"${foreignTable.as}#${queryKey}" ${sort.direction === types.SortDirection.ASC ? "ASC" : "DESC"}`;
+
+					resolverBuilder.push({
+						from: `${table.name}.${escapeColumnName(queryKey)}`,
+						as: `'${foreignTable.as}#${queryKey}'`,
+					});
+				}
+			}
+		}
+	}
+
+	return { orderByBuilder };
+};
+
 export const convertSearchRequestToQuery = (
-	request: types.IterateOptions | undefined,
+	request:
+		| { query: types.Query[]; sort?: types.Sort[] | types.Sort }
+		| undefined,
 	tables: Map<string, Table>,
 	rootTables: Table[],
 	options?: {
 		shape?: types.Shape | undefined;
-		stable?: boolean;
+		fetchAll?: boolean;
+		planner?: PlanningSession;
 	},
 ): { sql: string; bindable: any[] } => {
 	let unionBuilder = "";
@@ -1303,30 +1486,32 @@ export const convertSearchRequestToQuery = (
 
 	const selectsPerTable = selectAllFieldsFromTables(rootTables, options?.shape);
 	let bindableBuilder: any[] = [];
+
 	for (const [i, table] of rootTables.entries()) {
-		const { selects, joins: joinFromSelect } = selectsPerTable[i];
-		const selectQuery = generateSelectQuery(table, selects);
+		const { selects, joins, groupBy } = selectsPerTable[i];
+
 		try {
-			const { orderBy, query, bindable } = convertRequestToQuery(
-				"iterate",
-				request,
+			const { orderByBuilder } = buildOrderBy(
+				request?.sort,
 				tables,
 				table,
-				joinFromSelect,
+				joins,
+				selects,
 				[],
-				{
-					stable: options?.stable,
-				},
+				options,
 			);
-			unionBuilder += `${unionBuilder.length > 0 ? " UNION ALL " : ""} ${selectQuery} ${query}`;
-			orderByClause =
-				orderBy?.length > 0
-					? orderByClause.length > 0
-						? orderByClause + ", " + orderBy
-						: orderBy
-					: orderByClause;
-			matchedOnce = true;
-			bindableBuilder.push(...bindable);
+
+			if (!orderByClause && orderByBuilder) {
+				// assume all order by clauses will be the same
+				orderByClause =
+					orderByBuilder.length > 0
+						? orderByClause.length > 0
+							? orderByClause + ", " + orderByBuilder
+							: orderByBuilder
+						: orderByClause;
+			}
+
+			//orderByAddedOnce = true;
 		} catch (error) {
 			if (error instanceof MissingFieldError) {
 				lastError = error;
@@ -1334,6 +1519,33 @@ export const convertSearchRequestToQuery = (
 			}
 			throw error;
 		}
+
+		const selectQuery = generateSelectQuery(table, selects);
+
+		for (const flattenRequest of flattenQuery(request)) {
+			try {
+				const { query, bindable } = convertRequestToQuery(
+					"iterate",
+					flattenRequest,
+					tables,
+					table,
+					new Map(joins), // copy the map, else we might might do unececessary joins
+					[],
+					options,
+				);
+
+				unionBuilder += `${unionBuilder.length > 0 ? " UNION " : ""} ${selectQuery} ${query} ${groupBy ? "GROUP BY " + groupBy : ""}`;
+				matchedOnce = true;
+				bindableBuilder.push(...bindable);
+			} catch (error) {
+				if (error instanceof MissingFieldError) {
+					lastError = error;
+					orderByClause = "";
+					continue;
+				}
+				throw error;
+			}
+		}
 	}
 
 	if (!matchedOnce) {
@@ -1341,20 +1553,43 @@ export const convertSearchRequestToQuery = (
 	}
 
 	return {
-		sql: `${unionBuilder} ${orderByClause ? "ORDER BY " + orderByClause : ""} limit ? offset ?`,
+		sql: `${unionBuilder} ${orderByClause ? "ORDER BY " + orderByClause : ""} ${options?.fetchAll ? "" : "limit ? offset ?"}`,
 		bindable: bindableBuilder,
 	};
 };
 
-type SearchQueryParts = { query: string; orderBy: string; bindable: any[] };
-type CountQueryParts = { query: string; join: string; bindable: any[] };
+type SearchQueryParts = {
+	query: string;
+	/* orderBy: string; */
+	bindable: any[];
+	selects: string[];
+};
+type CountQueryParts = {
+	query: string;
+	join: string;
+	bindable: any[];
+	selects: string[];
+};
 
-function isIterateRequest(
-	request: any,
-	type: string,
-): request is types.IterateOptions | undefined {
-	return type === "iterate";
-}
+const getOrSetRootTable = (
+	joinBuilder: Map<string, JoinOrRootTable>,
+	table: Table,
+) => {
+	const refName = createQueryTableReferenceName(table);
+	let ref = joinBuilder.get(refName);
+	if (ref) {
+		return ref;
+	}
+	const join = {
+		// add the root as a join even though it is not, just so we can collect the columns it will be queried
+		table: table,
+		type: "root" as const,
+		as: table.name,
+		columns: [],
+	};
+	joinBuilder.set(refName, join);
+	return join;
+};
 
 const convertRequestToQuery = <
 	T extends "iterate" | "count" | "sum" | "delete",
@@ -1363,26 +1598,40 @@ const convertRequestToQuery = <
 	type: T,
 	request:
 		| (T extends "iterate"
-				? types.IterateOptions
+				? {
+						query?: types.Query[];
+						sort?: types.Sort[] | types.Sort;
+					}
 				: T extends "count"
-					? types.CountOptions
+					? {
+							query?: types.Query[];
+						}
 					: T extends "delete"
-						? types.DeleteOptions
-						: types.SumOptions)
+						? {
+								query?: types.Query[];
+							}
+						: {
+								query?: types.Query[];
+								key: string | string[];
+							})
 		| undefined,
 	tables: Map<string, Table>,
 	table: Table,
-	extraJoin?: Map<string, JoinTable>,
+	extraJoin?: Map<string, JoinOrRootTable>,
 	path: string[] = [],
 	options?: {
-		stable?: boolean;
+		fetchAll?: boolean;
+		planner?: PlanningSession;
 	},
 ): R => {
 	let whereBuilder = "";
 	let bindableBuilder: any[] = [];
-	let orderByBuilder: string | undefined = undefined;
+	/* let orderByBuilder: string | undefined = undefined; */
 	/* let tablesToSelect: string[] = [table.name]; */
-	let joinBuilder: Map<string, JoinTable> = extraJoin || new Map();
+	let joinBuilder: Map<string, JoinOrRootTable> = extraJoin || new Map();
+
+	getOrSetRootTable(joinBuilder, table);
+
 	const coercedQuery = types.toQuery(request?.query);
 	if (coercedQuery.length === 1) {
 		const { where, bindable } = convertQueryToSQLQuery(
@@ -1391,6 +1640,8 @@ const convertRequestToQuery = <
 			table,
 			joinBuilder,
 			path,
+			undefined,
+			0,
 		);
 		whereBuilder += where;
 		bindableBuilder.push(...bindable);
@@ -1401,14 +1652,19 @@ const convertRequestToQuery = <
 			table,
 			joinBuilder,
 			path,
+			undefined,
+			0,
 		);
 		whereBuilder += where;
 		bindableBuilder.push(...bindable);
 	}
 
-	if (isIterateRequest(request, type)) {
+	/* if (isIterateRequest(request, type)) {
 		let sort = request?.sort;
-		if (!sort && options?.stable) {
+		if (
+			(!sort || (Array.isArray(sort) && sort.length === 0)) &&
+			!options?.fetchAll
+		) {
 			sort =
 				table.primary && path.length === 0
 					? [{ key: [table.primary], direction: types.SortDirection.ASC }]
@@ -1429,61 +1685,97 @@ const convertRequestToQuery = <
 						undefined,
 						true,
 					);
-					for (const table of foreignTables) {
+
+					for (const foreignTable of foreignTables) {
 						if (once) {
 							orderByBuilder += ", ";
 						}
 						once = true;
-						orderByBuilder += `${table.as}.${queryKey} ${sort.direction === types.SortDirection.ASC ? "ASC" : "DESC"}`;
+
+						foreignTable.columns.push(queryKey); // add the sort key to the list of columns that will be used for this query
+
+						orderByBuilder += `${foreignTable.as}.${queryKey} ${sort.direction === types.SortDirection.ASC ? "ASC" : "DESC"}`;
 					}
 				}
-
-				/* orderByBuilder += request.sort
-					.map(
-						(sort) =>
-							`${table.name}.${sort.key} ${sort.direction === types.SortDirection.ASC ? "ASC" : "DESC"}`
-					)
-					.join(", "); */
 			}
 		}
-	}
+	} */
 	const where = whereBuilder.length > 0 ? "where " + whereBuilder : undefined;
 
 	if (extraJoin && extraJoin.size > 0) {
 		insertMapIntoMap(joinBuilder, extraJoin);
 	}
-	let join = buildJoin(joinBuilder, type === "iterate" ? true : false);
+	let { join } = buildJoin(joinBuilder, options);
 
 	const query = `${join ? join : ""} ${where ? where : ""}`;
 
 	return {
 		query,
-		orderBy: orderByBuilder,
+		/* orderBy: orderByBuilder, */
 		bindable: bindableBuilder,
 	} as R;
 };
 
 export const buildJoin = (
-	joinBuilder: Map<string, JoinTable>,
-	resolveAllColumns: boolean,
-) => {
-	let joinTypeDefault = resolveAllColumns
-		? /* "FULL OUTER JOIN" */ "LEFT OUTER JOIN"
-		: "JOIN";
+	joinBuilder: Map<string, JoinOrRootTable>,
+	options?: {
+		planner?: PlanningSession;
+	},
+): { join: string } => {
+	/* let joinTypeDefault = resolveAllColumns
+		? "CROSS JOIN"
+		: "JOIN"; */
 	let join = "";
+
+	for (const [_key, table] of joinBuilder) {
+		if (table.type !== "root") {
+			continue;
+		}
+		const out = _buildJoin(table, options);
+		join += out.join;
+	}
 	for (const [_key, table] of joinBuilder) {
+		if (table.type === "root") {
+			continue;
+		}
+		const out = _buildJoin(table, options);
+		join += out.join;
+	}
+	return { join };
+};
+
+const _buildJoin = (
+	table: JoinOrRootTable,
+	options?: {
+		planner?: PlanningSession;
+	},
+) => {
+	let join = "";
+	let indexedBy: string | undefined = undefined;
+	if (table.type !== "root") {
+		table!.columns.push(PARENT_TABLE_ID); // we unshift because we join on the parent id before where clause
+	}
+
+	if (table!.columns.length > 0) {
+		const usedColumns = removeDuplicatesOrdered(table!.columns);
+		indexedBy = options?.planner
+			? ` INDEXED BY ${options.planner.resolveIndex(table.table.name, usedColumns)} `
+			: "";
+	}
+
+	if (table.type !== "root") {
 		let nonInlinedParent =
 			table.table.parent && getNonInlinedTable(table.table.parent);
 		if (!nonInlinedParent) {
 			throw new Error("Unexpected: missing parent");
 		}
-
-		let joinType = table.table.referencedInArray
-			? /* "FULL OUTER JOIN" */ "LEFT OUTER JOIN"
-			: joinTypeDefault;
-		join += `${joinType} ${table.table.name} AS ${table.as} ON ${nonInlinedParent.name}.${nonInlinedParent.primary} = ${table.as}.${PARENT_TABLE_ID} `;
+		let joinType = table.type === "cross" ? "LEFT JOIN" : "LEFT JOIN";
+		join += ` ${joinType} ${table.table.name} AS ${table.as} ${indexedBy} ON ${nonInlinedParent.name}.${nonInlinedParent.primary} = ${table.as}.${PARENT_TABLE_ID} `;
+	} else if (indexedBy) {
+		join += indexedBy;
 	}
-	return join;
+
+	return { join };
 };
 
 const insertMapIntoMap = (map: Map<string, any>, insert: Map<string, any>) => {
@@ -1496,9 +1788,10 @@ export const convertQueryToSQLQuery = (
 	query: types.Query,
 	tables: Map<string, Table>,
 	table: Table,
-	joinBuilder: Map<string, JoinTable>,
-	path: string[] = [],
-	tableAlias: string | undefined = undefined,
+	joinBuilder: Map<string, JoinOrRootTable>,
+	path: string[],
+	tableAlias: string | undefined,
+	skipKeys: number,
 ): { where: string; bindable: any[] } => {
 	let whereBuilder = "";
 	let bindableBuilder: any[] = [];
@@ -1507,7 +1800,8 @@ export const convertQueryToSQLQuery = (
 	const handleAnd = (
 		queries: types.Query[],
 		path: string[],
-		tableAlias?: string,
+		tableAlias: string | undefined,
+		keysOffset: number,
 	) => {
 		for (const query of queries) {
 			const { where, bindable } = convertQueryToSQLQuery(
@@ -1517,6 +1811,7 @@ export const convertQueryToSQLQuery = (
 				joinBuilder,
 				path,
 				tableAlias,
+				keysOffset,
 			);
 			whereBuilder =
 				whereBuilder.length > 0 ? `(${whereBuilder}) AND (${where})` : where;
@@ -1532,16 +1827,18 @@ export const convertQueryToSQLQuery = (
 			joinBuilder,
 			path,
 			tableAlias,
+			skipKeys,
 		);
 		whereBuilder += where;
 		bindableBuilder.push(...bindable);
 	} else if (query instanceof types.Nested) {
 		let joinPrefix = "__" + String(tables.size);
-		path = [...path, query.path];
-		handleAnd(query.query, path, joinPrefix);
+		path = [...path, ...query.path];
+		let newSkipKeys = skipKeys + query.path.length;
+		handleAnd(query.query, path, joinPrefix, newSkipKeys);
 	} else if (query instanceof types.LogicalQuery) {
 		if (query instanceof types.And) {
-			handleAnd(query.and, path, tableAlias);
+			handleAnd(query.and, path, tableAlias, skipKeys);
 		} else if (query instanceof types.Or) {
 			for (const subquery of query.or) {
 				const { where, bindable } = convertQueryToSQLQuery(
@@ -1551,9 +1848,10 @@ export const convertQueryToSQLQuery = (
 					joinBuilder,
 					path,
 					tableAlias,
+					skipKeys,
 				);
 				whereBuilder =
-					whereBuilder.length > 0 ? `(${whereBuilder}) OR (${where})` : where;
+					whereBuilder.length > 0 ? `(${whereBuilder}) OR(${where})` : where;
 				bindableBuilder.push(...bindable);
 			}
 		} else if (query instanceof types.Not) {
@@ -1564,8 +1862,9 @@ export const convertQueryToSQLQuery = (
 				joinBuilder,
 				path,
 				tableAlias,
+				skipKeys,
 			);
-			whereBuilder = `NOT (${where})`;
+			whereBuilder = `NOT(${where})`;
 			bindableBuilder.push(...bindable);
 		} else {
 			throw new Error("Unsupported query type: " + query.constructor.name);
@@ -1584,38 +1883,54 @@ const cloneQuery = (query: types.StateFieldQuery) => {
 	return deserialize(serialize(query), types.StateFieldQuery);
 };
 
+type JoinOrRootTable = JoinTable | RootTable;
+
 type JoinTable = {
 	table: Table;
 	as: string;
+	type: "left" | "cross";
+	columns: string[];
+};
+
+type RootTable = {
+	type: "root";
+	table: Table;
+	as: string;
+	columns: string[];
 };
 
-const createTableReferenceName = (
+/* const createQueryTableReferenceName = (
 	table: Table,
 	alias: string | undefined,
-	fieldType: FieldType,
-	joinSize: number,
 ) => {
+	
 	if (
-		!alias &&
-		(fieldType instanceof VecKind ||
-			(fieldType instanceof OptionKind &&
-				fieldType.elementType instanceof VecKind))
+		!alias 
 	) {
-		let aliasSuffix = "_" + String(joinSize);
+		let aliasSuffix =
+			"_query"; //  "_" + String(joinSize); TODO this property will make every join unique, which is not wanted unless (ever?) since we can do OR in SQL which means we can do one join and perform AND/OR logic without joining multiple times to apply multiple conditions
 		alias = aliasSuffix;
 	}
 	const tableNameAs = alias ? alias + "_" + table.name : table.name;
 	return tableNameAs;
+}; */
+
+const createQueryTableReferenceName = (table: Table) => {
+	return table.parent == null ? table.name : "_query_" + table.name;
+};
+
+const createReconstructReferenceName = (table: Table) => {
+	return table.name; /* table.parent == null ? table.name : "_rec_" + table.name; */
 };
 
 const resolveTableToQuery = (
 	table: Table,
 	tables: Map<string, Table>,
-	join: Map<string, JoinTable>,
+	join: Map<string, JoinOrRootTable>,
 	path: string[],
 	alias: string | undefined,
 	searchSelf: boolean,
-) => {
+): { queryKey: string; foreignTables: JoinOrRootTable[] } => {
 	// we are matching in two ways.
 
 	// 1. joins
@@ -1632,12 +1947,19 @@ const resolveTableToQuery = (
 		if (field) {
 			return {
 				queryKey: field.name,
-				foreignTables: [{ table, as: table.name }],
+				foreignTables: [getOrSetRootTable(join, table)],
 			};
 		}
 	}
 
-	let currentTables: JoinTable[] = [{ table, as: alias || table.name }];
+	let currentTables: JoinTable[] = [
+		{
+			table,
+			as: alias || table.name,
+			type: "cross" as const,
+			columns: [],
+		},
+	];
 	let prevTables: JoinTable[] | undefined = undefined;
 
 	// outer:
@@ -1650,20 +1972,29 @@ const resolveTableToQuery = (
 			if (!field && currentTable.children.length > 0) {
 				// second arg is needed because of polymorphic fields we might end up here intentially to check what tables to query
 				throw new MissingFieldError(
-					`Property with key "${key}" is not found in the schema ${JSON.stringify(schema.fields.map((x) => x.key))}`,
+					`Property with key "${key}" is not found in the schema ${JSON.stringify(schema.fields.map((x) => x.key))} `,
 				);
 			}
 			for (const child of currentTable.children) {
-				const tableNameAs = createTableReferenceName(
+				const tableNameAs = createQueryTableReferenceName(
 					child,
-					alias,
+					/* alias */ /* ,
 					field.type,
-					join.size,
+					join.size, */
 				);
+
 				let isMatching =
 					child.parentPath![child.parentPath!.length - 1] === key;
 				if (isMatching) {
-					const tableWithAlias = { table: child, as: tableNameAs };
+					const tableWithAlias = {
+						columns: [],
+						table: child,
+						as: tableNameAs,
+						type:
+							currentTable.children.length > 1
+								? ("left" as const)
+								: ("cross" as const),
+					};
 					if (child.isSimpleValue) {
 						if (!child.inline) {
 							join.set(tableNameAs, tableWithAlias);
@@ -1727,9 +2058,10 @@ const convertStateFieldQuery = (
 	query: types.StateFieldQuery,
 	tables: Map<string, Table>,
 	table: Table,
-	join: Map<string, JoinTable>,
+	join: Map<string, JoinOrRootTable>,
 	path: string[],
-	tableAlias: string | undefined = undefined,
+	tableAlias: string | undefined,
+	skipKeys: number,
 ): { where: string; bindable: any[] } => {
 	// if field id represented as foreign table, do join and compare
 	const inlinedName = getInlineTableFieldName(query.key);
@@ -1738,11 +2070,15 @@ const convertStateFieldQuery = (
 	); /* stringArraysEquals(query.key, [...table.parentPath, x.name]) )*/
 	const isForeign = !tableField; // table.fields.find(x => x.name === query.key[query.key.length - 1])
 	if (isForeign) {
+		const tablePath: string[] = [...path];
+		for (let i = skipKeys; i < query.key.length; i++) {
+			tablePath.push(query.key[i]);
+		}
 		const { queryKey, foreignTables } = resolveTableToQuery(
 			table,
 			tables,
 			join,
-			[...path, ...query.key],
+			tablePath,
 			tableAlias,
 			false,
 		);
@@ -1755,6 +2091,7 @@ const convertStateFieldQuery = (
 			if (ftable.table === table) {
 				throw new Error("Unexpected");
 			}
+
 			const { where, bindable } = convertQueryToSQLQuery(
 				query,
 				tables,
@@ -1762,6 +2099,7 @@ const convertStateFieldQuery = (
 				join,
 				path,
 				ftable.as,
+				skipKeys,
 			);
 			whereBuilder.push(where);
 			bindableBuilder.push(bindable);
@@ -1772,6 +2110,12 @@ const convertStateFieldQuery = (
 		};
 	}
 
+	const columnAggregator = join.get(createQueryTableReferenceName(table))!;
+	if (!columnAggregator) {
+		throw new Error("Unexpected");
+	}
+	columnAggregator.columns.push(inlinedName);
+
 	let bindable: any[] = [];
 	const keyWithTable =
 		(tableAlias || table.name) + "." + escapeColumnName(inlinedName);
@@ -1780,10 +2124,10 @@ const convertStateFieldQuery = (
 		let statement = "";
 
 		if (query.method === types.StringMatchMethod.contains) {
-			statement = `${keyWithTable} LIKE ?`;
+			statement = `${keyWithTable} LIKE ? `;
 			bindable.push(`%${query.value}%`);
 		} else if (query.method === types.StringMatchMethod.prefix) {
-			statement = `${keyWithTable} LIKE ?`;
+			statement = `${keyWithTable} LIKE ? `;
 			bindable.push(`${query.value}%`);
 		} else if (query.method === types.StringMatchMethod.exact) {
 			statement = `${keyWithTable} = ?`;
@@ -1802,7 +2146,7 @@ const convertStateFieldQuery = (
 	} else if (query instanceof types.IntegerCompare) {
 		if (tableField!.type === "BLOB") {
 			// TODO perf
-			where = `hex(${keyWithTable}) LIKE ?`;
+			where = `hex(${keyWithTable}) LIKE ? `;
 			bindable.push(
 				`%${toHexString(new Uint8Array([Number(query.value.value)]))}%`,
 			);
@@ -1810,20 +2154,20 @@ const convertStateFieldQuery = (
 			if (query.compare === types.Compare.Equal) {
 				where = `${keyWithTable} = ?`;
 			} else if (query.compare === types.Compare.Greater) {
-				where = `${keyWithTable} > ?`;
+				where = `${keyWithTable} > ? `;
 			} else if (query.compare === types.Compare.Less) {
-				where = `${keyWithTable} < ?`;
+				where = `${keyWithTable} <?`;
 			} else if (query.compare === types.Compare.GreaterOrEqual) {
-				where = `${keyWithTable} >= ?`;
+				where = `${keyWithTable} >= ? `;
 			} else if (query.compare === types.Compare.LessOrEqual) {
-				where = `${keyWithTable} <= ?`;
+				where = `${keyWithTable} <= ? `;
 			} else {
-				throw new Error(`Unsupported compare type: ${query.compare}`);
+				throw new Error(`Unsupported compare type: ${query.compare} `);
 			}
 
 			if (unwrapNestedType(tableField.from!.type) === "u64") {
 				// shift left because that is how we insert the value
-				bindable.push(query.value.value);
+				bindable.push(u64ToI64(query.value.value));
 			} else {
 				bindable.push(query.value.value);
 			}
@@ -1838,3 +2182,14 @@ const convertStateFieldQuery = (
 	}
 	return { where, bindable };
 };
+
+const removeDuplicatesOrdered = (arr: string[]) => {
+	let seen = new Set();
+	return arr.filter((item) => {
+		if (seen.has(item)) {
+			return false;
+		}
+		seen.add(item);
+		return true;
+	});
+};