@peerbit/indexer-sqlite3 1.1.4 → 1.2.0

package/dist/src/schema.js

@@ -8,8 +8,9 @@ var __metadata = (this && this.__metadata) || function (k, v) {
     if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
 };
 import { FixedArrayKind, OptionKind, VecKind, WrappedType, deserialize, field as fieldDecalaration, getDependencies, getSchema, serialize, variant, } from "@dao-xyz/borsh";
-import { toHexString } from "@peerbit/crypto";
+import { fromHexString, toHexString } from "@peerbit/crypto";
 import * as types from "@peerbit/indexer-interface";
+import { flattenQuery } from "./query-planner.js";
 const SQLConversionMap = {
     u8: "INTEGER",
     u16: "INTEGER",
@@ -27,17 +28,27 @@ const SQLConversionMap = {
     Date: "TEXT",
 };
 const WRAPPED_SIMPLE_VALUE_VARIANT = "wrapped";
+let JSON_GROUP_ARRAY = "json_group_array";
+let JSON_OBJECT = "distinct json_object";
+export const u64ToI64 = (u64) => {
+    return (typeof u64 === "number" ? BigInt(u64) : u64) - 9223372036854775808n;
+};
+export const i64ToU64 = (i64) => (typeof i64 === "number" ? BigInt(i64) : i64) + 9223372036854775808n;
 export const convertToSQLType = (value, type) => {
     // add bigint when https://github.com/TryGhost/node-sqlite3/pull/1501 fixed
     if (value != null) {
         if (type === "bool") {
             return value ? 1 : 0;
         }
+        if (type === "u64") {
+            // shift to fit in i64
+            return u64ToI64(value);
+        }
     }
     return value;
 };
 const nullAsUndefined = (value) => (value === null ? undefined : value);
-export const escapeColumnName = (name) => `"${name}"`;
+export const escapeColumnName = (name, char = '"') => `${char}${name}${char}`;
 export class MissingFieldError extends Error {
     constructor(message) {
         super(message);
@@ -61,9 +72,13 @@ export const convertFromSQLType = (value, type) => {
             : nullAsUndefined(value);
     }
     if (type === "u64") {
-        return typeof value === "number" || typeof value === "string"
-            ? BigInt(value)
-            : nullAsUndefined(value);
+        if (typeof value === "number" || typeof value === "bigint") {
+            return i64ToU64(value); // TODO is not always value type bigint?
+        }
+        if (value == null) {
+            return nullAsUndefined(value);
+        }
+        throw new Error(`Unexpected value type for value ${value} expected number or bigint for u64 field`);
     }
     return nullAsUndefined(value);
 };
@@ -115,6 +130,7 @@ export const getSQLTable = (ctor, path, primary, inline, addJoinField, fromOptio
             referencedInArray: false,
             isSimpleValue: false,
             inline,
+            indices: new Set(),
         };
         ret.push(table);
         for (const dep of dependencies) {
@@ -146,12 +162,21 @@ const getNameOfClass = (ctor) => {
     return name;
 };
 export const getTableName = (path = [], clazz) => {
+    let pathKey = path.length > 0 ? path.join("__") + "__" : "";
+    if (typeof clazz !== "string") {
+        const tableName = clazz["__table_" + pathKey];
+        if (tableName) {
+            return tableName;
+        }
+    }
     let name = typeof clazz === "string" ? clazz : getNameOfClass(clazz);
     // prefix the generated table name so that the name is a valid SQL identifier (table name)
     // choose prefix which is readable and explains that this is a generated table name
     // leading _ to allow path to have numbers
-    const ret = (path.length > 0 ? path.join("__") + "__" : "") +
-        name.replace(/[^a-zA-Z0-9_]/g, "_");
+    const ret = pathKey + name.replace(/[^a-zA-Z0-9_]/g, "_");
+    if (typeof clazz !== "string") {
+        clazz["__table_" + pathKey] = ret;
+    }
     return ret;
 };
 export const CHILD_TABLE_ID = "__id";
@@ -174,12 +199,12 @@ export const getSQLFields = (tableName, path, ctor, primary, addJoinFieldFromPar
         ? addJoinFieldFromParent
         : (fields, contstraints) => {
             // we resolve primary field here since it might be unknown until this point
-            const primaryField = primary != null
+            const parentPrimaryField = primary != null
                 ? sqlFields.find((field) => field.name === primary)
                 : undefined;
-            const parentPrimaryFieldName = primaryField?.key || CHILD_TABLE_ID;
-            const parentPrimaryFieldType = primaryField
-                ? primaryField.type
+            const parentPrimaryFieldName = parentPrimaryField?.key || CHILD_TABLE_ID;
+            const parentPrimaryFieldType = parentPrimaryField
+                ? parentPrimaryField.type
                 : "INTEGER";
             fields.unshift({
                 name: CHILD_TABLE_ID,
@@ -188,6 +213,7 @@ export const getSQLFields = (tableName, path, ctor, primary, addJoinFieldFromPar
                 type: "INTEGER",
                 isPrimary: true,
                 from: undefined,
+                unwrappedType: undefined,
                 path: [CHILD_TABLE_ID],
             }, 
             // foreign key parent document
@@ -196,8 +222,9 @@ export const getSQLFields = (tableName, path, ctor, primary, addJoinFieldFromPar
                 key: PARENT_TABLE_ID,
                 definition: `${PARENT_TABLE_ID} ${parentPrimaryFieldType}`,
                 type: parentPrimaryFieldType,
+                from: parentPrimaryField?.from,
+                unwrappedType: parentPrimaryField?.unwrappedType,
                 isPrimary: false,
-                from: undefined,
                 path: [PARENT_TABLE_ID],
             });
             contstraints.push({
@@ -256,6 +283,7 @@ export const getSQLFields = (tableName, path, ctor, primary, addJoinFieldFromPar
                             type: "INTEGER",
                             isPrimary: false,
                             from: undefined,
+                            unwrappedType: undefined,
                             path: [ARRAY_INDEX_COLUMN],
                         },
                         ...table.fields.slice(2),
@@ -278,6 +306,7 @@ export const getSQLFields = (tableName, path, ctor, primary, addJoinFieldFromPar
             type: fieldType,
             isPrimary,
             from: field,
+            unwrappedType: unwrapNestedType(field.type),
             path: [...path.slice(1), key],
         });
     };
@@ -352,6 +381,7 @@ export const getSQLFields = (tableName, path, ctor, primary, addJoinFieldFromPar
                 type: "bool",
                 isPrimary: false,
                 from: undefined,
+                unwrappedType: undefined,
                 path: [...path.slice(1), key],
                 describesExistenceOfAnother: path[path.length - 1],
             });
@@ -438,7 +468,7 @@ const getTableFromValue = (parentTable, tables, field, value) => {
                 continue;
             }
             if (ctor) {
-                clazzName = getNameOfClass(ctor);
+                clazzName = ctor;
                 break;
             }
         }
@@ -527,14 +557,14 @@ export const insert = async (insertFn, obj, tables, table, fields, handleNestedC
                     for (const _field of subTable.fields) {
                         bindableValues.push(null);
                     }
-                    bindableValues[bindableValues.length - 1] = false; // assign the value "false" to the exist field column
+                    bindableValues[bindableValues.length - 1] = 0; // assign the value "false" to the exist field column
                     continue;
                 }
                 await insert((values, table) => {
                     if (table.inline) {
                         bindableValues.push(...values); // insert the bindable values into the parent bindable array
                         if (field.type instanceof OptionKind) {
-                            bindableValues.push(true); // assign the value "true" to the exist field column
+                            bindableValues.push(1); // assign the value "true" to the exist field column
                         }
                         return undefined;
                     }
@@ -618,16 +648,15 @@ const matchFieldInShape = (shape, path, field) => {
 };
 export const selectChildren = (childrenTable) => "select * from " + childrenTable.name + " where " + PARENT_TABLE_ID + " = ?";
 export const generateSelectQuery = (table, selects) => {
-    return `SELECT ${selects.map((x) => `${x.from} as ${x.as}`).join(", ")} FROM ${table.name}`;
+    return `select ${selects.map((x) => `${x.from} as ${x.as}`).join(", ")} FROM ${table.name}`;
 };
 export const selectAllFieldsFromTables = (tables, shape) => {
     const selectsPerTable = [];
     for (const table of tables) {
-        const { selects, join: joinFromSelect } = selectAllFieldsFromTable(table, shape);
-        selectsPerTable.push({ selects, joins: joinFromSelect });
+        const { selects, join: joinFromSelect, groupBy, } = selectAllFieldsFromTable(table, shape);
+        selectsPerTable.push({ selects, joins: joinFromSelect, groupBy });
     }
     // pad with empty selects to make sure all selects have the same length
-    /* const maxSelects = Math.max(...selectsPerTable.map(x => x.selects.length)); */
     let newSelects = [];
     for (const [i, selects] of selectsPerTable.entries()) {
         const newSelect = [];
@@ -642,10 +671,6 @@ export const selectAllFieldsFromTables = (tables, shape) => {
             }
         }
         newSelects.push(newSelect);
-        /* let pad = 0;
-        while (select.selects.length < maxSelects) {
-            select.selects.push({ from: "NULL", as: `'pad#${++pad}'` });
-        } */
     }
     // also return table name
     for (const [i, selects] of selectsPerTable.entries()) {
@@ -657,8 +682,58 @@ export const selectAllFieldsFromTable = (table, shape) => {
     let stack = [{ table, shape }];
     let join = new Map();
     const fieldResolvers = [];
+    let groupByParentId = false;
     for (const tableAndShape of stack) {
-        if (!tableAndShape.table.inline) {
+        if (tableAndShape.table.referencedInArray) {
+            let selectBuilder = `${JSON_GROUP_ARRAY}(${JSON_OBJECT}(`;
+            groupByParentId = true; // we need to group by the parent id as else we will not be returned with more than 1 result
+            let first = false;
+            const as = createReconstructReferenceName(tableAndShape.table);
+            for (const field of tableAndShape.table.fields) {
+                if ((field.isPrimary ||
+                    !tableAndShape.shape ||
+                    matchFieldInShape(tableAndShape.shape, [], field) ||
+                    // also always include the index field
+                    field.name === ARRAY_INDEX_COLUMN) &&
+                    field.name !== PARENT_TABLE_ID) {
+                    let resolveField = `${as}.${escapeColumnName(field.name)}`;
+                    // if field is bigint we need to convert it to string, so that later in a JSON.parse scenario it is not converted to a number, but remains a string until we can convert it back to a bigint manually
+                    if (field.unwrappedType === "u64") {
+                        resolveField = `CAST(${resolveField} AS TEXT)`;
+                    }
+                    // if field is blob we need to convert it to hex string
+                    if (field.type === "BLOB") {
+                        resolveField = `HEX(${resolveField})`;
+                    }
+                    if (first) {
+                        selectBuilder += `, `;
+                    }
+                    first = true;
+                    selectBuilder += `${escapeColumnName(field.name, "'")}, ${resolveField}`;
+                }
+            }
+            selectBuilder += `))  `; // FILTER (WHERE ${tableAndShape.table.name}.${tableAndShape.table.primary} IS NOT NULL)
+            fieldResolvers.push({
+                from: selectBuilder,
+                as,
+            });
+            join.set(createReconstructReferenceName(tableAndShape.table), {
+                as,
+                table: tableAndShape.table,
+                type: "left",
+                columns: [],
+            });
+        }
+        else if (!tableAndShape.table.inline) {
+            // we end up here when we have simple joins we want to make that are not arrays, and not inlined
+            if (tableAndShape.table.parent != null) {
+                join.set(createReconstructReferenceName(tableAndShape.table), {
+                    as: tableAndShape.table.name,
+                    table: tableAndShape.table,
+                    type: "left",
+                    columns: [],
+                });
+            }
             for (const field of tableAndShape.table.fields) {
                 if (field.isPrimary ||
                     !tableAndShape.shape ||
@@ -671,9 +746,6 @@ export const selectAllFieldsFromTable = (table, shape) => {
             }
         }
         for (const child of tableAndShape.table.children) {
-            if (child.referencedInArray) {
-                continue;
-            }
             let childShape = undefined;
             if (tableAndShape.shape) {
                 const parentPath = child.parentPath?.slice(1);
@@ -691,15 +763,16 @@ export const selectAllFieldsFromTable = (table, shape) => {
                             : maybeShape;
             }
             stack.push({ table: child, shape: childShape });
-            if (!child.inline) {
-                join.set(child.name, { as: child.name, table: child });
-            }
         }
     }
     if (fieldResolvers.length === 0) {
         throw new Error("No fields to resolve");
     }
     return {
+        groupBy: groupByParentId
+            ? `${table.name}.${escapeColumnName(table.primary)}` ||
+                undefined
+            : undefined,
         selects: fieldResolvers, //  `SELECT ${fieldResolvers.join(", ")} FROM ${table.name}`,
         join,
     };
@@ -734,14 +807,48 @@ export const resolveInstanceFromValue = async (fromTablePrefixedValues, tables,
                 ? maybeShape[0]
                 : maybeShape;
         if (isArray) {
-            let once = false;
+            /* let once = false; */
             let resolvedArr = [];
             for (const subtable of subTables) {
-                // TODO types
-                let rootTable = getNonInlinedTable(table);
-                const arr = await resolveChildren(fromTablePrefixedValues[getTablePrefixedField(rootTable, rootTable.primary, !tablePrefixed)], subtable);
-                if (arr) {
-                    once = true;
+                // check if the array already in the provided row
+                let arr = undefined;
+                const tableName = createReconstructReferenceName(subtable);
+                if (fromTablePrefixedValues[tableName]) {
+                    arr = JSON.parse(fromTablePrefixedValues[tableName]);
+                    arr = arr.filter((x) => x[subtable.primary] != null);
+                    // we need to go over all fields that are to be bigints and convert
+                    // them back to bigints
+                    // for blob fields we need to convert them back to Uint8Array
+                    for (const field of subtable.fields) {
+                        if (field.name === PARENT_TABLE_ID) {
+                            continue;
+                        }
+                        if (field.unwrappedType === "u64") {
+                            for (const item of arr) {
+                                item[field.name] = BigInt(item[field.name]);
+                            }
+                        }
+                        else if (field.type === "BLOB") {
+                            for (const item of arr) {
+                                item[field.name] = fromHexString(item[field.name]);
+                            }
+                        }
+                    }
+                }
+                else {
+                    if (subtable.children) {
+                        // TODO we only end up where when we resolve nested arrays,
+                        // which shoulld instead be resolved in a nested select (with json_group_array and json_object)
+                        let rootTable = getNonInlinedTable(table);
+                        const parentId = fromTablePrefixedValues[getTablePrefixedField(rootTable, rootTable.primary, !tablePrefixed)];
+                        arr = await resolveChildren(parentId, subtable);
+                    }
+                    else {
+                        arr = [];
+                    }
+                }
+                if (arr && arr.length > 0) {
+                    /* once = true; */
                     for (const element of arr) {
                         const resolved = await resolveInstanceFromValue(element, tables, subtable, // TODO fix
                         resolveChildren, false, subshape);
@@ -751,12 +858,7 @@ export const resolveInstanceFromValue = async (fromTablePrefixedValues, tables,
                     }
                 }
             }
-            if (!once) {
-                obj[field.key] = undefined;
-            }
-            else {
-                obj[field.key] = resolvedArr;
-            }
+            obj[field.key] = resolvedArr; // we can not do option(vec('T')) since we dont store the option type for Arrays (TODO)
         }
         else {
             // resolve nested object from row directly
@@ -845,14 +947,14 @@ export const fromRowToObj = (row, ctor) => {
     return Object.assign(Object.create(ctor.prototype), obj);
 };
 export const convertDeleteRequestToQuery = (request, tables, table) => {
-    const { query, bindable } = convertRequestToQuery("delete", request, tables, table);
+    const { query, bindable } = convertRequestToQuery("delete", { query: types.toQuery(request.query) }, tables, table);
     return {
         sql: `DELETE FROM ${table.name} WHERE ${table.primary} IN (SELECT ${table.primary} from ${table.name} ${query}) returning ${table.primary}`,
         bindable,
     };
 };
 export const convertSumRequestToQuery = (request, tables, table) => {
-    const { query, bindable } = convertRequestToQuery("sum", request, tables, table);
+    const { query, bindable } = convertRequestToQuery("sum", { query: types.toQuery(request.query), key: request.key }, tables, table);
     const inlineName = getInlineTableFieldName(request.key);
     const field = table.fields.find((x) => x.name === inlineName);
     if (unwrapNestedType(field.from.type) === "u64") {
@@ -865,12 +967,45 @@ export const convertSumRequestToQuery = (request, tables, table) => {
     };
 };
 export const convertCountRequestToQuery = (request, tables, table) => {
-    const { query, bindable } = convertRequestToQuery("count", request, tables, table);
+    const { query, bindable } = convertRequestToQuery("count", { query: request?.query ? types.toQuery(request.query) : undefined }, tables, table);
     return {
         sql: `SELECT count(*) as count FROM ${table.name} ${query}`,
         bindable,
     };
 };
+const buildOrderBy = (sort, tables, table, joinBuilder, resolverBuilder, path = [], options) => {
+    let orderByBuilder = undefined;
+    if ((!sort || (Array.isArray(sort) && sort.length === 0)) &&
+        !options?.fetchAll) {
+        sort =
+            table.primary && path.length === 0
+                ? [{ key: [table.primary], direction: types.SortDirection.ASC }]
+                : undefined;
+    }
+    if (sort) {
+        let sortArr = Array.isArray(sort) ? sort : [sort];
+        if (sortArr.length > 0) {
+            orderByBuilder = "";
+            let once = false;
+            for (const sort of sortArr) {
+                const { foreignTables, queryKey } = resolveTableToQuery(table, tables, joinBuilder, [...path, ...sort.key], undefined, true);
+                for (const foreignTable of foreignTables) {
+                    if (once) {
+                        orderByBuilder += ", ";
+                    }
+                    once = true;
+                    foreignTable.columns.push(queryKey); // add the sort key to the list of columns that will be used for this query
+                    orderByBuilder += `"${foreignTable.as}#${queryKey}" ${sort.direction === types.SortDirection.ASC ? "ASC" : "DESC"}`;
+                    resolverBuilder.push({
+                        from: `${table.name}.${escapeColumnName(queryKey)}`,
+                        as: `'${foreignTable.as}#${queryKey}'`,
+                    });
+                }
+            }
+        }
+    }
+    return { orderByBuilder };
+};
 export const convertSearchRequestToQuery = (request, tables, rootTables, options) => {
     let unionBuilder = "";
     let orderByClause = "";
@@ -879,21 +1014,19 @@ export const convertSearchRequestToQuery = (request, tables, rootTables, options
     const selectsPerTable = selectAllFieldsFromTables(rootTables, options?.shape);
     let bindableBuilder = [];
     for (const [i, table] of rootTables.entries()) {
-        const { selects, joins: joinFromSelect } = selectsPerTable[i];
-        const selectQuery = generateSelectQuery(table, selects);
+        const { selects, joins, groupBy } = selectsPerTable[i];
         try {
-            const { orderBy, query, bindable } = convertRequestToQuery("iterate", request, tables, table, joinFromSelect, [], {
-                stable: options?.stable,
-            });
-            unionBuilder += `${unionBuilder.length > 0 ? " UNION ALL " : ""} ${selectQuery} ${query}`;
-            orderByClause =
-                orderBy?.length > 0
-                    ? orderByClause.length > 0
-                        ? orderByClause + ", " + orderBy
-                        : orderBy
-                    : orderByClause;
-            matchedOnce = true;
-            bindableBuilder.push(...bindable);
+            const { orderByBuilder } = buildOrderBy(request?.sort, tables, table, joins, selects, [], options);
+            if (!orderByClause && orderByBuilder) {
+                // assume all order by clauses will be the same
+                orderByClause =
+                    orderByBuilder.length > 0
+                        ? orderByClause.length > 0
+                            ? orderByClause + ", " + orderByBuilder
+                            : orderByBuilder
+                        : orderByClause;
+            }
+            //orderByAddedOnce = true;
         }
         catch (error) {
             if (error instanceof MissingFieldError) {
@@ -902,138 +1035,209 @@ export const convertSearchRequestToQuery = (request, tables, rootTables, options
             }
             throw error;
         }
+        const selectQuery = generateSelectQuery(table, selects);
+        for (const flattenRequest of flattenQuery(request)) {
+            try {
+                const { query, bindable } = convertRequestToQuery("iterate", flattenRequest, tables, table, new Map(joins), // copy the map, else we might might do unececessary joins
+                [], options);
+                unionBuilder += `${unionBuilder.length > 0 ? " UNION " : ""} ${selectQuery} ${query} ${groupBy ? "GROUP BY " + groupBy : ""}`;
+                matchedOnce = true;
+                bindableBuilder.push(...bindable);
+            }
+            catch (error) {
+                if (error instanceof MissingFieldError) {
+                    lastError = error;
+                    orderByClause = "";
+                    continue;
+                }
+                throw error;
+            }
+        }
     }
     if (!matchedOnce) {
         throw lastError;
     }
     return {
-        sql: `${unionBuilder} ${orderByClause ? "ORDER BY " + orderByClause : ""} limit ? offset ?`,
+        sql: `${unionBuilder} ${orderByClause ? "ORDER BY " + orderByClause : ""} ${options?.fetchAll ? "" : "limit ? offset ?"}`,
         bindable: bindableBuilder,
     };
 };
-function isIterateRequest(request, type) {
-    return type === "iterate";
-}
+const getOrSetRootTable = (joinBuilder, table) => {
+    const refName = createQueryTableReferenceName(table);
+    let ref = joinBuilder.get(refName);
+    if (ref) {
+        return ref;
+    }
+    const join = {
+        // add the root as a join even though it is not, just so we can collect the columns it will be queried
+        table: table,
+        type: "root",
+        as: table.name,
+        columns: [],
+    };
+    joinBuilder.set(refName, join);
+    return join;
+};
 const convertRequestToQuery = (type, request, tables, table, extraJoin, path = [], options) => {
     let whereBuilder = "";
     let bindableBuilder = [];
-    let orderByBuilder = undefined;
+    /* let orderByBuilder: string | undefined = undefined; */
     /* let tablesToSelect: string[] = [table.name]; */
     let joinBuilder = extraJoin || new Map();
+    getOrSetRootTable(joinBuilder, table);
     const coercedQuery = types.toQuery(request?.query);
     if (coercedQuery.length === 1) {
-        const { where, bindable } = convertQueryToSQLQuery(coercedQuery[0], tables, table, joinBuilder, path);
+        const { where, bindable } = convertQueryToSQLQuery(coercedQuery[0], tables, table, joinBuilder, path, undefined, 0);
         whereBuilder += where;
         bindableBuilder.push(...bindable);
     }
     else if (coercedQuery.length > 1) {
-        const { where, bindable } = convertQueryToSQLQuery(new types.And(coercedQuery), tables, table, joinBuilder, path);
+        const { where, bindable } = convertQueryToSQLQuery(new types.And(coercedQuery), tables, table, joinBuilder, path, undefined, 0);
         whereBuilder += where;
         bindableBuilder.push(...bindable);
     }
-    if (isIterateRequest(request, type)) {
+    /* if (isIterateRequest(request, type)) {
         let sort = request?.sort;
-        if (!sort && options?.stable) {
+        if (
+            (!sort || (Array.isArray(sort) && sort.length === 0)) &&
+            !options?.fetchAll
+        ) {
             sort =
                 table.primary && path.length === 0
                     ? [{ key: [table.primary], direction: types.SortDirection.ASC }]
                     : undefined;
         }
+
         if (sort) {
             let sortArr = Array.isArray(sort) ? sort : [sort];
             if (sortArr.length > 0) {
                 orderByBuilder = "";
                 let once = false;
                 for (const sort of sortArr) {
-                    const { foreignTables, queryKey } = resolveTableToQuery(table, tables, joinBuilder, [...path, ...sort.key], undefined, true);
-                    for (const table of foreignTables) {
+                    const { foreignTables, queryKey } = resolveTableToQuery(
+                        table,
+                        tables,
+                        joinBuilder,
+                        [...path, ...sort.key],
+                        undefined,
+                        true,
+                    );
+
+                    for (const foreignTable of foreignTables) {
                         if (once) {
                             orderByBuilder += ", ";
                         }
                         once = true;
-                        orderByBuilder += `${table.as}.${queryKey} ${sort.direction === types.SortDirection.ASC ? "ASC" : "DESC"}`;
+
+                        foreignTable.columns.push(queryKey); // add the sort key to the list of columns that will be used for this query
+
+                        orderByBuilder += `${foreignTable.as}.${queryKey} ${sort.direction === types.SortDirection.ASC ? "ASC" : "DESC"}`;
                     }
                 }
-                /* orderByBuilder += request.sort
-                    .map(
-                        (sort) =>
-                            `${table.name}.${sort.key} ${sort.direction === types.SortDirection.ASC ? "ASC" : "DESC"}`
-                    )
-                    .join(", "); */
             }
         }
-    }
+    } */
     const where = whereBuilder.length > 0 ? "where " + whereBuilder : undefined;
     if (extraJoin && extraJoin.size > 0) {
         insertMapIntoMap(joinBuilder, extraJoin);
     }
-    let join = buildJoin(joinBuilder, type === "iterate" ? true : false);
+    let { join } = buildJoin(joinBuilder, options);
     const query = `${join ? join : ""} ${where ? where : ""}`;
     return {
         query,
-        orderBy: orderByBuilder,
+        /* orderBy: orderByBuilder, */
         bindable: bindableBuilder,
     };
 };
-export const buildJoin = (joinBuilder, resolveAllColumns) => {
-    let joinTypeDefault = resolveAllColumns
-        ? /* "FULL OUTER JOIN" */ "LEFT OUTER JOIN"
-        : "JOIN";
+export const buildJoin = (joinBuilder, options) => {
+    /* let joinTypeDefault = resolveAllColumns
+        ? "CROSS JOIN"
+        : "JOIN"; */
     let join = "";
     for (const [_key, table] of joinBuilder) {
+        if (table.type !== "root") {
+            continue;
+        }
+        const out = _buildJoin(table, options);
+        join += out.join;
+    }
+    for (const [_key, table] of joinBuilder) {
+        if (table.type === "root") {
+            continue;
+        }
+        const out = _buildJoin(table, options);
+        join += out.join;
+    }
+    return { join };
+};
+const _buildJoin = (table, options) => {
+    let join = "";
+    let indexedBy = undefined;
+    if (table.type !== "root") {
+        table.columns.push(PARENT_TABLE_ID); // we unshift because we join on the parent id before where clause
+    }
+    if (table.columns.length > 0) {
+        const usedColumns = removeDuplicatesOrdered(table.columns);
+        indexedBy = options?.planner
+            ? ` INDEXED BY ${options.planner.resolveIndex(table.table.name, usedColumns)} `
+            : "";
+    }
+    if (table.type !== "root") {
         let nonInlinedParent = table.table.parent && getNonInlinedTable(table.table.parent);
         if (!nonInlinedParent) {
             throw new Error("Unexpected: missing parent");
         }
-        let joinType = table.table.referencedInArray
-            ? /* "FULL OUTER JOIN" */ "LEFT OUTER JOIN"
-            : joinTypeDefault;
-        join += `${joinType} ${table.table.name} AS ${table.as} ON ${nonInlinedParent.name}.${nonInlinedParent.primary} = ${table.as}.${PARENT_TABLE_ID} `;
+        let joinType = table.type === "cross" ? "LEFT JOIN" : "LEFT JOIN";
+        join += ` ${joinType} ${table.table.name} AS ${table.as} ${indexedBy} ON ${nonInlinedParent.name}.${nonInlinedParent.primary} = ${table.as}.${PARENT_TABLE_ID} `;
     }
-    return join;
+    else if (indexedBy) {
+        join += indexedBy;
+    }
+    return { join };
 };
 const insertMapIntoMap = (map, insert) => {
     for (const [key, value] of insert) {
         map.set(key, value);
     }
 };
-export const convertQueryToSQLQuery = (query, tables, table, joinBuilder, path = [], tableAlias = undefined) => {
+export const convertQueryToSQLQuery = (query, tables, table, joinBuilder, path, tableAlias, skipKeys) => {
     let whereBuilder = "";
     let bindableBuilder = [];
     /* 	let tablesToSelect: string[] = []; */
-    const handleAnd = (queries, path, tableAlias) => {
+    const handleAnd = (queries, path, tableAlias, keysOffset) => {
         for (const query of queries) {
-            const { where, bindable } = convertQueryToSQLQuery(query, tables, table, joinBuilder, path, tableAlias);
+            const { where, bindable } = convertQueryToSQLQuery(query, tables, table, joinBuilder, path, tableAlias, keysOffset);
             whereBuilder =
                 whereBuilder.length > 0 ? `(${whereBuilder}) AND (${where})` : where;
             bindableBuilder.push(...bindable);
         }
     };
     if (query instanceof types.StateFieldQuery) {
-        const { where, bindable } = convertStateFieldQuery(query, tables, table, joinBuilder, path, tableAlias);
+        const { where, bindable } = convertStateFieldQuery(query, tables, table, joinBuilder, path, tableAlias, skipKeys);
         whereBuilder += where;
         bindableBuilder.push(...bindable);
     }
     else if (query instanceof types.Nested) {
         let joinPrefix = "__" + String(tables.size);
-        path = [...path, query.path];
-        handleAnd(query.query, path, joinPrefix);
+        path = [...path, ...query.path];
+        let newSkipKeys = skipKeys + query.path.length;
+        handleAnd(query.query, path, joinPrefix, newSkipKeys);
     }
     else if (query instanceof types.LogicalQuery) {
         if (query instanceof types.And) {
-            handleAnd(query.and, path, tableAlias);
+            handleAnd(query.and, path, tableAlias, skipKeys);
         }
         else if (query instanceof types.Or) {
             for (const subquery of query.or) {
-                const { where, bindable } = convertQueryToSQLQuery(subquery, tables, table, joinBuilder, path, tableAlias);
+                const { where, bindable } = convertQueryToSQLQuery(subquery, tables, table, joinBuilder, path, tableAlias, skipKeys);
                 whereBuilder =
-                    whereBuilder.length > 0 ? `(${whereBuilder}) OR (${where})` : where;
+                    whereBuilder.length > 0 ? `(${whereBuilder}) OR(${where})` : where;
                 bindableBuilder.push(...bindable);
             }
         }
         else if (query instanceof types.Not) {
-            const { where, bindable } = convertQueryToSQLQuery(query.not, tables, table, joinBuilder, path, tableAlias);
-            whereBuilder = `NOT (${where})`;
+            const { where, bindable } = convertQueryToSQLQuery(query.not, tables, table, joinBuilder, path, tableAlias, skipKeys);
+            whereBuilder = `NOT(${where})`;
             bindableBuilder.push(...bindable);
         }
         else {
@@ -1051,16 +1255,26 @@ export const convertQueryToSQLQuery = (query, tables, table, joinBuilder, path =
 const cloneQuery = (query) => {
     return deserialize(serialize(query), types.StateFieldQuery);
 };
-const createTableReferenceName = (table, alias, fieldType, joinSize) => {
-    if (!alias &&
-        (fieldType instanceof VecKind ||
-            (fieldType instanceof OptionKind &&
-                fieldType.elementType instanceof VecKind))) {
-        let aliasSuffix = "_" + String(joinSize);
+/* const createQueryTableReferenceName = (
+    table: Table,
+    alias: string | undefined,
+) => {
+    
+    if (
+        !alias
+    ) {
+        let aliasSuffix =
+            "_query"; //  "_" + String(joinSize); TODO this property will make every join unique, which is not wanted unless (ever?) since we can do OR in SQL which means we can do one join and perform AND/OR logic without joining multiple times to apply multiple conditions
         alias = aliasSuffix;
     }
     const tableNameAs = alias ? alias + "_" + table.name : table.name;
     return tableNameAs;
+}; */
+const createQueryTableReferenceName = (table) => {
+    return table.parent == null ? table.name : "_query_" + table.name;
+};
+const createReconstructReferenceName = (table) => {
+    return table.name; /* table.parent == null ? table.name : "_rec_" + table.name; */
 };
 const resolveTableToQuery = (table, tables, join, path, alias, searchSelf) => {
     // we are matching in two ways.
@@ -1076,11 +1290,18 @@ const resolveTableToQuery = (table, tables, join, path, alias, searchSelf) => {
         if (field) {
             return {
                 queryKey: field.name,
-                foreignTables: [{ table, as: table.name }],
+                foreignTables: [getOrSetRootTable(join, table)],
             };
         }
     }
-    let currentTables = [{ table, as: alias || table.name }];
+    let currentTables = [
+        {
+            table,
+            as: alias || table.name,
+            type: "cross",
+            columns: [],
+        },
+    ];
     let prevTables = undefined;
     // outer:
     for (const [_i, key] of path /* .slice(0, -1) */
@@ -1091,13 +1312,20 @@ const resolveTableToQuery = (table, tables, join, path, alias, searchSelf) => {
             const field = schema.fields.find((x) => x.key === key);
             if (!field && currentTable.children.length > 0) {
                 // second arg is needed because of polymorphic fields we might end up here intentially to check what tables to query
-                throw new MissingFieldError(`Property with key "${key}" is not found in the schema ${JSON.stringify(schema.fields.map((x) => x.key))}`);
+                throw new MissingFieldError(`Property with key "${key}" is not found in the schema ${JSON.stringify(schema.fields.map((x) => x.key))} `);
             }
             for (const child of currentTable.children) {
-                const tableNameAs = createTableReferenceName(child, alias, field.type, join.size);
+                const tableNameAs = createQueryTableReferenceName(child);
                 let isMatching = child.parentPath[child.parentPath.length - 1] === key;
                 if (isMatching) {
-                    const tableWithAlias = { table: child, as: tableNameAs };
+                    const tableWithAlias = {
+                        columns: [],
+                        table: child,
+                        as: tableNameAs,
+                        type: currentTable.children.length > 1
+                            ? "left"
+                            : "cross",
+                    };
                     if (child.isSimpleValue) {
                         if (!child.inline) {
                             join.set(tableNameAs, tableWithAlias);
@@ -1144,13 +1372,17 @@ const resolveTableToQuery = (table, tables, join, path, alias, searchSelf) => {
         : FOREIGN_VALUE_PROPERTY;
     return { queryKey, foreignTables };
 };
-const convertStateFieldQuery = (query, tables, table, join, path, tableAlias = undefined) => {
+const convertStateFieldQuery = (query, tables, table, join, path, tableAlias, skipKeys) => {
     // if field id represented as foreign table, do join and compare
     const inlinedName = getInlineTableFieldName(query.key);
     const tableField = table.fields.find((x) => x.name === inlinedName); /* stringArraysEquals(query.key, [...table.parentPath, x.name]) )*/
     const isForeign = !tableField; // table.fields.find(x => x.name === query.key[query.key.length - 1])
     if (isForeign) {
-        const { queryKey, foreignTables } = resolveTableToQuery(table, tables, join, [...path, ...query.key], tableAlias, false);
+        const tablePath = [...path];
+        for (let i = skipKeys; i < query.key.length; i++) {
+            tablePath.push(query.key[i]);
+        }
+        const { queryKey, foreignTables } = resolveTableToQuery(table, tables, join, tablePath, tableAlias, false);
         query = cloneQuery(query);
         query.key = [queryKey];
         let whereBuilder = [];
@@ -1159,7 +1391,7 @@ const convertStateFieldQuery = (query, tables, table, join, path, tableAlias = u
             if (ftable.table === table) {
                 throw new Error("Unexpected");
             }
-            const { where, bindable } = convertQueryToSQLQuery(query, tables, ftable.table, join, path, ftable.as);
+            const { where, bindable } = convertQueryToSQLQuery(query, tables, ftable.table, join, path, ftable.as, skipKeys);
             whereBuilder.push(where);
             bindableBuilder.push(bindable);
         }
@@ -1168,17 +1400,22 @@ const convertStateFieldQuery = (query, tables, table, join, path, tableAlias = u
             bindable: bindableBuilder.flat(),
         };
     }
+    const columnAggregator = join.get(createQueryTableReferenceName(table));
+    if (!columnAggregator) {
+        throw new Error("Unexpected");
+    }
+    columnAggregator.columns.push(inlinedName);
     let bindable = [];
     const keyWithTable = (tableAlias || table.name) + "." + escapeColumnName(inlinedName);
     let where;
     if (query instanceof types.StringMatch) {
         let statement = "";
         if (query.method === types.StringMatchMethod.contains) {
-            statement = `${keyWithTable} LIKE ?`;
+            statement = `${keyWithTable} LIKE ? `;
             bindable.push(`%${query.value}%`);
         }
         else if (query.method === types.StringMatchMethod.prefix) {
-            statement = `${keyWithTable} LIKE ?`;
+            statement = `${keyWithTable} LIKE ? `;
             bindable.push(`${query.value}%`);
         }
         else if (query.method === types.StringMatchMethod.exact) {
@@ -1199,7 +1436,7 @@ const convertStateFieldQuery = (query, tables, table, join, path, tableAlias = u
     else if (query instanceof types.IntegerCompare) {
         if (tableField.type === "BLOB") {
             // TODO perf
-            where = `hex(${keyWithTable}) LIKE ?`;
+            where = `hex(${keyWithTable}) LIKE ? `;
             bindable.push(`%${toHexString(new Uint8Array([Number(query.value.value)]))}%`);
         }
         else {
@@ -1207,23 +1444,23 @@ const convertStateFieldQuery = (query, tables, table, join, path, tableAlias = u
                 where = `${keyWithTable} = ?`;
             }
             else if (query.compare === types.Compare.Greater) {
-                where = `${keyWithTable} > ?`;
+                where = `${keyWithTable} > ? `;
             }
             else if (query.compare === types.Compare.Less) {
-                where = `${keyWithTable} < ?`;
+                where = `${keyWithTable} <?`;
             }
             else if (query.compare === types.Compare.GreaterOrEqual) {
-                where = `${keyWithTable} >= ?`;
+                where = `${keyWithTable} >= ? `;
             }
             else if (query.compare === types.Compare.LessOrEqual) {
-                where = `${keyWithTable} <= ?`;
+                where = `${keyWithTable} <= ? `;
             }
             else {
-                throw new Error(`Unsupported compare type: ${query.compare}`);
+                throw new Error(`Unsupported compare type: ${query.compare} `);
             }
             if (unwrapNestedType(tableField.from.type) === "u64") {
                 // shift left because that is how we insert the value
-                bindable.push(query.value.value);
+                bindable.push(u64ToI64(query.value.value));
             }
             else {
                 bindable.push(query.value.value);
@@ -1242,4 +1479,14 @@ const convertStateFieldQuery = (query, tables, table, join, path, tableAlias = u
     }
     return { where, bindable };
 };
+const removeDuplicatesOrdered = (arr) => {
+    let seen = new Set();
+    return arr.filter((item) => {
+        if (seen.has(item)) {
+            return false;
+        }
+        seen.add(item);
+        return true;
+    });
+};
 //# sourceMappingURL=schema.js.map