@carbonorm/carbonnode 6.0.14 → 6.0.18

package/src/executors/SqlExecutor.ts

@@ -16,15 +16,76 @@ import namedPlaceholders from 'named-placeholders';
 import type { PoolConnection } from 'mysql2/promise';
 import { Buffer } from 'buffer';
 import { Executor } from "./Executor";
-import {checkCache, setCache} from "../utils/cacheManager";
+import {checkCache, evictCacheEntry, setCache} from "../utils/cacheManager";
+import logSql, {
+    SqlAllowListStatus,
+} from "../utils/logSql";
 import { normalizeSingularRequest } from "../utils/normalizeSingularRequest";
 import {sortAndSerializeQueryObject} from "../utils/sortAndSerializeQueryObject";
 import { loadSqlAllowList, normalizeSql } from "../utils/sqlAllowList";
 import { getLogContext, LogLevel, logWithLevel } from "../utils/logLevel";
 
+const SQL_ALLOWLIST_BLOCKED_CODE = "SQL_ALLOWLIST_BLOCKED";
+
+export type SqlAllowListBlockedError = Error & {
+    code: typeof SQL_ALLOWLIST_BLOCKED_CODE;
+    tableName?: string;
+    method?: string;
+    normalizedSql: string;
+    allowListPath: string;
+    sqlAllowList: {
+        sql: string;
+        table: string | null;
+        method: string | null;
+        allowListPath: string;
+        canAdd: boolean;
+    };
+};
+
+const createSqlAllowListBlockedError = (args: {
+    tableName?: string;
+    method?: string;
+    normalizedSql: string;
+    allowListPath: string;
+}): SqlAllowListBlockedError => {
+    const error = new Error(
+        `SQL statement is not permitted by allowlist (${args.allowListPath}).`,
+    ) as SqlAllowListBlockedError;
+
+    error.name = "SqlAllowListBlockedError";
+    error.code = SQL_ALLOWLIST_BLOCKED_CODE;
+    error.tableName = args.tableName;
+    error.method = args.method;
+    error.normalizedSql = args.normalizedSql;
+    error.allowListPath = args.allowListPath;
+    error.sqlAllowList = {
+        sql: args.normalizedSql,
+        table: args.tableName ?? null,
+        method: args.method ?? null,
+        allowListPath: args.allowListPath,
+        canAdd: true,
+    };
+
+    return error;
+};
+
 export class SqlExecutor<
     G extends OrmGenerics
 > extends Executor<G> {
+    private resolveSqlLogMethod(method: iRestMethods, sql: string): string {
+        const token = sql.trim().split(/\s+/, 1)[0]?.toUpperCase();
+        if (token) return token;
+        switch (method) {
+            case C6C.GET:
+                return "SELECT";
+            case C6C.POST:
+                return "INSERT";
+            case C6C.PUT:
+                return "UPDATE";
+            default:
+                return "DELETE";
+        }
+    }
 
     async execute(): Promise<DetermineResponseDataType<G['RequestMethod'], G['RestTableInterface']>> {
         const { TABLE_NAME } = this.config.restModel;
@@ -487,8 +548,7 @@ export class SqlExecutor<
         const tableName = this.config.restModel.TABLE_NAME;
         const logContext = getLogContext(this.config, this.request);
         const cacheResults = method === C6C.GET
-            && !this.config.sqlAllowListPath
-            && (this.request as { cacheResults?: boolean })?.cacheResults !== false;
+            && (this.request.cacheResults ?? true);
 
         const cacheRequestData = cacheResults
             ? JSON.parse(JSON.stringify(this.request ?? {}))
@@ -498,29 +558,55 @@ export class SqlExecutor<
             ? sortAndSerializeQueryObject(tableName, cacheRequestData ?? {})
             : undefined;
 
+        const evictFromCache =
+            method === C6C.GET && cacheResults && cacheRequestData
+                ? () => evictCacheEntry(method, tableName, cacheRequestData)
+                : undefined;
+
         if (cacheResults) {
             const cachedRequest = checkCache<DetermineResponseDataType<G['RequestMethod'], G['RestTableInterface']>>(
                 method,
                 tableName,
                 cacheRequestData,
+                logContext
             );
             if (cachedRequest) {
-                return (await cachedRequest).data;
+                const cachedData = (await cachedRequest).data;
+                if (evictFromCache
+                    && cachedData
+                    && typeof cachedData === "object"
+                    && Array.isArray((cachedData as DetermineResponseDataType<'GET', G['RestTableInterface']>).rest)) {
+                    (cachedData as DetermineResponseDataType<'GET', G['RestTableInterface']>).evictFromCache = evictFromCache;
+                }
+                return cachedData;
             }
         }
 
         const sqlExecution = this.buildSqlExecutionContext(method, tableName, logContext);
+        const sqlMethod = this.resolveSqlLogMethod(method, sqlExecution.sql);
         const queryPromise = this.withConnection(async (conn) =>
-            this.executeQueryWithLifecycle(conn, method, sqlExecution, logContext),
+            this.executeQueryWithLifecycle(
+                conn,
+                method,
+                sqlExecution,
+                logContext,
+                sqlMethod
+            ),
         );
 
         if (!cacheResults || !cacheRequestData || !requestArgumentsSerialized) {
             return await queryPromise;
         }
 
-        const cacheRequest = queryPromise.then((data) =>
-            this.createCacheResponseEnvelope(method, tableName, data),
-        );
+        const cacheRequest = queryPromise.then((data) => {
+            if (evictFromCache
+                && data
+                && typeof data === "object"
+                && Array.isArray((data as DetermineResponseDataType<'GET', G['RestTableInterface']>).rest)) {
+                (data as DetermineResponseDataType<'GET', G['RestTableInterface']>).evictFromCache = evictFromCache;
+            }
+            return this.createCacheResponseEnvelope(method, tableName, data);
+        });
 
         setCache(method, tableName, cacheRequestData, {
             requestArgumentsSerialized,
@@ -640,6 +726,7 @@ export class SqlExecutor<
         method: iRestMethods,
         sqlExecution: iRestSqlExecutionContext,
         logContext: ReturnType<typeof getLogContext>,
+        sqlMethod: string
     ): Promise<DetermineResponseDataType<G['RequestMethod'], G['RestTableInterface']>> {
         const useTransaction = method !== C6C.GET;
         let committed = false;
@@ -655,7 +742,27 @@ export class SqlExecutor<
                 await conn.beginTransaction();
             }
 
-            await this.validateSqlAllowList(sqlExecution.sql);
+            let allowListStatus: SqlAllowListStatus = "not verified";
+            try {
+                allowListStatus = await this.validateSqlAllowList(sqlExecution.sql);
+            } catch (error) {
+                logSql({
+                    method: sqlMethod,
+                    sql: sqlExecution.sql,
+                    context: logContext,
+                    cacheStatus: this.request.cacheResults === false ? "ignored" : "miss",
+                    allowListStatus: "denied",
+                });
+                throw error;
+            }
+
+            logSql({
+                method: sqlMethod,
+                sql: sqlExecution.sql,
+                context: logContext,
+                cacheStatus: this.request.cacheResults === false ? "ignored" : "miss",
+                allowListStatus,
+            });
 
             await this.runLifecycleHooks<"beforeExecution">(
                 "beforeExecution",
@@ -729,17 +836,29 @@ export class SqlExecutor<
         }
     }
 
-    private async validateSqlAllowList(sql: string): Promise<void> {
+    private async validateSqlAllowList(sql: string): Promise<SqlAllowListStatus> {
         const allowListPath = this.config.sqlAllowListPath;
         if (!allowListPath) {
-            return;
+            return "not verified";
         }
 
         const allowList = await loadSqlAllowList(allowListPath);
         const normalized = normalizeSql(sql);
         if (!allowList.has(normalized)) {
-            throw new Error(`SQL statement is not permitted by allowlist (${allowListPath}).`);
+            throw createSqlAllowListBlockedError({
+                tableName:
+                    typeof this.config.restModel?.TABLE_NAME === "string"
+                        ? this.config.restModel.TABLE_NAME
+                        : undefined,
+                method:
+                    typeof this.config.requestMethod === "string"
+                        ? this.config.requestMethod
+                        : undefined,
+                normalizedSql: normalized,
+                allowListPath,
+            });
         }
+        return "allowed";
     }
 
 

package/src/orm/queries/DeleteQueryBuilder.ts

@@ -2,8 +2,6 @@ import { OrmGenerics } from "../../types/ormGenerics";
 import { SqlBuilderResult } from "../utils/sqlUtils";
 import { JoinBuilder } from "../builders/JoinBuilder";
 import { SelectQueryBuilder } from "./SelectQueryBuilder";
-import logSql from "../../utils/logSql";
-import {getLogContext} from "../../utils/logLevel";
 
 export class DeleteQueryBuilder<G extends OrmGenerics> extends JoinBuilder<G> {
     protected createSelectBuilder(request: any) {
@@ -27,8 +25,6 @@ export class DeleteQueryBuilder<G extends OrmGenerics> extends JoinBuilder<G> {
             sql += this.buildWhereClause(this.request.WHERE, params);
         }
 
-        logSql("DELETE", sql, getLogContext(this.config, this.request));
-
         return { sql, params };
     }
 }

package/src/orm/queries/PostQueryBuilder.ts

@@ -1,8 +1,6 @@
 import {C6C} from "../../constants/C6Constants";
 import {ConditionBuilder} from "../builders/ConditionBuilder";
 import {OrmGenerics} from "../../types/ormGenerics";
-import logSql from "../../utils/logSql";
-import {getLogContext} from "../../utils/logLevel";
 
 export class PostQueryBuilder<G extends OrmGenerics> extends ConditionBuilder<G>{
 
@@ -61,8 +59,6 @@ export class PostQueryBuilder<G extends OrmGenerics> extends ConditionBuilder<G>
             sql += ` ON DUPLICATE KEY UPDATE ${updateClause}`;
         }
 
-        logSql(verb, sql, getLogContext(this.config, this.request));
-
         return {sql, params};
     }
 }

package/src/orm/queries/SelectQueryBuilder.ts

@@ -1,8 +1,6 @@
 import {OrmGenerics} from "../../types/ormGenerics";
 import {PaginationBuilder} from "../builders/PaginationBuilder";
 import {SqlBuilderResult} from "../utils/sqlUtils";
-import logSql from "../../utils/logSql";
-import {getLogContext} from "../../utils/logLevel";
 
 export class SelectQueryBuilder<G extends OrmGenerics> extends PaginationBuilder<G>{
 
@@ -58,8 +56,6 @@ export class SelectQueryBuilder<G extends OrmGenerics> extends PaginationBuilder
             sql += ` LIMIT 100`;
         }
 
-        logSql("SELECT", sql, getLogContext(this.config, this.request));
-
         return { sql, params };
     }
 }

package/src/orm/queries/UpdateQueryBuilder.ts

@@ -3,8 +3,6 @@ import {OrmGenerics} from "../../types/ormGenerics";
 import { PaginationBuilder } from '../builders/PaginationBuilder';
 import {SqlBuilderResult} from "../utils/sqlUtils";
 import {SelectQueryBuilder} from "./SelectQueryBuilder";
-import logSql from "../../utils/logSql";
-import {getLogContext} from "../../utils/logLevel";
 
 export class UpdateQueryBuilder<G extends OrmGenerics> extends PaginationBuilder<G>{
     protected createSelectBuilder(request: any) {
@@ -56,8 +54,6 @@ export class UpdateQueryBuilder<G extends OrmGenerics> extends PaginationBuilder
             sql += this.buildPaginationClause(args.PAGINATION, params);
         }
 
-        logSql("UPDATE", sql, getLogContext(this.config, this.request));
-
         return { sql, params };
     }
 }

package/src/types/ormInterfaces.ts

@@ -140,7 +140,10 @@ export type C6RestResponse<
     session?: any;
     sql?: any;
 } & (Method extends 'GET'
-    ? { next?: () => Promise<DetermineResponseDataType<'GET', RestData, Overrides>> }
+    ? {
+        next?: () => Promise<DetermineResponseDataType<'GET', RestData, Overrides>>,
+        evictFromCache?: () => boolean
+    }
     : {
         affected: number,
         insertId?: number | string,

package/src/utils/cacheManager.ts

@@ -1,5 +1,6 @@
 import type {iCacheAPI, iCacheResponse} from "../types/ormInterfaces";
-import {LogLevel, logWithLevel, shouldLog} from "./logLevel";
+import {LogContext, LogLevel, logWithLevel, shouldLog} from "./logLevel";
+import logSql from "./logSql";
 
 // -----------------------------------------------------------------------------
 // Cache Storage
@@ -60,19 +61,26 @@ export function checkCache<ResponseDataType = any>(
     method: string,
     tableName: string | string[],
     requestData: any,
+    logContext: LogContext,
 ): Promise<iCacheResponse<ResponseDataType>> | false {
     const key = makeCacheKey(method, tableName, requestData);
     const cached = apiRequestCache.get(key);
 
-    if (!cached) return false;
+    if (!cached) {
+        console.log('apiRequestCache.size', apiRequestCache.size)
+        return false;
+    }
 
-    if (shouldLog(LogLevel.INFO, undefined)) {
-        console.groupCollapsed(
-            `%c API cache hit for ${method} ${tableName}`,
-            "color:#0c0",
-        );
-        console.log("Request Data:", requestData);
-        console.groupEnd();
+    if (shouldLog(LogLevel.INFO, logContext)) {
+        const sql = cached.response?.data?.sql?.sql ?? "";
+        const sqlMethod = sql.trim().split(/\s+/, 1)[0]?.toUpperCase() || method;
+        logSql({
+            allowListStatus: "not verified",
+            cacheStatus: "hit",
+            context: logContext,
+            method: sqlMethod,
+            sql
+        });
     }
 
     return cached.request;
@@ -90,3 +98,12 @@ export function setCache<ResponseDataType = any>(
     const key = makeCacheKey(method, tableName, requestData);
     apiRequestCache.set(key, cacheEntry);
 }
+
+export function evictCacheEntry(
+    method: string,
+    tableName: string | string[],
+    requestData: any,
+): boolean {
+    const key = makeCacheKey(method, tableName, requestData);
+    return apiRequestCache.delete(key);
+}

package/src/utils/logLevel.ts

@@ -135,10 +135,9 @@ export const applyLogLevelDefaults = (
 };
 
 export const getLogContext = (
-    config?: { logLevel?: number | null; verbose?: boolean | null },
-    request?: { debug?: boolean } | null,
-): LogContext | undefined => {
-    if (!config && !request) return undefined;
+    config: { logLevel?: number | null; verbose?: boolean | null },
+    request: { debug?: boolean } | null,
+): LogContext => {
     return {
         logLevel: config?.logLevel ?? undefined,
         verbose: config?.verbose ?? undefined,

package/src/utils/logSql.ts

@@ -2,19 +2,33 @@ import { getEnvBool } from "../variables/getEnv";
 import colorSql from "./colorSql";
 import { version } from "../../package.json";
 import versionToRgb from "./versionColor";
-import type {LogContext} from "./logLevel";
-import {LogLevel, shouldLog} from "./logLevel";
+import type { LogContext } from "./logLevel";
+import { LogLevel, shouldLog } from "./logLevel";
+
+export type SqlAllowListStatus = "allowed" | "denied" | "not verified";
+export type SqlCacheStatus = "hit" | "miss" | "ignored";
+
+export type LogSqlContextOptions = {
+    cacheStatus: SqlCacheStatus;
+    allowListStatus: SqlAllowListStatus;
+    method: string,
+    sql: string,
+    context?: LogContext,
+};
 
 const C = {
     SSR: "\x1b[95m", // bright magenta
     HTTP: "\x1b[94m", // bright blue
     SQL: "\x1b[96m", // bright cyan
+    WARN: "\x1b[93m", // yellow
+    ORANGE: "\x1b[38;2;255;165;0m", // orange (truecolor)
+    ERROR: "\x1b[91m", // red
     METHOD_COLORS: {
         SELECT: "\x1b[92m", // green
         INSERT: "\x1b[96m", // cyan
         REPLACE: "\x1b[96m", // cyan
         UPDATE: "\x1b[95m", // magenta
-        DELETE: "\x1b[91m", // red
+        DELETE: "\x1b[38;2;255;179;179m", // very light red (truecolor)
     },
     METHOD_FALLBACK: [
         "\x1b[92m", // green
@@ -24,6 +38,7 @@ const C = {
         "\x1b[94m", // blue
         "\x1b[97m", // white
     ],
+    GREY: "\x1b[90m", // light grey
     RESET: "\x1b[0m",
 };
 
@@ -47,13 +62,43 @@ function methodColor(method: string): string {
     return C.METHOD_FALLBACK[idx];
 }
 
-export default function logSql(method: string, sql: string, context?: LogContext): void {
-    if (!shouldLog(LogLevel.INFO, context)) return;
+const cacheLabel = (cacheStatus: SqlCacheStatus): string => {
+    switch (cacheStatus) {
+        case "hit":
+            return `${C.METHOD_COLORS.SELECT}[CACHE HIT]${C.RESET}`;
+        case "ignored":
+            return `${C.WARN}[CACHE IGNORED]${C.RESET}`;
+        default:
+            return `${C.ORANGE}[CACHE MISS]${C.RESET}`;
+    }
+};
+
+const allowListLabel = (status: SqlAllowListStatus): string => {
+    switch (status) {
+        case "allowed":
+            return `${C.METHOD_COLORS.SELECT}[VERIFIED]${C.RESET}`;
+        case "denied":
+            return `${C.ERROR}[DENIED]${C.RESET}`;
+        default:
+            return `${C.GREY}[NOT VERIFIED]${C.RESET}`;
+    }
+};
+
+export default function logSql(
+    options: LogSqlContextOptions,
+): void {
+    const method = options.method.toUpperCase();
+
+    if (!shouldLog(LogLevel.INFO, options.context)) return;
     const preText = getEnvBool("SSR", false)
         ? `${C.SSR}[SSR]${C.RESET} `
         : `${C.HTTP}[API]${C.RESET} `;
 
     const labelColor = methodColor(method);
     const versionColor = rgbAnsi(versionToRgb(version));
-    console.log(`${versionColor}[${version}]${C.RESET} ${preText}${labelColor}[${method}]${C.RESET} ${colorSql(sql)}`);
+    const cacheText = cacheLabel(options.cacheStatus);
+    const allowListText = allowListLabel(options.allowListStatus);
+    console.log(
+        `${versionColor}[${version}]${C.RESET} ${cacheText} ${allowListText} ${preText}${labelColor}[${method}]${C.RESET} ${colorSql(options.sql)}`,
+    );
 }

package/src/utils/sqlAllowList.ts

@@ -1,9 +1,95 @@
 import isNode from "../variables/isNode";
 
-const allowListCache = new Map<string, Set<string>>();
+type AllowListCacheEntry = {
+    allowList: Set<string>;
+    mtimeMs: number;
+    size: number;
+};
 
-export const normalizeSql = (sql: string): string =>
-    sql.replace(/\s+/g, " ").trim();
+const allowListCache = new Map<string, AllowListCacheEntry>();
+
+const ANSI_ESCAPE_REGEX = /\x1b\[[0-9;]*m/g;
+const COLLAPSED_BIND_ROW_REGEX = /\(\?\s*×\d+\)/g;
+
+function collapseBindGroups(sql: string): string {
+    let normalized = sql.replace(
+        /\(\s*(\?(?:\s*,\s*\?)*)\s*\)/g,
+        (_match, binds: string) => {
+            const bindCount = (binds.match(/\?/g) ?? []).length;
+            return `(? ×${bindCount})`;
+        },
+    );
+
+    normalized = normalized.replace(
+        /(\(\?\s*×\d+\))(?:\s*,\s*\1)+/g,
+        (_match, row) => `${row} ×*`,
+    );
+
+    normalized = normalized.replace(
+        /\b(VALUES|VALUE)\s+(\(\?\s*×\d+\))(?:\s*×\d+|\s*×\*)?/gi,
+        (_match, keyword: string, row: string) => `${keyword} ${row} ×*`,
+    );
+
+    normalized = normalized.replace(
+        /\bIN\s*\(\?\s*×\d+\)/gi,
+        "IN (? ×*)",
+    );
+
+    normalized = normalized.replace(
+        /\(\?\s*×\d+\)\s*×\d+/g,
+        (match) => {
+            const row = match.match(COLLAPSED_BIND_ROW_REGEX)?.[0];
+            return row ? `${row} ×*` : match;
+        },
+    );
+
+    return normalized;
+}
+
+function normalizeLimitOffset(sql: string): string {
+    return sql
+        .replace(/\bLIMIT\s+\d+\s*,\s*\d+\b/gi, "LIMIT ?, ?")
+        .replace(/\bLIMIT\s+\d+\s+OFFSET\s+\d+\b/gi, "LIMIT ? OFFSET ?")
+        .replace(/\bLIMIT\s+\d+\b/gi, "LIMIT ?")
+        .replace(/\bOFFSET\s+\d+\b/gi, "OFFSET ?");
+}
+
+function normalizeGeomFromTextLiterals(sql: string): string {
+    let normalized = sql.replace(
+        /ST_GEOMFROMTEXT\(\s*'POINT\([^']*\)'\s*,\s*(?:\d+|\?)\s*\)/gi,
+        "ST_GEOMFROMTEXT('POINT(? ?)', ?)",
+    );
+
+    normalized = normalized.replace(
+        /ST_GEOMFROMTEXT\(\s*'POLYGON\(\([^']*\)\)'\s*,\s*(?:\d+|\?)\s*\)/gi,
+        "ST_GEOMFROMTEXT('POLYGON((?))', ?)",
+    );
+
+    return normalized;
+}
+
+function normalizeGeoFunctionNames(sql: string): string {
+    return sql
+        .replace(/\bST_DISTANCE_SPHERE\b/gi, "ST_DISTANCE_SPHERE")
+        .replace(/\bST_GEOMFROMTEXT\b/gi, "ST_GEOMFROMTEXT")
+        .replace(/\bMBRCONTAINS\b/gi, "MBRCONTAINS");
+}
+
+function normalizeTokenPunctuationSpacing(sql: string): string {
+    return sql.replace(/`,\s*`/g, "`, `");
+}
+
+export const normalizeSql = (sql: string): string => {
+    let normalized = sql.replace(ANSI_ESCAPE_REGEX, " ");
+    normalized = normalized.replace(/\s+/g, " ").trim();
+    normalized = normalizeGeoFunctionNames(normalized);
+    normalized = normalizeTokenPunctuationSpacing(normalized);
+    normalized = collapseBindGroups(normalized);
+    normalized = normalizeLimitOffset(normalized);
+    normalized = normalizeGeomFromTextLiterals(normalized);
+    normalized = normalized.replace(/;\s*$/, "");
+    return normalized.replace(/\s+/g, " ").trim();
+};
 
 const parseAllowList = (raw: string, sourcePath: string): string[] => {
     let parsed: unknown;
@@ -30,15 +116,27 @@ const parseAllowList = (raw: string, sourcePath: string): string[] => {
 };
 
 export const loadSqlAllowList = async (allowListPath: string): Promise<Set<string>> => {
-    if (allowListCache.has(allowListPath)) {
-        return allowListCache.get(allowListPath)!;
-    }
-
     if (!isNode()) {
         throw new Error("SQL allowlist validation requires a Node runtime.");
     }
 
-    const {readFile} = await import("node:fs/promises");
+    const {readFile, stat} = await import("node:fs/promises");
+
+    let fileStat: { mtimeMs: number; size: number };
+    try {
+        fileStat = await stat(allowListPath);
+    } catch (error) {
+        throw new Error(`SQL allowlist file not found at ${allowListPath}.`);
+    }
+
+    const cached = allowListCache.get(allowListPath);
+    if (
+        cached &&
+        cached.mtimeMs === fileStat.mtimeMs &&
+        cached.size === fileStat.size
+    ) {
+        return cached.allowList;
+    }
 
     let raw: string;
     try {
@@ -49,7 +147,11 @@ export const loadSqlAllowList = async (allowListPath: string): Promise<Set<strin
 
     const sqlEntries = parseAllowList(raw, allowListPath);
     const allowList = new Set(sqlEntries);
-    allowListCache.set(allowListPath, allowList);
+    allowListCache.set(allowListPath, {
+        allowList,
+        mtimeMs: fileStat.mtimeMs,
+        size: fileStat.size,
+    });
     return allowList;
 };