@memberjunction/generic-database-provider 5.13.0 → 5.15.0

package/dist/index.js

@@ -1,3 +1,4 @@
 export { GenericDatabaseProvider } from './GenericDatabaseProvider.js';
 export { SqlLoggingSessionImpl } from './SqlLogger.js';
+export { QueryCompositionEngine } from './queryCompositionEngine.js';
 //# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,uBAAuB,EAA0B,MAAM,8BAA8B,CAAC;AAE/F,OAAO,EAAE,qBAAqB,EAAE,MAAM,gBAAgB,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,uBAAuB,EAA0B,MAAM,8BAA8B,CAAC;AAE/F,OAAO,EAAE,qBAAqB,EAAE,MAAM,gBAAgB,CAAC;AACvD,OAAO,EAAE,sBAAsB,EAAyC,MAAM,6BAA6B,CAAC"}

package/dist/queryCompositionEngine.d.ts

@@ -0,0 +1,278 @@
+import { DatabasePlatform, UserInfo, QueryDependencySpec } from "@memberjunction/core";
+/**
+ * Metadata about a single CTE generated during composition resolution.
+ */
+export interface CompositionCTEInfo {
+    /** ID of the referenced query */
+    QueryID: string;
+    /** Name of the referenced query */
+    QueryName: string;
+    /** Category path as written in the reference */
+    CategoryPath: string;
+    /** Generated CTE alias name */
+    CTEName: string;
+    /** Original SQL of the referenced query before parameter resolution */
+    OriginalSQL: string;
+    /** SQL after parameter values have been substituted */
+    ResolvedSQL: string;
+    /** Parameter values applied (key → resolved value) */
+    Parameters: Record<string, string>;
+}
+/**
+ * Result returned by the composition engine after resolving all {{query:"..."}} tokens.
+ */
+export interface CompositionResult {
+    /** The fully resolved SQL with CTEs prepended */
+    ResolvedSQL: string;
+    /** Metadata about each CTE generated */
+    CTEs: CompositionCTEInfo[];
+    /** Directed dependency graph: queryId → [dependsOnQueryIds] */
+    DependencyGraph: Map<string, string[]>;
+    /** Whether any composition tokens were found and resolved */
+    HasCompositions: boolean;
+    /** True if any resolved dependency query has UsesTemplate = true (depth-first, transitive) */
+    AnyDependencyUsesTemplates: boolean;
+}
+/**
+ * Parsed representation of a single {{query:"..."}} token found in SQL.
+ */
+interface ParsedCompositionToken {
+    /** The full original token text including {{ and }} */
+    FullToken: string;
+    /** Category path segments (everything before the query name) */
+    CategorySegments: string[];
+    /** The query name (last path segment) */
+    QueryName: string;
+    /** Full path as written (CategorySegments joined with /) */
+    FullPath: string;
+    /** Parsed parameter mappings */
+    Parameters: ParsedParameter[];
+}
+/**
+ * A single parameter from a composition reference.
+ */
+interface ParsedParameter {
+    /** Parameter name */
+    Name: string;
+    /** If quoted literal: the static value. Otherwise null. */
+    StaticValue: string | null;
+    /** If bare name: the pass-through parameter name from the outer query. Otherwise null. */
+    PassThroughName: string | null;
+}
+/**
+ * QueryCompositionEngine resolves {{query:"CategoryPath/QueryName(params)"}} tokens
+ * in query SQL into Common Table Expressions (CTEs). It handles:
+ *
+ * - Recursive resolution of nested compositions
+ * - Cycle detection via an in-progress set
+ * - Parameter modes: static literals and pass-through from outer query
+ * - Deduplication of identical query+params references
+ * - Platform-aware SQL resolution via QueryInfo.GetPlatformSQL()
+ *
+ * This engine runs BEFORE Nunjucks template processing, so regular {{param}}
+ * tokens are preserved for later substitution.
+ */
+export declare class QueryCompositionEngine {
+    /**
+     * Checks whether SQL contains any {{query:"..."}} composition tokens.
+     * Use this as a fast guard before calling ResolveComposition().
+     * Only considers tokens outside of SQL comments.
+     */
+    HasCompositionTokens(sql: string): boolean;
+    /**
+     * Parses all {{query:"..."}} tokens from SQL without resolving them.
+     * Useful for dependency extraction during the save pipeline.
+     * Only considers tokens outside of SQL comments.
+     *
+     * Uses MJLexer for structured tokenization instead of regex, providing
+     * full parsing of category paths, query names, and parameter lists.
+     *
+     * @param sql - The SQL text to parse
+     * @returns Array of parsed token metadata
+     */
+    ParseCompositionTokens(sql: string): ParsedCompositionToken[];
+    /**
+     * Resolves all {{query:"..."}} composition tokens in the given SQL into CTEs.
+     *
+     * @param sql - The SQL containing composition tokens
+     * @param platform - Target database platform for SQL resolution
+     * @param contextUser - User context for permission checks on referenced queries
+     * @param outerParams - Parameter values from the outer/parent query (for pass-through resolution)
+     * @param inlineDependencies - Optional inline dependency specs for transient query testing.
+     *        When provided, these are checked first before falling back to Metadata.Provider.Queries.
+     *        Inline dependencies skip governance validation (Reusable, IsApproved, UserCanRun).
+     * @returns CompositionResult with fully resolved SQL and provenance metadata
+     * @throws Error if a referenced query is not found, not composable, or creates a cycle
+     */
+    ResolveComposition(sql: string, platform: DatabasePlatform, contextUser: UserInfo, outerParams?: Record<string, string>, inlineDependencies?: QueryDependencySpec[]): CompositionResult;
+    /**
+     * Recursively resolves composition tokens in SQL, building up CTE entries.
+     */
+    private resolveTokensRecursive;
+    /**
+     * Looks up a query by category path + name, checking inline dependencies first,
+     * then falling back to the metadata provider.
+     *
+     * For inline dependencies, creates a synthetic QueryInfo with the SQL and flags set
+     * appropriately. Inline queries skip governance validation (Reusable, IsApproved, etc.)
+     * since they are inherently authorized by the caller.
+     */
+    private lookupQueryWithInline;
+    /**
+     * Finds a matching inline dependency spec for the given token.
+     */
+    private findInlineDependency;
+    /**
+     * Builds a synthetic QueryInfo from an inline dependency spec.
+     * Sets flags so that GetPlatformSQL returns the inline SQL directly.
+     */
+    private buildSyntheticQueryInfo;
+    /**
+     * Looks up a query by category path + name from the metadata provider only.
+     */
+    private lookupQueryFromMetadata;
+    /**
+     * Validates that a referenced query is eligible for composition.
+     */
+    private validateQueryComposable;
+    /**
+     * Resolves parameter values for a composition reference.
+     * Static values are used directly; pass-through values are looked up from outer params.
+     */
+    private resolveParameters;
+    /**
+     * Substitutes resolved parameter values into a query's SQL.
+     * - Static values: replaces {{paramName}} with 'value'
+     * - Pass-through values: renames {{paramName}} to {{outerParamName}} so
+     *   the downstream Nunjucks processor can resolve it from the outer query's parameters.
+     */
+    private substituteStaticParams;
+    /**
+     * Builds a deduplication key for a CTE based on query ID and sorted parameter values.
+     */
+    private buildDeduplicationKey;
+    /**
+     * Generates a SQL-safe CTE name from the query name + short hash for uniqueness.
+     */
+    private generateCTEName;
+    /**
+     * Resolves a DatabasePlatform string to the corresponding SQLDialect instance.
+     */
+    private getDialect;
+    /**
+     * Simple string hash for generating short, deterministic suffixes.
+     */
+    private simpleHash;
+    /**
+     * Assembles CTE entries into a WITH clause prepended to the main SQL.
+     *
+     * Handles the case where a dependency query's SQL itself contains a WITH clause
+     * (inner CTEs). SQL does not allow nested WITH clauses, so inner CTEs are "hoisted"
+     * out as sibling CTE definitions preceding the dependency's own CTE.
+     */
+    private assembleCTEs;
+    /**
+     * Extracts inner CTE definitions from SQL that starts with a WITH clause.
+     *
+     * Delegates to {@link SQLParser.ExtractCTEs} which uses AST parsing first
+     * (via node-sql-parser), falling back to a paren-depth regex approach when
+     * AST parsing fails (e.g. SQL contains Nunjucks template tokens).
+     *
+     * @param sql SQL starting with a WITH clause
+     * @param platform Database platform, used to select the AST dialect
+     */
+    private hoistInnerCTEs;
+    /**
+     * Strips a trailing ORDER BY clause from SQL that will be wrapped in a CTE.
+     *
+     * SQL Server disallows ORDER BY inside CTEs unless TOP, OFFSET, or FOR XML is present.
+     * PostgreSQL allows ORDER BY in CTEs, so no stripping is needed there.
+     *
+     * Uses a 4-tier strategy:
+     * 1. Fast exit — no ORDER keyword at all, or dialect allows ORDER BY in CTEs
+     * 2. AST path — parse (with Nunjucks preprocessing if needed), check if ORDER BY is legal
+     *    (TOP/OFFSET/FOR XML via AST nodes), null out orderby if not, regenerate.
+     *    Handles window functions, UNION/EXCEPT, subqueries, string literals, and Nunjucks templates.
+     * 3. Regex fallback — paren-depth heuristic for SQL the parser still can't handle
+     *    (e.g. STRING_AGG WITHIN GROUP)
+     * 4. OFFSET 0 ROWS injection — last resort when both AST and regex fail to strip ORDER BY.
+     *    Injects OFFSET 0 ROWS after the ORDER BY clause to make it legal in CTEs.
+     *    This is semantically neutral (returns all rows starting from 0) but switches
+     *    SQL Server into paging mode internally, which may affect query plan shape.
+     */
+    private stripTrailingOrderBy;
+    /**
+     * Attempts to strip the top-level ORDER BY clause using AST parsing.
+     * Tries direct parsing first, then MJPlaceholder-preprocessed parsing if the SQL
+     * contains MJ template syntax. Handles UNION/EXCEPT by walking the _next chain.
+     */
+    private stripOrderByViaAST;
+    /**
+     * Core AST stripping: parse, analyze, and regenerate SQL without ORDER BY.
+     */
+    private tryASTStrip;
+    /**
+     * Walks the _next chain (UNION/EXCEPT/INTERSECT) to find the statement
+     * that carries the ORDER BY clause.
+     */
+    private findOrderByStatement;
+    /**
+     * Nunjucks-aware ORDER BY stripping: preprocess templates into placeholder SQL,
+     * parse with AST to confirm top-level ORDER BY exists, then use the position-aware
+     * scanner on the original SQL to strip only the last top-level ORDER BY.
+     */
+    private tryNunjucksAwareStrip;
+    /**
+     * Checks AST properties to determine if ORDER BY is legal in a CTE context.
+     */
+    private isOrderByLegalInCTE;
+    /**
+     * Strips the last top-level ORDER BY clause using position-aware scanning.
+     * Skips strings, comments, Nunjucks tags, and tracks paren depth.
+     */
+    private stripLastTopLevelOrderBy;
+    /**
+     * Finds character positions of all ORDER BY keywords at the outermost level
+     * (paren depth 0, not inside strings, comments, or MJ template tokens).
+     *
+     * Uses MJLexer to skip MJ tokens ({{ }}, {% %}, {# #}), then scans only
+     * SQL_TEXT segments for ORDER BY keywords while tracking paren depth and
+     * respecting SQL string literals and comments.
+     */
+    private findTopLevelOrderByPositions;
+    /**
+     * Preprocesses Nunjucks templates into valid SQL for AST parsing.
+     * Uses MJPlaceholderSubstitution for context-aware placeholder generation.
+     */
+    private preprocessNunjucks;
+    /**
+     * Injects OFFSET 0 ROWS after the last top-level ORDER BY clause to make it
+     * legal in a CTE without changing the result set. Uses the position-aware scanner
+     * to find the correct insertion point after the ORDER BY columns.
+     */
+    private injectOffset0Rows;
+    /**
+     * Regex-based fallback for stripping trailing ORDER BY.
+     * Uses parenthesis depth counting to avoid stripping ORDER BY inside subqueries.
+     */
+    private stripOrderByViaRegex;
+    /**
+     * Strips SQL comments from the input string so that composition tokens
+     * inside comments are not treated as real references.
+     * Handles both single-line (-- ...) and multi-line block comments.
+     * Preserves string literals (single-quoted) to avoid stripping inside them.
+     */
+    private stripSQLComments;
+    /**
+     * Escapes {{ and }} inside SQL comments so that Nunjucks doesn't try to parse them.
+     * This is needed because dependency queries may carry comments containing
+     * {{query:"..."}} examples or documentation that would otherwise cause
+     * Nunjucks "expected variable end" errors.
+     *
+     * Only modifies content inside -- single-line and block comments.
+     * Leaves string literals and normal SQL untouched.
+     */
+    private escapeTemplateTokensInComments;
+}
+export {};
+//# sourceMappingURL=queryCompositionEngine.d.ts.map

package/dist/queryCompositionEngine.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"queryCompositionEngine.d.ts","sourceRoot":"","sources":["../src/queryCompositionEngine.ts"],"names":[],"mappings":"AAGA,OAAO,EAAuB,gBAAgB,EAAE,QAAQ,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAY5G;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAC/B,iCAAiC;IACjC,OAAO,EAAE,MAAM,CAAC;IAChB,mCAAmC;IACnC,SAAS,EAAE,MAAM,CAAC;IAClB,gDAAgD;IAChD,YAAY,EAAE,MAAM,CAAC;IACrB,+BAA+B;IAC/B,OAAO,EAAE,MAAM,CAAC;IAChB,uEAAuE;IACvE,WAAW,EAAE,MAAM,CAAC;IACpB,uDAAuD;IACvD,WAAW,EAAE,MAAM,CAAC;IACpB,sDAAsD;IACtD,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACtC;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAC9B,iDAAiD;IACjD,WAAW,EAAE,MAAM,CAAC;IACpB,wCAAwC;IACxC,IAAI,EAAE,kBAAkB,EAAE,CAAC;IAC3B,+DAA+D;IAC/D,eAAe,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IACvC,6DAA6D;IAC7D,eAAe,EAAE,OAAO,CAAC;IACzB,8FAA8F;IAC9F,0BAA0B,EAAE,OAAO,CAAC;CACvC;AAED;;GAEG;AACH,UAAU,sBAAsB;IAC5B,uDAAuD;IACvD,SAAS,EAAE,MAAM,CAAC;IAClB,gEAAgE;IAChE,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,yCAAyC;IACzC,SAAS,EAAE,MAAM,CAAC;IAClB,4DAA4D;IAC5D,QAAQ,EAAE,MAAM,CAAC;IACjB,gCAAgC;IAChC,UAAU,EAAE,eAAe,EAAE,CAAC;CACjC;AAED;;GAEG;AACH,UAAU,eAAe;IACrB,qBAAqB;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,2DAA2D;IAC3D,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,0FAA0F;IAC1F,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;CAClC;AA6BD;;;;;;;;;;;;GAYG;AACH,qBAAa,sBAAsB;IAC/B;;;;OAIG;IACI,oBAAoB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO;IAOjD;;;;;;;;;;OAUG;IACI,sBAAsB,CAAC,GAAG,EAAE,MAAM,GAAG,sBAAsB,EAAE;IA0BpE;;;;;;;;;;;;OAYG;IACI,kBAAkB,CACrB,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,gBAAgB,EAC1B,WAAW,EAAE,QAAQ,EACrB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EACpC,kBAAkB,CAAC,EAAE,mBAAmB,EAAE,GAC3C,iBAAiB;IA2CpB;;OAEG;IACH,OAAO,CAAC,sBAAsB;IA+H9B;;;;;;;OAOG;IACH,OAAO,CAAC,qBAAqB;IAwB7B;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAoB5B;;;OAGG;IACH,OAAO,CAAC,uBAAuB;IAY/B;;OAEG;IACH,OAAO,CAAC,uBAAuB;IAmC/B;;OAEG;IACH,OAAO,CAAC,uBAAuB;IA0B/B;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAoBzB;;;;;OAKG;IACH,OAAO,CAAC,sBAAsB;IAuB9B;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAS7B;;OAEG;IACH,OAAO,CAAC,eAAe;IAevB;;OAEG;IACH,OAAO,CAAC,UAAU;IAQlB;;OAEG;IACH,OAAO,CAAC,UAAU;IAUlB;;;;;;OAMG;IACH,OAAO,CAAC,YAAY;IAmCpB;;;;;;;;;OASG;IACH,OAAO,CAAC,cAAc;IAgBtB;;;;;;;;;;;;;;;;;OAiBG;IACH,OAAO,CAAC,oBAAoB;IAwB5B;;;;OAIG;IACH,OAAO,CAAC,kBAAkB;IAa1B;;OAEG;IACH,OAAO,CAAC,WAAW;IAqBnB;;;OAGG;IACH,OAAO,CAAC,oBAAoB;IAM5B;;;;OAIG;IACH,OAAO,CAAC,qBAAqB;IAoB7B;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAa3B;;;OAGG;IACH,OAAO,CAAC,wBAAwB;IAQhC;;;;;;;OAOG;IACH,OAAO,CAAC,4BAA4B;IAwDpC;;;OAGG;IACH,OAAO,CAAC,kBAAkB;IAI1B;;;;OAIG;IACH,OAAO,CAAC,iBAAiB;IASzB;;;OAGG;IACH,OAAO,CAAC,oBAAoB;IAgB5B;;;;;OAKG;IACH,OAAO,CAAC,gBAAgB;IAgDxB;;;;;;;;OAQG;IACH,OAAO,CAAC,8BAA8B;CAkDzC"}