@memberjunction/core-entities-server 6.2.0-edge.6 → 6.2.0-edge.8

package/dist/custom/MJQueryEntityServer.server.js

@@ -4,21 +4,56 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
     else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
     return c > 3 && r && Object.defineProperty(target, key, r), r;
 };
-import { BaseEntity, LogError, } from "@memberjunction/core";
+import { BaseEntity, BaseEntityResult, DatabaseProviderBase, LogError, ProviderBase, ProviderType, } from "@memberjunction/core";
 import { ComputeQueryFingerprintAsync, MJQueryEntityExtended, QueryEngine, } from "@memberjunction/core-entities";
 import { EscapeSQLString, Float32VectorToBase64, RegisterClass, MJGlobal, UUIDsEqual } from "@memberjunction/global";
-import { EmbedTextLocalHelper } from "./util.js";
-import { RunExtractionPipeline, CleanupQueryData, ConvertTSQLToPostgreSQL, } from "./query-extraction/index.js";
-import { SQLParser } from "@memberjunction/sql-parser";
-import { GetDialect } from "@memberjunction/sql-dialect";
-import { resolveDbPlatformFromEnv } from "@memberjunction/generic-database-provider";
+import { GenericDatabaseProvider, UserCache } from "@memberjunction/generic-database-provider";
+import { CompositionFullPath } from "@memberjunction/sql-parser";
+import { EmbedTextLocalHelper, IsLocalEmbeddingStaleHelper } from "./util.js";
+import { CleanupQueryData, ConvertTSQLToPostgreSQL, CollectDeclaredQueryChildren, HasDeclaredQueryChildren, FindReplacedRows, QueueReplacedRowDeletion, QueryChildNameKey, QueryEntityKey, NO_DECLARED_QUERY_CHILDREN, ComputeQueryExtraction, WriteExtractedChildRows, ApplyFieldFacts, RunDescriptionPass, SQLUsesTemplate, ReadCompositionTargetsAsync, MergeCompositionCandidates, } from "./query-extraction/index.js";
 import { ApplyApprovalRulesAsync, ApprovalRefusalResult } from "./approvalStamp.js";
+/** Whether `provider` can read views and create entity objects, which is all extraction's writes need. */
+function IsExtractionProvider(provider) {
+    if (provider === null || typeof provider !== 'object')
+        return false;
+    return typeof Reflect.get(provider, 'RunView') === 'function' && typeof Reflect.get(provider, 'GetEntityObject') === 'function';
+}
 /**
- * Server-side query entity with embedding generation, SQL extraction pipeline,
- * and dialect auto-conversion. Extends `MJQueryEntityExtended` which provides
- * child-relationship getters and business logic available on both client and server.
+ * Server-side query entity with embedding generation, SQL extraction, and dialect auto-conversion.
+ * Extends `MJQueryEntityExtended`, which provides the child-relationship getters and business
+ * logic available on both client and server.
+ *
+ * **Extraction.** A save of a new query, of a query whose SQL changed, or with
+ * {@link ExtractionOverride} `Force`, establishes the query's facts (`ComputeQueryExtraction`):
+ * its parameters, compositions and the entities it reads from MJ's parser, and its output fields
+ * from the database server's description of each render. Save-time rule errors and repeated
+ * output names refuse a new or edited query, with the reasons in the save's result. Once the save
+ * of the query row succeeds, the child rows are brought in line with the facts as the system user,
+ * keeping what people authored (`WriteExtractedChildRows`), and the query's `UsesTemplate`,
+ * `ExtractedSQLHash`, `ExtractionStatus` and `ExtractionMessage` record the run. Empty
+ * descriptions are then filled by an LLM after the save's transaction commits, unless the save
+ * skips AI actions.
  */
 let MJQueryEntityServer = class MJQueryEntityServer extends MJQueryEntityExtended {
+    constructor() {
+        super(...arguments);
+        /**
+         * What the next save does about extraction: `Auto` (the default) extracts when the query is new
+         * or its SQL changed; `Skip` does not extract, so the save-time rules are not applied and the
+         * child rows, the Extraction columns and `UsesTemplate` stay as they are (for bulk loaders and
+         * fixtures that write those themselves); `Force` extracts even when the SQL did not change.
+         * Reset to `Auto` after each save.
+         */
+        this.ExtractionOverride = 'Auto';
+        /** The SQL a re-extraction reads instead of the cached platform variant (set by {@link RerunExtraction}). */
+        this.platformSQLOverride = null;
+        /** The facts an explicit refresh already established, which its save writes instead of extracting again. */
+        this.presetExtraction = null;
+        /** Whether the running save is an explicit refresh ({@link ApplyRefreshAsync}). */
+        this.refreshing = false;
+        /** What the last save's child-row writes changed; `null` when it wrote none. */
+        this.lastWriteSummary = null;
+    }
     // ─── Embedding Methods ───────────────────────────────────────────────────────
     /**
      * Simple proxy to local helper method for embeddings. Needed for BaseEntity sub-classes that want to use embeddings built into BaseEntity
@@ -26,6 +61,10 @@ let MJQueryEntityServer = class MJQueryEntityServer extends MJQueryEntityExtende
     async EmbedTextLocal(textToEmbed) {
         return EmbedTextLocalHelper(this, textToEmbed);
     }
+    /** A stored vector from a model other than the default local one is re-embedded on save. */
+    async IsEmbeddingStale(embeddingModelID) {
+        return IsLocalEmbeddingStaleHelper(this, embeddingModelID);
+    }
     /**
      * Generates an embedding from composite text (Name + UserQuestion + Description) for richer semantic search.
      * Stores the vector in EmbeddingVector (JSON) and EmbeddingVectorBinary (float32 bytes, base64), and the
@@ -35,8 +74,17 @@ let MJQueryEntityServer = class MJQueryEntityServer extends MJQueryEntityExtende
      * embedder error — all three columns are cleared, so a query whose text changed never keeps a vector
      * computed from its old text. An embedder error is logged and does not fail the save; the query simply
      * has no semantic match until its next successful embedding.
+     *
+     * Does nothing when {@link SkipEmbeddings} is set (a bulk load computes them later), so the stored
+     * vector is left as it is.
+     *
+     * @param textChanged - false when the text is unchanged and only the stored vector's model is stale:
+     *   a failed embed then keeps the existing vector, which still matches the text.
      */
-    async GenerateCompositeEmbedding() {
+    async GenerateCompositeEmbedding(textChanged = true) {
+        if (this.SkipEmbeddings) {
+            return;
+        }
         const parts = [
             this.Name || '',
             this.UserQuestion || '',
@@ -59,7 +107,9 @@ let MJQueryEntityServer = class MJQueryEntityServer extends MJQueryEntityExtende
         catch (error) {
             LogError(`[MJQueryEntityServer] Embedding refresh failed for query "${this.Name}": ${error instanceof Error ? error.message : String(error)}`);
         }
-        this.clearEmbedding();
+        if (textChanged) {
+            this.clearEmbedding();
+        }
     }
     /** Clears the stored vector (both forms) and the model it was computed with. */
     clearEmbedding() {
@@ -67,8 +117,22 @@ let MJQueryEntityServer = class MJQueryEntityServer extends MJQueryEntityExtende
         this.EmbeddingVectorBinary = null;
         this.EmbeddingModelID = null;
     }
+    /** Re-embeds when the embedded text changed, or when the stored vector came from a model other than the default local one. */
+    async refreshEmbeddingAsync() {
+        if (this.SkipEmbeddings)
+            return;
+        const textChanged = !this.IsSaved || ['Name', 'Description', 'UserQuestion'].some(f => this.GetFieldByName(f).Dirty);
+        if (textChanged || (this.EmbeddingVector != null && await this.IsEmbeddingStale(this.EmbeddingModelID))) {
+            await this.GenerateCompositeEmbedding(textChanged);
+        }
+    }
     // ─── Save / Delete Overrides ─────────────────────────────────────────────────
     async Save(options) {
+        // Over a client provider (a process that loads this package but saves over GraphQL), the server
+        // the save reaches applies the approval rules and extracts; doing it here too would write the
+        // child rows twice, the second time from the client.
+        if (this.savesOverNetwork)
+            return super.Save(options);
         let approving;
         try {
             // Approving is a server act; a ReplayOnly save writes nothing, so it has nothing to stamp.
@@ -79,47 +143,185 @@ let MJQueryEntityServer = class MJQueryEntityServer extends MJQueryEntityExtende
             return false;
         }
         try {
-            const sqlField = this.GetFieldByName('SQL');
-            const nameField = this.GetFieldByName('Name');
-            const descriptionField = this.GetFieldByName('Description');
-            const userQuestionField = this.GetFieldByName('UserQuestion');
-            const shouldExtractData = !this.IsSaved || sqlField.Dirty;
-            const shouldGenerateEmbedding = !this.IsSaved || nameField.Dirty || descriptionField.Dirty || userQuestionField.Dirty;
-            // Generate embedding from composite text for better semantic search
-            if (shouldGenerateEmbedding) {
-                await this.GenerateCompositeEmbedding();
-            }
-            else if (!this.Description || this.Description.trim().length === 0) {
-                this.EmbeddingVector = null;
-                this.EmbeddingVectorBinary = null;
-                this.EmbeddingModelID = null;
-            }
-            // Save the query first without AI processing
-            const saveResult = await super.Save(options);
-            if (!saveResult) {
-                return false;
-            }
-            // Extract and sync parameters AFTER saving, outside of any transaction
-            if (shouldExtractData && this.SQL && this.SQL.trim().length > 0) {
-                await this.extractAndSyncDataAsync(options?.SkipEntityAIActions === true);
-                await this.autoConvertDialectsAsync();
-            }
-            else if (!this.SQL || this.SQL.trim().length === 0) {
-                this.UsesTemplate = false;
-                await this.cleanupEmptyQueryAsync();
-            }
-            // The parameter and dialect rows are rewritten after the Query row, so the approved
-            // fingerprint is taken last, from the definition as stored.
-            if (approving) {
-                await this.stampApprovalFingerprintAsync(options);
-            }
-            return true;
+            return await this.saveWithExtractionAsync(options, approving);
         }
         catch (e) {
             LogError('Failed to save query:', e);
             this.LatestResult?.Errors.push(e);
             return false;
         }
+        finally {
+            this.ExtractionOverride = 'Auto';
+            this.platformSQLOverride = null;
+            this.presetExtraction = null;
+            this.refreshing = false;
+        }
+    }
+    /** Whether this object saves through a network (client) provider rather than to a database. */
+    get savesOverNetwork() {
+        const provider = this.ProviderToUse;
+        return provider instanceof ProviderBase && provider.ProviderType === ProviderType.Network;
+    }
+    /**
+     * The save: facts first (a refusal writes nothing), then the query row (a graph save with any
+     * declared children), then the child rows extraction derives, then the approval fingerprint.
+     */
+    async saveWithExtractionAsync(options, approving) {
+        const declared = await this.PrepareDeclaredChildrenAsync(options);
+        const plan = this.planExtraction(options, HasDeclaredQueryChildren(declared));
+        const extraction = plan.Run ? await this.computeExtractionAsync(plan.Mode) : null;
+        if (extraction?.Refusal) {
+            this.RegisterResultHistoryEntry(this.refusalResult(extraction.Refusal));
+            return false;
+        }
+        this.applyRunColumns(extraction);
+        if (extraction)
+            this.applyFactsToDeclaredFields(extraction);
+        await this.refreshEmbeddingAsync();
+        const postCommit = this.capturePostCommitToken();
+        this.lastWriteSummary = null;
+        if (!await super.Save(options)) {
+            return false;
+        }
+        if (extraction) {
+            await this.writeExtractionAsync(extraction, options, postCommit);
+            // A refresh brings the derived rows in line with the SQL; it never writes SQL, a dialect variant's included.
+            if (!this.refreshing)
+                await this.autoConvertDialectsAsync();
+        }
+        else if (!this.hasSQL) {
+            await this.cleanupEmptyQueryAsync();
+        }
+        // The child and dialect rows are rewritten after the Query row, so the approved fingerprint is
+        // taken last, from the definition as stored.
+        if (approving) {
+            await this.stampApprovalFingerprintAsync(options);
+        }
+        return true;
+    }
+    get hasSQL() {
+        return !!this.SQL && this.SQL.trim().length > 0;
+    }
+    /**
+     * Whether this save extracts, and under which rules: a new or edited query extracts under the
+     * save-time rules; a save that writes declared child rows, or a `Force` override, re-extracts an
+     * unchanged query under the refresh rules, so the facts of rows it writes are corrected again.
+     */
+    planExtraction(options, writesDeclaredChildren) {
+        const edited = !this.IsSaved || this.GetFieldByName('SQL').Dirty;
+        if (options?.ReplayOnly || !this.hasSQL || this.ExtractionOverride === 'Skip') {
+            return { Run: false, Mode: 'Save' };
+        }
+        return { Run: edited || writesDeclaredChildren || this.ExtractionOverride === 'Force', Mode: edited ? 'Save' : 'Refresh' };
+    }
+    /**
+     * Sets the columns a save records about the SQL before the row is written. `UsesTemplate` is a
+     * lexical fact and follows every SQL change; it is never changed by a failed extraction. A save
+     * with {@link ExtractionOverride} `Skip` leaves it as the caller set it, like the other
+     * extraction results.
+     */
+    applyRunColumns(extraction) {
+        if (extraction) {
+            this.UsesTemplate = extraction.UsesTemplate;
+            this.ExtractedSQLHash = extraction.Hash;
+            this.ExtractionStatus = extraction.Status;
+            this.ExtractionMessage = extraction.Messages.length > 0 ? extraction.Messages.join('\n') : null;
+        }
+        else if (!this.hasSQL) {
+            this.UsesTemplate = false;
+        }
+        else if (this.ExtractionOverride !== 'Skip' && (!this.IsSaved || this.GetFieldByName('SQL').Dirty)) {
+            this.UsesTemplate = this.lexicalUsesTemplate();
+        }
+    }
+    /**
+     * Gives each field row this save declares the facts extraction established for it, before the
+     * graph is written. Facts follow the SQL and the database server whatever a caller declares, so
+     * a declaration that carries facts (types, nullability, sequence, source) for another platform is
+     * corrected before it is written instead of being written and then corrected, and a new declared
+     * row, which may declare only its name and annotations, gets the facts its columns require.
+     */
+    applyFactsToDeclaredFields(extraction) {
+        if (!extraction.Fields)
+            return;
+        const facts = new Map(extraction.Fields.map(f => [f.Name.toLowerCase(), f]));
+        for (const row of this.DeclaredFields?.Items ?? []) {
+            const fact = facts.get((row.Name ?? '').toLowerCase());
+            if (fact)
+                ApplyFieldFacts(row, fact);
+        }
+    }
+    /** Whether the SQL holds a template tag outside comments, read by the lexer. */
+    lexicalUsesTemplate() {
+        return SQLUsesTemplate(this.SQL ?? '', this.extractionPlatform);
+    }
+    /** The save's result entry for a refused save, carrying the reasons. */
+    refusalResult(reason) {
+        const result = new BaseEntityResult();
+        result.Success = false;
+        result.Type = this.IsSaved ? 'update' : 'create';
+        result.Message = reason;
+        result.StartedAt = new Date();
+        result.EndedAt = new Date();
+        return result;
+    }
+    /**
+     * The declared-rows-win hook, run before the save writes anything.
+     *
+     * A caller can save a query together with child rows it declares, through the `Parameters`,
+     * `Fields` and `Entities` collections. When a NEW declared row has the key of a stored row with a
+     * different ID whose `DetectionMethod` is not `Manual` (a row extraction created), the stored row
+     * is deleted inside the same graph save, before the declared row is inserted. The declared row
+     * keeps the ID the caller gave it, which is what makes a metadata file that names its child IDs
+     * converge instead of colliding with rows extraction wrote earlier. A stored `Manual` row is never
+     * deleted here; a declared row that collides with one fails the save on the database's own
+     * uniqueness rule where there is one.
+     *
+     * Extraction then runs after `super.Save()`, sees the declared rows already written, and treats
+     * them like any stored row: their facts are corrected from the SQL, their authored annotations
+     * kept.
+     *
+     * @param options - The save's options. A `ReplayOnly` save, or one that skips related
+     *   collections, writes no declared rows, so nothing is declared.
+     * @returns The keys of the child rows this save declares.
+     */
+    async PrepareDeclaredChildrenAsync(options) {
+        if (options?.ReplayOnly || options?.SkipRelatedCollections) {
+            return NO_DECLARED_QUERY_CHILDREN;
+        }
+        const collections = { Parameters: this.DeclaredParameters, Fields: this.DeclaredFields, Entities: this.DeclaredEntities };
+        const declared = CollectDeclaredQueryChildren(collections);
+        if (this.IsSaved && HasDeclaredQueryChildren(declared)) {
+            await this.queueReplacedRowDeletionsAsync();
+        }
+        return declared;
+    }
+    /**
+     * Reads the stored child rows from the database (never a cache, which inside a transaction can
+     * miss rows written earlier in it) and queues the extraction-created rows each new declared row
+     * replaces for deletion in the graph save.
+     */
+    async queueReplacedRowDeletionsAsync() {
+        const filter = `QueryID='${EscapeSQLString(this.ID)}'`;
+        const [parameters, fields, entities] = await this.RunViewProviderToUse.RunViews([
+            { EntityName: 'MJ: Query Parameters', ExtraFilter: filter, ResultType: 'entity_object', BypassCache: true, IgnoreMaxRows: true },
+            { EntityName: 'MJ: Query Fields', ExtraFilter: filter, ResultType: 'entity_object', BypassCache: true, IgnoreMaxRows: true },
+            { EntityName: 'MJ: Query Entities', ExtraFilter: filter, ResultType: 'entity_object', BypassCache: true, IgnoreMaxRows: true },
+        ], this.ContextCurrentUser);
+        if (!parameters?.Success || !fields?.Success || !entities?.Success) {
+            throw new Error(`could not read the query's stored child rows: ${parameters?.ErrorMessage || fields?.ErrorMessage || entities?.ErrorMessage}`);
+        }
+        const newItems = (items) => items.filter(i => !i.IsSaved);
+        const byName = (row) => QueryChildNameKey(row.Name);
+        for (const row of FindReplacedRows(newItems(this.DeclaredParameters.Items), parameters.Results, byName)) {
+            QueueReplacedRowDeletion(this.DeclaredParameters, row);
+        }
+        for (const row of FindReplacedRows(newItems(this.DeclaredFields.Items), fields.Results, byName)) {
+            QueueReplacedRowDeletion(this.DeclaredFields, row);
+        }
+        for (const row of FindReplacedRows(newItems(this.DeclaredEntities.Items), entities.Results, QueryEntityKey)) {
+            QueueReplacedRowDeletion(this.DeclaredEntities, row);
+        }
     }
     /**
      * Stores the fingerprint of the definition just approved: this row's SQL, template flag and
@@ -151,11 +353,7 @@ let MJQueryEntityServer = class MJQueryEntityServer extends MJQueryEntityExtende
     }
     async Delete(options) {
         try {
-            const deleteResult = await super.Delete(options);
-            if (!deleteResult) {
-                return false;
-            }
-            return deleteResult;
+            return await super.Delete(options);
         }
         catch (e) {
             LogError('Failed to delete query:', e);
@@ -163,131 +361,251 @@ let MJQueryEntityServer = class MJQueryEntityServer extends MJQueryEntityExtende
             return false;
         }
     }
-    // ─── Pipeline Delegation ─────────────────────────────────────────────────────
+    // ─── Extraction ──────────────────────────────────────────────────────────────
     /**
-     * Returns the database platform of the connected database.
-     * Uses the DB_PLATFORM env var (same source as MJServer's provider selection).
+     * The platform the query is extracted for: the platform of the provider this entity saves
+     * through, so two providers on different platforms in one process each extract for their own.
      */
-    get currentPlatform() {
-        return resolveDbPlatformFromEnv() ?? 'sqlserver';
+    get extractionPlatform() {
+        const provider = this.ProviderToUse;
+        return provider instanceof DatabaseProviderBase ? provider.PlatformKey : 'sqlserver';
+    }
+    /** The provider as extraction reads and writes through it; `null` when it cannot read views and create entity objects. */
+    get extractionProvider() {
+        const provider = this.ProviderToUse;
+        return IsExtractionProvider(provider) ? provider : null;
     }
     /**
-     * Resolves the SQL to use for extraction based on the current database platform.
-     *
-     * Resolution order:
-     *   1. MJ: Query SQLs record matching the current platform's dialect → use that SQL
-     *   2. Base Query.SQL field → fall back if no dialect variant exists
-     *
-     * After resolution, validates the SQL is compatible with the current platform's
-     * dialect by attempting a parse. Logs a warning if the SQL appears to be for a
-     * different dialect (e.g., T-SQL bracket identifiers on a PostgreSQL database).
+     * The user derived child rows are read and written as: the system user, once the caller's save
+     * of the query is authorized (the rows mirror SQL the caller was allowed to change, like CodeGen
+     * output). The caller when no system user is loaded.
      */
-    resolveExtractionSQL() {
-        const platform = this.currentPlatform;
-        // 1. Check for a dialect-specific QuerySQL record
-        const platformSQL = this.GetPlatformSQL(platform);
-        // If GetPlatformSQL returned something different from the base SQL, use it
-        const resolvedSQL = platformSQL || this.SQL;
-        if (!resolvedSQL || resolvedSQL.trim().length === 0) {
-            return '';
-        }
-        // Validate dialect compatibility
-        this.validateDialectCompatibility(resolvedSQL, platform);
-        return resolvedSQL;
+    get extractionUser() {
+        if (this.refreshing)
+            return this.ContextCurrentUser;
+        return UserCache.Instance?.GetSystemUser?.() ?? this.ContextCurrentUser;
+    }
+    /** The SQL extraction reads: a re-extraction's override, else the query's SQL for the platform. */
+    get extractionSQL() {
+        return this.platformSQLOverride ?? (this.GetPlatformSQL(this.extractionPlatform) || this.SQL || '');
+    }
+    /** Establishes the query's facts (nothing is written); an explicit refresh's facts when it already has them. */
+    async computeExtractionAsync(mode) {
+        if (this.presetExtraction)
+            return this.presetExtraction;
+        const provider = this.ProviderToUse;
+        await this.ensureQueryEngineLoadedAsync();
+        const targets = await this.readCompositionTargetsAsync();
+        return ComputeQueryExtraction({
+            QueryID: this.IsSaved ? this.ID : null,
+            QueryName: this.Name,
+            QueryPath: CompositionFullPath({ Name: this.Name, CategoryPath: this.CategoryID ? this.BuildCategoryPath(this.CategoryID) : '' }),
+            SQL: this.extractionSQL,
+            Platform: this.extractionPlatform,
+            Mode: mode,
+            IsExternal: !!this.ExternalDataSourceID,
+            KnownParameters: await this.knownParametersAsync(),
+            ParameterHints: this.ParameterHints,
+            ContextUser: this.ContextCurrentUser,
+            Entities: provider instanceof DatabaseProviderBase ? provider.Entities : [],
+            Queries: MergeCompositionCandidates(targets.Candidates, QueryEngine.Instance.Queries),
+            CompositionDependencies: targets.InlineDependencies,
+            Describe: provider instanceof GenericDatabaseProvider ? (sql) => provider.DescribeStatement(sql, this.ContextCurrentUser) : null,
+        });
     }
     /**
-     * Best-effort dialect validation: attempts to parse the SQL with the current platform's
-     * dialect. If parsing throws, logs a warning — the SQL may be for a different dialect.
-     *
-     * This is a heuristic, not a guarantee: node-sql-parser is permissive on many
-     * dialect-specific functions (ISNULL, GETDATE pass in both dialects), so a passing
-     * parse does not prove the SQL is correct for this dialect. But a failing parse is a
-     * strong signal of incompatibility.
+     * The queries the SQL composes, read on this save's provider with every cache bypassed, so a
+     * target saved earlier in the same transaction (one metadata push creating both queries) is
+     * found before the shared query cache hears of it. Nothing is read for SQL that composes
+     * nothing, or on a provider that cannot run views.
      */
-    validateDialectCompatibility(sql, platform) {
+    async readCompositionTargetsAsync() {
+        const provider = this.ProviderToUse;
+        if (!(provider instanceof DatabaseProviderBase))
+            return { Candidates: [], InlineDependencies: [] };
+        const engine = QueryEngine.Instance;
+        const cachedIDs = new Set(engine.Queries.map(q => q.ID.toLowerCase()));
+        return ReadCompositionTargetsAsync(this.extractionSQL, this.extractionPlatform, provider, this.extractionUser, cachedIDs, engine.SQLDialects);
+    }
+    /** Loads QueryEngine when this process has not yet (a CLI or sync context); compositions resolve against it. */
+    async ensureQueryEngineLoadedAsync() {
+        if (QueryEngine.Instance.Loaded)
+            return;
         try {
-            const dialect = GetDialect(platform);
-            // Attempt to parse — will throw on strong dialect markers
-            // (e.g., [bracket] identifiers fail on PG, :: casts fail on T-SQL)
-            SQLParser.ExtractSelectColumns(sql, dialect);
+            await QueryEngine.Instance.Config(false, this.ContextCurrentUser);
         }
-        catch {
-            const otherDialect = platform === 'postgresql' ? 'T-SQL (SQL Server)' : 'PostgreSQL';
-            console.warn(`[MJQueryEntityServer] Query "${this.Name}" — SQL may be incompatible with the ` +
-                `current database platform "${platform}". The SQL appears to use ${otherDialect} syntax. ` +
-                `Consider adding a dialect-specific variant via MJ: Query SQLs.`);
+        catch (e) {
+            LogError(`[MJQueryEntityServer] QueryEngine could not be loaded for extraction of "${this.Name}": ${e instanceof Error ? e.message : String(e)}`);
         }
     }
     /**
-     * Builds the QuerySyncContext from this entity instance for use by the extraction pipeline.
-     * Resolves the SQL to the appropriate dialect variant for the connected database.
+     * The parameter rows extraction knows before it runs: the stored rows, read from the database
+     * (never the cache), overlaid by the rows this save declares.
      */
-    buildSyncContext(skipAIEnrichment = false) {
-        return {
-            queryID: this.ID,
-            queryName: this.Name,
-            sql: this.resolveExtractionSQL(),
-            isSaved: this.IsSaved,
-            contextUser: this.ContextCurrentUser,
-            metadataProvider: this.ProviderToUse,
-            runViewProvider: this.RunViewProviderToUse,
-            platform: this.currentPlatform,
-            parameterHints: this.ParameterHints,
-            SkipAIEnrichment: skipAIEnrichment,
-        };
+    async knownParametersAsync() {
+        const byName = new Map();
+        for (const row of await this.storedParametersAsync())
+            byName.set(row.Name.toLowerCase(), row);
+        for (const row of this.DeclaredParameters?.Items ?? [])
+            byName.set(row.Name.toLowerCase(), row);
+        return [...byName.values()].map(row => ({
+            Name: row.Name, Type: row.Type, IsRequired: row.IsRequired === true, IsServerSet: row.IsServerSet === true,
+            SampleValue: row.SampleValue, DefaultValue: row.DefaultValue, ValidationFilters: row.ValidationFilters,
+            IsAuthored: row.DetectionMethod === 'Manual',
+        }));
+    }
+    async storedParametersAsync() {
+        const provider = this.extractionProvider;
+        if (!this.IsSaved || !provider)
+            return [];
+        const result = await provider.RunView({
+            EntityName: 'MJ: Query Parameters', ExtraFilter: `QueryID='${EscapeSQLString(this.ID)}'`, ResultType: 'entity_object', BypassCache: true,
+        }, this.extractionUser);
+        if (!result.Success)
+            throw new Error(`could not read the query's stored parameters: ${result.ErrorMessage}`);
+        return result.Results ?? [];
     }
     /**
-     * Re-runs the extraction pipeline on this query. Called by MJQuerySQLEntityServer
-     * when a dialect variant matching the current platform is saved — this handles
-     * the timing issue where the parent Query is saved before its QuerySQL children
-     * exist in the cache.
+     * Writes the derived child rows as the system user, inside the caller's transaction when there
+     * is one, then queues the description pass. A failed write is recorded on the query
+     * (`ExtractionStatus` `Error`) rather than failing a save whose row is already written.
+     */
+    async writeExtractionAsync(extraction, options, postCommit) {
+        const provider = this.extractionProvider;
+        try {
+            if (!provider)
+                throw new Error('the query is not saved through a database provider');
+            const summary = await WriteExtractedChildRows({ QueryID: this.ID, Provider: provider, User: this.extractionUser, EntityName: id => this.entityNameOf(id) }, extraction);
+            this.lastWriteSummary = summary;
+            this.queueDescriptionPass(summary, options, postCommit);
+        }
+        catch (e) {
+            // An explicit refresh runs in its own transaction, which a failed write rolls back.
+            if (this.refreshing)
+                throw e;
+            await this.recordWriteFailureAsync(e);
+        }
+    }
+    /** An entity's name from the saving provider's catalog, for reporting; the ID when it is not there. */
+    entityNameOf(entityID) {
+        const provider = this.ProviderToUse;
+        const entities = provider instanceof DatabaseProviderBase ? provider.Entities : [];
+        return entities.find(e => UUIDsEqual(e.ID, entityID))?.Name ?? entityID;
+    }
+    async recordWriteFailureAsync(error) {
+        const reason = `The extracted child rows could not be written: ${error instanceof Error ? error.message : String(error)}`;
+        LogError(`[MJQueryEntityServer] Query "${this.Name}": ${reason}`);
+        this.ExtractionStatus = 'Error';
+        this.ExtractionMessage = this.ExtractionMessage ? `${this.ExtractionMessage}\n${reason}` : reason;
+        if (!await super.Save()) {
+            LogError(`[MJQueryEntityServer] Query "${this.Name}": the extraction failure could not be recorded: ${this.LatestResult?.CompleteMessage}`);
+        }
+    }
+    /** The post-commit token of the caller's transaction, captured before the save writes anything. */
+    capturePostCommitToken() {
+        const provider = this.ProviderToUse;
+        return provider instanceof DatabaseProviderBase ? provider.CapturePostCommitToken() : undefined;
+    }
+    /**
+     * Queues the LLM description pass to run after the caller's transaction commits, when some row
+     * has no description and the save does not skip AI actions. No LLM call is made inside the
+     * transaction. A save-triggered extraction that changed no child row does not queue it: the
+     * rows are as the last run left them, so a repeated save (a re-push of unchanged metadata) does
+     * not call the LLM again, and a pass that failed is not retried, and recorded, on every save.
+     * An explicit refresh that asks for the pass always queues it.
+     */
+    queueDescriptionPass(summary, options, postCommit) {
+        const provider = this.ProviderToUse;
+        const undescribed = summary.FieldsWithoutDescription.length + summary.ParametersWithoutDescription.length;
+        if (options?.SkipEntityAIActions || undescribed === 0 || !(provider instanceof DatabaseProviderBase))
+            return;
+        if (!this.refreshing && summary.Changes.length === 0)
+            return;
+        const request = { QueryID: this.ID, QueryName: this.Name, SQL: this.extractionSQL, Provider: provider, User: this.extractionUser };
+        provider.RunAfterCommit(() => RunDescriptionPass(request), `query description pass for "${this.Name}"`, postCommit);
+    }
+    /**
+     * Re-extracts this query: a save with {@link ExtractionOverride} `Force`. Called by
+     * `MJQuerySQLEntityServer` when a dialect variant for the current platform is saved.
      *
-     * @param skipAIEnrichment Skip the LLM enrichment stage, as `SkipEntityAIActions` does for Save.
+     * @param skipAIEnrichment Skip the description pass, as `SkipEntityAIActions` does for a save.
+     * @param platformSQL The SQL to extract from (the variant just saved), instead of the cached
+     *   platform variant, which inside a transaction may not hold it yet.
+     */
+    async RerunExtraction(skipAIEnrichment = false, platformSQL) {
+        this.ExtractionOverride = 'Force';
+        this.platformSQLOverride = platformSQL ?? null;
+        if (!await this.Save({ SkipEntityAIActions: skipAIEnrichment })) {
+            LogError(`[MJQueryEntityServer] Re-extraction of query "${this.Name}" failed: ${this.LatestResult?.CompleteMessage}`);
+        }
+    }
+    // ─── Explicit refresh ────────────────────────────────────────────────────────
+    /**
+     * Establishes this saved query's facts under the refresh rules (a save-time rule error is
+     * recorded, never a refusal), as the save would, and writes nothing. The facts are read on the
+     * provider this entity was loaded with, for its platform.
      */
-    async RerunExtraction(skipAIEnrichment = false) {
-        await this.extractAndSyncDataAsync(skipAIEnrichment);
+    async ComputeRefreshExtractionAsync() {
+        return this.computeExtractionAsync('Refresh');
     }
     /**
-     * Runs the extraction pipeline and saves the UsesTemplate flag.
-     * On error, sets UsesTemplate=false and attempts to save so the query remains usable.
+     * The child-row changes `extraction` would make, worked out against the rows stored now and
+     * read as this entity's context user. Nothing is written.
+     */
+    async PreviewRefreshAsync(extraction, entityName) {
+        const provider = this.extractionProvider;
+        if (!provider)
+            throw new Error('the query is not loaded through a database provider');
+        return WriteExtractedChildRows({ QueryID: this.ID, Provider: provider, User: this.ContextCurrentUser, DryRun: true, EntityName: entityName ?? (id => this.entityNameOf(id)) }, extraction);
+    }
+    /**
+     * Writes `extraction` (from {@link ComputeRefreshExtractionAsync}): a save with
+     * {@link ExtractionOverride} `Force` that records the run on the query and brings the child
+     * rows in line with the facts. Unlike a save-triggered extraction, the child rows are written
+     * as this entity's context user, so each write is checked against that user's permissions, and
+     * a failed write fails the refresh instead of being recorded on the query. SQL, dialect
+     * variants and the embedding are never written.
      *
-     * @param skipAIEnrichment Skip the LLM enrichment stage (the save asked to skip AI actions).
+     * @param withAI Queue the LLM description pass for rows still without a description.
+     * @returns What the child-row writes changed.
+     * @throws Error when the save or a child-row write fails; the caller's transaction should be rolled back.
      */
-    async extractAndSyncDataAsync(skipAIEnrichment) {
+    async ApplyRefreshAsync(extraction, withAI) {
+        this.ExtractionOverride = 'Force';
+        this.presetExtraction = extraction;
+        this.refreshing = true;
+        // A refresh changes the derived rows, not the text the embedding is computed from.
+        const skipEmbeddings = this.SkipEmbeddings;
+        this.SkipEmbeddings = true;
         try {
-            const ctx = this.buildSyncContext(skipAIEnrichment);
-            const pipelineResult = await RunExtractionPipeline(ctx);
-            this.UsesTemplate = pipelineResult.UsesTemplate;
-            // Save the query again to persist UsesTemplate and any changes from AI processing
-            const updateResult = await super.Save();
-            if (!updateResult) {
-                LogError('Failed to save query after AI processing completed');
+            if (!await this.Save({ SkipEntityAIActions: !withAI })) {
+                throw new Error(this.LatestResult?.CompleteMessage || `the refresh of query "${this.Name}" could not be saved`);
             }
         }
-        catch (e) {
-            LogError('Error in async AI processing:', e);
-            this.UsesTemplate = false;
-            try {
-                await super.Save();
-            }
-            catch (saveError) {
-                LogError('Failed to save query after AI processing error:', saveError);
-            }
+        finally {
+            this.SkipEmbeddings = skipEmbeddings;
         }
+        return this.lastWriteSummary;
     }
     /**
-     * Cleans up all extraction data when SQL is empty, then saves the updated UsesTemplate flag.
+     * Cleans up all extraction data when SQL is empty.
      */
     async cleanupEmptyQueryAsync() {
+        const provider = this.extractionProvider;
+        if (!provider)
+            return;
         try {
-            const ctx = this.buildSyncContext();
+            const ctx = {
+                queryID: this.ID,
+                queryName: this.Name,
+                sql: '',
+                isSaved: this.IsSaved,
+                contextUser: this.extractionUser,
+                metadataProvider: provider,
+                runViewProvider: provider,
+                platform: this.extractionPlatform,
+            };
             await CleanupQueryData(ctx);
-            // Also remove dialect records (not managed by the pipeline)
-            await this.removeAllQuerySQLRecordsAsync();
-            const updateResult = await super.Save();
-            if (!updateResult) {
-                LogError('Failed to save query after cleanup');
-            }
         }
         catch (e) {
             LogError('Error in async cleanup:', e);
@@ -352,9 +670,23 @@ let MJQueryEntityServer = class MJQueryEntityServer extends MJQueryEntityExtende
      * Creates or updates a QuerySQL record for the given dialect.
      */
     async upsertQuerySQLRecord(targetDialect, convertedSQL) {
-        const md = this.ProviderToUse;
-        // Look up existing record from QueryEngine cache instead of a RunView call
-        const existing = QueryEngine.Instance.QuerySQLs.find(qs => UUIDsEqual(qs.QueryID, this.ID) && UUIDsEqual(qs.SQLDialectID, targetDialect.ID));
+        const md = this.extractionProvider;
+        if (!md)
+            return;
+        // Read through this entity's own provider rather than QueryEngine's cache. A cached record is
+        // bound to whichever provider loaded the engine, so saving it here would write outside the
+        // caller's transaction, blind to what that transaction has written.
+        const existingResult = await md.RunView({
+            EntityName: 'MJ: Query SQLs',
+            ExtraFilter: `QueryID='${EscapeSQLString(this.ID)}' AND SQLDialectID='${EscapeSQLString(targetDialect.ID)}'`,
+            ResultType: 'entity_object',
+            BypassCache: true,
+        }, this.ContextCurrentUser);
+        if (!existingResult.Success) {
+            console.warn(`Query "${this.Name}" - Could not read QuerySQL record for dialect "${targetDialect.Name}": ${existingResult.ErrorMessage}`);
+            return;
+        }
+        const existing = existingResult.Results?.[0];
         let record;
         if (existing) {
             record = existing;
@@ -370,23 +702,6 @@ let MJQueryEntityServer = class MJQueryEntityServer extends MJQueryEntityExtende
             console.warn(`Query "${this.Name}" - Failed to save QuerySQL record for dialect "${targetDialect.Name}"`);
         }
     }
-    /**
-     * Removes all QuerySQL records for this query. Called during cleanup operations.
-     */
-    async removeAllQuerySQLRecordsAsync() {
-        try {
-            if (!this.IsSaved)
-                return;
-            const records = QueryEngine.Instance.QuerySQLs.filter(qs => UUIDsEqual(qs.QueryID, this.ID));
-            const deletePromises = records.map(r => r.Delete());
-            if (deletePromises.length > 0) {
-                await Promise.all(deletePromises);
-            }
-        }
-        catch (e) {
-            console.warn(`Query "${this.Name}" - Failed to remove QuerySQL records:`, e);
-        }
-    }
 };
 MJQueryEntityServer = __decorate([
     RegisterClass(BaseEntity, 'MJ: Queries')