@duckcodeailabs/dql-cli 1.12.7 → 1.12.9

package/dist/local-runtime.js

@@ -1,5 +1,6 @@
 import { execFileSync, execSync } from "node:child_process";
 import { createHash } from "node:crypto";
+import { gzipSync } from "node:zlib";
 import { createServer } from "node:http";
 import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, statSync, watch, writeFileSync, } from "node:fs";
 import { createRequire } from "node:module";
@@ -20,7 +21,7 @@ import { fetchLatestPublishedDqlVersion, resolveDqlRuntimeVersionStatus } from '
 import { resolveRetrievalHealthStatus } from './retrieval-health.js';
 import { createDqlAgentProviderRunner, resolveAgentFollowUpContext } from './llm/providers/dql-agent-provider.js';
 import { listRemoteMcpSettings, saveRemoteMcpSettings } from './llm/mcp-config.js';
-import { ClaudeProvider, ConversationStore, advanceThreadState, buildConversationSnapshot, conversationHistoryFromContext, recallRelevantTurns, renderConversationEnvelopeForPrompt, GeminiProvider, MemoryStore, OllamaProvider, OpenAIProvider, buildBlockBusinessFingerprint, buildBlockSqlFingerprints, buildAnalysisQuestionPlan, composeSemanticQueryForQuestion, aggregationIntegrityIssuesForSql, buildLocalContextPack, applyContextPackCompatibility, toAgentRetrievalEvidence, prepareConversationPath, defaultMemoryPath, ensureDefaultMemoryFiles, ensureAgentProjectReady, isAgentProjectIndexReady, currentMetadataFingerprint, ensureMetadataCatalogFresh, readIndexedDomainKnowledge, readIndexedKnowledge360, compactSemanticRuntimeFailure, classifyAnalyticalFailure, propose, proposePlan, recordGovernedCorrection, HintStore, defaultHintIndexPath, ensureHintIndexFresh, listHintsFromGit, getHintEvaluationFromGit, getCorrectionTraceFromGit, inspectGovernedHint, editGovernedHintCandidate, reopenGovernedHint, retireHint, supersedeHint, hintsConflict, mineJoinPatterns, reviewGovernedHint, AgentRunEngine, SqliteAgentRunStore, defaultAgentRunGates, createLlmAgentRunPlanner, createHybridRouter, computeResultStats, buildDeterministicDashboardStory, synthesizeAnswer, streamOrGenerate, narrateResult, buildProposePreview, buildFromPrompt, internalRelationIdsInSql, defaultAgentRunStorePath, defaultAgentRunSqlitePath, resolveLocalOwner, resolveProposeConfig, recordQueryRun, recordRuntimeSchemaSnapshot, latestRuntimeSchemaSnapshotForProject, loadSkills, migrateLegacySkills, configuredSkillsPath, skillsDir, draftDomainSkillBootstrap, buildDomainSkillBootstrapPrompt, mergeDomainSkillBootstrapEnrichment, writeSkill, deleteSkill, deriveGeneratedDraftSlug, deriveAnalyticalRepair, reindexProject, invalidateAgentProjectState, recordAgentRuntimeVersion, resolveDomainContextEnvelope, projectEmbeddingProvider, isHashedEmbeddingProvider, clearProjectEmbeddingCache, upgradeVectorIndexForProject, openMetadataCatalog, defaultKgPath, planAppFromPrompt, KGStore, planResearch, loadSemanticMetrics, cascadeTraceToEvidenceRouteSteps, createCascadeAnswerResult, createCascadeTrace, routeReasoningEffort, routeForCascadeAnswerTier, clampReasoningEffort, bumpReasoningEffort, resolveThinkingMode, coerceThinkingMode, upsertGeneratedDqlArtifactDraft, loadAgentSemanticLayer, isTrustedConversationTurn, analyticalError, withAnalyticalErrorOrigin, withAnalyticalErrorOriginSync, } from '@duckcodeailabs/dql-agent';
+import { ClaudeProvider, ConversationStore, advanceThreadState, buildConversationSnapshot, conversationHistoryFromContext, recallRelevantTurns, renderConversationEnvelopeForPrompt, GeminiProvider, MemoryStore, OllamaProvider, OpenAIProvider, buildBlockBusinessFingerprint, buildBlockSqlFingerprints, buildAnalysisQuestionPlan, composeSemanticQueryForQuestion, aggregationIntegrityIssuesForSql, buildLocalContextPack, applyContextPackCompatibility, toAgentRetrievalEvidence, prepareConversationPath, defaultMemoryPath, ensureDefaultMemoryFiles, ensureAgentProjectReady, isAgentProjectIndexReady, currentMetadataFingerprint, ensureMetadataCatalogFresh, readIndexedDomainKnowledge, readIndexedKnowledge360, compactSemanticRuntimeFailure, classifyAnalyticalFailure, propose, proposePlan, recordGovernedCorrection, HintStore, defaultHintIndexPath, ensureHintIndexFresh, listHintsFromGit, getHintEvaluationFromGit, getCorrectionTraceFromGit, inspectGovernedHint, editGovernedHintCandidate, reopenGovernedHint, retireHint, supersedeHint, hintsConflict, mineJoinPatterns, reviewGovernedHint, AgentRunEngine, SqliteAgentRunStore, defaultAgentRunGates, createLlmAgentRunPlanner, createHybridRouter, computeResultStats, buildDeterministicDashboardStory, synthesizeAnswer, streamOrGenerate, narrateResult, buildProposePreview, buildFromPrompt, internalRelationIdsInSql, defaultAgentRunStorePath, defaultAgentRunSqlitePath, resolveLocalOwner, resolveProposeConfig, recordQueryRun, recordRuntimeSchemaSnapshot, latestRuntimeSchemaSnapshotForProject, loadSkills, migrateLegacySkills, configuredSkillsPath, skillsDir, draftDomainSkillBootstrap, buildDomainSkillBootstrapPrompt, mergeDomainSkillBootstrapEnrichment, writeSkill, deleteSkill, deriveGeneratedDraftSlug, deriveAnalyticalRepair, reindexProject, invalidateAgentProjectState, recordAgentRuntimeVersion, resolveDomainContextEnvelope, projectEmbeddingProvider, isHashedEmbeddingProvider, clearProjectEmbeddingCache, upgradeVectorIndexForProject, openMetadataCatalog, defaultKgPath, planAppFromPrompt, KGStore, planResearch, loadSemanticMetrics, cascadeTraceToEvidenceRouteSteps, createCascadeAnswerResult, createCascadeTrace, routeReasoningEffort, routeForCascadeAnswerTier, clampReasoningEffort, bumpReasoningEffort, resolveThinkingMode, coerceThinkingMode, upsertGeneratedDqlArtifactDraft, loadAgentSemanticLayer, isTrustedConversationTurn, resolveInternalRelationIds, analyticalError, withAnalyticalErrorOrigin, withAnalyticalErrorOriginSync, } from '@duckcodeailabs/dql-agent';
 import { addSqlResultFilter, filterableResultColumns, replaceBlockStudioSql } from './sql-result-filter.js';
 import { gatherProposeEnrichment } from './propose-enrich.js';
 import { handleAppsApi, proposeAppAiBuild, recommendVisualization } from './apps-api.js';
@@ -526,6 +527,73 @@ function recordConversationTurn(store, threadId, run) {
         // Conversation persistence is additive; a failed write must not fail the run.
     }
 }
+/**
+ * Presentation projection of a run, for anything that SHIPS it — thread history
+ * and the live SSE stream alike.
+ *
+ * A stored run is dominated by DUPLICATED diagnostics rather than content. On a
+ * real 2.4 MB run: `diagnosticReceipt` is held twice — once top-level and once
+ * inside `artifacts[0].payload`, byte-identical — and each copy embeds its own
+ * full `steps` and `artifacts`, which embed the payload again. The part the UI
+ * actually renders (`payload.result`) was 3 KB.
+ *
+ * Everything the thread view reads is preserved, including
+ * `diagnosticReceipt.failure.message`; only the self-referential nesting and the
+ * unread `considered` block are dropped. The complete record stays available
+ * from the existing `GET /api/agent-runs/:id` run-state route.
+ */
+export function slimAgentRunForTransport(run) {
+    const slimReceipt = (receipt) => {
+        const record = agentRunRecord(receipt);
+        if (!record)
+            return receipt;
+        // `steps`/`artifacts` inside the receipt duplicate the run's own.
+        const { steps: _steps, artifacts: _artifacts, ...rest } = record;
+        return rest;
+    };
+    const artifacts = (run.artifacts ?? []).map((artifact) => {
+        const payload = agentRunRecord(artifact.payload);
+        if (!payload)
+            return artifact;
+        const { considered: _considered, ...restPayload } = payload;
+        return {
+            ...artifact,
+            payload: {
+                ...restPayload,
+                ...(payload.diagnosticReceipt ? { diagnosticReceipt: slimReceipt(payload.diagnosticReceipt) } : {}),
+            },
+        };
+    });
+    // `steps[].artifacts` is a THIRD copy of the run's own artifacts, and one
+    // progress event carries a FOURTH as its payload. Both are dropped rather than
+    // truncated: the data itself is unchanged and still on `run.artifacts`.
+    const MAX_EVENT_PAYLOAD_BYTES = 8 * 1024;
+    const steps = (run.steps ?? []).map((step) => {
+        const record = agentRunRecord(step);
+        if (!record?.artifacts)
+            return step;
+        const { artifacts: _stepArtifacts, ...rest } = record;
+        return rest;
+    });
+    const events = (run.events ?? []).slice(-20).map((event) => {
+        const record = agentRunRecord(event);
+        if (!record?.payload)
+            return event;
+        const encoded = JSON.stringify(record.payload) ?? '';
+        if (encoded.length <= MAX_EVENT_PAYLOAD_BYTES)
+            return event;
+        return { ...record, payload: { omitted: 'oversized', bytes: encoded.length } };
+    });
+    return {
+        ...run,
+        artifacts,
+        steps,
+        events,
+        ...(run.diagnosticReceipt
+            ? { diagnosticReceipt: slimReceipt(run.diagnosticReceipt) }
+            : {}),
+    };
+}
 export function conversationTurnInputFromRun(run) {
     const artifact = run.artifacts.find((candidate) => candidate.kind === 'answer')
         ?? run.artifacts.find((candidate) => candidate.kind === 'research_run')
@@ -4702,7 +4770,97 @@ export async function startLocalServer(opts) {
         response.write(`event: ${event}\n`);
         response.write(`data: ${serializeJSON(data)}\n\n`);
     };
+    /**
+     * Gzip text responses.
+     *
+     * Nothing was compressed before: the main bundle went out at 592 KB where gzip
+     * makes it 154 KB (73% smaller), and JSON API responses — which compress even
+     * better — were also sent raw. One wrapper here rather than a change at
+     * hundreds of `res.end(...)` call sites.
+     *
+     * STREAMING IS EXEMPT. Buffering an SSE response in order to compress it would
+     * defeat streaming entirely, so any response that calls `res.write()` opts
+     * itself out permanently, as does `text/event-stream`.
+     */
+    const MIN_COMPRESS_BYTES = 1024;
+    const COMPRESSIBLE = /^(?:text\/(?!event-stream)|application\/(?:json|javascript|xml|wasm)|image\/svg)/i;
+    const withCompression = (request, response) => {
+        if (!/\bgzip\b/i.test(String(request.headers['accept-encoding'] ?? '')))
+            return response;
+        const writeHead = response.writeHead.bind(response);
+        const end = response.end.bind(response);
+        const write = response.write.bind(response);
+        let status = 200;
+        let headers = {};
+        let captured = false;
+        let streaming = false;
+        response.writeHead = ((code, hdrs) => {
+            status = code;
+            headers = hdrs ?? {};
+            captured = true;
+            return response;
+        });
+        response.write = ((...args) => {
+            if (captured && !streaming) {
+                streaming = true;
+                writeHead(status, headers);
+            }
+            return write(...args);
+        });
+        response.end = ((...args) => {
+            const body = args[0];
+            const contentType = String(headers['Content-Type'] ?? '');
+            const encodable = captured
+                && !streaming
+                && (typeof body === 'string' || Buffer.isBuffer(body))
+                && COMPRESSIBLE.test(contentType);
+            if (!encodable) {
+                if (captured && !streaming) {
+                    streaming = true;
+                    writeHead(status, headers);
+                }
+                return end(...args);
+            }
+            const raw = Buffer.isBuffer(body) ? body : Buffer.from(body, 'utf8');
+            streaming = true;
+            // VALIDATOR CACHING. API responses carried no ETag or Last-Modified, so a
+            // browser could never revalidate: every panel switch re-downloaded every
+            // read in full. An ETag with `Cache-Control: no-cache` means the browser
+            // always ASKS but the server answers 304 with no body when nothing
+            // changed — freshness is unchanged, the bytes are not re-sent.
+            const cacheableRead = request.method === 'GET' && status === 200;
+            const etag = cacheableRead
+                ? `W/"${createHash('sha1').update(raw).digest('base64url')}"`
+                : undefined;
+            if (etag && String(request.headers['if-none-match'] ?? '') === etag) {
+                writeHead(304, {
+                    ETag: etag,
+                    'Cache-Control': headers['Cache-Control'] ?? 'no-cache',
+                    Vary: 'Accept-Encoding, Origin',
+                });
+                return end();
+            }
+            const validatorHeaders = etag
+                ? { ETag: etag, 'Cache-Control': headers['Cache-Control'] ?? 'no-cache' }
+                : {};
+            if (raw.byteLength < MIN_COMPRESS_BYTES) {
+                writeHead(status, { ...headers, ...validatorHeaders });
+                return end(raw);
+            }
+            const gz = gzipSync(raw);
+            writeHead(status, {
+                ...headers,
+                ...validatorHeaders,
+                'Content-Encoding': 'gzip',
+                'Content-Length': gz.byteLength,
+                Vary: 'Accept-Encoding, Origin',
+            });
+            return end(gz);
+        });
+        return response;
+    };
     const server = createServer((req, res) => {
+        res = withCompression(req, res);
         const requestUrl = req.url || '/';
         const url = new URL(requestUrl, 'http://127.0.0.1');
         const path = url.pathname || '/';
@@ -5693,7 +5851,12 @@ export async function startLocalServer(opts) {
                                 writeAgentRunSse(res, 'agent-run-answer-delta', { runId: parsed.request.runId, delta });
                             });
                             recordConversationTurn(conversationStore, parsed.request.threadId, run);
-                            writeAgentRunSse(res, 'agent-run-complete', run);
+                            // The terminal frame carried the whole stored run — 2.4 MB on a real
+                            // question, of which the renderable part was ~3 KB. The rest was the
+                            // same artifacts repeated four times over. The client renders from
+                            // the projection and fetches the full record from the run-state
+                            // route only if something needs it.
+                            writeAgentRunSse(res, 'agent-run-complete', slimAgentRunForTransport(run));
                             res.end();
                             return;
                         }
@@ -6865,6 +7028,26 @@ export async function startLocalServer(opts) {
                             res.end(serializeJSON({ ok: true }));
                             return;
                         }
+                        // Rename / pin. Both are housekeeping on an existing conversation, so
+                        // neither moves `updated_at` — see the store methods.
+                        if (req.method === 'PATCH' && !action) {
+                            const body = await readJSON(req).catch(() => null);
+                            const record = agentRunRecord(body) ?? {};
+                            const title = agentRunString(record.title);
+                            const favorite = typeof record.favorite === 'boolean' ? record.favorite : undefined;
+                            if (title === undefined && favorite === undefined) {
+                                res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
+                                res.end(serializeJSON({ error: 'Provide a title or a favorite flag.' }));
+                                return;
+                            }
+                            if (title !== undefined)
+                                store.renameThread(threadId, title);
+                            if (favorite !== undefined)
+                                store.setThreadFavorite(threadId, favorite);
+                            res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
+                            res.end(serializeJSON({ thread: store.getThread(threadId) }));
+                            return;
+                        }
                         if (req.method === 'DELETE' && !action) {
                             store.deleteThread(threadId);
                             res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
@@ -6914,7 +7097,13 @@ export async function startLocalServer(opts) {
                                 if (!turn.agentRunId)
                                     return [];
                                 const run = agentRunStore.get(turn.agentRunId);
-                                return run ? [run] : [];
+                                // Thread history is a PRESENTATION payload. A stored run averages
+                                // ~700 KB and can exceed 2 MB, almost none of it renderable, so
+                                // fetching 50 of them meant tens of megabytes to read, parse,
+                                // re-serialize and ship before the sidebar could draw — the
+                                // reported ~10s load. `GET /api/agent-runs/:id` serves the full
+                                // record when the inspector actually needs it.
+                                return run ? [slimAgentRunForTransport(run)] : [];
                             });
                             res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
                             res.end(serializeJSON({
@@ -12728,10 +12917,26 @@ export async function startLocalServer(opts) {
                         res.end(serializeJSON({ cellType: cell.type, result: null }));
                         return;
                     }
-                    const internalRelationError = internalDqlRelationIdValidationError(executableSql, cell.type === 'dql' ? 'DQL query' : 'Notebook query');
+                    // DECODE, THEN VALIDATE. A leaked `source::db.schema.table` is an internal
+                    // retrieval key whose suffix IS the physical relation, so rejecting the
+                    // cell and asking the user (or another AI round trip) to repair it was
+                    // busywork for a mechanical substitution. Anything still unresolvable
+                    // after this — a BARE `source::orders`, which names no database or
+                    // schema — genuinely cannot be decoded and still fails here.
+                    let decodedSql = resolveInternalRelationIds(executableSql, EMPTY_SCHEMA_GROUNDING);
+                    let bareResolution = null;
+                    if (internalDqlRelationIdValidationError(decodedSql, 'x')) {
+                        bareResolution = await resolveBareInternalRelationIds(decodedSql, executor, cellConnection);
+                        decodedSql = bareResolution.sql;
+                    }
+                    const internalRelationError = internalDqlRelationIdValidationError(decodedSql, cell.type === 'dql' ? 'DQL query' : 'Notebook query', 
+                    // A probe that came back empty is PROOF the relation is not visible
+                    // to this connection, so the refusal states that instead of sending
+                    // the user to an AI repair that cannot conjure a missing table.
+                    bareResolution ? { probed: true, ambiguous: bareResolution.ambiguous } : undefined);
                     if (internalRelationError)
                         throw new DqlInternalRelationIdError(internalRelationError);
-                    const prepared = prepareLocalExecution(executableSql, cellConnection, projectRoot, projectConfig);
+                    const prepared = prepareLocalExecution(decodedSql, cellConnection, projectRoot, projectConfig);
                     const app = loadRuntimeApp(projectRoot, typeof body.appId === 'string' ? body.appId : activePersonaAppId());
                     assertAppAccess({ app, domain: resolved.domain ?? app?.domain, level: 'execute' });
                     const semanticExecution = semanticCompose?.compiled
@@ -12791,7 +12996,10 @@ export async function startLocalServer(opts) {
                         tests: plan?.tests,
                         result: normalized,
                         executionTarget: executionTargetDescriptor(body),
-                        compiledSql: semanticCompose?.compiled?.sql ?? executableSql,
+                        // Report the statement that ACTUALLY ran. `executableSql` is the
+                        // pre-decode text, so showing it would put a `source::` identity in
+                        // front of the user that the warehouse never saw.
+                        compiledSql: semanticCompose?.compiled?.sql ?? decodedSql,
                         engine: semanticCompose?.compiled?.engine,
                         ...(semanticExecution ? {
                             semanticTrace: semanticExecution.semanticTrace,
@@ -15478,6 +15686,8 @@ const AGENT_PREVIEW_FORBIDDEN_SQL = [
 function withAnalyticalCompilationOrigin(stage, fn) {
     return withAnalyticalErrorOriginSync({ origin: 'dql_compilation', stage }, fn);
 }
+/** No dbt grounding — decoding an internal id needs none for a qualified suffix. */
+const EMPTY_SCHEMA_GROUNDING = { tables: [], joinKeys: [], byKey: new Map() };
 export function clampAnalyticalRowBound(rowLimit) {
     return Number.isFinite(rowLimit) ? Math.max(1, Math.min(10_000, Math.floor(rowLimit))) : 200;
 }
@@ -15966,11 +16176,19 @@ function readOnlySqlValidationError(sql, subject, dialect = 'duckdb') {
     }
     return null;
 }
-function internalDqlRelationIdValidationError(sql, subject) {
+function internalDqlRelationIdValidationError(sql, subject, probe) {
     const internalIds = internalRelationIdsInSql(stripSqlStringsAndComments(sql));
     if (internalIds.length === 0)
         return null;
-    return `${subject} contains internal DQL graph relation identifier${internalIds.length === 1 ? '' : 's'} ${internalIds.join(', ')}. Use the physical database.schema.table relation instead, or choose Ask AI to fix from the notebook error.`;
+    const plural = internalIds.length === 1 ? '' : 's';
+    const head = `${subject} contains internal DQL graph relation identifier${plural} ${internalIds.join(', ')}.`;
+    if (probe?.ambiguous.length) {
+        return `${head} ${probe.ambiguous.join(', ')} matched more than one schema in this connection, so the name alone does not identify a relation. Qualify it as database.schema.table.`;
+    }
+    if (probe?.probed) {
+        return `${head} DQL checked this connection and no relation with that name is visible to it. Qualify it as database.schema.table, or connect to the source that holds it.`;
+    }
+    return `${head} Use the physical database.schema.table relation instead, or choose Ask AI to fix from the notebook error.`;
 }
 function parseHintLessonInput(value) {
     if (!value || typeof value !== 'object' || Array.isArray(value))
@@ -23114,6 +23332,58 @@ ${predicates.map((predicate) => `    ${predicate}`).join(' OR\n')}
 ORDER BY table_schema, table_name, ordinal_position
 LIMIT 400`;
 }
+/**
+ * Resolve a BARE internal graph identity (`source::orders` — a name with no
+ * database or schema) against the warehouse.
+ *
+ * A qualified identity decodes mechanically, but a bare one carries no
+ * location, so the old behaviour was to reject the cell and hand the user an
+ * "Ask AI to fix" button. That was an LLM round trip for a lookup: the name is
+ * unambiguous whenever exactly one relation in the connection answers to it.
+ * Probe for it, and only refuse when the answer is genuinely zero or many —
+ * at which point the refusal can say which it was.
+ *
+ * Runs ONLY after a decode has already failed, so the happy path pays nothing.
+ */
+export async function resolveBareInternalRelationIds(sql, executor, connection) {
+    const bare = internalRelationIdsInSql(stripSqlStringsAndComments(sql))
+        .filter((id) => !id.slice(id.indexOf('::') + 2).includes('.'));
+    if (bare.length === 0)
+        return { sql, resolved: [], ambiguous: [] };
+    const names = bare.map((id) => id.slice(id.indexOf('::') + 2));
+    const probeSql = buildNamedRelationProbeSql(names);
+    if (!probeSql)
+        return { sql, resolved: [], ambiguous: [] };
+    let rows = [];
+    try {
+        const probe = await executor.executeQuery(probeSql, [], {}, connection);
+        rows = (probe?.rows ?? []);
+    }
+    catch {
+        // The probe is best-effort. A warehouse that cannot answer it leaves the
+        // identity unresolved, which is the same outcome as not probing at all.
+        return { sql, resolved: [], ambiguous: [] };
+    }
+    const resolved = [];
+    const ambiguous = [];
+    let out = sql;
+    for (const id of bare) {
+        const name = id.slice(id.indexOf('::') + 2).toLowerCase();
+        const matches = new Set(rows
+            .filter((row) => String(row.table_name ?? '').toLowerCase() === name)
+            .map((row) => `${String(row.table_schema ?? '')}.${String(row.table_name ?? '')}`));
+        if (matches.size !== 1) {
+            if (matches.size > 1)
+                ambiguous.push(id);
+            continue;
+        }
+        const target = [...matches][0];
+        const idPattern = id.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+        out = out.replace(new RegExp(`\\b${idPattern}`, 'g'), target);
+        resolved.push({ from: id, to: target });
+    }
+    return { sql: out, resolved, ambiguous };
+}
 export function buildRuntimeSchemaSearchSql(question) {
     const terms = Array.from(agentSchemaTokens(question))
         .map((term) => term.toLowerCase().replace(/[^a-z0-9_]/g, ''))