@duckcodeailabs/dql-cli 1.11.4 → 1.11.6

package/dist/local-runtime.js

@@ -5,7 +5,7 @@ import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, s
 import { createRequire } from "node:module";
 import { homedir } from "node:os";
 import { fileURLToPath } from "node:url";
-import { dirname, extname, join, normalize, relative, resolve, } from "node:path";
+import { dirname, extname, join, normalize, relative, resolve, sep, } from "node:path";
 import { Readable } from "node:stream";
 import { buildMixedSourceWarehouseFallbackSql, findMentionedNotebookDataset, planMixedSourceNotebookSql, planMixedSourceSql, } from "./mixed-source-sql.js";
 import { resolveNpmInvocation } from './npm-runtime.js';
@@ -19,7 +19,7 @@ import { fetchLatestPublishedDqlVersion, resolveDqlRuntimeVersionStatus } from '
 import { resolveRetrievalHealthStatus } from './retrieval-health.js';
 import { createDqlAgentProviderRunner, resolveAgentFollowUpContext } from './llm/providers/dql-agent-provider.js';
 import { listRemoteMcpSettings, saveRemoteMcpSettings } from './llm/mcp-config.js';
-import { ClaudeProvider, ConversationStore, advanceThreadState, buildConversationSnapshot, recallRelevantTurns, GeminiProvider, MemoryStore, OllamaProvider, OpenAIProvider, buildBlockBusinessFingerprint, buildBlockSqlFingerprints, buildAnalysisQuestionPlan, composeSemanticQueryForQuestion, aggregationIntegrityIssuesForSql, buildLocalContextPack, applyContextPackCompatibility, toAgentRetrievalEvidence, defaultConversationPath, defaultMemoryPath, ensureDefaultMemoryFiles, ensureAgentProjectReady, isAgentProjectIndexReady, currentMetadataFingerprint, ensureMetadataCatalogFresh, readIndexedDomainKnowledge, readIndexedKnowledge360, compactSemanticRuntimeFailure, classifyAnalyticalFailure, propose, proposePlan, recordCorrectionTrace, emitCorrectionEvalCase, mineJoinPatterns, reviewHint, AgentRunEngine, SqliteAgentRunStore, defaultAgentRunGates, createLlmAgentRunPlanner, createHybridRouter, computeResultStats, buildDeterministicDashboardStory, synthesizeAnswer, streamOrGenerate, narrateResult, buildProposePreview, buildFromPrompt, defaultAgentRunStorePath, defaultAgentRunSqlitePath, resolveLocalOwner, resolveProposeConfig, recordQueryRun, recordRuntimeSchemaSnapshot, latestRuntimeSchemaSnapshotForProject, loadSkills, migrateLegacySkills, configuredSkillsPath, skillsDir, draftDomainSkillBootstrap, buildDomainSkillBootstrapPrompt, mergeDomainSkillBootstrapEnrichment, writeSkill, deleteSkill, deriveGeneratedDraftSlug, deriveAnalyticalRepair, reindexProject, invalidateAgentProjectState, recordAgentRuntimeVersion, resolveDomainContextEnvelope, defaultKgPath, planAppFromPrompt, KGStore, planResearch, loadSemanticMetrics, cascadeTraceToEvidenceRouteSteps, createCascadeAnswerResult, createCascadeTrace, routeReasoningEffort, routeForCascadeAnswerTier, clampReasoningEffort, bumpReasoningEffort, resolveThinkingMode, coerceThinkingMode, upsertGeneratedDqlArtifactDraft, loadAgentSemanticLayer, } from '@duckcodeailabs/dql-agent';
+import { ClaudeProvider, ConversationStore, advanceThreadState, buildConversationSnapshot, recallRelevantTurns, GeminiProvider, MemoryStore, OllamaProvider, OpenAIProvider, buildBlockBusinessFingerprint, buildBlockSqlFingerprints, buildAnalysisQuestionPlan, composeSemanticQueryForQuestion, aggregationIntegrityIssuesForSql, buildLocalContextPack, applyContextPackCompatibility, toAgentRetrievalEvidence, defaultConversationPath, defaultMemoryPath, ensureDefaultMemoryFiles, ensureAgentProjectReady, isAgentProjectIndexReady, currentMetadataFingerprint, ensureMetadataCatalogFresh, readIndexedDomainKnowledge, readIndexedKnowledge360, compactSemanticRuntimeFailure, classifyAnalyticalFailure, propose, proposePlan, recordCorrectionTrace, listHintsFromGit, evaluateHint, emitCorrectionEvalCase, mineJoinPatterns, reviewHint, AgentRunEngine, SqliteAgentRunStore, defaultAgentRunGates, createLlmAgentRunPlanner, createHybridRouter, computeResultStats, buildDeterministicDashboardStory, synthesizeAnswer, streamOrGenerate, narrateResult, buildProposePreview, buildFromPrompt, defaultAgentRunStorePath, defaultAgentRunSqlitePath, resolveLocalOwner, resolveProposeConfig, recordQueryRun, recordRuntimeSchemaSnapshot, latestRuntimeSchemaSnapshotForProject, loadSkills, migrateLegacySkills, configuredSkillsPath, skillsDir, draftDomainSkillBootstrap, buildDomainSkillBootstrapPrompt, mergeDomainSkillBootstrapEnrichment, writeSkill, deleteSkill, deriveGeneratedDraftSlug, deriveAnalyticalRepair, reindexProject, invalidateAgentProjectState, recordAgentRuntimeVersion, resolveDomainContextEnvelope, defaultKgPath, planAppFromPrompt, KGStore, planResearch, loadSemanticMetrics, cascadeTraceToEvidenceRouteSteps, createCascadeAnswerResult, createCascadeTrace, routeReasoningEffort, routeForCascadeAnswerTier, clampReasoningEffort, bumpReasoningEffort, resolveThinkingMode, coerceThinkingMode, upsertGeneratedDqlArtifactDraft, loadAgentSemanticLayer, } from '@duckcodeailabs/dql-agent';
 import { gatherProposeEnrichment } from './propose-enrich.js';
 import { handleAppsApi, proposeAppAiBuild, recommendVisualization } from './apps-api.js';
 import { getActiveProvider, getEffectiveProviderConfig, isProviderSettingsId, listProviderSettings, saveProviderSettings, } from './settings/provider-settings.js';
@@ -2003,7 +2003,13 @@ export async function startLocalServer(opts) {
                                 : 'Semantic compilation details', governedAnswer, undefined, needsClarification ? 'not_applicable' : 'review_required')]
                         : governedAnswer.dqlArtifact && !isProviderError && !isGroundingGap && !isModelDeclined && !isPolicyBlocked
                             ? [agentRunArtifact('dql_block_draft', 'DQL draft (review required)', governedAnswer.dqlArtifact, undefined, 'review_required')]
-                            : [])
+                            // Every refusal still owes the user an account of itself. Emitting no
+                            // artifact leaves the answer as a bare sentence with a Copy button:
+                            // the inspector can only be opened FROM an artifact, so "How it was
+                            // answered", the DQL draft and the compiled SQL all become
+                            // unreachable exactly when the user most needs to see why it stopped
+                            // and carry the query into a notebook.
+                            : [agentRunArtifact('answer', 'No answer was accepted', governedAnswer, undefined, needsClarification ? 'not_applicable' : 'blocked')])
                     : [agentRunArtifact('answer', isCertified ? 'Certified answer' : isSemantic ? 'Governed semantic answer' : isExploratory ? 'Exploratory DBT-grounded answer' : 'Review-required answer', governedAnswer, governedAnswer.sourceCertifiedBlock ?? governedAnswer.block?.name, isCertified ? 'certified' : isSemantic ? 'governed' : 'review_required')],
             evaluations: [
                 agentRunEvaluation('route-decision', 'Route decision', true, 'info', routeDecision?.reason ?? 'Routed request to governed answer.', {
@@ -6353,6 +6359,128 @@ export async function startLocalServer(opts) {
             // the approval, so the derived candidate is approved immediately unless the
             // caller opts out. Advisory only: never overrides certified routing. The
             // multi-tenant review workflow + automated distillation stay a cloud feature.
+            // Hint review queue. Corrections are captured as CANDIDATES; a hint only
+            // starts shaping other people's answers once a human approves it here.
+            if (req.method === 'GET' && path === '/api/agent/hints') {
+                try {
+                    const status = url.searchParams.get('status');
+                    const all = listHintsFromGit(projectRoot);
+                    const hints = status ? all.filter((hint) => hint.status === status) : all;
+                    res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
+                    res.end(serializeJSON({
+                        hints: hints.sort((left, right) => right.updatedAt.localeCompare(left.updatedAt)).slice(0, 200),
+                        counts: {
+                            candidate: all.filter((hint) => hint.status === 'candidate').length,
+                            approved: all.filter((hint) => hint.status === 'approved').length,
+                            rejected: all.filter((hint) => hint.status === 'rejected').length,
+                        },
+                    }));
+                }
+                catch (error) {
+                    res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
+                    res.end(serializeJSON({ hints: [], counts: { candidate: 0, approved: 0, rejected: 0 }, error: error instanceof Error ? error.message : String(error) }));
+                }
+                return;
+            }
+            const hintReviewMatch = /^\/api\/agent\/hints\/([^/]+)\/review$/.exec(path);
+            if (req.method === 'POST' && hintReviewMatch) {
+                try {
+                    const hintId = decodeURIComponent(hintReviewMatch[1]);
+                    const body = await readJSON(req).catch(() => null);
+                    const decision = body?.decision === 'approved' || body?.decision === 'rejected' ? body.decision : null;
+                    if (!decision) {
+                        res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
+                        res.end(serializeJSON({ error: "decision must be 'approved' or 'rejected'." }));
+                        return;
+                    }
+                    const reviewer = typeof body?.reviewer === 'string' && body.reviewer.trim()
+                        ? body.reviewer.trim()
+                        : (resolveLocalOwner(projectRoot) ?? 'local');
+                    const note = typeof body?.note === 'string' && body.note.trim() ? body.note.trim() : undefined;
+                    const snapshotId = projectSnapshot().snapshotId;
+                    // dbt-first v3 requires a PASSED evaluation artifact recorded against the
+                    // same snapshot before a candidate may be approved. Approving from the
+                    // review queue therefore runs the hint's required evaluation first: the
+                    // corrected SQL is re-validated against the current project so a hint can
+                    // never be promoted on evidence that has gone stale.
+                    if (decision === 'approved') {
+                        const pending = listHintsFromGit(projectRoot).find((entry) => entry.id === hintId);
+                        if (pending?.status === 'candidate' && pending.requiredEvaluation && !pending.evaluationId) {
+                            // The evaluation must be a real check, not a rubber stamp: the
+                            // corrected SQL has to parse and reference at least one relation
+                            // against the CURRENT project. A correction that no longer analyses
+                            // cannot be promoted into everyone's prompt.
+                            const validation = (() => {
+                                const sql = pending.correctedSql?.trim();
+                                if (!sql)
+                                    return { ok: false, error: 'The correction carries no SQL to verify.' };
+                                try {
+                                    const analysis = analyzeSqlReferences(sql);
+                                    if (!analysis.tables.length)
+                                        return { ok: false, error: 'Corrected SQL references no known relation.' };
+                                    return { ok: true };
+                                }
+                                catch (error) {
+                                    return { ok: false, error: error instanceof Error ? error.message : String(error) };
+                                }
+                            })();
+                            try {
+                                evaluateHint(projectRoot, {
+                                    hintId,
+                                    snapshotId,
+                                    evaluator: reviewer,
+                                    evaluation: pending.requiredEvaluation,
+                                    checks: [{
+                                            name: 'corrected-sql-validates',
+                                            passed: validation.ok,
+                                            evidence: validation.ok ? 'Corrected SQL passed local context validation.' : validation.error,
+                                        }],
+                                    evidence: [`snapshot: ${snapshotId}`, `reviewer: ${reviewer}`],
+                                });
+                            }
+                            catch (evaluationError) {
+                                res.writeHead(409, { 'Content-Type': 'application/json; charset=utf-8' });
+                                res.end(serializeJSON({ ok: false, error: evaluationError instanceof Error ? evaluationError.message : String(evaluationError) }));
+                                return;
+                            }
+                        }
+                    }
+                    const reviewed = reviewHint(projectRoot, { hintId, decision, reviewer, snapshotId, ...(note ? { note } : {}) });
+                    if (!reviewed) {
+                        res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
+                        res.end(serializeJSON({ ok: false, error: 'Hint not found.' }));
+                        return;
+                    }
+                    const hint = reviewed.hint;
+                    // Keep the mirrored advisory memory in step with the decision, so an
+                    // approved lesson is recalled and a rejected one stops being recalled.
+                    try {
+                        const memory = new MemoryStore(defaultMemoryPath(projectRoot));
+                        memory.upsert({
+                            id: `mem_${hint.id}`,
+                            scope: 'project',
+                            title: hint.title,
+                            content: hint.guidance,
+                            tags: [hint.scope.metric, hint.scope.domain, hint.scope.dbtModel].filter((x) => Boolean(x)),
+                            source: 'correction',
+                            confidence: 0.9,
+                            importance: 0.85,
+                            enabled: decision === 'approved',
+                        });
+                        memory.close();
+                    }
+                    catch {
+                        /* best-effort */
+                    }
+                    res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
+                    res.end(serializeJSON({ ok: true, hint }));
+                }
+                catch (error) {
+                    res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
+                    res.end(serializeJSON({ ok: false, error: error instanceof Error ? error.message : String(error) }));
+                }
+                return;
+            }
             if (req.method === 'POST' && path === '/api/agent/learnings/correction') {
                 const body = await readJSON(req).catch(() => null);
                 const question = body && typeof body.question === 'string' ? body.question.trim() : '';
@@ -6376,6 +6504,29 @@ export async function startLocalServer(opts) {
                 const rationale = typeof body.rationale === 'string' && body.rationale.trim() ? body.rationale.trim() : undefined;
                 const author = typeof body.author === 'string' ? body.author : (resolveLocalOwner(projectRoot) ?? undefined);
                 try {
+                    // dbt-first manifest v3 requires reviewable provenance before a correction
+                    // may be recorded at all. The client cannot know most of it, so derive it
+                    // here: without this the endpoint rejects every correction on exactly the
+                    // governed projects the learning loop is meant to serve.
+                    const correctionEvidence = Array.isArray(body.evidence)
+                        ? body.evidence.map(String).filter(Boolean)
+                        : [
+                            `question: ${question}`,
+                            ...(wrongSql ? [`previous SQL: ${wrongSql.replace(/\s+/g, ' ').slice(0, 400)}`] : []),
+                            `corrected SQL: ${correctedSql.replace(/\s+/g, ' ').slice(0, 400)}`,
+                            ...(rationale ? [`rationale: ${rationale}`] : []),
+                        ];
+                    const correctionSnapshotId = typeof body.snapshotId === 'string' && body.snapshotId.trim()
+                        ? body.snapshotId.trim()
+                        : projectSnapshot().snapshotId;
+                    const correctionFailedRoute = typeof body.failedRoute === 'string' && body.failedRoute.trim()
+                        ? body.failedRoute.trim()
+                        : 'generated_answer';
+                    // Must match the name emitCorrectionEvalCase writes, so the required
+                    // evaluation and the emitted regression case are the same artifact.
+                    const correctionEvaluation = typeof body.requiredEvaluation === 'string' && body.requiredEvaluation.trim()
+                        ? body.requiredEvaluation.trim()
+                        : `correction: ${question.slice(0, 60)}`;
                     const { trace, hint } = recordCorrectionTrace(projectRoot, {
                         question,
                         scope,
@@ -6384,19 +6535,29 @@ export async function startLocalServer(opts) {
                         correctedSql,
                         rationale,
                         author,
+                        failedRoute: correctionFailedRoute,
+                        evidence: correctionEvidence,
+                        snapshotId: correctionSnapshotId,
+                        requiredEvaluation: correctionEvaluation,
                         hintTitle: typeof body.title === 'string' && body.title.trim() ? body.title.trim() : undefined,
                         hintGuidance: typeof body.guidance === 'string' && body.guidance.trim() ? body.guidance.trim() : undefined,
                         tags: Array.isArray(body.tags) ? body.tags.map(String) : undefined,
                     });
+                    // Capture always, approve deliberately. A hint is injected into the
+                    // generation prompt as a "human-approved correction" and shapes every
+                    // future answer in scope — one analyst's local fix must not silently
+                    // become a team-wide rule across thousands of models. The trace and the
+                    // candidate are always recorded, so nothing is lost; approval is a
+                    // one-click review in Settings. Callers may still opt in explicitly.
                     let approvedHint = hint;
-                    if (body.approve !== false) {
-                        reviewHint(projectRoot, { hintId: hint.id, decision: 'approved', reviewer: author ?? 'local', note: 'Self-approved (OSS single-user).' });
+                    if (body.approve === true) {
+                        reviewHint(projectRoot, { hintId: hint.id, decision: 'approved', reviewer: author ?? 'local', note: 'Approved on capture.' });
                         approvedHint = { ...hint, status: 'approved' };
                     }
                     // W4.3 — turn the correction into a durable regression eval case so the wrong
                     // answer can never silently return. Best-effort; never blocks the correction.
                     try {
-                        emitCorrectionEvalCase(projectRoot, { question, correctedSql });
+                        emitCorrectionEvalCase(projectRoot, { question, correctedSql, name: correctionEvaluation });
                     }
                     catch {
                         /* best-effort */
@@ -6413,7 +6574,10 @@ export async function startLocalServer(opts) {
                             source: 'correction',
                             confidence: 0.9,
                             importance: 0.85,
-                            enabled: true,
+                            // A candidate is evidence, not a rule. Mirroring it as an ENABLED
+                            // memory would inject it into prompts through the side door and
+                            // defeat the approval gate; it is enabled when the hint is approved.
+                            enabled: approvedHint.status === 'approved',
                         });
                         memory.close();
                     }
@@ -6935,6 +7099,55 @@ export async function startLocalServer(opts) {
                 }
                 return;
             }
+            // DELETE /api/notebooks?path=notebooks/x.dqlnb — remove a notebook file.
+            // Deliberately narrow: only .dqlnb/.dql files under notebooks/, resolved and
+            // re-checked against the project root so a crafted path cannot escape it.
+            if (req.method === 'DELETE' && path === '/api/notebooks') {
+                try {
+                    const relativePath = url.searchParams.get('path')?.trim();
+                    if (!relativePath) {
+                        res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
+                        res.end(serializeJSON({ ok: false, error: 'Missing notebook path.' }));
+                        return;
+                    }
+                    const normalized = relativePath.replace(/\\/g, '/').replace(/^\.\//, '');
+                    if (!normalized.startsWith('notebooks/') || !/\.(dqlnb|dql)$/i.test(normalized) || normalized.includes('..')) {
+                        res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
+                        res.end(serializeJSON({ ok: false, error: 'Only notebook files under notebooks/ can be deleted.' }));
+                        return;
+                    }
+                    const absolute = resolve(projectRoot, normalized);
+                    const rootPrefix = `${resolve(projectRoot)}${sep}`;
+                    if (!absolute.startsWith(rootPrefix)) {
+                        res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
+                        res.end(serializeJSON({ ok: false, error: 'Path escapes the project root.' }));
+                        return;
+                    }
+                    if (!existsSync(absolute)) {
+                        res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
+                        res.end(serializeJSON({ ok: false, error: 'Notebook not found.' }));
+                        return;
+                    }
+                    rmSync(absolute, { force: true });
+                    // Drop the run-state sidecar too, or the next notebook created with the
+                    // same slug inherits the deleted one's cached results.
+                    const sidecar = absolute.replace(/\.(dqlnb|dql)$/i, '.run.json');
+                    if (sidecar !== absolute)
+                        rmSync(sidecar, { force: true });
+                    res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
+                    res.end(serializeJSON({ ok: true, path: normalized }));
+                }
+                catch (error) {
+                    if (error instanceof DQLAccessDeniedError) {
+                        res.writeHead(403, { 'Content-Type': 'application/json; charset=utf-8' });
+                        res.end(serializeJSON({ ok: false, error: error.message, code: 'unauthorized' }));
+                        return;
+                    }
+                    res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
+                    res.end(serializeJSON({ ok: false, error: error instanceof Error ? error.message : String(error) }));
+                }
+                return;
+            }
             if (req.method === 'PUT' && path === '/api/notebook-content') {
                 try {
                     const body = await readJSON(req);
@@ -15598,10 +15811,28 @@ export function validateBlockStudioSource(source, semanticLayer) {
                 message: 'Semantic block is not runnable yet. Select a metric and complete any required time settings.',
             }
             : {
-                severity: 'warning',
+                // A block that DECLARES a query section and leaves it blank is broken,
+                // not merely incomplete: it can never execute, yet it saved cleanly
+                // because this was only a warning. A bare skeleton with no query
+                // section at all stays a warning so draft authoring is unbroken.
+                severity: /query\s*=\s*"""/i.test(source) ? 'error' : 'warning',
                 code: 'sql_missing',
-                message: 'No executable SQL found in the block source.',
+                message: /query\s*=\s*"""/i.test(source)
+                    ? 'The block declares a query section but it is empty. Add the SQL, or remove the query section and define metrics and dimensions instead.'
+                    : 'No executable SQL found in the block source.',
+            });
+    }
+    // The mirror image: a semantic block is metrics x dimensions and must not carry
+    // a query. Both together is the corrupt shape the analyzer lets through silently.
+    if (semanticConfig.blockType === 'semantic') {
+        const semanticQueryBody = source.match(/query\s*=\s*"""([\s\S]*?)"""/i)?.[1] ?? '';
+        if (semanticQueryBody.trim()) {
+            diagnostics.push({
+                severity: 'error',
+                code: 'semantic_block_has_query',
+                message: 'Semantic blocks are defined by metrics and dimensions and must not contain a query section. Move the SQL into a custom block.',
             });
+        }
     }
     const parameterInvocation = prepareBlockInvocation({ source, surface: 'block_studio' });
     if (parameterInvocation.errors.length > 0) {