@duckcodeailabs/dql-cli 1.8.2 → 1.8.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/doctor.d.ts.map +1 -1
- package/dist/commands/doctor.js +62 -2
- package/dist/commands/doctor.js.map +1 -1
- package/dist/llm/cancellation.d.ts +13 -0
- package/dist/llm/cancellation.d.ts.map +1 -0
- package/dist/llm/cancellation.js +19 -0
- package/dist/llm/cancellation.js.map +1 -0
- package/dist/llm/providers/dql-agent-provider.d.ts.map +1 -1
- package/dist/llm/providers/dql-agent-provider.js +5 -0
- package/dist/llm/providers/dql-agent-provider.js.map +1 -1
- package/dist/local-runtime.d.ts +32 -1
- package/dist/local-runtime.d.ts.map +1 -1
- package/dist/local-runtime.js +207 -17
- package/dist/local-runtime.js.map +1 -1
- package/dist/package.json +10 -10
- package/dist/providers/subscription-cli.d.ts.map +1 -1
- package/dist/providers/subscription-cli.js +16 -0
- package/dist/providers/subscription-cli.js.map +1 -1
- package/dist/retrieval-health.d.ts +29 -0
- package/dist/retrieval-health.d.ts.map +1 -0
- package/dist/retrieval-health.js +88 -0
- package/dist/retrieval-health.js.map +1 -0
- package/dist/version-status.d.ts +28 -0
- package/dist/version-status.d.ts.map +1 -0
- package/dist/version-status.js +118 -0
- package/dist/version-status.js.map +1 -0
- package/package.json +10 -10
package/dist/local-runtime.js
CHANGED
|
@@ -14,9 +14,12 @@ import { loadSemanticLayerFromDir, normalizeDqlArtifactReference, serializeMetri
|
|
|
14
14
|
import { load as loadYaml } from 'js-yaml';
|
|
15
15
|
import { listBlockTemplates } from './block-templates.js';
|
|
16
16
|
import { getRunner as getLLMRunner } from './llm/index.js';
|
|
17
|
+
import { rethrowIfCancelled } from './llm/cancellation.js';
|
|
18
|
+
import { fetchLatestPublishedDqlVersion, resolveDqlRuntimeVersionStatus } from './version-status.js';
|
|
19
|
+
import { resolveRetrievalHealthStatus } from './retrieval-health.js';
|
|
17
20
|
import { createDqlAgentProviderRunner, resolveAgentFollowUpContext } from './llm/providers/dql-agent-provider.js';
|
|
18
21
|
import { listRemoteMcpSettings, saveRemoteMcpSettings } from './llm/mcp-config.js';
|
|
19
|
-
import { ClaudeProvider, ConversationStore, advanceThreadState, buildConversationSnapshot, recallRelevantTurns, GeminiProvider, MemoryStore, OllamaProvider, OpenAIProvider, buildBlockBusinessFingerprint, buildBlockSqlFingerprints, buildAnalysisQuestionPlan, buildLocalContextPack, toAgentRetrievalEvidence, defaultConversationPath, defaultMemoryPath, ensureDefaultMemoryFiles, ensureAgentProjectReady, isAgentProjectIndexReady, currentMetadataFingerprint, ensureMetadataCatalogFresh, readIndexedDomainKnowledge, readIndexedKnowledge360, propose, proposePlan, recordCorrectionTrace, emitCorrectionEvalCase, mineJoinPatterns, reviewHint, AgentRunEngine,
|
|
22
|
+
import { ClaudeProvider, ConversationStore, advanceThreadState, buildConversationSnapshot, recallRelevantTurns, GeminiProvider, MemoryStore, OllamaProvider, OpenAIProvider, buildBlockBusinessFingerprint, buildBlockSqlFingerprints, buildAnalysisQuestionPlan, buildLocalContextPack, toAgentRetrievalEvidence, defaultConversationPath, defaultMemoryPath, ensureDefaultMemoryFiles, ensureAgentProjectReady, isAgentProjectIndexReady, currentMetadataFingerprint, ensureMetadataCatalogFresh, readIndexedDomainKnowledge, readIndexedKnowledge360, propose, proposePlan, recordCorrectionTrace, emitCorrectionEvalCase, mineJoinPatterns, reviewHint, AgentRunEngine, SqliteAgentRunStore, defaultAgentRunGates, createLlmAgentRunPlanner, createHybridRouter, computeResultStats, buildDeterministicDashboardStory, synthesizeAnswer, streamOrGenerate, narrateResult, buildProposePreview, buildFromPrompt, defaultAgentRunStorePath, defaultAgentRunSqlitePath, resolveLocalOwner, resolveProposeConfig, recordQueryRun, recordRuntimeSchemaSnapshot, latestRuntimeSchemaSnapshotForProject, loadSkills, migrateLegacySkills, configuredSkillsPath, skillsDir, draftDomainSkillBootstrap, buildDomainSkillBootstrapPrompt, mergeDomainSkillBootstrapEnrichment, writeSkill, deleteSkill, deriveGeneratedDraftSlug, reindexProject, invalidateAgentProjectState, recordAgentRuntimeVersion, resolveDomainContextEnvelope, defaultKgPath, planAppFromPrompt, KGStore, planResearch, loadSemanticMetrics, cascadeTraceToEvidenceRouteSteps, createCascadeAnswerResult, createCascadeTrace, routeReasoningEffort, routeForCascadeAnswerTier, clampReasoningEffort, bumpReasoningEffort, resolveThinkingMode, coerceThinkingMode, upsertGeneratedDqlArtifactDraft, } from '@duckcodeailabs/dql-agent';
|
|
20
23
|
import { gatherProposeEnrichment } from './propose-enrich.js';
|
|
21
24
|
import { handleAppsApi, proposeAppAiBuild, recommendVisualization } from './apps-api.js';
|
|
22
25
|
import { getActiveProvider, getEffectiveProviderConfig, isProviderSettingsId, listProviderSettings, saveProviderSettings, } from './settings/provider-settings.js';
|
|
@@ -164,17 +167,26 @@ export function parseAgentRunRequestBody(body) {
|
|
|
164
167
|
}
|
|
165
168
|
const AGENT_LOOKUP_DEADLINE_MS = 45_000;
|
|
166
169
|
const AGENT_RESEARCH_DEADLINE_MS = 120_000;
|
|
170
|
+
/** Env-tunable run deadline (e.g. slow subscription-CLI providers), clamped to sane bounds. */
|
|
171
|
+
function resolveAgentDeadlineMs(envKey, fallback, env = process.env) {
|
|
172
|
+
const configured = Number(env[envKey]);
|
|
173
|
+
if (!Number.isFinite(configured) || configured <= 0)
|
|
174
|
+
return fallback;
|
|
175
|
+
return Math.max(15_000, Math.min(600_000, Math.floor(configured)));
|
|
176
|
+
}
|
|
167
177
|
/**
|
|
168
178
|
* PERF-002: one wall-clock budget follows the request through routing, provider
|
|
169
179
|
* calls, repair, and execution. Ordinary Ask never inherits Research's budget
|
|
170
180
|
* merely because it spans two tables; explicit/deep investigation does.
|
|
171
181
|
*/
|
|
172
|
-
export function agentRunDeadlineMs(request) {
|
|
182
|
+
export function agentRunDeadlineMs(request, env = process.env) {
|
|
183
|
+
const lookupDeadline = resolveAgentDeadlineMs('DQL_AGENT_LOOKUP_DEADLINE_MS', AGENT_LOOKUP_DEADLINE_MS, env);
|
|
184
|
+
const researchDeadline = resolveAgentDeadlineMs('DQL_AGENT_RESEARCH_DEADLINE_MS', AGENT_RESEARCH_DEADLINE_MS, env);
|
|
173
185
|
if (request.requestedMode === 'research' || request.analysisDepth === 'deep') {
|
|
174
|
-
return
|
|
186
|
+
return researchDeadline;
|
|
175
187
|
}
|
|
176
188
|
const plan = buildAnalysisQuestionPlan(request.question);
|
|
177
|
-
return plan.needsResearchWorkspace ?
|
|
189
|
+
return plan.needsResearchWorkspace ? researchDeadline : lookupDeadline;
|
|
178
190
|
}
|
|
179
191
|
export function shouldSynthesizeAgentRunAnswer(governedAnswer) {
|
|
180
192
|
if (governedAnswer.kind === 'no_answer')
|
|
@@ -215,7 +227,11 @@ function businessNarrativeGaps(warnings) {
|
|
|
215
227
|
// source of prior turns (survives refresh); the client-built context remains
|
|
216
228
|
// the fallback for embedders that never send a threadId.
|
|
217
229
|
async function conversationContextFromThread(store, threadId, clientContext, question) {
|
|
218
|
-
|
|
230
|
+
// Token-budget backstop: six verbatim turns with per-field caps. Older turns
|
|
231
|
+
// remain reachable through the rolling summary + semantic recall below —
|
|
232
|
+
// carrying more raw prose mostly slows every provider call on follow-ups.
|
|
233
|
+
const clampTurnText = (value) => value && value.length > 1_200 ? `${value.slice(0, 1_200)}…` : value;
|
|
234
|
+
const turns = store.recentTurns(threadId, 6).map((turn) => {
|
|
219
235
|
const contract = turn.contract ?? {};
|
|
220
236
|
const topN = contract.topN;
|
|
221
237
|
const topNValue = typeof topN === 'number'
|
|
@@ -225,8 +241,8 @@ async function conversationContextFromThread(store, threadId, clientContext, que
|
|
|
225
241
|
: undefined;
|
|
226
242
|
return compactConversationRecord({
|
|
227
243
|
id: turn.id,
|
|
228
|
-
question: turn.question,
|
|
229
|
-
answerSummary: turn.answerSummary,
|
|
244
|
+
question: clampTurnText(turn.question),
|
|
245
|
+
answerSummary: clampTurnText(turn.answerSummary),
|
|
230
246
|
sourceCertifiedBlock: turn.sourceCertifiedBlock,
|
|
231
247
|
route: turn.route,
|
|
232
248
|
trustLabel: turn.trustLabel,
|
|
@@ -425,6 +441,8 @@ export async function startLocalServer(opts) {
|
|
|
425
441
|
const loopback = bindHost === '127.0.0.1' || bindHost === 'localhost' || bindHost === '::1';
|
|
426
442
|
const authToken = opts.authToken ?? process.env.DQL_SERVER_TOKEN;
|
|
427
443
|
const runtimeVersion = readDqlRuntimeVersion();
|
|
444
|
+
// Warm the latest-version cache in the background (2s cap, 24h cache; offline → unknown).
|
|
445
|
+
void fetchLatestPublishedDqlVersion();
|
|
428
446
|
const allowedOrigins = new Set((opts.allowedOrigins ?? (process.env.DQL_ALLOWED_ORIGINS ?? '').split(','))
|
|
429
447
|
.map((value) => value.trim().replace(/\/$/, ''))
|
|
430
448
|
.filter(Boolean));
|
|
@@ -1093,7 +1111,12 @@ export async function startLocalServer(opts) {
|
|
|
1093
1111
|
await runner.run({
|
|
1094
1112
|
provider: resolvedProvider,
|
|
1095
1113
|
messages: [
|
|
1096
|
-
|
|
1114
|
+
// No-thread fallback path: bound the raw client history so follow-ups
|
|
1115
|
+
// cannot inflate every provider call (8 turns × 4k chars).
|
|
1116
|
+
...(request.history ?? []).slice(-8).map((message) => ({
|
|
1117
|
+
role: message.role,
|
|
1118
|
+
content: message.text.length > 4_000 ? `${message.text.slice(0, 4_000)}…` : message.text,
|
|
1119
|
+
})),
|
|
1097
1120
|
...(isRepair
|
|
1098
1121
|
? [{ role: 'assistant', content: `The prior attempt needs repair without changing the original question or requested output grain: ${repair?.repairHint}` }]
|
|
1099
1122
|
: []),
|
|
@@ -1429,6 +1452,9 @@ export async function startLocalServer(opts) {
|
|
|
1429
1452
|
applySmartVisualization(governedAnswer, request.question);
|
|
1430
1453
|
}
|
|
1431
1454
|
catch (error) {
|
|
1455
|
+
// Deadline/cancellation propagates to the engine, which renders the
|
|
1456
|
+
// graceful "bounded execution deadline" outcome — never a provider error.
|
|
1457
|
+
rethrowIfCancelled(error, request.signal);
|
|
1432
1458
|
const message = formatAgentRunInfrastructureError(error, 'AI answer provider');
|
|
1433
1459
|
return {
|
|
1434
1460
|
summary: message,
|
|
@@ -1456,7 +1482,7 @@ export async function startLocalServer(opts) {
|
|
|
1456
1482
|
const isExploratory = Boolean(governedAnswer.exploratoryCandidate);
|
|
1457
1483
|
const isExecutionFailure = agentAnswerHasExecutionFailure(governedAnswer);
|
|
1458
1484
|
const isGroundingGap = governedAnswer.kind === 'no_answer'
|
|
1459
|
-
&& governedAnswer.refusalCode === 'grounding_gap'
|
|
1485
|
+
&& (governedAnswer.refusalCode === 'grounding_gap' || governedAnswer.refusalCode === 'modeling_gap')
|
|
1460
1486
|
&& !isExploratory;
|
|
1461
1487
|
const isProviderError = governedAnswer.kind === 'no_answer' && governedAnswer.refusalCode === 'provider_error';
|
|
1462
1488
|
// AGT-004: a rejected attribution/export/proof policy is a deliberate
|
|
@@ -1786,6 +1812,7 @@ export async function startLocalServer(opts) {
|
|
|
1786
1812
|
}
|
|
1787
1813
|
}
|
|
1788
1814
|
catch (error) {
|
|
1815
|
+
rethrowIfCancelled(error, request.signal);
|
|
1789
1816
|
researchWorkspaceError = formatNotebookResearchStorageError(error);
|
|
1790
1817
|
}
|
|
1791
1818
|
}
|
|
@@ -2355,7 +2382,13 @@ export async function startLocalServer(opts) {
|
|
|
2355
2382
|
getEvidence: buildAgentRunEvidence,
|
|
2356
2383
|
getCatalogContext: buildRankedAgentRunCatalogContext,
|
|
2357
2384
|
});
|
|
2358
|
-
|
|
2385
|
+
// P0: one row per run with retention + old-run compaction. The legacy JSON
|
|
2386
|
+
// store rewrote the entire file (123 MB observed) twice per answered question;
|
|
2387
|
+
// existing history is imported once and the JSON renamed to *.migrated.
|
|
2388
|
+
const agentRunStore = new SqliteAgentRunStore({
|
|
2389
|
+
path: defaultAgentRunSqlitePath(projectRoot),
|
|
2390
|
+
legacyJsonPath: defaultAgentRunStorePath(projectRoot),
|
|
2391
|
+
});
|
|
2359
2392
|
// A run may outlive its streaming browser connection, so cancellation is
|
|
2360
2393
|
// server-owned and keyed by run id rather than relying on fetch abort alone.
|
|
2361
2394
|
const activeAgentRunControllers = new Map();
|
|
@@ -2577,7 +2610,7 @@ export async function startLocalServer(opts) {
|
|
|
2577
2610
|
const preflight = repairExploratorySqlBeforeExecution(candidate.sql, schemaContext, question);
|
|
2578
2611
|
const boundedSql = applyRequestedTopNToExploratorySql(preflight.sql, requestedTopN);
|
|
2579
2612
|
const repairs = boundedSql === preflight.sql
|
|
2580
|
-
? preflight.repairs
|
|
2613
|
+
? [...preflight.repairs]
|
|
2581
2614
|
: [...preflight.repairs, `Applied the requested overall top-${requestedTopN} bound before exploratory execution.`];
|
|
2582
2615
|
if (preflight.blockedReason) {
|
|
2583
2616
|
return {
|
|
@@ -2591,19 +2624,62 @@ export async function startLocalServer(opts) {
|
|
|
2591
2624
|
if (!analysis.parsed) {
|
|
2592
2625
|
return { proofs: [], sql: boundedSql, repairs, error: 'DQL could not parse the exploratory SQL to validate its join predicates.' };
|
|
2593
2626
|
}
|
|
2627
|
+
const probeableJoins = probeableExploratoryJoins(analysis.joins, analysis.ctes);
|
|
2594
2628
|
// A normal customer → order → order-item → product question needs three
|
|
2595
2629
|
// joins. Keep the lane bounded, but do not reject this common dbt-star path
|
|
2596
|
-
// solely because the old two-join demo limit was too small.
|
|
2597
|
-
|
|
2630
|
+
// solely because the old two-join demo limit was too small. CTE-internal
|
|
2631
|
+
// joins are counted physically (they appear in analysis.joins with their
|
|
2632
|
+
// physical endpoints); joins ON a CTE are restructuring, not new paths.
|
|
2633
|
+
if (probeableJoins.length > 4) {
|
|
2598
2634
|
return { proofs: [], sql: boundedSql, repairs, error: 'This exploratory query has more than four joins and requires an analyst to review the relationship path.' };
|
|
2599
2635
|
}
|
|
2600
|
-
if (
|
|
2636
|
+
if (probeableJoins.some((join) => !join.leftRelation || !join.rightRelation)) {
|
|
2601
2637
|
return { proofs: [], sql: boundedSql, repairs, error: 'This exploratory query has a join endpoint DQL could not resolve for bounded validation.' };
|
|
2602
2638
|
}
|
|
2639
|
+
// Structural binding: when the candidate carries a declared draft join path,
|
|
2640
|
+
// every SQL join must ride one of its edges on the declared key pair. A join
|
|
2641
|
+
// outside the declared path (or on different keys) is not executed — it is a
|
|
2642
|
+
// review-required modeling question, not a runtime guess.
|
|
2643
|
+
const declaredEdges = candidate.exploratoryPath?.edges ?? [];
|
|
2644
|
+
const joinEdges = new Map();
|
|
2645
|
+
if (declaredEdges.length > 0) {
|
|
2646
|
+
for (const [index, join] of probeableJoins.entries()) {
|
|
2647
|
+
const edge = declaredEdges.find((value) => declaredExploratoryRelationMatches(join.leftRelation, value.fromRelation) && declaredExploratoryRelationMatches(join.rightRelation, value.toRelation)
|
|
2648
|
+
|| declaredExploratoryRelationMatches(join.leftRelation, value.toRelation) && declaredExploratoryRelationMatches(join.rightRelation, value.fromRelation));
|
|
2649
|
+
if (!edge) {
|
|
2650
|
+
return {
|
|
2651
|
+
proofs: [],
|
|
2652
|
+
sql: boundedSql,
|
|
2653
|
+
repairs,
|
|
2654
|
+
error: `The join between ${join.leftRelation} and ${join.rightRelation} is not part of the declared relationship path, so DQL did not execute it. Declare the relationship in the DQL model to enable it.`,
|
|
2655
|
+
};
|
|
2656
|
+
}
|
|
2657
|
+
const keyMatches = edge.keys.some((key) => exploratoryColumnsMatch(join.leftColumn, key.from) && exploratoryColumnsMatch(join.rightColumn, key.to)
|
|
2658
|
+
|| exploratoryColumnsMatch(join.leftColumn, key.to) && exploratoryColumnsMatch(join.rightColumn, key.from));
|
|
2659
|
+
if (!keyMatches) {
|
|
2660
|
+
return {
|
|
2661
|
+
proofs: [],
|
|
2662
|
+
sql: boundedSql,
|
|
2663
|
+
repairs,
|
|
2664
|
+
error: `The join ${join.leftColumn} = ${join.rightColumn} uses different keys than relationship "${edge.relationshipId}" declares (${edge.keys.map((key) => `${key.from}=${key.to}`).join(', ')}). Review the declared keys before executing.`,
|
|
2665
|
+
};
|
|
2666
|
+
}
|
|
2667
|
+
joinEdges.set(index, edge);
|
|
2668
|
+
}
|
|
2669
|
+
const exercised = new Set([...joinEdges.values()].map((edge) => edge.relationshipId));
|
|
2670
|
+
for (const edge of declaredEdges) {
|
|
2671
|
+
if (!exercised.has(edge.relationshipId)) {
|
|
2672
|
+
repairs.push(`Declared relationship "${edge.relationshipId}" was not exercised by this query.`);
|
|
2673
|
+
}
|
|
2674
|
+
}
|
|
2675
|
+
}
|
|
2676
|
+
if (probeableJoins.length < analysis.joins.length) {
|
|
2677
|
+
repairs.push(`Skipped join probes for ${analysis.joins.length - probeableJoins.length} join(s) on derived (CTE) relations; the final query execution validates them.`);
|
|
2678
|
+
}
|
|
2603
2679
|
const activeConnection = requireActiveConnection();
|
|
2604
2680
|
const proofs = [];
|
|
2605
2681
|
try {
|
|
2606
|
-
for (const join of
|
|
2682
|
+
for (const [index, join] of probeableJoins.entries()) {
|
|
2607
2683
|
const proofSql = buildExploratoryJoinProbeSql({
|
|
2608
2684
|
leftRelation: join.leftRelation,
|
|
2609
2685
|
leftColumn: join.leftColumn,
|
|
@@ -2613,6 +2689,14 @@ export async function startLocalServer(opts) {
|
|
|
2613
2689
|
const prepared = prepareLocalExecution(proofSql, activeConnection, projectRoot, projectConfig);
|
|
2614
2690
|
const raw = await executor.executeQuery(prepared.sql, [], runtimeVariables({}), prepared.connection);
|
|
2615
2691
|
proofs.push({ summary: summarizeExploratoryJoinProbe(raw.rows, join.leftRelation, join.leftColumn, join.rightRelation, join.rightColumn) });
|
|
2692
|
+
// Gating (not observational): a structural contradiction between the
|
|
2693
|
+
// UNFILTERED key overlap and the declared relationship stops automatic
|
|
2694
|
+
// execution. A legitimately empty *filtered* business result is still an
|
|
2695
|
+
// answer — these gates only inspect the raw key samples.
|
|
2696
|
+
const contradiction = exploratoryProbeContradiction(raw.rows, join, joinEdges.get(index));
|
|
2697
|
+
if (contradiction) {
|
|
2698
|
+
return { proofs, sql: boundedSql, repairs, error: contradiction };
|
|
2699
|
+
}
|
|
2616
2700
|
}
|
|
2617
2701
|
const result = await executeGeneratedSqlForAgent(boundedSql);
|
|
2618
2702
|
return { result, proofs, sql: boundedSql, repairs };
|
|
@@ -3733,8 +3817,19 @@ export async function startLocalServer(opts) {
|
|
|
3733
3817
|
}
|
|
3734
3818
|
}
|
|
3735
3819
|
if (req.method === 'GET' && path === '/api/health') {
|
|
3820
|
+
// REL-002: expose runtime/build identity so a server started before a
|
|
3821
|
+
// rebuild or running a different version than the project pin is visibly
|
|
3822
|
+
// stale. Uses only the cached latest-version lookup — never blocks.
|
|
3823
|
+
const versionStatus = resolveDqlRuntimeVersionStatus({ projectRoot, runningVersion: runtimeVersion });
|
|
3824
|
+
void fetchLatestPublishedDqlVersion();
|
|
3825
|
+
const healthValueGrounding = resolveAgentRuntimeValueGrounding(projectConfig);
|
|
3826
|
+
const retrievalHealth = resolveRetrievalHealthStatus({
|
|
3827
|
+
projectRoot,
|
|
3828
|
+
valueGroundingMode: healthValueGrounding.mode,
|
|
3829
|
+
searchSafeColumnCount: healthValueGrounding.searchSafeColumns.size,
|
|
3830
|
+
});
|
|
3736
3831
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3737
|
-
res.end(serializeJSON({ status: 'ok', version: runtimeVersion }));
|
|
3832
|
+
res.end(serializeJSON({ status: 'ok', version: runtimeVersion, versionStatus, retrievalHealth }));
|
|
3738
3833
|
return;
|
|
3739
3834
|
}
|
|
3740
3835
|
if (req.method === 'GET' && path === '/api/onboarding/launch') {
|
|
@@ -4543,6 +4638,11 @@ export async function startLocalServer(opts) {
|
|
|
4543
4638
|
const conversationStore = parsed.request.threadId ? getConversationStore() : null;
|
|
4544
4639
|
if (conversationStore && parsed.request.threadId && conversationStore.getThread(parsed.request.threadId)) {
|
|
4545
4640
|
parsed.request.conversationContext = await conversationContextFromThread(conversationStore, parsed.request.threadId, parsed.request.conversationContext, parsed.request.question);
|
|
4641
|
+
// The persisted thread is authoritative for prior turns. Raw client
|
|
4642
|
+
// history would duplicate the same conversation into the prompt a
|
|
4643
|
+
// second time — dropping it keeps follow-up prompts bounded.
|
|
4644
|
+
if (parsed.request.history?.length)
|
|
4645
|
+
parsed.request.history = [];
|
|
4546
4646
|
}
|
|
4547
4647
|
const wantsStream = url.searchParams.get('stream') === '1' || url.searchParams.get('stream') === 'true';
|
|
4548
4648
|
const runId = parsed.request.runId;
|
|
@@ -13108,6 +13208,16 @@ export function buildExploratoryJoinProbeSql(input) {
|
|
|
13108
13208
|
SELECT right_key, COUNT(*) AS match_count
|
|
13109
13209
|
FROM matches
|
|
13110
13210
|
GROUP BY right_key
|
|
13211
|
+
),
|
|
13212
|
+
left_key_counts AS (
|
|
13213
|
+
SELECT join_key, COUNT(*) AS key_count
|
|
13214
|
+
FROM left_sample
|
|
13215
|
+
GROUP BY join_key
|
|
13216
|
+
),
|
|
13217
|
+
right_key_counts AS (
|
|
13218
|
+
SELECT join_key, COUNT(*) AS key_count
|
|
13219
|
+
FROM right_sample
|
|
13220
|
+
GROUP BY join_key
|
|
13111
13221
|
)
|
|
13112
13222
|
SELECT
|
|
13113
13223
|
(SELECT COUNT(*) FROM left_sample) AS left_sample_rows,
|
|
@@ -13116,7 +13226,87 @@ SELECT
|
|
|
13116
13226
|
(SELECT COUNT(*) FROM left_sample AS l WHERE NOT EXISTS (SELECT 1 FROM right_sample AS r WHERE r.join_key = l.join_key)) AS unmatched_left_sample_rows,
|
|
13117
13227
|
(SELECT COUNT(*) FROM right_sample AS r WHERE NOT EXISTS (SELECT 1 FROM left_sample AS l WHERE l.join_key = r.join_key)) AS unmatched_right_sample_rows,
|
|
13118
13228
|
(SELECT COALESCE(MAX(match_count), 0) FROM left_counts) AS max_matches_per_left_key,
|
|
13119
|
-
(SELECT COALESCE(MAX(match_count), 0) FROM right_counts) AS max_matches_per_right_key
|
|
13229
|
+
(SELECT COALESCE(MAX(match_count), 0) FROM right_counts) AS max_matches_per_right_key,
|
|
13230
|
+
(SELECT COALESCE(MAX(key_count), 0) FROM left_key_counts) AS max_left_rows_per_key,
|
|
13231
|
+
(SELECT COALESCE(MAX(key_count), 0) FROM right_key_counts) AS max_right_rows_per_key`;
|
|
13232
|
+
}
|
|
13233
|
+
/**
|
|
13234
|
+
* CTE names are query-internal derived relations, not warehouse tables. A join
|
|
13235
|
+
* whose endpoint is a CTE (e.g. `WITH joy_items AS (…) … JOIN joy_items`) must
|
|
13236
|
+
* be excluded from declared-path enforcement and from join probes — probing
|
|
13237
|
+
* `FROM "joy_items"` throws a DuckDB catalog error for a table that was never
|
|
13238
|
+
* supposed to exist. The physical joins INSIDE the CTE are still present in
|
|
13239
|
+
* the parsed join list and get validated/probed on their own.
|
|
13240
|
+
*/
|
|
13241
|
+
export function probeableExploratoryJoins(joins, ctes) {
|
|
13242
|
+
const cteNames = new Set(ctes.map((name) => name.split('.').at(-1)?.toLowerCase() ?? name.toLowerCase()));
|
|
13243
|
+
if (cteNames.size === 0)
|
|
13244
|
+
return joins;
|
|
13245
|
+
return joins.filter((join) => ![join.leftRelation, join.rightRelation].some((relation) => {
|
|
13246
|
+
const bare = relation?.replace(/["`\[\]]/g, '').split('.').at(-1)?.toLowerCase();
|
|
13247
|
+
return Boolean(bare && cteNames.has(bare));
|
|
13248
|
+
}));
|
|
13249
|
+
}
|
|
13250
|
+
function declaredExploratoryRelationMatches(sqlRelation, declaredRelation) {
|
|
13251
|
+
if (!declaredRelation)
|
|
13252
|
+
return false;
|
|
13253
|
+
const normalize = (value) => value.replace(/["`\[\]]/g, '').trim();
|
|
13254
|
+
return exploratoryRelationsMatch(normalize(sqlRelation), normalize(declaredRelation));
|
|
13255
|
+
}
|
|
13256
|
+
function exploratoryColumnsMatch(sqlColumn, declaredColumn) {
|
|
13257
|
+
const normalize = (value) => value.replace(/["`\[\]]/g, '').split('.').at(-1)?.trim().toLowerCase() ?? value.toLowerCase();
|
|
13258
|
+
return normalize(sqlColumn) === normalize(declaredColumn);
|
|
13259
|
+
}
|
|
13260
|
+
/**
|
|
13261
|
+
* Gate a join probe against the declared relationship. Returns an error string
|
|
13262
|
+
* when the UNFILTERED key samples structurally contradict the declaration:
|
|
13263
|
+
* - zero key overlap while both sides have rows (wrong/mistyped key), or
|
|
13264
|
+
* - duplicate keys on the declared "one" side of a *_to_one / one_to_* edge.
|
|
13265
|
+
* Sampling caveat: gates only fire on definitive contradictions, never on low
|
|
13266
|
+
* match rates, so a sparse but real relationship still executes.
|
|
13267
|
+
*/
|
|
13268
|
+
export function exploratoryProbeContradiction(rows, join, edge) {
|
|
13269
|
+
const row = Array.isArray(rows) && rows[0] && typeof rows[0] === 'object'
|
|
13270
|
+
? rows[0]
|
|
13271
|
+
: {};
|
|
13272
|
+
const number = (key) => {
|
|
13273
|
+
const value = Number(row[key]);
|
|
13274
|
+
return Number.isFinite(value) ? value : 0;
|
|
13275
|
+
};
|
|
13276
|
+
const leftRows = number('left_sample_rows');
|
|
13277
|
+
const rightRows = number('right_sample_rows');
|
|
13278
|
+
const joined = number('joined_rows');
|
|
13279
|
+
if (leftRows > 0 && rightRows > 0 && joined === 0) {
|
|
13280
|
+
return `The join key ${join.leftColumn} = ${join.rightColumn} produced no matching rows between ${join.leftRelation} and ${join.rightRelation} (unfiltered sample). The declared key pair is likely wrong — review the relationship before executing this analysis.`;
|
|
13281
|
+
}
|
|
13282
|
+
if (!edge)
|
|
13283
|
+
return undefined;
|
|
13284
|
+
// Orient the declared cardinality onto the parsed join: which physical side is
|
|
13285
|
+
// the declared "one" side? Duplicated keys there contradict the declaration.
|
|
13286
|
+
const leftIsFrom = Boolean(join.leftRelation && declaredExploratoryRelationMatches(join.leftRelation, edge.fromRelation));
|
|
13287
|
+
const rightIsFrom = Boolean(join.rightRelation && declaredExploratoryRelationMatches(join.rightRelation, edge.fromRelation));
|
|
13288
|
+
if (leftIsFrom === rightIsFrom)
|
|
13289
|
+
return undefined; // orientation unresolved — do not guess
|
|
13290
|
+
const oneSide = edge.cardinality === 'many_to_one' ? 'to' : edge.cardinality === 'one_to_many' ? 'from' : edge.cardinality === 'one_to_one' ? 'both' : undefined;
|
|
13291
|
+
if (!oneSide)
|
|
13292
|
+
return undefined;
|
|
13293
|
+
// Per-side key duplication measured WITHIN each sample (max rows sharing one
|
|
13294
|
+
// key value). Duplicates inside a sample are definitive — a subset of a table
|
|
13295
|
+
// cannot show duplicates the full table does not have. The pairwise
|
|
13296
|
+
// max_matches_* columns are NOT used here: they inflate whenever the
|
|
13297
|
+
// legitimate "many" side repeats a key value.
|
|
13298
|
+
const leftDuplicated = number('max_left_rows_per_key') > 1;
|
|
13299
|
+
const rightDuplicated = number('max_right_rows_per_key') > 1;
|
|
13300
|
+
const fromSideIsLeft = leftIsFrom;
|
|
13301
|
+
const duplicatedSides = [
|
|
13302
|
+
...(leftDuplicated ? [fromSideIsLeft ? 'from' : 'to'] : []),
|
|
13303
|
+
...(rightDuplicated ? [fromSideIsLeft ? 'to' : 'from'] : []),
|
|
13304
|
+
];
|
|
13305
|
+
const violated = oneSide === 'both' ? duplicatedSides.length > 0 : duplicatedSides.includes(oneSide);
|
|
13306
|
+
if (violated) {
|
|
13307
|
+
return `Relationship "${edge.relationshipId}" declares ${edge.cardinality}, but the sampled join keys show duplicates on the declared unique side. Executing would duplicate rows — validate and correct the relationship first.`;
|
|
13308
|
+
}
|
|
13309
|
+
return undefined;
|
|
13120
13310
|
}
|
|
13121
13311
|
function summarizeExploratoryJoinProbe(rows, leftRelation, leftColumn, rightRelation, rightColumn) {
|
|
13122
13312
|
const row = Array.isArray(rows) && rows[0] && typeof rows[0] === 'object'
|