@duckcodeailabs/dql-cli 1.6.31 → 1.6.33
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/apps-api.d.ts.map +1 -1
- package/dist/apps-api.js +3 -1
- package/dist/apps-api.js.map +1 -1
- package/dist/args.d.ts +18 -0
- package/dist/args.d.ts.map +1 -1
- package/dist/args.js +37 -0
- package/dist/args.js.map +1 -1
- package/dist/assets/dql-notebook/assets/index-B6x1cRJo.js +5698 -0
- package/dist/assets/dql-notebook/index.html +1 -1
- package/dist/commands/agent.d.ts +47 -1
- package/dist/commands/agent.d.ts.map +1 -1
- package/dist/commands/agent.js +635 -27
- package/dist/commands/agent.js.map +1 -1
- package/dist/commands/app.d.ts.map +1 -1
- package/dist/commands/app.js +3 -1
- package/dist/commands/app.js.map +1 -1
- package/dist/commands/diff.d.ts.map +1 -1
- package/dist/commands/diff.js +111 -22
- package/dist/commands/diff.js.map +1 -1
- package/dist/commands/eval-judge.d.ts +37 -0
- package/dist/commands/eval-judge.d.ts.map +1 -0
- package/dist/commands/eval-judge.js +99 -0
- package/dist/commands/eval-judge.js.map +1 -0
- package/dist/commands/eval.d.ts +27 -1
- package/dist/commands/eval.d.ts.map +1 -1
- package/dist/commands/eval.js +143 -6
- package/dist/commands/eval.js.map +1 -1
- package/dist/commands/mcp.js +27 -3
- package/dist/commands/mcp.js.map +1 -1
- package/dist/index.js +6 -0
- package/dist/index.js.map +1 -1
- package/dist/llm/analytics-tools.d.ts.map +1 -1
- package/dist/llm/analytics-tools.js +12 -7
- package/dist/llm/analytics-tools.js.map +1 -1
- package/dist/llm/answer-loop-tools.d.ts +7 -0
- package/dist/llm/answer-loop-tools.d.ts.map +1 -0
- package/dist/llm/answer-loop-tools.js +67 -0
- package/dist/llm/answer-loop-tools.js.map +1 -0
- package/dist/llm/proposal-metadata.d.ts +5 -0
- package/dist/llm/proposal-metadata.d.ts.map +1 -0
- package/dist/llm/proposal-metadata.js +27 -0
- package/dist/llm/proposal-metadata.js.map +1 -0
- package/dist/llm/providers/claude-agent-sdk.d.ts +6 -0
- package/dist/llm/providers/claude-agent-sdk.d.ts.map +1 -1
- package/dist/llm/providers/claude-agent-sdk.js +12 -2
- package/dist/llm/providers/claude-agent-sdk.js.map +1 -1
- package/dist/llm/providers/claude-code.d.ts +7 -1
- package/dist/llm/providers/claude-code.d.ts.map +1 -1
- package/dist/llm/providers/claude-code.js +11 -2
- package/dist/llm/providers/claude-code.js.map +1 -1
- package/dist/llm/providers/dql-agent-provider.d.ts +29 -0
- package/dist/llm/providers/dql-agent-provider.d.ts.map +1 -1
- package/dist/llm/providers/dql-agent-provider.js +757 -26
- package/dist/llm/providers/dql-agent-provider.js.map +1 -1
- package/dist/llm/tools.d.ts.map +1 -1
- package/dist/llm/tools.js +34 -137
- package/dist/llm/tools.js.map +1 -1
- package/dist/llm/types.d.ts +67 -2
- package/dist/llm/types.d.ts.map +1 -1
- package/dist/local-runtime.d.ts +160 -5
- package/dist/local-runtime.d.ts.map +1 -1
- package/dist/local-runtime.js +1759 -187
- package/dist/local-runtime.js.map +1 -1
- package/dist/package.json +10 -10
- package/dist/providers/oauth/claude-oauth.d.ts +3 -3
- package/dist/providers/oauth/claude-oauth.d.ts.map +1 -1
- package/dist/providers/oauth/claude-oauth.js +3 -3
- package/dist/providers/oauth/claude-oauth.js.map +1 -1
- package/dist/providers/oauth/codex-oauth.d.ts +2 -3
- package/dist/providers/oauth/codex-oauth.d.ts.map +1 -1
- package/dist/providers/oauth/codex-oauth.js +6 -3
- package/dist/providers/oauth/codex-oauth.js.map +1 -1
- package/dist/settings/provider-settings.d.ts +1 -0
- package/dist/settings/provider-settings.d.ts.map +1 -1
- package/dist/settings/provider-settings.js +1 -1
- package/dist/settings/provider-settings.js.map +1 -1
- package/package.json +11 -11
- package/dist/assets/dql-notebook/assets/index-BpPs6_rz.js +0 -5648
package/dist/local-runtime.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { execFileSync, execSync } from 'node:child_process';
|
|
2
|
+
import { createHash } from 'node:crypto';
|
|
2
3
|
import { createServer } from 'node:http';
|
|
3
4
|
import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, watch, writeFileSync } from 'node:fs';
|
|
4
5
|
import { createRequire } from 'node:module';
|
|
@@ -7,19 +8,19 @@ import { dirname, extname, join, normalize, relative, resolve } from 'node:path'
|
|
|
7
8
|
import Anthropic from '@anthropic-ai/sdk';
|
|
8
9
|
import OpenAI from 'openai';
|
|
9
10
|
import { buildExecutionPlan, createWelcomeNotebook, deserializeNotebook, getConnectorFormSchemas, hasSemanticRefs, resolveSemanticRefs, } from '@duckcodeailabs/dql-notebook';
|
|
10
|
-
import { loadSemanticLayerFromDir, resolveSemanticLayerAsync, getDialect, Parser, buildLineageGraph, buildManifest, findAppDocuments, findDashboardsForApp, isBlockIdRef, loadAppDocument, loadDashboardDocument, analyzeImpact, buildTrustChain, detectDomainFlows, getDomainTrustOverview, queryLineage, queryBusiness360, queryCompleteLineagePaths, LineageGraph, canonicalize, canonicalizeNotebook, diffDQL, diffNotebook, writeDomainDeclaration, deleteDomainDeclaration, domainFolderSlug, } from '@duckcodeailabs/dql-core';
|
|
11
|
+
import { loadSemanticLayerFromDir, normalizeDqlArtifactReference, serializeMetricDefinitionToYaml, resolveSemanticLayerAsync, getDialect, Parser, buildLineageGraph, buildManifest, findAppDocuments, findDashboardsForApp, isBlockIdRef, loadAppDocument, loadDashboardDocument, analyzeImpact, buildTrustChain, detectDomainFlows, getDomainTrustOverview, queryLineage, queryBusiness360, queryCompleteLineagePaths, LineageGraph, canonicalize, canonicalizeNotebook, diffDQL, diffNotebook, writeDomainDeclaration, deleteDomainDeclaration, domainFolderSlug, } from '@duckcodeailabs/dql-core';
|
|
11
12
|
import { load as loadYaml } from 'js-yaml';
|
|
12
13
|
import { listBlockTemplates } from './block-templates.js';
|
|
13
14
|
import { getRunner as getLLMRunner } from './llm/index.js';
|
|
14
15
|
import { createDqlAgentProviderRunner } from './llm/providers/dql-agent-provider.js';
|
|
15
16
|
import { listRemoteMcpSettings, saveRemoteMcpSettings } from './llm/mcp-config.js';
|
|
16
|
-
import { ClaudeProvider, GeminiProvider, MemoryStore, OllamaProvider, OpenAIProvider, buildBlockBusinessFingerprint, buildBlockSqlFingerprints, buildLocalContextPack, defaultMemoryPath, ensureDefaultMemoryFiles, ensureMetadataCatalogFresh, propose, proposePlan, recordCorrectionTrace, reviewHint, AgentRunEngine, FileAgentRunStore, defaultAgentRunGates, createLlmAgentRunPlanner, createHybridRouter, synthesizeAnswer, streamOrGenerate, narrateResult, normalizeAnthropicBaseUrl, buildProposePreview, buildFromPrompt, defaultAgentRunStorePath, resolveLocalOwner, resolveProposeConfig, recordQueryRun, recordRuntimeSchemaSnapshot, loadSkills, writeSkill, deleteSkill, reindexProject, defaultKgPath, planApp, planResearch, loadSemanticMetrics, routeReasoningEffort, clampReasoningEffort, bumpReasoningEffort, } from '@duckcodeailabs/dql-agent';
|
|
17
|
+
import { ClaudeProvider, ConversationStore, advanceThreadState, buildConversationSnapshot, recallRelevantTurns, GeminiProvider, MemoryStore, OllamaProvider, OpenAIProvider, buildBlockBusinessFingerprint, buildBlockSqlFingerprints, buildLocalContextPack, defaultConversationPath, defaultMemoryPath, ensureDefaultMemoryFiles, ensureMetadataCatalogFresh, propose, proposePlan, recordCorrectionTrace, reviewHint, AgentRunEngine, FileAgentRunStore, defaultAgentRunGates, createLlmAgentRunPlanner, createHybridRouter, computeResultStats, synthesizeAnswer, streamOrGenerate, narrateResult, normalizeAnthropicBaseUrl, buildProposePreview, buildFromPrompt, defaultAgentRunStorePath, resolveLocalOwner, resolveProposeConfig, recordQueryRun, recordRuntimeSchemaSnapshot, loadSkills, writeSkill, deleteSkill, deriveGeneratedDraftSlug, reindexProject, defaultKgPath, planApp, planResearch, loadSemanticMetrics, routeReasoningEffort, routeForCascadeAnswerTier, clampReasoningEffort, bumpReasoningEffort, upsertGeneratedDqlArtifactDraft, } from '@duckcodeailabs/dql-agent';
|
|
17
18
|
import { gatherProposeEnrichment } from './propose-enrich.js';
|
|
18
19
|
import { handleAppsApi, proposeAppAiBuild, recommendVisualization } from './apps-api.js';
|
|
19
|
-
import { getActiveProvider, getEffectiveProviderConfig, listProviderSettings, saveProviderSettings, } from './settings/provider-settings.js';
|
|
20
|
+
import { getActiveProvider, getEffectiveProviderConfig, isProviderSettingsId, listProviderSettings, saveProviderSettings, } from './settings/provider-settings.js';
|
|
20
21
|
import { ClaudeCodeCliProvider, CodexCliProvider } from './providers/subscription-cli.js';
|
|
21
|
-
import { ClaudeOAuthManager, claudeOAuthConnected, CLAUDE_OAUTH_MODELS, CLAUDE_OAUTH_DEFAULT_MODEL, } from './providers/oauth/claude-oauth.js';
|
|
22
|
-
import { CodexOAuthManager, codexOAuthConnected, CODEX_OAUTH_MODELS, CODEX_OAUTH_DEFAULT_MODEL, } from './providers/oauth/codex-oauth.js';
|
|
22
|
+
import { ClaudeOAuthManager, ClaudeOAuthProvider, claudeOAuthConnected, CLAUDE_OAUTH_MODELS, CLAUDE_OAUTH_DEFAULT_MODEL, } from './providers/oauth/claude-oauth.js';
|
|
23
|
+
import { CodexOAuthManager, CodexOAuthProvider, codexOAuthConnected, CODEX_OAUTH_MODELS, CODEX_OAUTH_DEFAULT_MODEL, } from './providers/oauth/codex-oauth.js';
|
|
23
24
|
import { DQLAccessDeniedError, activePersonaAppId, assertAppAccess, loadRuntimeApp, runtimeVariables, } from './governance-runtime.js';
|
|
24
25
|
import { LocalAppStorage, LocalNotebookResearchStorage, defaultLocalAppsDbPath, defaultNotebookResearchDbPath } from '@duckcodeailabs/dql-project';
|
|
25
26
|
import { Certifier, ENTERPRISE_RULES, evaluateInvariants, hasInvariantViolation, } from '@duckcodeailabs/dql-governance';
|
|
@@ -78,6 +79,16 @@ function parseAgentRunRequestedMode(value) {
|
|
|
78
79
|
? value
|
|
79
80
|
: undefined;
|
|
80
81
|
}
|
|
82
|
+
function parseAgentRunReasoningEffort(value) {
|
|
83
|
+
return value === 'low' || value === 'medium' || value === 'high'
|
|
84
|
+
? value
|
|
85
|
+
: undefined;
|
|
86
|
+
}
|
|
87
|
+
function parseAgentRunAnalysisDepth(value) {
|
|
88
|
+
return value === 'quick' || value === 'deep'
|
|
89
|
+
? value
|
|
90
|
+
: undefined;
|
|
91
|
+
}
|
|
81
92
|
function parseAgentRunSelectedObject(value) {
|
|
82
93
|
const record = agentRunRecord(value);
|
|
83
94
|
if (!record)
|
|
@@ -105,7 +116,7 @@ function parseAgentRunHistory(value) {
|
|
|
105
116
|
});
|
|
106
117
|
return history.length > 0 ? history.slice(-20) : undefined;
|
|
107
118
|
}
|
|
108
|
-
function parseAgentRunRequestBody(body) {
|
|
119
|
+
export function parseAgentRunRequestBody(body) {
|
|
109
120
|
const record = agentRunRecord(body);
|
|
110
121
|
if (!record)
|
|
111
122
|
return { error: 'Invalid JSON body.' };
|
|
@@ -128,11 +139,181 @@ function parseAgentRunRequestBody(body) {
|
|
|
128
139
|
signals: signals,
|
|
129
140
|
selectedObject,
|
|
130
141
|
workspaceContext,
|
|
142
|
+
conversationContext: agentRunRecord(record.conversationContext),
|
|
131
143
|
history: parseAgentRunHistory(record.history),
|
|
144
|
+
threadId: agentRunString(record.threadId),
|
|
132
145
|
runId: agentRunString(record.runId),
|
|
146
|
+
reasoningEffort: parseAgentRunReasoningEffort(record.reasoningEffort),
|
|
147
|
+
analysisDepth: parseAgentRunAnalysisDepth(record.analysisDepth) ?? parseAgentRunAnalysisDepth(record.depth),
|
|
133
148
|
},
|
|
134
149
|
};
|
|
135
150
|
}
|
|
151
|
+
export function shouldSynthesizeAgentRunAnswer(governedAnswer) {
|
|
152
|
+
if (governedAnswer.kind === 'no_answer')
|
|
153
|
+
return false;
|
|
154
|
+
if (governedAnswer.kind === 'certified' || governedAnswer.certification === 'certified')
|
|
155
|
+
return false;
|
|
156
|
+
const finalText = (governedAnswer.answer ?? governedAnswer.text ?? '').trim();
|
|
157
|
+
if (governedAnswer.dqlArtifact && finalText)
|
|
158
|
+
return false;
|
|
159
|
+
return true;
|
|
160
|
+
}
|
|
161
|
+
// ── Server-side conversation threads ────────────────────────────────────────
|
|
162
|
+
// When a run carries a threadId, the persisted thread is the authoritative
|
|
163
|
+
// source of prior turns (survives refresh); the client-built context remains
|
|
164
|
+
// the fallback for embedders that never send a threadId.
|
|
165
|
+
async function conversationContextFromThread(store, threadId, clientContext, question) {
|
|
166
|
+
const turns = store.recentTurns(threadId, 8).map((turn) => {
|
|
167
|
+
const contract = turn.contract ?? {};
|
|
168
|
+
const topN = contract.topN;
|
|
169
|
+
const topNValue = typeof topN === 'number'
|
|
170
|
+
? topN
|
|
171
|
+
: topN && typeof topN === 'object' && typeof topN.n === 'number'
|
|
172
|
+
? topN.n
|
|
173
|
+
: undefined;
|
|
174
|
+
return compactConversationRecord({
|
|
175
|
+
id: turn.id,
|
|
176
|
+
question: turn.question,
|
|
177
|
+
answerSummary: turn.answerSummary,
|
|
178
|
+
sourceCertifiedBlock: turn.sourceCertifiedBlock,
|
|
179
|
+
route: turn.route,
|
|
180
|
+
trustLabel: turn.trustLabel,
|
|
181
|
+
certification: turn.certification,
|
|
182
|
+
contextPackId: turn.contextPackId,
|
|
183
|
+
dqlArtifact: turn.dqlArtifact,
|
|
184
|
+
cascade: turn.cascade,
|
|
185
|
+
sourceSql: turn.sql,
|
|
186
|
+
requestedFilters: conversationStringArray(contract.filters),
|
|
187
|
+
requestedDimensions: conversationStringArray(contract.dimensions),
|
|
188
|
+
requestedMeasures: conversationStringArray(contract.measures),
|
|
189
|
+
topN: topNValue,
|
|
190
|
+
result: turn.result
|
|
191
|
+
? compactConversationRecord({
|
|
192
|
+
columns: turn.result.columns,
|
|
193
|
+
rowsSample: turn.result.rowsSample,
|
|
194
|
+
dimensionValues: turn.result.dimensionValues,
|
|
195
|
+
measureColumns: turn.result.measureColumns,
|
|
196
|
+
rowCount: turn.result.rowCount,
|
|
197
|
+
})
|
|
198
|
+
: undefined,
|
|
199
|
+
});
|
|
200
|
+
}).filter((turn) => Boolean(turn));
|
|
201
|
+
const thread = store.getThread(threadId);
|
|
202
|
+
// Bounded structured snapshot (working state + rolling summary + topic relation)
|
|
203
|
+
// for the answer loop's conversation-state prompt section.
|
|
204
|
+
const serverSnapshot = buildConversationSnapshot(store, threadId, { question });
|
|
205
|
+
if (serverSnapshot && question) {
|
|
206
|
+
// Semantic recall over OLDER turns (the recent window is already verbatim).
|
|
207
|
+
serverSnapshot.recalledTurns = await recallRelevantTurns(store, threadId, question, {
|
|
208
|
+
limit: 3,
|
|
209
|
+
excludeTurnIds: serverSnapshot.recentTurns.map((turn) => turn.id),
|
|
210
|
+
});
|
|
211
|
+
}
|
|
212
|
+
return {
|
|
213
|
+
...(clientContext ?? {}),
|
|
214
|
+
conversationStateVersion: 1,
|
|
215
|
+
threadId,
|
|
216
|
+
...(serverSnapshot ? { serverSnapshot } : {}),
|
|
217
|
+
...(thread?.rollingSummary ? { conversationSummary: thread.rollingSummary } : {}),
|
|
218
|
+
...(turns.length > 0 ? { turns, activeTurnId: turns[turns.length - 1].id } : {}),
|
|
219
|
+
};
|
|
220
|
+
}
|
|
221
|
+
/** Best-effort: persist a completed run as a conversation turn (never throws). */
|
|
222
|
+
function recordConversationTurn(store, threadId, run) {
|
|
223
|
+
if (!store || !threadId)
|
|
224
|
+
return;
|
|
225
|
+
try {
|
|
226
|
+
if (!store.getThread(threadId))
|
|
227
|
+
return;
|
|
228
|
+
const turn = store.appendTurn(threadId, conversationTurnInputFromRun(run));
|
|
229
|
+
// Fold the turn into the thread's working state + rolling summary (incremental).
|
|
230
|
+
advanceThreadState(store, threadId, turn);
|
|
231
|
+
}
|
|
232
|
+
catch {
|
|
233
|
+
// Conversation persistence is additive; a failed write must not fail the run.
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
function conversationTurnInputFromRun(run) {
|
|
237
|
+
const artifact = run.artifacts.find((candidate) => candidate.kind === 'answer')
|
|
238
|
+
?? run.artifacts.find((candidate) => candidate.kind === 'research_run')
|
|
239
|
+
?? run.artifacts[0];
|
|
240
|
+
const payload = agentRunRecord(artifact?.payload);
|
|
241
|
+
const result = agentRunRecord(payload?.result);
|
|
242
|
+
const columns = conversationResultColumns(result?.columns);
|
|
243
|
+
const rows = Array.isArray(result?.rows)
|
|
244
|
+
? result.rows.filter((row) => Boolean(row && typeof row === 'object' && !Array.isArray(row))).slice(0, 8)
|
|
245
|
+
: [];
|
|
246
|
+
const rowsSample = rows.map((row) => columns.map((column) => row[column]));
|
|
247
|
+
const contextPack = agentRunRecord(payload?.contextPack);
|
|
248
|
+
const questionPlan = agentRunRecord(contextPack?.questionPlan);
|
|
249
|
+
const requestedShape = agentRunRecord(questionPlan?.requestedShape);
|
|
250
|
+
const rowCountRaw = result?.rowCount;
|
|
251
|
+
const measureColumns = conversationMeasureColumns(columns, requestedShape, rows);
|
|
252
|
+
return {
|
|
253
|
+
question: run.question,
|
|
254
|
+
answerSummary: run.answer ?? run.summary,
|
|
255
|
+
route: run.route,
|
|
256
|
+
trustLabel: agentRunString(payload?.trustLabel) ?? run.trustState,
|
|
257
|
+
certification: agentRunString(payload?.certification),
|
|
258
|
+
sourceCertifiedBlock: agentRunString(payload?.sourceCertifiedBlock)
|
|
259
|
+
?? (artifact?.kind === 'answer' ? agentRunString(artifact.ref) : undefined),
|
|
260
|
+
contextPackId: agentRunString(payload?.contextPackId),
|
|
261
|
+
sql: agentRunString(payload?.proposedSql) ?? agentRunString(payload?.sql),
|
|
262
|
+
dqlArtifact: agentRunRecord(payload?.dqlArtifact),
|
|
263
|
+
cascade: agentRunRecord(payload?.cascade),
|
|
264
|
+
result: columns.length > 0 || rows.length > 0
|
|
265
|
+
? {
|
|
266
|
+
columns,
|
|
267
|
+
rowsSample,
|
|
268
|
+
dimensionValues: conversationDimensionValues(columns, rows),
|
|
269
|
+
measureColumns,
|
|
270
|
+
rowCount: typeof rowCountRaw === 'number' ? rowCountRaw : rows.length || undefined,
|
|
271
|
+
}
|
|
272
|
+
: undefined,
|
|
273
|
+
contract: requestedShape,
|
|
274
|
+
};
|
|
275
|
+
}
|
|
276
|
+
function conversationResultColumns(value) {
|
|
277
|
+
if (!Array.isArray(value))
|
|
278
|
+
return [];
|
|
279
|
+
return value
|
|
280
|
+
.map((column) => typeof column === 'string'
|
|
281
|
+
? column
|
|
282
|
+
: agentRunString(column?.name) ?? '')
|
|
283
|
+
.filter(Boolean)
|
|
284
|
+
.slice(0, 24);
|
|
285
|
+
}
|
|
286
|
+
function conversationMeasureColumns(columns, requestedShape, rows) {
|
|
287
|
+
const requested = conversationStringArray(requestedShape?.measures) ?? [];
|
|
288
|
+
const numericColumns = columns.filter((column) => rows.some((row) => typeof row[column] === 'number' && Number.isFinite(row[column])));
|
|
289
|
+
const metricNamedColumns = columns.filter((column) => /\b(revenue|sales|amount|total|count|average|avg|sum|spend|cost|margin|profit|value|points?|score|quantity|units?|rate|volume)\b/i.test(column.replace(/_/g, ' ')));
|
|
290
|
+
const unique = Array.from(new Set([...requested, ...numericColumns, ...metricNamedColumns]
|
|
291
|
+
.map((value) => value.trim())
|
|
292
|
+
.filter(Boolean)));
|
|
293
|
+
return unique.length > 0 ? unique.slice(0, 24) : undefined;
|
|
294
|
+
}
|
|
295
|
+
/** Distinct string values per low-cardinality column — the deictic-resolution fuel. */
|
|
296
|
+
function conversationDimensionValues(columns, rows) {
|
|
297
|
+
if (columns.length === 0 || rows.length === 0)
|
|
298
|
+
return undefined;
|
|
299
|
+
const out = {};
|
|
300
|
+
for (const column of columns.slice(0, 8)) {
|
|
301
|
+
const values = Array.from(new Set(rows.map((row) => row[column]).filter((value) => typeof value === 'string' && value.trim().length > 0))).slice(0, 24);
|
|
302
|
+
if (values.length > 0 && values.length <= 24)
|
|
303
|
+
out[column] = values;
|
|
304
|
+
}
|
|
305
|
+
return Object.keys(out).length > 0 ? out : undefined;
|
|
306
|
+
}
|
|
307
|
+
function conversationStringArray(value) {
|
|
308
|
+
if (!Array.isArray(value))
|
|
309
|
+
return undefined;
|
|
310
|
+
const values = value.filter((item) => typeof item === 'string' && item.trim().length > 0);
|
|
311
|
+
return values.length > 0 ? values : undefined;
|
|
312
|
+
}
|
|
313
|
+
function compactConversationRecord(record) {
|
|
314
|
+
const entries = Object.entries(record).filter(([, value]) => value !== undefined && value !== null);
|
|
315
|
+
return entries.length > 0 ? Object.fromEntries(entries) : undefined;
|
|
316
|
+
}
|
|
136
317
|
export async function startLocalServer(opts) {
|
|
137
318
|
const { rootDir, executor, connection: rawConnection, preferredPort, projectRoot = process.cwd() } = opts;
|
|
138
319
|
const bindHost = opts.host ?? process.env.DQL_HOST ?? '127.0.0.1';
|
|
@@ -378,7 +559,10 @@ export async function startLocalServer(opts) {
|
|
|
378
559
|
let governedAnswer;
|
|
379
560
|
let providerError;
|
|
380
561
|
const isRepair = (repair?.attempt ?? 0) > 0 && Boolean(repair?.repairHint);
|
|
381
|
-
const reasoningEffort =
|
|
562
|
+
const reasoningEffort = request.reasoningEffort
|
|
563
|
+
? resolveRequestedRunReasoningEffort(projectRoot, resolvedProvider, request.reasoningEffort, isRepair)
|
|
564
|
+
: resolveRunReasoningEffort(projectRoot, resolvedProvider, route, isRepair);
|
|
565
|
+
const analysisDepth = request.analysisDepth ?? (route === 'research' ? 'deep' : undefined);
|
|
382
566
|
const contextEnvelope = {
|
|
383
567
|
mode: 'agent_run',
|
|
384
568
|
selectedObject: request.selectedObject,
|
|
@@ -386,24 +570,37 @@ export async function startLocalServer(opts) {
|
|
|
386
570
|
instruction: [
|
|
387
571
|
'Route through the governed DQL answer loop.',
|
|
388
572
|
'Prefer certified DQL blocks when they exactly cover the question.',
|
|
389
|
-
'Generated
|
|
573
|
+
'Generated DQL artifacts remain review-required; SQL is only the bounded preview/compiled evidence.',
|
|
390
574
|
'If the question needs investigation, return the clearest answer and next review action without certifying generated work.',
|
|
391
575
|
...(isRepair ? [`This is a repair attempt — fix the previous failure: ${repair?.repairHint}`] : []),
|
|
392
576
|
].join(' '),
|
|
393
577
|
};
|
|
394
578
|
const controller = new AbortController();
|
|
579
|
+
// Best-effort active warehouse dialect so Lane-2 semantic compiles emit
|
|
580
|
+
// dialect-correct SQL (e.g. DATE_TRUNC / identifier quoting). Absent when no
|
|
581
|
+
// connection is configured — the compiler then uses its default dialect.
|
|
582
|
+
let semanticDriver;
|
|
583
|
+
try {
|
|
584
|
+
semanticDriver = requireActiveConnection().driver;
|
|
585
|
+
}
|
|
586
|
+
catch {
|
|
587
|
+
semanticDriver = undefined;
|
|
588
|
+
}
|
|
395
589
|
await runner.run({
|
|
396
590
|
provider: resolvedProvider,
|
|
397
591
|
messages: [
|
|
398
592
|
...(request.history ?? []).map((message) => ({ role: message.role, content: message.text })),
|
|
399
593
|
{ role: 'user', content: isRepair ? `${request.question}\n\nFix the previous attempt: ${repair?.repairHint}` : request.question },
|
|
400
594
|
],
|
|
595
|
+
conversationContext: request.conversationContext,
|
|
401
596
|
upstream: {
|
|
402
597
|
cellId: `agent-run:${request.selectedObject?.kind ?? 'workspace'}:${request.selectedObject?.id ?? request.runId ?? 'auto'}`,
|
|
403
598
|
sql: JSON.stringify(contextEnvelope, null, 2),
|
|
404
599
|
},
|
|
405
600
|
reasoningEffort,
|
|
601
|
+
...(analysisDepth ? { analysisDepth } : {}),
|
|
406
602
|
projectRoot,
|
|
603
|
+
...(semanticDriver ? { semanticDriver } : {}),
|
|
407
604
|
executeCertifiedBlock: executeCertifiedBlockForAgent,
|
|
408
605
|
executeGeneratedSql: executeGeneratedSqlForAgent,
|
|
409
606
|
getSchemaContext: getSchemaContextForAgent,
|
|
@@ -420,6 +617,22 @@ export async function startLocalServer(opts) {
|
|
|
420
617
|
}
|
|
421
618
|
return governedAnswer;
|
|
422
619
|
}
|
|
620
|
+
function resolvedRunRouteFromAnswer(governedAnswer) {
|
|
621
|
+
return routeForCascadeAnswerTier(governedAnswer.route?.tier);
|
|
622
|
+
}
|
|
623
|
+
function groundingGapRepairHint(governedAnswer) {
|
|
624
|
+
const details = governedAnswer.refusalDetails;
|
|
625
|
+
if (!details)
|
|
626
|
+
return governedAnswer.text;
|
|
627
|
+
const offending = details.offending;
|
|
628
|
+
const tokens = [
|
|
629
|
+
details.code ? `code=${details.code}` : undefined,
|
|
630
|
+
offending?.relation ? `relation=${offending.relation}` : undefined,
|
|
631
|
+
offending?.column ? `column=${offending.column}` : undefined,
|
|
632
|
+
details.message ? `message=${details.message}` : undefined,
|
|
633
|
+
].filter((token) => Boolean(token));
|
|
634
|
+
return tokens.length > 0 ? tokens.join('; ') : governedAnswer.text;
|
|
635
|
+
}
|
|
423
636
|
const answerRunExecutor = async ({ request, route, routeDecision, attempt, repairHint, emitAnswerDelta }) => {
|
|
424
637
|
let governedAnswer;
|
|
425
638
|
try {
|
|
@@ -447,14 +660,20 @@ export async function startLocalServer(opts) {
|
|
|
447
660
|
],
|
|
448
661
|
};
|
|
449
662
|
}
|
|
663
|
+
const resolvedRoute = resolvedRunRouteFromAnswer(governedAnswer) ?? route;
|
|
450
664
|
const isCertified = governedAnswer.certification === 'certified' || governedAnswer.kind === 'certified';
|
|
451
|
-
const
|
|
665
|
+
const isGroundingGap = governedAnswer.kind === 'no_answer' && governedAnswer.refusalCode === 'grounding_gap';
|
|
666
|
+
const isProviderError = governedAnswer.kind === 'no_answer' && governedAnswer.refusalCode === 'provider_error';
|
|
667
|
+
const groundingRepairHint = isGroundingGap ? groundingGapRepairHint(governedAnswer) : undefined;
|
|
668
|
+
// A provider outage is a retryable infrastructure failure, not a question the
|
|
669
|
+
// user needs to clarify — surface it as blocked so the UI offers a retry.
|
|
670
|
+
const needsClarification = governedAnswer.kind === 'no_answer' && !isGroundingGap && !isProviderError;
|
|
452
671
|
const sql = governedAnswer.proposedSql ?? governedAnswer.sql;
|
|
453
|
-
|
|
454
|
-
//
|
|
455
|
-
//
|
|
672
|
+
const runnableSql = governedAnswer.kind === 'no_answer' ? undefined : sql;
|
|
673
|
+
// Synthesis is a legacy polish pass. Certified/no-answer paths and lanes
|
|
674
|
+
// that already produced DQL-first final prose keep the fast path.
|
|
456
675
|
let synthesizedAnswer;
|
|
457
|
-
if (
|
|
676
|
+
if (shouldSynthesizeAgentRunAnswer(governedAnswer)) {
|
|
458
677
|
try {
|
|
459
678
|
const provider = await createBlockStudioAssistProvider(projectRoot);
|
|
460
679
|
if (provider) {
|
|
@@ -481,32 +700,55 @@ export async function startLocalServer(opts) {
|
|
|
481
700
|
synthesizedAnswer = undefined;
|
|
482
701
|
}
|
|
483
702
|
}
|
|
484
|
-
const status = needsClarification ? 'needs_clarification' : isCertified ? 'completed' : 'needs_review';
|
|
485
|
-
const trustState = needsClarification ? 'not_applicable' : isCertified ? 'certified' : 'review_required';
|
|
486
|
-
const stopReason = needsClarification ? 'needs_clarification' : isCertified ? 'certified_answer_found' : 'human_review_required';
|
|
703
|
+
const status = isProviderError ? 'blocked' : needsClarification ? 'needs_clarification' : isCertified ? 'completed' : 'needs_review';
|
|
704
|
+
const trustState = isProviderError ? 'blocked' : needsClarification ? 'not_applicable' : isCertified ? 'certified' : 'review_required';
|
|
705
|
+
const stopReason = isProviderError ? 'blocked' : needsClarification ? 'needs_clarification' : isCertified ? 'certified_answer_found' : 'human_review_required';
|
|
487
706
|
const nextActions = needsClarification
|
|
488
707
|
? [{ id: 'clarify', label: 'Clarify question', route: 'generated_answer' }]
|
|
489
708
|
: [
|
|
490
|
-
|
|
709
|
+
{ id: 'create-block', label: governedAnswer.dqlArtifact ? 'Review DQL draft' : 'Create DQL draft', route: 'dql_block_draft', artifactKind: 'dql_block_draft' },
|
|
491
710
|
{ id: 'research-gap', label: 'Research deeper', route: 'research' },
|
|
492
|
-
{ id: '
|
|
711
|
+
...(runnableSql ? [{ id: 'insert-sql', label: 'Insert SQL preview', route: 'sql_cell', artifactKind: 'sql_cell' }] : []),
|
|
493
712
|
];
|
|
494
713
|
return {
|
|
714
|
+
resolvedRoute,
|
|
715
|
+
answerTier: governedAnswer.route?.tier,
|
|
495
716
|
summary: governedAnswer.route?.label ?? (isCertified ? 'Answered from certified DQL context.' : 'Answered with review-required generated analysis.'),
|
|
496
717
|
answer: synthesizedAnswer ?? governedAnswer.answer ?? governedAnswer.text,
|
|
497
718
|
status,
|
|
498
719
|
trustState,
|
|
499
720
|
stopReason,
|
|
500
|
-
artifacts:
|
|
501
|
-
|
|
721
|
+
artifacts: governedAnswer.kind === 'no_answer'
|
|
722
|
+
// A refusal still keeps the DQL draft the answer loop produced (when any),
|
|
723
|
+
// so the "Review DQL draft" next-action isn't a dead link and the user can
|
|
724
|
+
// see the SQL that was about to run. Provider outages carry no draft.
|
|
725
|
+
? (governedAnswer.dqlArtifact && !isProviderError
|
|
726
|
+
? [agentRunArtifact('dql_block_draft', 'DQL draft (review required)', governedAnswer.dqlArtifact, undefined, 'review_required')]
|
|
727
|
+
: [])
|
|
502
728
|
: [agentRunArtifact('answer', isCertified ? 'Certified answer' : 'Review-required answer', governedAnswer, governedAnswer.sourceCertifiedBlock ?? governedAnswer.block?.name, isCertified ? 'certified' : 'review_required')],
|
|
503
729
|
evaluations: [
|
|
504
|
-
agentRunEvaluation('route-decision', 'Route decision', true, 'info', routeDecision?.reason ?? 'Routed request to governed answer.'
|
|
730
|
+
agentRunEvaluation('route-decision', 'Route decision', true, 'info', routeDecision?.reason ?? 'Routed request to governed answer.', {
|
|
731
|
+
plannedRoute: route,
|
|
732
|
+
resolvedRoute,
|
|
733
|
+
aiRoute: governedAnswer.route,
|
|
734
|
+
}),
|
|
505
735
|
agentRunEvaluation('trust-boundary', 'Trust boundary', isCertified, isCertified ? 'info' : 'warning', isCertified
|
|
506
736
|
? 'The answer came from certified DQL context.'
|
|
507
737
|
: needsClarification
|
|
508
738
|
? 'The answer loop needs more context before producing a governed answer.'
|
|
509
739
|
: 'The answer is generated or semantic-layer backed and remains review-required.', governedAnswer.route),
|
|
740
|
+
...(isGroundingGap ? [
|
|
741
|
+
{
|
|
742
|
+
...agentRunEvaluation('grounding-gap', 'Metadata grounding', false, 'warning', 'The answer loop found a metadata grounding gap that can be retried with wider context.', {
|
|
743
|
+
refusalCode: governedAnswer.refusalCode,
|
|
744
|
+
refusalDetails: governedAnswer.refusalDetails,
|
|
745
|
+
validationWarnings: governedAnswer.validationWarnings,
|
|
746
|
+
route: governedAnswer.route,
|
|
747
|
+
}),
|
|
748
|
+
suggestedRepair: groundingRepairHint,
|
|
749
|
+
repairAction: { kind: 'retry', hint: groundingRepairHint },
|
|
750
|
+
},
|
|
751
|
+
] : []),
|
|
510
752
|
...(governedAnswer.executionError ? [
|
|
511
753
|
agentRunEvaluation('execution-error', 'Execution error', false, 'warning', governedAnswer.executionError),
|
|
512
754
|
] : []),
|
|
@@ -529,8 +771,10 @@ export async function startLocalServer(opts) {
|
|
|
529
771
|
const provider = await createBlockStudioAssistProvider(projectRoot);
|
|
530
772
|
if (provider) {
|
|
531
773
|
const system = buildConversationSystemPrompt(kind, isGeneralKnowledge, catalogContext, request.audience ?? 'analyst');
|
|
774
|
+
const conversationMemory = renderConversationMemoryForPrompt(request.conversationContext);
|
|
532
775
|
const messages = [
|
|
533
776
|
{ role: 'system', content: system },
|
|
777
|
+
...(conversationMemory ? [{ role: 'system', content: conversationMemory }] : []),
|
|
534
778
|
...(request.history ?? []).slice(-6).map((message) => ({ role: message.role, content: message.text })),
|
|
535
779
|
{ role: 'user', content: request.question },
|
|
536
780
|
];
|
|
@@ -545,7 +789,8 @@ export async function startLocalServer(opts) {
|
|
|
545
789
|
text = undefined;
|
|
546
790
|
}
|
|
547
791
|
if (!text) {
|
|
548
|
-
text =
|
|
792
|
+
text = buildConversationContextRecap(request.conversationContext)
|
|
793
|
+
?? buildConversationalFallback(kind, isGeneralKnowledge, request.question, suggestions);
|
|
549
794
|
emitAnswerDelta?.(text);
|
|
550
795
|
}
|
|
551
796
|
return {
|
|
@@ -564,7 +809,8 @@ export async function startLocalServer(opts) {
|
|
|
564
809
|
conversation: conversationRunExecutor,
|
|
565
810
|
certified_answer: answerRunExecutor,
|
|
566
811
|
generated_answer: answerRunExecutor,
|
|
567
|
-
research: async (
|
|
812
|
+
research: async (researchContext) => {
|
|
813
|
+
const { runId, request, routeDecision, emit } = researchContext;
|
|
568
814
|
const metrics = loadSemanticMetrics(projectRoot);
|
|
569
815
|
let blocks = collectPlanBlocks(projectRoot, { certifiedOnly: true });
|
|
570
816
|
const usedCertifiedOnly = blocks.length > 0;
|
|
@@ -585,6 +831,13 @@ export async function startLocalServer(opts) {
|
|
|
585
831
|
// When the user explicitly picked research, investigate — don't collapse to one step.
|
|
586
832
|
forceInvestigate: request.requestedMode === 'research',
|
|
587
833
|
});
|
|
834
|
+
// Direct governed answer ("the metric answers this directly"): don't wrap it in
|
|
835
|
+
// a review-required research dossier — run the query through the answer loop and
|
|
836
|
+
// return the executed result table + DQL artifact. A dossier is only for genuine
|
|
837
|
+
// multi-step investigation (plan.done === false) or explicit research mode.
|
|
838
|
+
if (plan.done && !plan.followUp && request.requestedMode !== 'research') {
|
|
839
|
+
return answerRunExecutor(researchContext);
|
|
840
|
+
}
|
|
588
841
|
const needsClarification = Boolean(plan.followUp);
|
|
589
842
|
const notebookPath = agentRunNotebookPath(request, runId);
|
|
590
843
|
const researchIntent = agentRunResearchIntent(request);
|
|
@@ -650,9 +903,15 @@ export async function startLocalServer(opts) {
|
|
|
650
903
|
researchWorkspaceError = formatNotebookResearchStorageError(error);
|
|
651
904
|
}
|
|
652
905
|
}
|
|
653
|
-
const
|
|
654
|
-
|
|
655
|
-
const
|
|
906
|
+
const researchResultPreview = researchRun?.resultPreview;
|
|
907
|
+
const researchResultData = coerceNarrateResultData(researchResultPreview);
|
|
908
|
+
const researchResultRecord = agentRunRecord(researchResultPreview);
|
|
909
|
+
// A query that ran and matched 0 rows STILL executed — treat it as a clean,
|
|
910
|
+
// grounded execution (not "no result"), so an empty answer is surfaced as
|
|
911
|
+
// "0 rows matched" rather than silently downgraded to review-required.
|
|
912
|
+
const researchDidExecute = Boolean(researchResultData) || Boolean(researchResultRecord && Array.isArray(researchResultRecord.rows));
|
|
913
|
+
const researchZeroRows = !researchResultData && researchDidExecute;
|
|
914
|
+
const researchExecutedCleanly = researchDidExecute && !researchWorkspaceError && researchRun?.status !== 'error';
|
|
656
915
|
const narration = !needsClarification && researchResultData
|
|
657
916
|
? await narrateForAgentRun({
|
|
658
917
|
question: request.question,
|
|
@@ -665,18 +924,22 @@ export async function startLocalServer(opts) {
|
|
|
665
924
|
const summary = needsClarification
|
|
666
925
|
? 'Needs clarification before running deeper research.'
|
|
667
926
|
: narration?.summary
|
|
668
|
-
?? (
|
|
669
|
-
? '
|
|
670
|
-
: researchRun?.status === '
|
|
671
|
-
? 'Saved a research dossier
|
|
672
|
-
:
|
|
673
|
-
? '
|
|
674
|
-
:
|
|
675
|
-
? 'Prepared a
|
|
676
|
-
:
|
|
927
|
+
?? (researchZeroRows
|
|
928
|
+
? 'The query executed cleanly against real data and matched 0 rows.'
|
|
929
|
+
: researchRun?.status === 'ready'
|
|
930
|
+
? 'Saved a grounded research dossier with context evidence and next review actions.'
|
|
931
|
+
: researchRun?.status === 'error'
|
|
932
|
+
? 'Saved a research dossier, but the preview needs review before promotion.'
|
|
933
|
+
: researchWorkspaceError
|
|
934
|
+
? 'Prepared a grounded research plan; durable research storage is unavailable in this runtime.'
|
|
935
|
+
: plan.done
|
|
936
|
+
? 'Prepared a direct grounded-answer plan.'
|
|
937
|
+
: 'Prepared a grounded research plan over real DQL assets.');
|
|
677
938
|
return {
|
|
678
939
|
summary,
|
|
679
|
-
answer: plan.followUp?.question ?? narration?.summary
|
|
940
|
+
answer: plan.followUp?.question ?? narration?.summary
|
|
941
|
+
?? (researchZeroRows ? 'The query executed cleanly and matched 0 rows.' : undefined)
|
|
942
|
+
?? researchRun?.summary,
|
|
680
943
|
status: needsClarification ? 'needs_clarification' : 'needs_review',
|
|
681
944
|
trustState: needsClarification ? 'not_applicable' : (researchExecutedCleanly ? 'grounded' : 'review_required'),
|
|
682
945
|
stopReason: needsClarification ? 'needs_clarification' : 'human_review_required',
|
|
@@ -706,15 +969,17 @@ export async function startLocalServer(opts) {
|
|
|
706
969
|
? 'Research workspace storage is unavailable; the plan remains available in this agent run.'
|
|
707
970
|
: 'No durable research record was created because the run needs clarification first.', { notebookPath, researchRunId: researchRun?.id, error: researchWorkspaceError }),
|
|
708
971
|
agentRunEvaluation('result-executed', 'Executed against data', researchExecutedCleanly, researchExecutedCleanly ? 'info' : 'warning', researchExecutedCleanly
|
|
709
|
-
?
|
|
710
|
-
|
|
972
|
+
? (researchZeroRows
|
|
973
|
+
? 'The query executed cleanly against real data and matched 0 rows.'
|
|
974
|
+
: 'The query executed cleanly against real data and returned rows.')
|
|
975
|
+
: 'No executed result was available; the output stays exploratory pending review.', { rowCount: Array.isArray(researchResultRecord?.rows) ? researchResultRecord.rows.length : 0 }),
|
|
711
976
|
],
|
|
712
977
|
nextActions: needsClarification
|
|
713
978
|
? [{ id: 'answer-follow-up', label: 'Answer follow-up', route: 'research' }]
|
|
714
979
|
: [
|
|
715
980
|
...(researchRun?.id ? [{ id: 'open-research', label: 'Open research dossier', artifactKind: 'research_run' }] : []),
|
|
716
|
-
|
|
717
|
-
{ id: '
|
|
981
|
+
{ id: 'create-block', label: 'Review DQL draft', route: 'dql_block_draft', artifactKind: 'dql_block_draft' },
|
|
982
|
+
...(researchRun?.generatedSql || researchRun?.reviewedSql ? [{ id: 'insert-sql', label: 'Insert SQL preview', route: 'sql_cell', artifactKind: 'sql_cell' }] : []),
|
|
718
983
|
],
|
|
719
984
|
};
|
|
720
985
|
},
|
|
@@ -729,12 +994,67 @@ export async function startLocalServer(opts) {
|
|
|
729
994
|
agentRunEvaluation('review-boundary', 'Review boundary', true, 'warning', 'Generated SQL must be reviewed before it becomes certified analytics.'),
|
|
730
995
|
],
|
|
731
996
|
nextActions: [
|
|
732
|
-
{ id: 'insert-sql', label: 'Insert SQL
|
|
733
|
-
{ id: 'create-block', label: '
|
|
997
|
+
{ id: 'insert-sql', label: 'Insert SQL preview', artifactKind: 'sql_cell' },
|
|
998
|
+
{ id: 'create-block', label: 'Review as DQL draft', route: 'dql_block_draft', artifactKind: 'dql_block_draft' },
|
|
734
999
|
],
|
|
735
1000
|
};
|
|
736
1001
|
},
|
|
737
1002
|
dql_block_draft: async ({ request, routeDecision, attempt, repairHint }) => {
|
|
1003
|
+
const contextRecord = agentRunRecord(request.conversationContext);
|
|
1004
|
+
const carriedDqlArtifact = normalizeDqlArtifactReference(contextRecord?.dqlArtifact)
|
|
1005
|
+
?? normalizeDqlArtifactReference(request.workspaceContext?.dqlArtifact);
|
|
1006
|
+
if (carriedDqlArtifact && (attempt ?? 0) === 0) {
|
|
1007
|
+
const sourceQuestion = agentRunString(contextRecord?.sourceQuestion) ?? request.question;
|
|
1008
|
+
const session = await createDqlArtifactGenerationSessionForProject(projectRoot, {
|
|
1009
|
+
question: sourceQuestion,
|
|
1010
|
+
dqlArtifact: carriedDqlArtifact,
|
|
1011
|
+
inputMode: 'upload',
|
|
1012
|
+
inputPath: carriedDqlArtifact.sourcePath,
|
|
1013
|
+
domain: agentRunWorkspaceValue(request, 'domain'),
|
|
1014
|
+
owner: agentRunWorkspaceValue(request, 'owner'),
|
|
1015
|
+
tags: ['agent-run', 'review-required'],
|
|
1016
|
+
contextPackId: agentRunString(contextRecord?.contextPackId),
|
|
1017
|
+
routeIntent: agentRunString(contextRecord?.route),
|
|
1018
|
+
sourceBlock: carriedDqlArtifact.sourcePath,
|
|
1019
|
+
}, semanticLayer);
|
|
1020
|
+
const candidate = session.candidates[0];
|
|
1021
|
+
const validation = candidate?.validation && typeof candidate.validation === 'object'
|
|
1022
|
+
? candidate.validation
|
|
1023
|
+
: undefined;
|
|
1024
|
+
const ready = validation?.valid !== false;
|
|
1025
|
+
const payload = {
|
|
1026
|
+
target: 'block',
|
|
1027
|
+
name: candidate?.name ?? carriedDqlArtifact.name ?? 'DQL block draft',
|
|
1028
|
+
path: candidate?.draftSave?.path ?? candidate?.savedPath,
|
|
1029
|
+
importId: session.id,
|
|
1030
|
+
candidateId: candidate?.id,
|
|
1031
|
+
source: candidate?.dqlSource,
|
|
1032
|
+
dqlSource: candidate?.dqlSource,
|
|
1033
|
+
dqlArtifact: carriedDqlArtifact,
|
|
1034
|
+
certifierVerdict: {
|
|
1035
|
+
ready,
|
|
1036
|
+
diagnostics: validation?.diagnostics ?? [],
|
|
1037
|
+
},
|
|
1038
|
+
generation: session.generation,
|
|
1039
|
+
};
|
|
1040
|
+
return {
|
|
1041
|
+
summary: ready
|
|
1042
|
+
? 'Saved the generated DQL artifact as a review-required block draft.'
|
|
1043
|
+
: 'Saved the generated DQL artifact as a block draft with review blockers or warnings.',
|
|
1044
|
+
artifacts: [agentRunArtifact('dql_block_draft', payload.name, payload, payload.path)],
|
|
1045
|
+
evaluations: [
|
|
1046
|
+
agentRunEvaluation('route-decision', 'Route decision', true, 'info', routeDecision?.reason ?? 'Routed request to DQL block draft generation.'),
|
|
1047
|
+
agentRunEvaluation('dql-artifact-reuse', 'Used generated DQL artifact', true, 'info', 'Persisted the existing DQL artifact without regenerating it from SQL.', { importId: session.id, path: payload.path }),
|
|
1048
|
+
agentRunEvaluation('certification-boundary', 'Certification boundary', ready, 'warning', ready
|
|
1049
|
+
? 'The draft has no automatic validation blockers, but certification still requires human review.'
|
|
1050
|
+
: 'The draft has validation blockers that must be resolved before certification review.', payload),
|
|
1051
|
+
],
|
|
1052
|
+
nextActions: [
|
|
1053
|
+
{ id: 'open-review', label: 'Open review checklist', artifactKind: 'dql_block_draft' },
|
|
1054
|
+
{ id: 'build-app', label: 'Build app from block', route: 'app_build', artifactKind: 'app_draft' },
|
|
1055
|
+
],
|
|
1056
|
+
};
|
|
1057
|
+
}
|
|
738
1058
|
const result = await buildAgentPromptArtifact(request, 'block', { attempt, repairHint });
|
|
739
1059
|
const ready = result.target === 'block' ? result.certifierVerdict.ready : false;
|
|
740
1060
|
return {
|
|
@@ -902,9 +1222,27 @@ export async function startLocalServer(opts) {
|
|
|
902
1222
|
getCatalogContext: () => buildAgentRunCatalogContext(),
|
|
903
1223
|
});
|
|
904
1224
|
const agentRunStore = new FileAgentRunStore({ path: defaultAgentRunStorePath(projectRoot) });
|
|
1225
|
+
// Server-side conversation threads: persisted multi-turn state (survives refresh).
|
|
1226
|
+
// Lazy so a failed native-sqlite load never blocks the runtime; conversation
|
|
1227
|
+
// persistence degrades to the per-request context instead.
|
|
1228
|
+
let conversationStoreInstance;
|
|
1229
|
+
const getConversationStore = () => {
|
|
1230
|
+
if (conversationStoreInstance !== undefined)
|
|
1231
|
+
return conversationStoreInstance;
|
|
1232
|
+
try {
|
|
1233
|
+
conversationStoreInstance = new ConversationStore(defaultConversationPath(projectRoot));
|
|
1234
|
+
}
|
|
1235
|
+
catch {
|
|
1236
|
+
conversationStoreInstance = null;
|
|
1237
|
+
}
|
|
1238
|
+
return conversationStoreInstance;
|
|
1239
|
+
};
|
|
905
1240
|
const agentRunEngine = new AgentRunEngine({
|
|
906
1241
|
store: agentRunStore,
|
|
907
|
-
executors:
|
|
1242
|
+
executors: {
|
|
1243
|
+
...agentRunExecutors,
|
|
1244
|
+
...(opts.agentRunExecutors ?? {}),
|
|
1245
|
+
},
|
|
908
1246
|
gates: defaultAgentRunGates,
|
|
909
1247
|
planner: agentRunPlanner,
|
|
910
1248
|
router: agentRunRouter,
|
|
@@ -1063,25 +1401,42 @@ export async function startLocalServer(opts) {
|
|
|
1063
1401
|
};
|
|
1064
1402
|
};
|
|
1065
1403
|
const getSchemaContextForAgent = async (question) => {
|
|
1404
|
+
const scanRuntimeSchema = async () => {
|
|
1405
|
+
if (!connection)
|
|
1406
|
+
return { ranked: [], snapshot: [] };
|
|
1407
|
+
const result = await executor.executeQuery(`SELECT table_schema, table_name, column_name, data_type
|
|
1408
|
+
FROM information_schema.columns
|
|
1409
|
+
WHERE UPPER(table_schema) NOT IN ('INFORMATION_SCHEMA', 'PG_CATALOG')
|
|
1410
|
+
ORDER BY table_schema, table_name, ordinal_position
|
|
1411
|
+
LIMIT 2000`, [], runtimeVariables({}), connection);
|
|
1412
|
+
return {
|
|
1413
|
+
ranked: buildAgentSchemaContext(question, result.rows),
|
|
1414
|
+
snapshot: buildAgentSchemaContext(question, result.rows, { includeUnscored: true, limit: 500 }),
|
|
1415
|
+
};
|
|
1416
|
+
};
|
|
1066
1417
|
const catalogContext = await buildAgentSchemaContextFromCatalog(projectRoot, question).catch(() => []);
|
|
1067
1418
|
if (catalogContext.length > 0) {
|
|
1068
1419
|
if (!connection)
|
|
1069
1420
|
return catalogContext;
|
|
1070
|
-
const
|
|
1071
|
-
|
|
1421
|
+
const runtimeScan = shouldAugmentAgentRuntimeSchema(question)
|
|
1422
|
+
? await scanRuntimeSchema().catch(() => undefined)
|
|
1423
|
+
: undefined;
|
|
1424
|
+
const runtimeContext = runtimeScan?.ranked ?? [];
|
|
1425
|
+
const merged = mergeAgentSchemaContexts(catalogContext, runtimeContext);
|
|
1426
|
+
const enriched = await enrichAgentSchemaContextWithValueMatches(question, merged, executor, connection);
|
|
1427
|
+
recordAgentRuntimeSchemaSnapshot(projectRoot, !runtimeScan?.snapshot.length
|
|
1428
|
+
? enriched
|
|
1429
|
+
: mergeAgentSchemaSampleValues(runtimeScan.snapshot, enriched), runtimeContext.length > 0
|
|
1430
|
+
? 'full information_schema runtime scan for composite Ask AI question'
|
|
1431
|
+
: 'catalog enriched runtime schema');
|
|
1072
1432
|
return enriched;
|
|
1073
1433
|
}
|
|
1074
1434
|
if (!connection)
|
|
1075
1435
|
return [];
|
|
1076
1436
|
try {
|
|
1077
|
-
const
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
ORDER BY table_schema, table_name, ordinal_position
|
|
1081
|
-
LIMIT 2000`, [], runtimeVariables({}), connection);
|
|
1082
|
-
const schemaContext = buildAgentSchemaContext(question, result.rows);
|
|
1083
|
-
const enriched = await enrichAgentSchemaContextWithValueMatches(question, schemaContext, executor, connection);
|
|
1084
|
-
recordAgentRuntimeSchemaSnapshot(projectRoot, enriched, 'information_schema runtime scan');
|
|
1437
|
+
const schemaContext = await scanRuntimeSchema();
|
|
1438
|
+
const enriched = await enrichAgentSchemaContextWithValueMatches(question, schemaContext.ranked, executor, connection);
|
|
1439
|
+
recordAgentRuntimeSchemaSnapshot(projectRoot, schemaContext.snapshot, 'full information_schema runtime scan');
|
|
1085
1440
|
return enriched;
|
|
1086
1441
|
}
|
|
1087
1442
|
catch {
|
|
@@ -1147,6 +1502,7 @@ export async function startLocalServer(opts) {
|
|
|
1147
1502
|
sql: JSON.stringify(contextEnvelope, null, 2),
|
|
1148
1503
|
},
|
|
1149
1504
|
reasoningEffort,
|
|
1505
|
+
analysisDepth: 'deep',
|
|
1150
1506
|
projectRoot,
|
|
1151
1507
|
executeCertifiedBlock: executeCertifiedBlockForAgent,
|
|
1152
1508
|
executeGeneratedSql: executeGeneratedSqlForAgent,
|
|
@@ -1223,6 +1579,21 @@ export async function startLocalServer(opts) {
|
|
|
1223
1579
|
intents: 0,
|
|
1224
1580
|
notebooks: 0,
|
|
1225
1581
|
},
|
|
1582
|
+
reviewMetrics: {
|
|
1583
|
+
totalReviewCount: 0,
|
|
1584
|
+
openReviewCount: 0,
|
|
1585
|
+
terminalReviewCount: 0,
|
|
1586
|
+
draftCreatedCount: 0,
|
|
1587
|
+
certifiedCount: 0,
|
|
1588
|
+
completedCount: 0,
|
|
1589
|
+
rejectedCount: 0,
|
|
1590
|
+
draftCreationRate: null,
|
|
1591
|
+
certifyConversionRate: null,
|
|
1592
|
+
medianOpenReviewAgeMs: null,
|
|
1593
|
+
medianTimeToDraftMs: null,
|
|
1594
|
+
medianTimeToCertificationMs: null,
|
|
1595
|
+
medianTimeToTerminalMs: null,
|
|
1596
|
+
},
|
|
1226
1597
|
limit: input.limit,
|
|
1227
1598
|
offset: input.offset ?? 0,
|
|
1228
1599
|
});
|
|
@@ -1265,6 +1636,7 @@ export async function startLocalServer(opts) {
|
|
|
1265
1636
|
const context = input.context === undefined ? run.context : input.context;
|
|
1266
1637
|
let generatedSql = notebookResearchString(input.generatedSql) ?? run.generatedSql;
|
|
1267
1638
|
let reviewedSql = notebookResearchString(input.reviewedSql) ?? run.reviewedSql;
|
|
1639
|
+
let dqlArtifact = normalizeDqlArtifactReference(input.dqlArtifact) ?? run.dqlArtifact;
|
|
1268
1640
|
const startedAt = new Date().toISOString();
|
|
1269
1641
|
storage.updateRun(run.id, {
|
|
1270
1642
|
domain,
|
|
@@ -1275,6 +1647,7 @@ export async function startLocalServer(opts) {
|
|
|
1275
1647
|
context,
|
|
1276
1648
|
generatedSql,
|
|
1277
1649
|
reviewedSql,
|
|
1650
|
+
dqlArtifact,
|
|
1278
1651
|
status: 'running',
|
|
1279
1652
|
reviewStatus: 'needs_review',
|
|
1280
1653
|
error: '',
|
|
@@ -1291,8 +1664,8 @@ export async function startLocalServer(opts) {
|
|
|
1291
1664
|
intent,
|
|
1292
1665
|
researchPattern: notebookResearchIntentPattern(intent),
|
|
1293
1666
|
},
|
|
1294
|
-
strictness: '
|
|
1295
|
-
limit:
|
|
1667
|
+
strictness: 'exploratory',
|
|
1668
|
+
limit: 160,
|
|
1296
1669
|
}).catch((error) => {
|
|
1297
1670
|
const message = error instanceof Error ? error.message : String(error);
|
|
1298
1671
|
return {
|
|
@@ -1334,6 +1707,7 @@ export async function startLocalServer(opts) {
|
|
|
1334
1707
|
}, null, 2),
|
|
1335
1708
|
},
|
|
1336
1709
|
reasoningEffort: resolveRunReasoningEffort(projectRoot, resolvedProvider, 'research', false),
|
|
1710
|
+
analysisDepth: 'deep',
|
|
1337
1711
|
projectRoot,
|
|
1338
1712
|
executeCertifiedBlock: executeCertifiedBlockForAgent,
|
|
1339
1713
|
executeGeneratedSql: executeGeneratedSqlForAgent,
|
|
@@ -1353,8 +1727,9 @@ export async function startLocalServer(opts) {
|
|
|
1353
1727
|
generationWarnings.push(`AI SQL generation did not return a governed answer. Metadata context was saved for review.${providerError ? ` ${providerError}` : ''}`);
|
|
1354
1728
|
}
|
|
1355
1729
|
else {
|
|
1730
|
+
dqlArtifact = normalizeDqlArtifactReference(governedAnswer.dqlArtifact) ?? dqlArtifact;
|
|
1356
1731
|
generatedSql = notebookResearchString(governedAnswer.proposedSql) ?? notebookResearchString(governedAnswer.sql);
|
|
1357
|
-
if (!generatedSql) {
|
|
1732
|
+
if (!generatedSql && !dqlArtifact) {
|
|
1358
1733
|
generationWarnings.push('AI returned a governed answer without SQL. Metadata context was saved; add reviewed SQL before DQL promotion.');
|
|
1359
1734
|
}
|
|
1360
1735
|
}
|
|
@@ -1416,7 +1791,7 @@ export async function startLocalServer(opts) {
|
|
|
1416
1791
|
const evidence = {
|
|
1417
1792
|
trustStatus: {
|
|
1418
1793
|
label: governedAnswer?.trustLabel
|
|
1419
|
-
?? (reviewedSql ? 'Reviewed notebook SQL' : generatedSql ? 'AI-generated research SQL' : 'Metadata-grounded research plan'),
|
|
1794
|
+
?? (reviewedSql ? 'Reviewed notebook SQL' : generatedSql ? 'AI-generated research SQL' : dqlArtifact ? 'AI-generated DQL artifact' : 'Metadata-grounded research plan'),
|
|
1420
1795
|
reviewRequired: true,
|
|
1421
1796
|
},
|
|
1422
1797
|
contextPackId: contextPack.id,
|
|
@@ -1455,9 +1830,11 @@ export async function startLocalServer(opts) {
|
|
|
1455
1830
|
?? notebookResearchSummary(question, resultPreview, previewError);
|
|
1456
1831
|
const recommendation = previewError
|
|
1457
1832
|
? 'Review the SQL, selected metadata, and connection context before rerunning.'
|
|
1458
|
-
:
|
|
1459
|
-
? 'Review the
|
|
1460
|
-
:
|
|
1833
|
+
: dqlArtifact && !reviewedSql
|
|
1834
|
+
? 'Review the DQL artifact, parameter choices, grain, and evidence before promoting this research into a DQL draft block.'
|
|
1835
|
+
: sqlForPreview
|
|
1836
|
+
? 'Review the SQL, parameter choices, grain, and evidence before promoting this research into a DQL draft block.'
|
|
1837
|
+
: 'Review the selected metadata context, then paste reviewed SQL or configure an AI provider before DQL draft promotion.';
|
|
1461
1838
|
const researchPlan = buildNotebookResearchPlan({
|
|
1462
1839
|
run,
|
|
1463
1840
|
evidence,
|
|
@@ -1465,6 +1842,7 @@ export async function startLocalServer(opts) {
|
|
|
1465
1842
|
previewError,
|
|
1466
1843
|
generatedSql,
|
|
1467
1844
|
reviewedSql,
|
|
1845
|
+
dqlArtifact,
|
|
1468
1846
|
routeDecision,
|
|
1469
1847
|
});
|
|
1470
1848
|
return storage.updateRun(run.id, {
|
|
@@ -1482,6 +1860,7 @@ export async function startLocalServer(opts) {
|
|
|
1482
1860
|
researchPlan,
|
|
1483
1861
|
generatedSql,
|
|
1484
1862
|
reviewedSql,
|
|
1863
|
+
dqlArtifact,
|
|
1485
1864
|
display: display && display.ok ? display.display : undefined,
|
|
1486
1865
|
contextPackId: contextPack.id,
|
|
1487
1866
|
routeDecision,
|
|
@@ -1508,10 +1887,48 @@ export async function startLocalServer(opts) {
|
|
|
1508
1887
|
}
|
|
1509
1888
|
};
|
|
1510
1889
|
const promoteNotebookResearchToDql = async (storage, run, input = {}) => {
|
|
1511
|
-
const
|
|
1512
|
-
|
|
1513
|
-
|
|
1890
|
+
const reviewedSql = notebookResearchString(run.reviewedSql);
|
|
1891
|
+
const dqlArtifact = normalizeDqlArtifactReference(run.dqlArtifact);
|
|
1892
|
+
const sql = reviewedSql ?? notebookResearchString(run.generatedSql);
|
|
1514
1893
|
const sourcePath = `${run.notebookPath}${run.sourceCellId ? `#${run.sourceCellId}` : ''}`;
|
|
1894
|
+
if (!reviewedSql && dqlArtifact) {
|
|
1895
|
+
const session = await createDqlArtifactGenerationSessionForProject(projectRoot, {
|
|
1896
|
+
question: run.question,
|
|
1897
|
+
dqlArtifact,
|
|
1898
|
+
inputMode: 'upload',
|
|
1899
|
+
inputPath: dqlArtifact.sourcePath ?? sourcePath,
|
|
1900
|
+
domain: notebookResearchString(input.domain) ?? run.domain,
|
|
1901
|
+
owner: notebookResearchString(input.owner) ?? run.owner,
|
|
1902
|
+
tags: ['notebook-research', 'review-required', ...(Array.isArray(input.tags) ? input.tags : [])],
|
|
1903
|
+
generatedSql: notebookResearchString(run.generatedSql),
|
|
1904
|
+
contextPackId: run.contextPackId,
|
|
1905
|
+
routeIntent: run.intent,
|
|
1906
|
+
sourceBlock: dqlArtifact.sourcePath,
|
|
1907
|
+
}, semanticLayer);
|
|
1908
|
+
const draftPath = session.candidates.find((candidate) => candidate.draftSave?.path)?.draftSave?.path
|
|
1909
|
+
?? session.candidates.find((candidate) => candidate.savedPath)?.savedPath;
|
|
1910
|
+
const promotion = buildNotebookDqlPromotionSummary(session, draftPath);
|
|
1911
|
+
const updated = storage.markPromoted(run.id, {
|
|
1912
|
+
draftBlockPath: draftPath,
|
|
1913
|
+
dqlImportId: session.id,
|
|
1914
|
+
dqlCandidateIds: session.candidates.map((candidate) => candidate.id),
|
|
1915
|
+
dqlPromotion: promotion,
|
|
1916
|
+
}) ?? run;
|
|
1917
|
+
const planned = storage.updateRun(updated.id, {
|
|
1918
|
+
researchPlan: buildNotebookResearchPlan({
|
|
1919
|
+
run: updated,
|
|
1920
|
+
evidence: updated.evidence,
|
|
1921
|
+
resultPreview: updated.resultPreview,
|
|
1922
|
+
generatedSql: updated.generatedSql,
|
|
1923
|
+
reviewedSql: updated.reviewedSql,
|
|
1924
|
+
dqlArtifact: updated.dqlArtifact,
|
|
1925
|
+
routeDecision: updated.routeDecision,
|
|
1926
|
+
}),
|
|
1927
|
+
}) ?? updated;
|
|
1928
|
+
return { run: planned, session };
|
|
1929
|
+
}
|
|
1930
|
+
if (!sql)
|
|
1931
|
+
throw new Error('Notebook research needs reviewed SQL, generated SQL, or a DQL artifact before DQL draft promotion.');
|
|
1515
1932
|
const session = await createDqlGenerationSessionForProject(projectRoot, {
|
|
1516
1933
|
inputMode: 'upload',
|
|
1517
1934
|
sources: [{ path: sourcePath.endsWith('.sql') ? sourcePath : `${sourcePath}.sql`, content: sql }],
|
|
@@ -1537,6 +1954,7 @@ export async function startLocalServer(opts) {
|
|
|
1537
1954
|
resultPreview: updated.resultPreview,
|
|
1538
1955
|
generatedSql: updated.generatedSql,
|
|
1539
1956
|
reviewedSql: updated.reviewedSql,
|
|
1957
|
+
dqlArtifact: updated.dqlArtifact,
|
|
1540
1958
|
routeDecision: updated.routeDecision,
|
|
1541
1959
|
}),
|
|
1542
1960
|
}) ?? updated;
|
|
@@ -1778,7 +2196,7 @@ export async function startLocalServer(opts) {
|
|
|
1778
2196
|
try {
|
|
1779
2197
|
const tablesResult = await executor.executeQuery(`SELECT table_schema, table_name
|
|
1780
2198
|
FROM information_schema.tables
|
|
1781
|
-
WHERE table_schema NOT IN ('
|
|
2199
|
+
WHERE UPPER(table_schema) NOT IN ('INFORMATION_SCHEMA', 'PG_CATALOG')`, [], {}, activeConnection);
|
|
1782
2200
|
tableMapping = buildSemanticTableMapping(semanticLayer, tablesResult.rows);
|
|
1783
2201
|
}
|
|
1784
2202
|
catch {
|
|
@@ -2061,10 +2479,14 @@ export async function startLocalServer(opts) {
|
|
|
2061
2479
|
res.end(serializeJSON({ error: parsed.error ?? 'Invalid agent run request.' }));
|
|
2062
2480
|
return;
|
|
2063
2481
|
}
|
|
2064
|
-
// The
|
|
2065
|
-
//
|
|
2066
|
-
|
|
2067
|
-
|
|
2482
|
+
// The answer-loop cascade now proves certified/semantic/generated tiers
|
|
2483
|
+
// after retrieval and execution. Avoid pre-routing ordinary Ask runs from
|
|
2484
|
+
// token-overlap signals; explicit callers may still provide signals.
|
|
2485
|
+
// Thread-scoped runs: the persisted thread supplies prior turns (server
|
|
2486
|
+
// state wins; the client-built context stays the no-threadId fallback).
|
|
2487
|
+
const conversationStore = parsed.request.threadId ? getConversationStore() : null;
|
|
2488
|
+
if (conversationStore && parsed.request.threadId && conversationStore.getThread(parsed.request.threadId)) {
|
|
2489
|
+
parsed.request.conversationContext = await conversationContextFromThread(conversationStore, parsed.request.threadId, parsed.request.conversationContext, parsed.request.question);
|
|
2068
2490
|
}
|
|
2069
2491
|
const wantsStream = url.searchParams.get('stream') === '1' || url.searchParams.get('stream') === 'true';
|
|
2070
2492
|
if (wantsStream) {
|
|
@@ -2079,11 +2501,13 @@ export async function startLocalServer(opts) {
|
|
|
2079
2501
|
}, (delta) => {
|
|
2080
2502
|
writeAgentRunSse(res, 'agent-run-answer-delta', { runId: parsed.request.runId, delta });
|
|
2081
2503
|
});
|
|
2504
|
+
recordConversationTurn(conversationStore, parsed.request.threadId, run);
|
|
2082
2505
|
writeAgentRunSse(res, 'agent-run-complete', run);
|
|
2083
2506
|
res.end();
|
|
2084
2507
|
return;
|
|
2085
2508
|
}
|
|
2086
2509
|
const run = await agentRunEngine.run(parsed.request);
|
|
2510
|
+
recordConversationTurn(conversationStore, parsed.request.threadId, run);
|
|
2087
2511
|
res.writeHead(201, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2088
2512
|
res.end(serializeJSON({ run }));
|
|
2089
2513
|
}
|
|
@@ -2115,6 +2539,7 @@ export async function startLocalServer(opts) {
|
|
|
2115
2539
|
const notebookPath = (record && agentRunString(record.notebookPath))
|
|
2116
2540
|
?? `notebooks/certification-requests/${Date.now()}.dqlnb`;
|
|
2117
2541
|
const generatedSql = record ? agentRunString(record.generatedSql) : undefined;
|
|
2542
|
+
const dqlArtifact = record ? normalizeDqlArtifactReference(record.dqlArtifact) : undefined;
|
|
2118
2543
|
try {
|
|
2119
2544
|
const storage = openNotebookResearchStorage();
|
|
2120
2545
|
try {
|
|
@@ -2126,9 +2551,11 @@ export async function startLocalServer(opts) {
|
|
|
2126
2551
|
domain: record ? agentRunString(record.domain) : undefined,
|
|
2127
2552
|
owner: record ? agentRunString(record.owner) : undefined,
|
|
2128
2553
|
generatedSql,
|
|
2554
|
+
dqlArtifact,
|
|
2129
2555
|
context: {
|
|
2130
2556
|
surface: 'stakeholder_request_certification',
|
|
2131
2557
|
requestedContext: agentRunRecord(record?.context) ?? null,
|
|
2558
|
+
dqlArtifact: dqlArtifact ?? null,
|
|
2132
2559
|
},
|
|
2133
2560
|
});
|
|
2134
2561
|
res.writeHead(201, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
@@ -2166,6 +2593,23 @@ export async function startLocalServer(opts) {
|
|
|
2166
2593
|
// DRAFT proposals (each with its stored Certifier verdict) so the notebook
|
|
2167
2594
|
// "Get Started" surface can route them into human review. dryRun preview —
|
|
2168
2595
|
// nothing is written or certified by this call.
|
|
2596
|
+
if (req.method === 'GET' && path === '/api/agent-runs/tier-distribution') {
|
|
2597
|
+
try {
|
|
2598
|
+
const store = getConversationStore();
|
|
2599
|
+
const limitParam = Number(url.searchParams.get('limit'));
|
|
2600
|
+
const limit = Number.isFinite(limitParam) && limitParam > 0 ? limitParam : undefined;
|
|
2601
|
+
const distribution = store
|
|
2602
|
+
? store.tierDistribution({ ...(limit ? { limit } : {}) })
|
|
2603
|
+
: { total: 0, byRouteTier: {}, byTerminalLane: {} };
|
|
2604
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2605
|
+
res.end(JSON.stringify(distribution));
|
|
2606
|
+
}
|
|
2607
|
+
catch (error) {
|
|
2608
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2609
|
+
res.end(JSON.stringify({ error: error instanceof Error ? error.message : String(error) }));
|
|
2610
|
+
}
|
|
2611
|
+
return;
|
|
2612
|
+
}
|
|
2169
2613
|
if ((req.method === 'GET' || req.method === 'POST') && path === '/api/propose') {
|
|
2170
2614
|
try {
|
|
2171
2615
|
let owner;
|
|
@@ -2196,6 +2640,69 @@ export async function startLocalServer(opts) {
|
|
|
2196
2640
|
}
|
|
2197
2641
|
return;
|
|
2198
2642
|
}
|
|
2643
|
+
// Composting preview: mine certified block clusters for semantic metric
|
|
2644
|
+
// candidates. Read-only; returns draft YAML + PR body text for review.
|
|
2645
|
+
if ((req.method === 'GET' || req.method === 'POST') && path === '/api/propose/composting') {
|
|
2646
|
+
try {
|
|
2647
|
+
let owner;
|
|
2648
|
+
let limit;
|
|
2649
|
+
let minSupport;
|
|
2650
|
+
if (req.method === 'POST') {
|
|
2651
|
+
const body = await readJSON(req).catch(() => ({}));
|
|
2652
|
+
if (typeof body?.owner === 'string')
|
|
2653
|
+
owner = body.owner;
|
|
2654
|
+
if (typeof body?.limit === 'number' && Number.isFinite(body.limit) && body.limit > 0)
|
|
2655
|
+
limit = body.limit;
|
|
2656
|
+
if (typeof body?.minSupport === 'number' && Number.isFinite(body.minSupport) && body.minSupport > 1)
|
|
2657
|
+
minSupport = body.minSupport;
|
|
2658
|
+
}
|
|
2659
|
+
else {
|
|
2660
|
+
const ownerParam = url.searchParams.get('owner');
|
|
2661
|
+
if (ownerParam)
|
|
2662
|
+
owner = ownerParam;
|
|
2663
|
+
const limitParam = Number(url.searchParams.get('limit'));
|
|
2664
|
+
if (Number.isFinite(limitParam) && limitParam > 0)
|
|
2665
|
+
limit = limitParam;
|
|
2666
|
+
const minSupportParam = Number(url.searchParams.get('minSupport'));
|
|
2667
|
+
if (Number.isFinite(minSupportParam) && minSupportParam > 1)
|
|
2668
|
+
minSupport = minSupportParam;
|
|
2669
|
+
}
|
|
2670
|
+
const changeset = buildSemanticCompostingChangeset(projectRoot, { owner, limit, minSupport });
|
|
2671
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2672
|
+
res.end(serializeJSON(changeset));
|
|
2673
|
+
}
|
|
2674
|
+
catch (error) {
|
|
2675
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2676
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
2677
|
+
}
|
|
2678
|
+
return;
|
|
2679
|
+
}
|
|
2680
|
+
// Composting generation: writes only approved metric candidate ids under
|
|
2681
|
+
// semantic-layer/metrics/_drafts plus a PR-body artifact. Nothing certified.
|
|
2682
|
+
if (req.method === 'POST' && path === '/api/propose/composting/generate') {
|
|
2683
|
+
try {
|
|
2684
|
+
const body = (await readJSON(req).catch(() => ({})));
|
|
2685
|
+
const ids = Array.isArray(body?.ids)
|
|
2686
|
+
? body.ids.filter((id) => typeof id === 'string' && id.trim().length > 0)
|
|
2687
|
+
: [];
|
|
2688
|
+
if (ids.length === 0) {
|
|
2689
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2690
|
+
res.end(serializeJSON({ error: 'Provide non-empty { ids } to generate semantic composting drafts.' }));
|
|
2691
|
+
return;
|
|
2692
|
+
}
|
|
2693
|
+
const owner = typeof body?.owner === 'string' ? body.owner : undefined;
|
|
2694
|
+
const limit = typeof body?.limit === 'number' && Number.isFinite(body.limit) && body.limit > 0 ? body.limit : undefined;
|
|
2695
|
+
const minSupport = typeof body?.minSupport === 'number' && Number.isFinite(body.minSupport) && body.minSupport > 1 ? body.minSupport : undefined;
|
|
2696
|
+
const result = generateSemanticCompostingDrafts(projectRoot, ids, { owner, limit, minSupport });
|
|
2697
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2698
|
+
res.end(serializeJSON(result));
|
|
2699
|
+
}
|
|
2700
|
+
catch (error) {
|
|
2701
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2702
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
2703
|
+
}
|
|
2704
|
+
return;
|
|
2705
|
+
}
|
|
2199
2706
|
// Deterministic PLAN only (classify → plan). Writes NOTHING. Same data the
|
|
2200
2707
|
// readiness endpoint embeds, exposed standalone for the approve gate.
|
|
2201
2708
|
if (req.method === 'POST' && path === '/api/propose/plan') {
|
|
@@ -2724,6 +3231,117 @@ export async function startLocalServer(opts) {
|
|
|
2724
3231
|
}
|
|
2725
3232
|
return;
|
|
2726
3233
|
}
|
|
3234
|
+
// ── Conversation threads (server-side session persistence) ────────────
|
|
3235
|
+
if (path === '/api/agent/threads' || path.startsWith('/api/agent/threads/')) {
|
|
3236
|
+
const store = getConversationStore();
|
|
3237
|
+
if (!store) {
|
|
3238
|
+
res.writeHead(503, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3239
|
+
res.end(serializeJSON({ error: 'Conversation store is unavailable.' }));
|
|
3240
|
+
return;
|
|
3241
|
+
}
|
|
3242
|
+
try {
|
|
3243
|
+
if (req.method === 'GET' && path === '/api/agent/threads') {
|
|
3244
|
+
const limit = Number(url.searchParams.get('limit') ?? '50');
|
|
3245
|
+
const includeArchived = url.searchParams.get('archived') === '1';
|
|
3246
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3247
|
+
res.end(serializeJSON({ threads: store.listThreads({ limit: Number.isFinite(limit) ? limit : 50, includeArchived }) }));
|
|
3248
|
+
return;
|
|
3249
|
+
}
|
|
3250
|
+
if (req.method === 'POST' && path === '/api/agent/threads') {
|
|
3251
|
+
const body = await readJSON(req).catch(() => null);
|
|
3252
|
+
const record = agentRunRecord(body) ?? {};
|
|
3253
|
+
const thread = store.createThread({
|
|
3254
|
+
surface: agentRunString(record.surface),
|
|
3255
|
+
title: agentRunString(record.title),
|
|
3256
|
+
notebookPath: agentRunString(record.notebookPath),
|
|
3257
|
+
});
|
|
3258
|
+
res.writeHead(201, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3259
|
+
res.end(serializeJSON({ thread }));
|
|
3260
|
+
return;
|
|
3261
|
+
}
|
|
3262
|
+
// Cross-thread turn search ("search conversations") — must precede :id.
|
|
3263
|
+
if (req.method === 'GET' && path === '/api/agent/threads/search') {
|
|
3264
|
+
const query = url.searchParams.get('q') ?? '';
|
|
3265
|
+
const limit = Number(url.searchParams.get('limit') ?? '10');
|
|
3266
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3267
|
+
res.end(serializeJSON({
|
|
3268
|
+
turns: query.trim()
|
|
3269
|
+
? store.searchTurns({ query, limit: Number.isFinite(limit) ? limit : 10 })
|
|
3270
|
+
: [],
|
|
3271
|
+
}));
|
|
3272
|
+
return;
|
|
3273
|
+
}
|
|
3274
|
+
const threadMatch = path.match(/^\/api\/agent\/threads\/([^/]+)(?:\/(archive|promote))?$/);
|
|
3275
|
+
if (threadMatch) {
|
|
3276
|
+
const threadId = decodeURIComponent(threadMatch[1]);
|
|
3277
|
+
const action = threadMatch[2];
|
|
3278
|
+
const thread = store.getThread(threadId);
|
|
3279
|
+
if (!thread) {
|
|
3280
|
+
res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3281
|
+
res.end(serializeJSON({ error: 'Unknown thread id.' }));
|
|
3282
|
+
return;
|
|
3283
|
+
}
|
|
3284
|
+
if (req.method === 'POST' && action === 'archive') {
|
|
3285
|
+
store.archiveThread(threadId);
|
|
3286
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3287
|
+
res.end(serializeJSON({ ok: true }));
|
|
3288
|
+
return;
|
|
3289
|
+
}
|
|
3290
|
+
// Explicit promotion — the ONLY path from conversation into durable
|
|
3291
|
+
// memory ("remember this"). Deliberate user act, tagged for audit.
|
|
3292
|
+
if (req.method === 'POST' && action === 'promote') {
|
|
3293
|
+
const body = await readJSON(req).catch(() => null);
|
|
3294
|
+
const record = agentRunRecord(body) ?? {};
|
|
3295
|
+
const turnId = agentRunString(record.turnId);
|
|
3296
|
+
const turn = turnId
|
|
3297
|
+
? store.recentTurns(threadId, 200).find((candidate) => candidate.id === turnId)
|
|
3298
|
+
: undefined;
|
|
3299
|
+
if (!turn) {
|
|
3300
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3301
|
+
res.end(serializeJSON({ error: 'turnId is required and must belong to this thread.' }));
|
|
3302
|
+
return;
|
|
3303
|
+
}
|
|
3304
|
+
const scopeRaw = agentRunString(record.scope);
|
|
3305
|
+
const scope = scopeRaw === 'notebook' || scopeRaw === 'project' || scopeRaw === 'user' ? scopeRaw : 'project';
|
|
3306
|
+
const memory = new MemoryStore(defaultMemoryPath(projectRoot));
|
|
3307
|
+
try {
|
|
3308
|
+
const saved = memory.upsert({
|
|
3309
|
+
scope,
|
|
3310
|
+
title: agentRunString(record.title) ?? turn.question.slice(0, 120),
|
|
3311
|
+
content: `Q: ${turn.question}\nA: ${turn.answerSummary ?? turn.answerText ?? '(no answer summary)'}`,
|
|
3312
|
+
tags: ['conversation', threadId],
|
|
3313
|
+
source: 'conversation',
|
|
3314
|
+
confidence: 0.6,
|
|
3315
|
+
importance: 0.5,
|
|
3316
|
+
enabled: true,
|
|
3317
|
+
});
|
|
3318
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3319
|
+
res.end(serializeJSON({ ok: true, memory: saved }));
|
|
3320
|
+
}
|
|
3321
|
+
finally {
|
|
3322
|
+
memory.close();
|
|
3323
|
+
}
|
|
3324
|
+
return;
|
|
3325
|
+
}
|
|
3326
|
+
if (req.method === 'GET' && !action) {
|
|
3327
|
+
const limit = Number(url.searchParams.get('limit') ?? '50');
|
|
3328
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3329
|
+
res.end(serializeJSON({
|
|
3330
|
+
thread,
|
|
3331
|
+
turns: store.recentTurns(threadId, Number.isFinite(limit) ? limit : 50),
|
|
3332
|
+
}));
|
|
3333
|
+
return;
|
|
3334
|
+
}
|
|
3335
|
+
}
|
|
3336
|
+
res.writeHead(405, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3337
|
+
res.end(serializeJSON({ error: 'Method not allowed' }));
|
|
3338
|
+
}
|
|
3339
|
+
catch (error) {
|
|
3340
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3341
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
3342
|
+
}
|
|
3343
|
+
return;
|
|
3344
|
+
}
|
|
2727
3345
|
if (req.method === 'GET' && path === '/api/agent/memory') {
|
|
2728
3346
|
const memory = new MemoryStore(defaultMemoryPath(projectRoot));
|
|
2729
3347
|
try {
|
|
@@ -3247,6 +3865,7 @@ export async function startLocalServer(opts) {
|
|
|
3247
3865
|
const sourceCellId = notebookResearchString(body.sourceCellId) ?? notebookResearchSourceCellId(sourceCell);
|
|
3248
3866
|
const sourceCellName = notebookResearchString(body.sourceCellName) ?? notebookResearchSourceCellName(sourceCell);
|
|
3249
3867
|
const sourceCellFingerprint = notebookResearchString(body.sourceCellFingerprint) ?? notebookResearchSourceCellFingerprint(sourceCell);
|
|
3868
|
+
const dqlArtifact = normalizeDqlArtifactReference(body.dqlArtifact);
|
|
3250
3869
|
if (!notebookPath || !question) {
|
|
3251
3870
|
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3252
3871
|
res.end(serializeJSON({ error: 'notebookPath and question are required.' }));
|
|
@@ -3266,6 +3885,7 @@ export async function startLocalServer(opts) {
|
|
|
3266
3885
|
context: body.context,
|
|
3267
3886
|
generatedSql: notebookResearchString(body.generatedSql),
|
|
3268
3887
|
reviewedSql: notebookResearchString(body.reviewedSql),
|
|
3888
|
+
dqlArtifact,
|
|
3269
3889
|
});
|
|
3270
3890
|
const run = body.run === true
|
|
3271
3891
|
? await runNotebookResearch(storage, created, {
|
|
@@ -3276,6 +3896,7 @@ export async function startLocalServer(opts) {
|
|
|
3276
3896
|
context: body.context,
|
|
3277
3897
|
generatedSql: notebookResearchString(body.generatedSql),
|
|
3278
3898
|
reviewedSql: notebookResearchString(body.reviewedSql),
|
|
3899
|
+
dqlArtifact,
|
|
3279
3900
|
})
|
|
3280
3901
|
: created;
|
|
3281
3902
|
res.writeHead(201, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
@@ -3318,8 +3939,8 @@ export async function startLocalServer(opts) {
|
|
|
3318
3939
|
intent,
|
|
3319
3940
|
researchPattern: notebookResearchIntentPattern(intent),
|
|
3320
3941
|
},
|
|
3321
|
-
strictness: '
|
|
3322
|
-
limit:
|
|
3942
|
+
strictness: 'exploratory',
|
|
3943
|
+
limit: 160,
|
|
3323
3944
|
});
|
|
3324
3945
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3325
3946
|
res.end(serializeJSON(notebookResearchContextPreview(contextPack)));
|
|
@@ -3543,6 +4164,7 @@ export async function startLocalServer(opts) {
|
|
|
3543
4164
|
routeDecision: body.routeDecision,
|
|
3544
4165
|
generatedSql: notebookResearchString(body.generatedSql),
|
|
3545
4166
|
reviewedSql: notebookResearchString(body.reviewedSql),
|
|
4167
|
+
dqlArtifact: normalizeDqlArtifactReference(body.dqlArtifact),
|
|
3546
4168
|
warnings: Array.isArray(body.warnings) ? body.warnings.filter((item) => typeof item === 'string') : undefined,
|
|
3547
4169
|
reviewStatus: notebookResearchReviewStatus(body.reviewStatus),
|
|
3548
4170
|
dqlPromotionAction: notebookResearchPromotionAction(body.dqlPromotionAction),
|
|
@@ -3555,6 +4177,7 @@ export async function startLocalServer(opts) {
|
|
|
3555
4177
|
previewError: updated.error,
|
|
3556
4178
|
generatedSql: updated.generatedSql,
|
|
3557
4179
|
reviewedSql: updated.reviewedSql,
|
|
4180
|
+
dqlArtifact: updated.dqlArtifact,
|
|
3558
4181
|
routeDecision: updated.routeDecision,
|
|
3559
4182
|
}),
|
|
3560
4183
|
}) ?? updated;
|
|
@@ -3574,6 +4197,7 @@ export async function startLocalServer(opts) {
|
|
|
3574
4197
|
context: body.context,
|
|
3575
4198
|
generatedSql: notebookResearchString(body.generatedSql),
|
|
3576
4199
|
reviewedSql: notebookResearchString(body.reviewedSql),
|
|
4200
|
+
dqlArtifact: normalizeDqlArtifactReference(body.dqlArtifact),
|
|
3577
4201
|
});
|
|
3578
4202
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3579
4203
|
res.end(serializeJSON({ run: withNotebookResearchChecklist(updated) }));
|
|
@@ -5886,7 +6510,7 @@ export async function startLocalServer(opts) {
|
|
|
5886
6510
|
// Build table mapping: resolve semantic model names to actual DB table names
|
|
5887
6511
|
let tableMapping;
|
|
5888
6512
|
try {
|
|
5889
|
-
const tablesResult = await executor.executeQuery(`SELECT table_schema, table_name FROM information_schema.tables WHERE table_schema NOT IN ('
|
|
6513
|
+
const tablesResult = await executor.executeQuery(`SELECT table_schema, table_name FROM information_schema.tables WHERE UPPER(table_schema) NOT IN ('INFORMATION_SCHEMA', 'PG_CATALOG')`, [], {}, targetConnection);
|
|
5890
6514
|
const dbTableNames = new Set();
|
|
5891
6515
|
const schemaQualified = new Map();
|
|
5892
6516
|
for (const row of tablesResult.rows) {
|
|
@@ -5977,7 +6601,7 @@ export async function startLocalServer(opts) {
|
|
|
5977
6601
|
const driver = targetConnection.driver;
|
|
5978
6602
|
let tableMapping;
|
|
5979
6603
|
try {
|
|
5980
|
-
const tablesResult = await executor.executeQuery(`SELECT table_schema, table_name FROM information_schema.tables WHERE table_schema NOT IN ('
|
|
6604
|
+
const tablesResult = await executor.executeQuery(`SELECT table_schema, table_name FROM information_schema.tables WHERE UPPER(table_schema) NOT IN ('INFORMATION_SCHEMA', 'PG_CATALOG')`, [], {}, targetConnection);
|
|
5981
6605
|
const schemaQualified = new Map();
|
|
5982
6606
|
for (const row of tablesResult.rows) {
|
|
5983
6607
|
const schema = String(row['table_schema'] ?? '');
|
|
@@ -6729,16 +7353,27 @@ export function resolveDefaultLLMProvider(projectRoot) {
|
|
|
6729
7353
|
* answer-loop runner — never the MCP `claudeCodeRunner`, which doesn't emit a governed
|
|
6730
7354
|
* answer envelope. Everything else uses the Settings-resolved default runner.
|
|
6731
7355
|
*/
|
|
6732
|
-
function resolveGovernedAnswerRunner(projectRoot) {
|
|
7356
|
+
export function resolveGovernedAnswerRunner(projectRoot) {
|
|
6733
7357
|
const active = getActiveProvider(projectRoot);
|
|
6734
|
-
if (active
|
|
7358
|
+
if (isGovernedAnswerProviderId(active)) {
|
|
6735
7359
|
return { provider: active, runner: createDqlAgentProviderRunner(active) };
|
|
6736
7360
|
}
|
|
6737
7361
|
const resolved = resolveDefaultLLMProvider(projectRoot);
|
|
6738
7362
|
if (!resolved)
|
|
6739
7363
|
return null;
|
|
6740
|
-
|
|
6741
|
-
|
|
7364
|
+
if (isGovernedAnswerProviderId(resolved)) {
|
|
7365
|
+
return { provider: resolved, runner: createDqlAgentProviderRunner(resolved) };
|
|
7366
|
+
}
|
|
7367
|
+
return null;
|
|
7368
|
+
}
|
|
7369
|
+
function isGovernedAnswerProviderId(value) {
|
|
7370
|
+
return value === 'anthropic'
|
|
7371
|
+
|| value === 'openai'
|
|
7372
|
+
|| value === 'gemini'
|
|
7373
|
+
|| value === 'ollama'
|
|
7374
|
+
|| value === 'custom-openai'
|
|
7375
|
+
|| value === 'claude-code'
|
|
7376
|
+
|| value === 'codex';
|
|
6742
7377
|
}
|
|
6743
7378
|
/** Map a runner provider id to the settings id whose reasoning ceiling applies. */
|
|
6744
7379
|
function reasoningSettingsIdFor(provider) {
|
|
@@ -6783,6 +7418,11 @@ function resolveRunReasoningEffort(projectRoot, provider, route, isRepair) {
|
|
|
6783
7418
|
desired = bumpReasoningEffort(desired);
|
|
6784
7419
|
return ceiling ? clampReasoningEffort(desired, ceiling) : desired;
|
|
6785
7420
|
}
|
|
7421
|
+
function resolveRequestedRunReasoningEffort(projectRoot, provider, requested, isRepair) {
|
|
7422
|
+
const ceiling = getEffectiveProviderConfig(projectRoot, reasoningSettingsIdFor(provider)).reasoningEffort;
|
|
7423
|
+
const desired = isRepair ? bumpReasoningEffort(requested) : requested;
|
|
7424
|
+
return ceiling ? clampReasoningEffort(desired, ceiling) : desired;
|
|
7425
|
+
}
|
|
6786
7426
|
function loadAppDashboard(projectRoot, appId, dashboardId) {
|
|
6787
7427
|
for (const p of findAppDocuments(projectRoot)) {
|
|
6788
7428
|
const { document: app } = loadAppDocument(p);
|
|
@@ -7107,6 +7747,11 @@ export function buildProposeReadiness(projectRoot, projectConfig = loadProjectCo
|
|
|
7107
7747
|
if (proposal.certification.errors.length === 0)
|
|
7108
7748
|
readyForReview += 1;
|
|
7109
7749
|
}
|
|
7750
|
+
const reviewableProposals = buildReviewableProposals(projectRoot, manifestPath, projectConfig, summary.proposals, options.owner);
|
|
7751
|
+
const reviewAges = reviewableProposals
|
|
7752
|
+
.map((proposal) => proposal.review.reviewAgeHours)
|
|
7753
|
+
.filter((value) => typeof value === 'number' && Number.isFinite(value));
|
|
7754
|
+
const estimatedReviewMinutes = reviewableProposals.reduce((sum, proposal) => sum + proposal.review.estimatedReviewMinutes, 0);
|
|
7110
7755
|
return {
|
|
7111
7756
|
ready: true,
|
|
7112
7757
|
summary: {
|
|
@@ -7121,10 +7766,594 @@ export function buildProposeReadiness(projectRoot, projectConfig = loadProjectCo
|
|
|
7121
7766
|
readyForReview,
|
|
7122
7767
|
blockingTotal,
|
|
7123
7768
|
warningTotal,
|
|
7769
|
+
reviewTelemetry: {
|
|
7770
|
+
existingDrafts: reviewableProposals.filter((proposal) => proposal.review.draftExists).length,
|
|
7771
|
+
medianReviewAgeHours: median(reviewAges),
|
|
7772
|
+
readyForReviewRate: ratio(readyForReview, summary.proposalsRanked),
|
|
7773
|
+
estimatedReviewMinutes,
|
|
7774
|
+
},
|
|
7124
7775
|
},
|
|
7125
7776
|
plan,
|
|
7126
|
-
proposals:
|
|
7777
|
+
proposals: reviewableProposals,
|
|
7778
|
+
};
|
|
7779
|
+
}
|
|
7780
|
+
function buildReviewableProposals(projectRoot, dbtManifestPath, projectConfig, proposals, owner) {
|
|
7781
|
+
const manifest = safeBuildProjectManifest(projectRoot);
|
|
7782
|
+
return proposals.map((proposal, index) => {
|
|
7783
|
+
const preview = buildProposePreview(projectRoot, dbtManifestPath, proposal.slug, {
|
|
7784
|
+
config: projectConfig.propose,
|
|
7785
|
+
owner,
|
|
7786
|
+
});
|
|
7787
|
+
const draftPath = proposal.path ?? resolveProposeDraftPath(projectRoot, proposal.domain, proposal.slug);
|
|
7788
|
+
const draftFile = join(projectRoot, draftPath);
|
|
7789
|
+
const draftExists = existsSync(draftFile);
|
|
7790
|
+
const stats = draftExists ? statSync(draftFile) : undefined;
|
|
7791
|
+
const firstSeenAt = stats ? stableFsTime(stats.birthtimeMs, stats.ctimeMs) : undefined;
|
|
7792
|
+
const lastUpdatedAt = stats ? stableFsTime(stats.mtimeMs, stats.ctimeMs) : undefined;
|
|
7793
|
+
const reviewAgeHours = firstSeenAt ? hoursSince(firstSeenAt) : undefined;
|
|
7794
|
+
const blockingCount = proposal.certification.errors.length;
|
|
7795
|
+
const warningCount = proposal.certification.warnings.length;
|
|
7796
|
+
const outputs = preview?.outputs ?? proposal.inference.declaredOutputs;
|
|
7797
|
+
const status = blockingCount === 0
|
|
7798
|
+
? 'ready_for_review'
|
|
7799
|
+
: draftExists
|
|
7800
|
+
? 'draft_exists'
|
|
7801
|
+
: 'new';
|
|
7802
|
+
return {
|
|
7803
|
+
...proposal,
|
|
7804
|
+
review: {
|
|
7805
|
+
queueRank: index + 1,
|
|
7806
|
+
status,
|
|
7807
|
+
priorityScore: proposal.ranking.score,
|
|
7808
|
+
blockingCount,
|
|
7809
|
+
warningCount,
|
|
7810
|
+
estimatedReviewMinutes: estimateProposalReviewMinutes(blockingCount, warningCount, Boolean(preview?.sqlPreview)),
|
|
7811
|
+
draftPath,
|
|
7812
|
+
draftExists,
|
|
7813
|
+
firstSeenAt,
|
|
7814
|
+
lastUpdatedAt,
|
|
7815
|
+
reviewAgeHours,
|
|
7816
|
+
certifyCommand: `dql certify --from-draft ${quoteCliArg(draftPath)} --owner you@example.com`,
|
|
7817
|
+
payload: {
|
|
7818
|
+
question: preview?.examples?.[0] ?? proposal.inference.examples[0]?.question ?? `Review ${proposal.slug}`,
|
|
7819
|
+
model: proposal.model,
|
|
7820
|
+
domain: proposal.domain,
|
|
7821
|
+
sqlPreview: preview?.sqlPreview,
|
|
7822
|
+
outputs,
|
|
7823
|
+
grain: preview?.grain ?? proposal.inference.grain,
|
|
7824
|
+
pattern: preview?.pattern ?? proposal.inference.pattern,
|
|
7825
|
+
evidence: proposal.evidence,
|
|
7826
|
+
resultSample: { status: 'not_run', rows: [] },
|
|
7827
|
+
nearestCertifiedBlock: findNearestCertifiedBlock(manifest, proposal, outputs),
|
|
7828
|
+
},
|
|
7829
|
+
},
|
|
7830
|
+
};
|
|
7831
|
+
});
|
|
7832
|
+
}
|
|
7833
|
+
function safeBuildProjectManifest(projectRoot) {
|
|
7834
|
+
try {
|
|
7835
|
+
return buildManifest({ projectRoot });
|
|
7836
|
+
}
|
|
7837
|
+
catch {
|
|
7838
|
+
return { blocks: {} };
|
|
7839
|
+
}
|
|
7840
|
+
}
|
|
7841
|
+
function resolveProposeDraftPath(projectRoot, domain, slug) {
|
|
7842
|
+
const safeDomain = domain
|
|
7843
|
+
.trim()
|
|
7844
|
+
.toLowerCase()
|
|
7845
|
+
.replace(/[^a-z0-9/_-]+/g, '-')
|
|
7846
|
+
.replace(/^[-/]+|[-/]+$/g, '');
|
|
7847
|
+
return safeDomain && existsSync(join(projectRoot, 'domains', safeDomain))
|
|
7848
|
+
? `domains/${safeDomain}/blocks/_drafts/${slug}.dql`
|
|
7849
|
+
: `blocks/_drafts/${slug}.dql`;
|
|
7850
|
+
}
|
|
7851
|
+
function findNearestCertifiedBlock(manifest, proposal, outputs) {
|
|
7852
|
+
const outputSet = new Set(outputs.map((output) => output.toLowerCase()));
|
|
7853
|
+
let best;
|
|
7854
|
+
for (const block of Object.values(manifest.blocks ?? {})) {
|
|
7855
|
+
if (String(block.status ?? '').toLowerCase() !== 'certified')
|
|
7856
|
+
continue;
|
|
7857
|
+
const blockOutputs = (block.declaredOutputs ?? []).map((output) => output.toLowerCase());
|
|
7858
|
+
const sharedOutputs = blockOutputs.filter((output) => outputSet.has(output));
|
|
7859
|
+
const sameDomain = block.domain === proposal.domain;
|
|
7860
|
+
const overlapScore = sharedOutputs.length * 10 + (sameDomain ? 5 : 0);
|
|
7861
|
+
if (overlapScore <= 0)
|
|
7862
|
+
continue;
|
|
7863
|
+
if (!best || overlapScore > best.overlapScore) {
|
|
7864
|
+
best = {
|
|
7865
|
+
name: block.name,
|
|
7866
|
+
path: block.filePath,
|
|
7867
|
+
domain: block.domain,
|
|
7868
|
+
overlapScore,
|
|
7869
|
+
sharedOutputs,
|
|
7870
|
+
};
|
|
7871
|
+
}
|
|
7872
|
+
}
|
|
7873
|
+
return best;
|
|
7874
|
+
}
|
|
7875
|
+
function estimateProposalReviewMinutes(blockingCount, warningCount, hasSqlPreview) {
|
|
7876
|
+
const base = hasSqlPreview ? 12 : 18;
|
|
7877
|
+
return base + blockingCount * 8 + warningCount * 4;
|
|
7878
|
+
}
|
|
7879
|
+
function stableFsTime(primaryMs, fallbackMs) {
|
|
7880
|
+
const ms = Number.isFinite(primaryMs) && primaryMs > 0 ? primaryMs : fallbackMs;
|
|
7881
|
+
return Number.isFinite(ms) && ms > 0 ? new Date(ms).toISOString() : undefined;
|
|
7882
|
+
}
|
|
7883
|
+
function hoursSince(iso) {
|
|
7884
|
+
const ts = Date.parse(iso);
|
|
7885
|
+
if (!Number.isFinite(ts))
|
|
7886
|
+
return 0;
|
|
7887
|
+
return Number(Math.max(0, (Date.now() - ts) / 3_600_000).toFixed(2));
|
|
7888
|
+
}
|
|
7889
|
+
function median(values) {
|
|
7890
|
+
if (values.length === 0)
|
|
7891
|
+
return null;
|
|
7892
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
7893
|
+
const mid = Math.floor(sorted.length / 2);
|
|
7894
|
+
const value = sorted.length % 2 === 1
|
|
7895
|
+
? sorted[mid]
|
|
7896
|
+
: (sorted[mid - 1] + sorted[mid]) / 2;
|
|
7897
|
+
return Number(value.toFixed(2));
|
|
7898
|
+
}
|
|
7899
|
+
function ratio(numerator, denominator) {
|
|
7900
|
+
if (denominator === 0)
|
|
7901
|
+
return null;
|
|
7902
|
+
return Number((numerator / denominator).toFixed(4));
|
|
7903
|
+
}
|
|
7904
|
+
function quoteCliArg(value) {
|
|
7905
|
+
if (/^[A-Za-z0-9_./:-]+$/.test(value))
|
|
7906
|
+
return value;
|
|
7907
|
+
return `'${value.replace(/'/g, "'\\''")}'`;
|
|
7908
|
+
}
|
|
7909
|
+
/**
|
|
7910
|
+
* Mine certified block clusters for recurring metric definitions and render a
|
|
7911
|
+
* reviewable semantic-layer changeset. This is read-only: callers must confirm
|
|
7912
|
+
* candidate ids through generateSemanticCompostingDrafts before files are written.
|
|
7913
|
+
*/
|
|
7914
|
+
export function buildSemanticCompostingChangeset(projectRoot, options = {}) {
|
|
7915
|
+
let manifest;
|
|
7916
|
+
try {
|
|
7917
|
+
manifest = buildManifest({ projectRoot });
|
|
7918
|
+
}
|
|
7919
|
+
catch (error) {
|
|
7920
|
+
return {
|
|
7921
|
+
ready: false,
|
|
7922
|
+
reason: error instanceof Error ? error.message : String(error),
|
|
7923
|
+
summary: {
|
|
7924
|
+
certifiedBlocksScanned: 0,
|
|
7925
|
+
eligibleMetricClusters: 0,
|
|
7926
|
+
candidatesRanked: 0,
|
|
7927
|
+
existingDrafts: 0,
|
|
7928
|
+
donorBlocksUsed: 0,
|
|
7929
|
+
minSupport: Math.max(2, Math.floor(options.minSupport ?? 2)),
|
|
7930
|
+
},
|
|
7931
|
+
candidates: [],
|
|
7932
|
+
prBody: '',
|
|
7933
|
+
};
|
|
7934
|
+
}
|
|
7935
|
+
const minSupport = Math.max(2, Math.floor(options.minSupport ?? 2));
|
|
7936
|
+
const limit = Number.isFinite(options.limit) && Number(options.limit) > 0 ? Math.floor(Number(options.limit)) : 10;
|
|
7937
|
+
const owner = options.owner?.trim() || resolveLocalOwner(projectRoot, { persist: false });
|
|
7938
|
+
const existingMetricNames = new Set(Object.entries(manifest.metrics ?? {})
|
|
7939
|
+
.filter(([, metric]) => !isSemanticDraftMetricPath(metric.filePath))
|
|
7940
|
+
.map(([name]) => name.toLowerCase()));
|
|
7941
|
+
const certifiedBlocks = Object.values(manifest.blocks ?? {})
|
|
7942
|
+
.filter((block) => String(block.status ?? '').toLowerCase() === 'certified');
|
|
7943
|
+
const groups = new Map();
|
|
7944
|
+
for (const block of certifiedBlocks) {
|
|
7945
|
+
const table = singleTableDependency(block);
|
|
7946
|
+
if (!table)
|
|
7947
|
+
continue;
|
|
7948
|
+
const where = extractCompostingWhereClause(block.sql);
|
|
7949
|
+
const normalizedFilter = where ? normalizeCompostingSql(where) : undefined;
|
|
7950
|
+
const metrics = extractCompostingAggregateMetrics(block.sql);
|
|
7951
|
+
if (metrics.length === 0)
|
|
7952
|
+
continue;
|
|
7953
|
+
const donor = {
|
|
7954
|
+
name: block.name,
|
|
7955
|
+
path: block.filePath,
|
|
7956
|
+
domain: block.domain,
|
|
7957
|
+
outputs: blockOutputNames(block),
|
|
7958
|
+
filters: where ? splitCompostingFilters(where) : [],
|
|
7959
|
+
};
|
|
7960
|
+
for (const metric of metrics) {
|
|
7961
|
+
const normalizedExpression = normalizeCompostingSql(metric.expression);
|
|
7962
|
+
const domain = block.domain?.trim() || 'misc';
|
|
7963
|
+
const key = [
|
|
7964
|
+
domain.toLowerCase(),
|
|
7965
|
+
table.toLowerCase(),
|
|
7966
|
+
metric.type,
|
|
7967
|
+
normalizedExpression,
|
|
7968
|
+
normalizedFilter ?? '',
|
|
7969
|
+
].join('\0');
|
|
7970
|
+
const existing = groups.get(key);
|
|
7971
|
+
if (existing) {
|
|
7972
|
+
existing.aliases.push(metric.alias ?? '');
|
|
7973
|
+
existing.donors.set(block.filePath, donor);
|
|
7974
|
+
}
|
|
7975
|
+
else {
|
|
7976
|
+
groups.set(key, {
|
|
7977
|
+
domain,
|
|
7978
|
+
table,
|
|
7979
|
+
type: metric.type,
|
|
7980
|
+
expression: metric.expression,
|
|
7981
|
+
normalizedExpression,
|
|
7982
|
+
filter: where,
|
|
7983
|
+
normalizedFilter,
|
|
7984
|
+
aliases: [metric.alias ?? ''],
|
|
7985
|
+
donors: new Map([[block.filePath, donor]]),
|
|
7986
|
+
});
|
|
7987
|
+
}
|
|
7988
|
+
}
|
|
7989
|
+
}
|
|
7990
|
+
const usedNames = new Set(existingMetricNames);
|
|
7991
|
+
const candidates = Array.from(groups.values())
|
|
7992
|
+
.filter((group) => group.donors.size >= minSupport)
|
|
7993
|
+
.sort((a, b) => b.donors.size - a.donors.size || a.normalizedExpression.localeCompare(b.normalizedExpression))
|
|
7994
|
+
.map((group) => buildCompostingMetricCandidate(projectRoot, group, owner, usedNames))
|
|
7995
|
+
.filter((candidate) => Boolean(candidate))
|
|
7996
|
+
.slice(0, limit);
|
|
7997
|
+
const donorBlocksUsed = new Set(candidates.flatMap((candidate) => candidate.donorBlocks.map((donor) => donor.path))).size;
|
|
7998
|
+
return {
|
|
7999
|
+
ready: true,
|
|
8000
|
+
summary: {
|
|
8001
|
+
certifiedBlocksScanned: certifiedBlocks.length,
|
|
8002
|
+
eligibleMetricClusters: Array.from(groups.values()).filter((group) => group.donors.size >= minSupport).length,
|
|
8003
|
+
candidatesRanked: candidates.length,
|
|
8004
|
+
existingDrafts: candidates.filter((candidate) => candidate.draftExists).length,
|
|
8005
|
+
donorBlocksUsed,
|
|
8006
|
+
minSupport,
|
|
8007
|
+
},
|
|
8008
|
+
candidates,
|
|
8009
|
+
prBody: renderCompostingPrBody(candidates),
|
|
8010
|
+
};
|
|
8011
|
+
}
|
|
8012
|
+
export function generateSemanticCompostingDrafts(projectRoot, candidateIds, options = {}) {
|
|
8013
|
+
const changeset = buildSemanticCompostingChangeset(projectRoot, options);
|
|
8014
|
+
if (!changeset.ready) {
|
|
8015
|
+
return {
|
|
8016
|
+
ready: false,
|
|
8017
|
+
reason: changeset.reason,
|
|
8018
|
+
draftsWritten: 0,
|
|
8019
|
+
draftsSkipped: 0,
|
|
8020
|
+
candidates: [],
|
|
8021
|
+
paths: [],
|
|
8022
|
+
prBody: '',
|
|
8023
|
+
};
|
|
8024
|
+
}
|
|
8025
|
+
const wanted = candidateIds && candidateIds.length > 0 ? new Set(candidateIds) : undefined;
|
|
8026
|
+
const selected = wanted ? changeset.candidates.filter((candidate) => wanted.has(candidate.id)) : changeset.candidates;
|
|
8027
|
+
const paths = [];
|
|
8028
|
+
let draftsWritten = 0;
|
|
8029
|
+
let draftsSkipped = 0;
|
|
8030
|
+
for (const candidate of selected) {
|
|
8031
|
+
const absolutePath = join(projectRoot, candidate.draftPath);
|
|
8032
|
+
if (existsSync(absolutePath)) {
|
|
8033
|
+
draftsSkipped += 1;
|
|
8034
|
+
}
|
|
8035
|
+
else {
|
|
8036
|
+
mkdirSync(dirname(absolutePath), { recursive: true });
|
|
8037
|
+
writeFileSync(absolutePath, candidate.yaml, 'utf-8');
|
|
8038
|
+
draftsWritten += 1;
|
|
8039
|
+
}
|
|
8040
|
+
paths.push(candidate.draftPath);
|
|
8041
|
+
}
|
|
8042
|
+
let prBodyPath;
|
|
8043
|
+
const prBody = renderCompostingPrBody(selected);
|
|
8044
|
+
if (selected.length > 0) {
|
|
8045
|
+
prBodyPath = 'semantic-layer/metrics/_drafts/PR_BODY.md';
|
|
8046
|
+
const absolutePrBodyPath = join(projectRoot, prBodyPath);
|
|
8047
|
+
mkdirSync(dirname(absolutePrBodyPath), { recursive: true });
|
|
8048
|
+
writeFileSync(absolutePrBodyPath, prBody, 'utf-8');
|
|
8049
|
+
}
|
|
8050
|
+
return {
|
|
8051
|
+
ready: true,
|
|
8052
|
+
draftsWritten,
|
|
8053
|
+
draftsSkipped,
|
|
8054
|
+
candidates: selected,
|
|
8055
|
+
paths,
|
|
8056
|
+
prBody,
|
|
8057
|
+
prBodyPath,
|
|
8058
|
+
};
|
|
8059
|
+
}
|
|
8060
|
+
function buildCompostingMetricCandidate(projectRoot, group, owner, usedNames) {
|
|
8061
|
+
const baseName = preferredCompostingMetricName(group);
|
|
8062
|
+
const name = dedupeCompostingName(baseName, usedNames);
|
|
8063
|
+
if (!name)
|
|
8064
|
+
return undefined;
|
|
8065
|
+
usedNames.add(name.toLowerCase());
|
|
8066
|
+
const label = titleFromSlug(name);
|
|
8067
|
+
const donors = Array.from(group.donors.values()).sort((a, b) => a.path.localeCompare(b.path));
|
|
8068
|
+
const support = donors.length;
|
|
8069
|
+
const domainSlug = slugForPath(group.domain);
|
|
8070
|
+
const draftPath = `semantic-layer/metrics/_drafts/${domainSlug}/${name}.yaml`;
|
|
8071
|
+
const recurringFilters = group.filter ? splitCompostingFilters(group.filter) : [];
|
|
8072
|
+
const metric = {
|
|
8073
|
+
name,
|
|
8074
|
+
label,
|
|
8075
|
+
description: `Draft metric composted from ${support} certified DQL blocks. Review the expression, filter, grain, and owner before certification.`,
|
|
8076
|
+
domain: group.domain,
|
|
8077
|
+
status: 'draft',
|
|
8078
|
+
sql: group.expression,
|
|
8079
|
+
type: group.type,
|
|
8080
|
+
table: group.table,
|
|
8081
|
+
filter: group.filter,
|
|
8082
|
+
tags: ['composted', 'human-review'],
|
|
8083
|
+
owner,
|
|
8084
|
+
source: {
|
|
8085
|
+
provider: 'dql',
|
|
8086
|
+
objectType: 'composted_metric',
|
|
8087
|
+
objectId: `composted:${name}`,
|
|
8088
|
+
objectName: label,
|
|
8089
|
+
importedAt: new Date().toISOString(),
|
|
8090
|
+
extra: {
|
|
8091
|
+
support,
|
|
8092
|
+
donorBlocks: donors.map((donor) => donor.name),
|
|
8093
|
+
donorPaths: donors.map((donor) => donor.path),
|
|
8094
|
+
normalizedExpression: group.normalizedExpression,
|
|
8095
|
+
normalizedFilter: group.normalizedFilter,
|
|
8096
|
+
},
|
|
8097
|
+
},
|
|
7127
8098
|
};
|
|
8099
|
+
return {
|
|
8100
|
+
id: `metric:${name}:${stableShortHash(`${group.domain}|${group.table}|${group.normalizedExpression}|${group.normalizedFilter ?? ''}`)}`,
|
|
8101
|
+
kind: 'metric',
|
|
8102
|
+
name,
|
|
8103
|
+
label,
|
|
8104
|
+
description: metric.description,
|
|
8105
|
+
domain: group.domain,
|
|
8106
|
+
table: group.table,
|
|
8107
|
+
type: group.type,
|
|
8108
|
+
sql: group.expression,
|
|
8109
|
+
filter: group.filter,
|
|
8110
|
+
status: 'draft',
|
|
8111
|
+
support,
|
|
8112
|
+
donorBlocks: donors,
|
|
8113
|
+
draftPath,
|
|
8114
|
+
draftExists: existsSync(join(projectRoot, draftPath)),
|
|
8115
|
+
yaml: serializeMetricDefinitionToYaml(metric),
|
|
8116
|
+
provenance: {
|
|
8117
|
+
normalizedExpression: group.normalizedExpression,
|
|
8118
|
+
normalizedFilter: group.normalizedFilter,
|
|
8119
|
+
recurringFilters,
|
|
8120
|
+
},
|
|
8121
|
+
review: {
|
|
8122
|
+
priorityScore: support * 10 + recurringFilters.length * 2,
|
|
8123
|
+
rationale: `Recurring ${group.type} metric found in ${support} certified blocks${group.filter ? ' with the same filter' : ''}.`,
|
|
8124
|
+
},
|
|
8125
|
+
};
|
|
8126
|
+
}
|
|
8127
|
+
function singleTableDependency(block) {
|
|
8128
|
+
const dependencies = Array.from(new Set((block.tableDependencies ?? []).filter((table) => table && !table.startsWith('ref:'))));
|
|
8129
|
+
if (dependencies.length === 1)
|
|
8130
|
+
return dependencies[0];
|
|
8131
|
+
const rawRefs = Array.from(new Set((block.rawTableRefs ?? []).filter(Boolean)));
|
|
8132
|
+
return rawRefs.length === 1 ? rawRefs[0] : undefined;
|
|
8133
|
+
}
|
|
8134
|
+
function blockOutputNames(block) {
|
|
8135
|
+
if (Array.isArray(block.declaredOutputs) && block.declaredOutputs.length > 0)
|
|
8136
|
+
return block.declaredOutputs;
|
|
8137
|
+
if (Array.isArray(block.outputContract) && block.outputContract.length > 0)
|
|
8138
|
+
return block.outputContract.map((output) => output.name);
|
|
8139
|
+
if (Array.isArray(block.outputs) && block.outputs.length > 0)
|
|
8140
|
+
return block.outputs.map((output) => output.name);
|
|
8141
|
+
return [];
|
|
8142
|
+
}
|
|
8143
|
+
function isSemanticDraftMetricPath(filePath) {
|
|
8144
|
+
return Boolean(filePath?.replaceAll('\\', '/').includes('semantic-layer/metrics/_drafts/'));
|
|
8145
|
+
}
|
|
8146
|
+
function extractCompostingAggregateMetrics(sql) {
|
|
8147
|
+
const select = extractCompostingSelectClause(sql);
|
|
8148
|
+
if (!select)
|
|
8149
|
+
return [];
|
|
8150
|
+
return splitCompostingTopLevel(select, ',').flatMap((item) => {
|
|
8151
|
+
const parsed = parseCompostingSelectItem(item);
|
|
8152
|
+
if (!parsed)
|
|
8153
|
+
return [];
|
|
8154
|
+
return [parsed];
|
|
8155
|
+
});
|
|
8156
|
+
}
|
|
8157
|
+
function extractCompostingSelectClause(sql) {
|
|
8158
|
+
const source = stripSqlTerminator(sql);
|
|
8159
|
+
const selectMatch = /\bselect\b/i.exec(source);
|
|
8160
|
+
if (!selectMatch)
|
|
8161
|
+
return undefined;
|
|
8162
|
+
const fromIndex = findTopLevelKeyword(source, 'from', selectMatch.index + selectMatch[0].length);
|
|
8163
|
+
if (fromIndex < 0)
|
|
8164
|
+
return undefined;
|
|
8165
|
+
return source.slice(selectMatch.index + selectMatch[0].length, fromIndex).trim();
|
|
8166
|
+
}
|
|
8167
|
+
function parseCompostingSelectItem(item) {
|
|
8168
|
+
const { expression, alias } = splitCompostingAlias(item.trim());
|
|
8169
|
+
const normalizedExpression = expression.trim();
|
|
8170
|
+
const distinctCount = normalizedExpression.match(/^count\s*\(\s*distinct\s+[\s\S]+\)$/i);
|
|
8171
|
+
if (distinctCount) {
|
|
8172
|
+
return { expression: normalizedExpression, alias, type: 'count_distinct' };
|
|
8173
|
+
}
|
|
8174
|
+
const count = normalizedExpression.match(/^count\s*\([\s\S]*\)$/i);
|
|
8175
|
+
if (count)
|
|
8176
|
+
return { expression: normalizedExpression, alias, type: 'count' };
|
|
8177
|
+
const aggregate = normalizedExpression.match(/^(sum|avg|min|max)\s*\([\s\S]+\)$/i);
|
|
8178
|
+
if (!aggregate)
|
|
8179
|
+
return undefined;
|
|
8180
|
+
return {
|
|
8181
|
+
expression: normalizedExpression,
|
|
8182
|
+
alias,
|
|
8183
|
+
type: aggregate[1].toLowerCase(),
|
|
8184
|
+
};
|
|
8185
|
+
}
|
|
8186
|
+
function splitCompostingAlias(item) {
|
|
8187
|
+
const asMatch = item.match(/^([\s\S]+?)\s+as\s+("[^"]+"|`[^`]+`|\[[^\]]+\]|[A-Za-z_][A-Za-z0-9_$]*)$/i);
|
|
8188
|
+
if (asMatch)
|
|
8189
|
+
return { expression: asMatch[1].trim(), alias: stripIdentifierQuotes(asMatch[2]) };
|
|
8190
|
+
return { expression: item.trim() };
|
|
8191
|
+
}
|
|
8192
|
+
function extractCompostingWhereClause(sql) {
|
|
8193
|
+
const source = stripSqlTerminator(sql);
|
|
8194
|
+
const whereIndex = findTopLevelKeyword(source, 'where', 0);
|
|
8195
|
+
if (whereIndex < 0)
|
|
8196
|
+
return undefined;
|
|
8197
|
+
const start = whereIndex + 'where'.length;
|
|
8198
|
+
const endCandidates = ['group by', 'having', 'order by', 'limit', 'qualify']
|
|
8199
|
+
.map((keyword) => findTopLevelKeyword(source, keyword, start))
|
|
8200
|
+
.filter((index) => index >= 0);
|
|
8201
|
+
const end = endCandidates.length > 0 ? Math.min(...endCandidates) : source.length;
|
|
8202
|
+
const where = source.slice(start, end).trim();
|
|
8203
|
+
return where || undefined;
|
|
8204
|
+
}
|
|
8205
|
+
function splitCompostingFilters(where) {
|
|
8206
|
+
return splitCompostingTopLevel(where, 'and')
|
|
8207
|
+
.map((part) => part.trim())
|
|
8208
|
+
.filter(Boolean);
|
|
8209
|
+
}
|
|
8210
|
+
function splitCompostingTopLevel(input, delimiter) {
|
|
8211
|
+
const out = [];
|
|
8212
|
+
let start = 0;
|
|
8213
|
+
let depth = 0;
|
|
8214
|
+
let quote;
|
|
8215
|
+
for (let i = 0; i < input.length; i += 1) {
|
|
8216
|
+
const char = input[i];
|
|
8217
|
+
if (quote) {
|
|
8218
|
+
if (char === quote && input[i - 1] !== '\\')
|
|
8219
|
+
quote = undefined;
|
|
8220
|
+
continue;
|
|
8221
|
+
}
|
|
8222
|
+
if (char === '"' || char === "'" || char === '`') {
|
|
8223
|
+
quote = char;
|
|
8224
|
+
continue;
|
|
8225
|
+
}
|
|
8226
|
+
if (char === '(')
|
|
8227
|
+
depth += 1;
|
|
8228
|
+
else if (char === ')')
|
|
8229
|
+
depth = Math.max(0, depth - 1);
|
|
8230
|
+
if (depth !== 0)
|
|
8231
|
+
continue;
|
|
8232
|
+
if (delimiter === ',' && char === ',') {
|
|
8233
|
+
out.push(input.slice(start, i).trim());
|
|
8234
|
+
start = i + 1;
|
|
8235
|
+
}
|
|
8236
|
+
else if (delimiter === 'and'
|
|
8237
|
+
&& input.slice(i, i + 3).toLowerCase() === 'and'
|
|
8238
|
+
&& !/[A-Za-z0-9_]/.test(input[i - 1] ?? '')
|
|
8239
|
+
&& !/[A-Za-z0-9_]/.test(input[i + 3] ?? '')) {
|
|
8240
|
+
out.push(input.slice(start, i).trim());
|
|
8241
|
+
start = i + 3;
|
|
8242
|
+
i += 2;
|
|
8243
|
+
}
|
|
8244
|
+
}
|
|
8245
|
+
out.push(input.slice(start).trim());
|
|
8246
|
+
return out.filter(Boolean);
|
|
8247
|
+
}
|
|
8248
|
+
function findTopLevelKeyword(input, keyword, fromIndex) {
|
|
8249
|
+
const lower = input.toLowerCase();
|
|
8250
|
+
const target = keyword.toLowerCase();
|
|
8251
|
+
let depth = 0;
|
|
8252
|
+
let quote;
|
|
8253
|
+
for (let i = Math.max(0, fromIndex); i <= input.length - target.length; i += 1) {
|
|
8254
|
+
const char = input[i];
|
|
8255
|
+
if (quote) {
|
|
8256
|
+
if (char === quote && input[i - 1] !== '\\')
|
|
8257
|
+
quote = undefined;
|
|
8258
|
+
continue;
|
|
8259
|
+
}
|
|
8260
|
+
if (char === '"' || char === "'" || char === '`') {
|
|
8261
|
+
quote = char;
|
|
8262
|
+
continue;
|
|
8263
|
+
}
|
|
8264
|
+
if (char === '(') {
|
|
8265
|
+
depth += 1;
|
|
8266
|
+
continue;
|
|
8267
|
+
}
|
|
8268
|
+
if (char === ')') {
|
|
8269
|
+
depth = Math.max(0, depth - 1);
|
|
8270
|
+
continue;
|
|
8271
|
+
}
|
|
8272
|
+
if (depth !== 0)
|
|
8273
|
+
continue;
|
|
8274
|
+
if (lower.slice(i, i + target.length) === target
|
|
8275
|
+
&& !/[A-Za-z0-9_]/.test(input[i - 1] ?? '')
|
|
8276
|
+
&& !/[A-Za-z0-9_]/.test(input[i + target.length] ?? '')) {
|
|
8277
|
+
return i;
|
|
8278
|
+
}
|
|
8279
|
+
}
|
|
8280
|
+
return -1;
|
|
8281
|
+
}
|
|
8282
|
+
function preferredCompostingMetricName(group) {
|
|
8283
|
+
const aliases = group.aliases.map((alias) => slugForIdentifier(alias)).filter(Boolean);
|
|
8284
|
+
const counts = new Map();
|
|
8285
|
+
for (const alias of aliases)
|
|
8286
|
+
counts.set(alias, (counts.get(alias) ?? 0) + 1);
|
|
8287
|
+
const preferredAlias = Array.from(counts.entries()).sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))[0]?.[0];
|
|
8288
|
+
if (preferredAlias)
|
|
8289
|
+
return preferredAlias;
|
|
8290
|
+
const expressionName = slugForIdentifier(group.expression.replace(/\b(sum|avg|min|max|count|distinct)\b/gi, ''));
|
|
8291
|
+
return expressionName ? `${expressionName}_${group.type}` : `${group.type}_metric`;
|
|
8292
|
+
}
|
|
8293
|
+
function dedupeCompostingName(baseName, usedNames) {
|
|
8294
|
+
const base = slugForIdentifier(baseName).slice(0, 72).replace(/_+$/g, '');
|
|
8295
|
+
if (!base)
|
|
8296
|
+
return undefined;
|
|
8297
|
+
if (!usedNames.has(base.toLowerCase()))
|
|
8298
|
+
return base;
|
|
8299
|
+
let index = 2;
|
|
8300
|
+
while (usedNames.has(`${base}_${index}`.toLowerCase()))
|
|
8301
|
+
index += 1;
|
|
8302
|
+
return `${base}_${index}`;
|
|
8303
|
+
}
|
|
8304
|
+
function normalizeCompostingSql(value) {
|
|
8305
|
+
return value
|
|
8306
|
+
.replace(/["`]/g, '')
|
|
8307
|
+
.replace(/\s+/g, ' ')
|
|
8308
|
+
.trim()
|
|
8309
|
+
.toLowerCase();
|
|
8310
|
+
}
|
|
8311
|
+
function slugForIdentifier(value) {
|
|
8312
|
+
return value
|
|
8313
|
+
.trim()
|
|
8314
|
+
.replace(/["`[\]]/g, '')
|
|
8315
|
+
.replace(/([a-z0-9])([A-Z])/g, '$1_$2')
|
|
8316
|
+
.toLowerCase()
|
|
8317
|
+
.replace(/[^a-z0-9]+/g, '_')
|
|
8318
|
+
.replace(/^_+|_+$/g, '');
|
|
8319
|
+
}
|
|
8320
|
+
function slugForPath(value) {
|
|
8321
|
+
return slugForIdentifier(value) || 'misc';
|
|
8322
|
+
}
|
|
8323
|
+
function stripIdentifierQuotes(value) {
|
|
8324
|
+
return value.replace(/^["`\[]|["`\]]$/g, '').trim();
|
|
8325
|
+
}
|
|
8326
|
+
function titleFromSlug(value) {
|
|
8327
|
+
return value.split('_').filter(Boolean).map((part) => part.charAt(0).toUpperCase() + part.slice(1)).join(' ');
|
|
8328
|
+
}
|
|
8329
|
+
function stableShortHash(value) {
|
|
8330
|
+
return createHash('sha1').update(value).digest('hex').slice(0, 10);
|
|
8331
|
+
}
|
|
8332
|
+
function renderCompostingPrBody(candidates) {
|
|
8333
|
+
const lines = [
|
|
8334
|
+
'# Semantic Composting Changeset',
|
|
8335
|
+
'',
|
|
8336
|
+
'This changeset proposes draft semantic metrics mined from recurring logic in certified DQL blocks.',
|
|
8337
|
+
'Review each metric expression, filter, owner, grain compatibility, and downstream impact before marking it certified.',
|
|
8338
|
+
'',
|
|
8339
|
+
];
|
|
8340
|
+
if (candidates.length === 0) {
|
|
8341
|
+
lines.push('No recurring certified-block metric clusters met the current support threshold.');
|
|
8342
|
+
return `${lines.join('\n')}\n`;
|
|
8343
|
+
}
|
|
8344
|
+
for (const candidate of candidates) {
|
|
8345
|
+
lines.push(`## ${candidate.label}`);
|
|
8346
|
+
lines.push('');
|
|
8347
|
+
lines.push(`- Draft path: \`${candidate.draftPath}\``);
|
|
8348
|
+
lines.push(`- Domain/table: \`${candidate.domain}\` / \`${candidate.table}\``);
|
|
8349
|
+
lines.push(`- Expression: \`${candidate.sql}\``);
|
|
8350
|
+
if (candidate.filter)
|
|
8351
|
+
lines.push(`- Filter: \`${candidate.filter}\``);
|
|
8352
|
+
lines.push(`- Support: ${candidate.support} certified block(s)`);
|
|
8353
|
+
lines.push(`- Donors: ${candidate.donorBlocks.map((donor) => `\`${donor.path}\``).join(', ')}`);
|
|
8354
|
+
lines.push('');
|
|
8355
|
+
}
|
|
8356
|
+
return `${lines.join('\n')}\n`;
|
|
7128
8357
|
}
|
|
7129
8358
|
export async function generateProposeDrafts(projectRoot, slugs, projectConfig = loadProjectConfig(projectRoot), options = {}) {
|
|
7130
8359
|
const manifestPath = resolveDbtManifestPath(projectRoot, projectConfig);
|
|
@@ -8084,7 +9313,7 @@ async function introspectSchema(executor, connection) {
|
|
|
8084
9313
|
try {
|
|
8085
9314
|
const catalogRows = await executor.executeQuery(`SELECT table_schema, table_name, table_type
|
|
8086
9315
|
FROM information_schema.tables
|
|
8087
|
-
WHERE table_schema NOT IN ('
|
|
9316
|
+
WHERE UPPER(table_schema) NOT IN ('INFORMATION_SCHEMA', 'PG_CATALOG')
|
|
8088
9317
|
ORDER BY table_schema, table_name`, [], {}, connection);
|
|
8089
9318
|
tables = catalogRows.rows.map((row) => {
|
|
8090
9319
|
const schema = String(row['table_schema'] ?? row['TABLE_SCHEMA'] ?? 'default');
|
|
@@ -8095,7 +9324,7 @@ async function introspectSchema(executor, connection) {
|
|
|
8095
9324
|
});
|
|
8096
9325
|
const columnRows = await executor.executeQuery(`SELECT table_schema, table_name, column_name, data_type
|
|
8097
9326
|
FROM information_schema.columns
|
|
8098
|
-
WHERE table_schema NOT IN ('
|
|
9327
|
+
WHERE UPPER(table_schema) NOT IN ('INFORMATION_SCHEMA', 'PG_CATALOG')
|
|
8099
9328
|
ORDER BY table_schema, table_name, ordinal_position`, [], {}, connection);
|
|
8100
9329
|
columnsByPath = columnRows.rows.reduce((map, row) => {
|
|
8101
9330
|
const schema = String(row['table_schema'] ?? row['TABLE_SCHEMA'] ?? 'default');
|
|
@@ -8259,6 +9488,20 @@ function parseBlockStudioArrayField(source, key) {
|
|
|
8259
9488
|
function parseBlockStudioStringField(source, key) {
|
|
8260
9489
|
return source.match(new RegExp(`\\b${key}\\s*=\\s*"([^"]*)"`, 'i'))?.[1] ?? undefined;
|
|
8261
9490
|
}
|
|
9491
|
+
function parseSemanticRequestedFilters(source) {
|
|
9492
|
+
return parseBlockStudioArrayField(source, 'requested_filters')
|
|
9493
|
+
.map((entry) => {
|
|
9494
|
+
const match = entry.match(/^([^=<>!]+)\s*=\s*(.+)$/);
|
|
9495
|
+
if (!match)
|
|
9496
|
+
return undefined;
|
|
9497
|
+
const dimension = match[1]?.trim();
|
|
9498
|
+
const value = match[2]?.trim();
|
|
9499
|
+
if (!dimension || !value)
|
|
9500
|
+
return undefined;
|
|
9501
|
+
return { dimension, operator: 'equals', values: [value] };
|
|
9502
|
+
})
|
|
9503
|
+
.filter((filter) => Boolean(filter));
|
|
9504
|
+
}
|
|
8262
9505
|
function parseSemanticBlockConfig(source) {
|
|
8263
9506
|
const blockType = (parseBlockStudioStringField(source, 'type') ?? 'custom').toLowerCase() === 'semantic'
|
|
8264
9507
|
? 'semantic'
|
|
@@ -8266,6 +9509,7 @@ function parseSemanticBlockConfig(source) {
|
|
|
8266
9509
|
const metric = parseBlockStudioStringField(source, 'metric');
|
|
8267
9510
|
const metrics = parseBlockStudioArrayField(source, 'metrics');
|
|
8268
9511
|
const dimensions = parseBlockStudioArrayField(source, 'dimensions');
|
|
9512
|
+
const filters = parseSemanticRequestedFilters(source);
|
|
8269
9513
|
const timeDimension = parseBlockStudioStringField(source, 'time_dimension');
|
|
8270
9514
|
const granularity = parseBlockStudioStringField(source, 'granularity');
|
|
8271
9515
|
const limitMatch = source.match(/\blimit\s*=\s*(\d+)/i);
|
|
@@ -8274,6 +9518,7 @@ function parseSemanticBlockConfig(source) {
|
|
|
8274
9518
|
metric,
|
|
8275
9519
|
metrics,
|
|
8276
9520
|
dimensions,
|
|
9521
|
+
filters,
|
|
8277
9522
|
timeDimension,
|
|
8278
9523
|
granularity,
|
|
8279
9524
|
limit: limitMatch ? Number.parseInt(limitMatch[1], 10) : undefined,
|
|
@@ -8285,7 +9530,7 @@ export async function resolveSemanticTableMapping(executor, connection, semantic
|
|
|
8285
9530
|
try {
|
|
8286
9531
|
const tablesResult = await executor.executeQuery(`SELECT table_schema, table_name
|
|
8287
9532
|
FROM information_schema.tables
|
|
8288
|
-
WHERE table_schema NOT IN ('
|
|
9533
|
+
WHERE UPPER(table_schema) NOT IN ('INFORMATION_SCHEMA', 'PG_CATALOG')
|
|
8289
9534
|
ORDER BY table_schema, table_name
|
|
8290
9535
|
LIMIT 2000`, [], {}, connection);
|
|
8291
9536
|
return buildSemanticTableMapping(semanticLayer, tablesResult.rows);
|
|
@@ -8419,6 +9664,7 @@ function composeSemanticBlockSql(source, semanticLayer, options) {
|
|
|
8419
9664
|
? composeRuntimeSemanticQuery({
|
|
8420
9665
|
metrics,
|
|
8421
9666
|
dimensions: config.dimensions,
|
|
9667
|
+
filters: config.filters,
|
|
8422
9668
|
timeDimension: config.timeDimension && config.granularity
|
|
8423
9669
|
? { name: config.timeDimension, granularity: config.granularity }
|
|
8424
9670
|
: undefined,
|
|
@@ -8433,6 +9679,7 @@ function composeSemanticBlockSql(source, semanticLayer, options) {
|
|
|
8433
9679
|
: semanticLayer.composeQuery({
|
|
8434
9680
|
metrics,
|
|
8435
9681
|
dimensions: config.dimensions,
|
|
9682
|
+
filters: config.filters,
|
|
8436
9683
|
timeDimension: config.timeDimension && config.granularity
|
|
8437
9684
|
? { name: config.timeDimension, granularity: config.granularity }
|
|
8438
9685
|
: undefined,
|
|
@@ -8599,6 +9846,101 @@ export function validateBlockStudioSource(source, semanticLayer) {
|
|
|
8599
9846
|
executableSql,
|
|
8600
9847
|
};
|
|
8601
9848
|
}
|
|
9849
|
+
export async function createDqlArtifactGenerationSessionForProject(projectRoot, options, semanticLayer) {
|
|
9850
|
+
const dqlArtifact = normalizeDqlArtifactReference(options.dqlArtifact);
|
|
9851
|
+
if (!dqlArtifact)
|
|
9852
|
+
throw new Error('A DQL artifact with source is required for DQL draft promotion.');
|
|
9853
|
+
const question = notebookResearchString(options.question) ?? dqlArtifact.name ?? 'Generated DQL artifact';
|
|
9854
|
+
const slug = deriveGeneratedDraftSlug(dqlArtifact.name ?? question);
|
|
9855
|
+
const domain = notebookResearchString(options.domain) ?? 'misc';
|
|
9856
|
+
const owner = notebookResearchString(options.owner) ?? '';
|
|
9857
|
+
const tags = Array.from(new Set(['notebook-research', 'review-required', 'dql-artifact', ...(Array.isArray(options.tags) ? options.tags : [])]
|
|
9858
|
+
.map((tag) => notebookResearchString(tag))
|
|
9859
|
+
.filter((tag) => Boolean(tag))));
|
|
9860
|
+
const proposedContractId = `${domain}.Unknown.${slug}`;
|
|
9861
|
+
const draft = upsertGeneratedDqlArtifactDraft(projectRoot, {
|
|
9862
|
+
slug,
|
|
9863
|
+
question,
|
|
9864
|
+
proposedContractId,
|
|
9865
|
+
owner,
|
|
9866
|
+
proposedDomain: domain,
|
|
9867
|
+
dqlArtifact,
|
|
9868
|
+
sourceQuestion: question,
|
|
9869
|
+
sourceBlock: options.sourceBlock,
|
|
9870
|
+
contextPackId: options.contextPackId,
|
|
9871
|
+
routeIntent: options.routeIntent,
|
|
9872
|
+
outputs: Array.from(new Set([...(dqlArtifact.dimensions ?? []), ...(dqlArtifact.metrics ?? [])])),
|
|
9873
|
+
});
|
|
9874
|
+
const now = new Date().toISOString();
|
|
9875
|
+
const uniqueSuffix = createHash('sha1').update(`${now}:${draft.path}:${dqlArtifact.source}`).digest('hex').slice(0, 12);
|
|
9876
|
+
const importId = `imp_dql_artifact_${uniqueSuffix}`;
|
|
9877
|
+
const candidateId = `cand_${slug.replace(/[^a-z0-9_]+/g, '_').replace(/^_+|_+$/g, '') || 'dql_artifact'}_${uniqueSuffix.slice(0, 6)}`;
|
|
9878
|
+
const sourcePath = dqlArtifact.sourcePath ?? options.inputPath ?? draft.path;
|
|
9879
|
+
const savedSource = readFileSync(join(projectRoot, draft.path), 'utf-8');
|
|
9880
|
+
const candidate = {
|
|
9881
|
+
id: candidateId,
|
|
9882
|
+
sourceKind: 'raw-sql-file',
|
|
9883
|
+
sourcePath,
|
|
9884
|
+
name: dqlArtifact.name ?? slug,
|
|
9885
|
+
domain,
|
|
9886
|
+
description: question,
|
|
9887
|
+
owner,
|
|
9888
|
+
tags,
|
|
9889
|
+
grain: dqlArtifact.timeDimension ? `${dqlArtifact.timeDimension.name}.${dqlArtifact.timeDimension.granularity}` : undefined,
|
|
9890
|
+
outputs: Array.from(new Set([...(dqlArtifact.dimensions ?? []), ...(dqlArtifact.metrics ?? [])])),
|
|
9891
|
+
dimensions: dqlArtifact.dimensions,
|
|
9892
|
+
allowedFilters: dqlArtifact.filters?.map((filter) => filter.dimension),
|
|
9893
|
+
parameterPolicy: dqlArtifact.filters?.map((filter) => ({ name: filter.dimension, policy: 'dynamic' })),
|
|
9894
|
+
sql: notebookResearchString(options.generatedSql) ?? '',
|
|
9895
|
+
dqlSource: savedSource,
|
|
9896
|
+
validation: validateBlockStudioSource(savedSource, semanticLayer),
|
|
9897
|
+
preview: null,
|
|
9898
|
+
lineage: {
|
|
9899
|
+
sourceTables: [],
|
|
9900
|
+
parameters: [],
|
|
9901
|
+
warnings: [],
|
|
9902
|
+
statementIndex: 1,
|
|
9903
|
+
totalStatements: 1,
|
|
9904
|
+
},
|
|
9905
|
+
confidence: 0.95,
|
|
9906
|
+
splitStrategy: 'manual',
|
|
9907
|
+
warnings: [],
|
|
9908
|
+
conversionNotes: ['Persisted the DQL artifact returned by the governed answer; no SQL-to-DQL regeneration was required.'],
|
|
9909
|
+
aiAssistance: [],
|
|
9910
|
+
reviewStatus: 'draft',
|
|
9911
|
+
savedPath: draft.path,
|
|
9912
|
+
generationMode: 'deterministic',
|
|
9913
|
+
generationProvider: 'dql-artifact',
|
|
9914
|
+
llmContext: `Persisted generated DQL artifact for: ${question}`,
|
|
9915
|
+
evidence: [],
|
|
9916
|
+
draftSave: { status: 'saved', path: draft.path, savedAt: now },
|
|
9917
|
+
recommendedAction: 'create_new',
|
|
9918
|
+
similarityMatches: [],
|
|
9919
|
+
};
|
|
9920
|
+
const session = {
|
|
9921
|
+
id: importId,
|
|
9922
|
+
sourceKind: 'raw-sql-file',
|
|
9923
|
+
inputPath: sourcePath,
|
|
9924
|
+
inputMode: options.inputMode ?? 'upload',
|
|
9925
|
+
sourceFiles: [sourcePath],
|
|
9926
|
+
createdAt: now,
|
|
9927
|
+
updatedAt: now,
|
|
9928
|
+
defaults: { domain, owner, tags },
|
|
9929
|
+
candidateIds: [candidate.id],
|
|
9930
|
+
mode: 'ai-import',
|
|
9931
|
+
generation: {
|
|
9932
|
+
provider: 'dql-artifact',
|
|
9933
|
+
aiEnabled: false,
|
|
9934
|
+
contextObjectCount: 0,
|
|
9935
|
+
createdDrafts: 1,
|
|
9936
|
+
warnings: [],
|
|
9937
|
+
},
|
|
9938
|
+
candidates: [candidate],
|
|
9939
|
+
};
|
|
9940
|
+
writeBlockStudioImportSession(projectRoot, session);
|
|
9941
|
+
await refreshLocalMetadataCatalog(projectRoot);
|
|
9942
|
+
return session;
|
|
9943
|
+
}
|
|
8602
9944
|
export async function createDqlGenerationSessionForProject(projectRoot, options, semanticLayer) {
|
|
8603
9945
|
const deterministicOnly = isDeterministicDqlGenerationProvider(options.provider);
|
|
8604
9946
|
const requestedProvider = !deterministicOnly && isProviderSettingsId(options.provider) ? options.provider : undefined;
|
|
@@ -9597,7 +10939,13 @@ function inferDqlGenerationPattern(sql) {
|
|
|
9597
10939
|
return 'trend';
|
|
9598
10940
|
if (groupFields.length === 1 && /_id$|_key$/i.test(groupFields[0]))
|
|
9599
10941
|
return 'entity_rollup';
|
|
9600
|
-
|
|
10942
|
+
// Entity profile = a non-aggregated, un-grouped row lookup filtered by an
|
|
10943
|
+
// identifier (…WHERE <something>_id = / IN …). Structural + domain-agnostic —
|
|
10944
|
+
// no hard-coded entity vocabulary.
|
|
10945
|
+
const hasAggregate = /\b(sum|count|avg|min|max|median|percentile|rank)\s*\(/i.test(sql);
|
|
10946
|
+
const hasGroupBy = /\bgroup\s+by\b/i.test(sql);
|
|
10947
|
+
const filtersByIdentifier = /\bwhere\b[\s\S]*?\b[A-Za-z_][A-Za-z0-9_]*(?:_id|_key|id)\b\s*(?:=|in\b)/i.test(sql);
|
|
10948
|
+
if (!hasAggregate && !hasGroupBy && filtersByIdentifier) {
|
|
9601
10949
|
return 'entity_profile';
|
|
9602
10950
|
}
|
|
9603
10951
|
return 'custom';
|
|
@@ -10195,10 +11543,15 @@ async function createBlockStudioAssistProvider(projectRoot, requestedProvider) {
|
|
|
10195
11543
|
provider = new OpenAIProvider({ apiKey: config.apiKey, baseUrl: config.baseUrl, model: config.model, allowNoApiKey: true });
|
|
10196
11544
|
break;
|
|
10197
11545
|
case 'claude-code':
|
|
10198
|
-
|
|
11546
|
+
// OAuth-first: use the browser-login token when connected; fall back to the CLI passthrough.
|
|
11547
|
+
provider = claudeOAuthConnected(projectRoot)
|
|
11548
|
+
? new ClaudeOAuthProvider({ projectRoot, model: config.model })
|
|
11549
|
+
: new ClaudeCodeCliProvider({ model: config.model });
|
|
10199
11550
|
break;
|
|
10200
11551
|
case 'codex':
|
|
10201
|
-
provider =
|
|
11552
|
+
provider = codexOAuthConnected(projectRoot)
|
|
11553
|
+
? new CodexOAuthProvider({ projectRoot, model: config.model })
|
|
11554
|
+
: new CodexCliProvider({ model: config.model });
|
|
10202
11555
|
break;
|
|
10203
11556
|
default:
|
|
10204
11557
|
return null;
|
|
@@ -10212,8 +11565,7 @@ function agentResultToSynthesisPreview(result) {
|
|
|
10212
11565
|
const columns = Array.isArray(result.columns)
|
|
10213
11566
|
? result.columns.map((col) => typeof col === 'string' ? col : col?.name ?? String(col))
|
|
10214
11567
|
: [];
|
|
10215
|
-
const
|
|
10216
|
-
const rows = rawRows.map((row) => {
|
|
11568
|
+
const normalizeRow = (row) => {
|
|
10217
11569
|
if (row && typeof row === 'object' && !Array.isArray(row))
|
|
10218
11570
|
return row;
|
|
10219
11571
|
// Array-shaped rows → zip with column names.
|
|
@@ -10223,69 +11575,20 @@ function agentResultToSynthesisPreview(result) {
|
|
|
10223
11575
|
return obj;
|
|
10224
11576
|
}
|
|
10225
11577
|
return { value: row };
|
|
10226
|
-
}
|
|
11578
|
+
};
|
|
11579
|
+
// Normalize ALL rows so verified statistics cover the full result — the
|
|
11580
|
+
// narrator sees only a 20-row sample, and sample-derived aggregate claims
|
|
11581
|
+
// ("spend ranges from X to Y") were the last hallucination surface.
|
|
11582
|
+
const allRows = Array.isArray(result.rows) ? result.rows.map(normalizeRow) : [];
|
|
11583
|
+
const rows = allRows.slice(0, 20);
|
|
11584
|
+
const resolvedColumns = columns.length > 0 ? columns : Object.keys(rows[0] ?? {});
|
|
10227
11585
|
return {
|
|
10228
|
-
columns:
|
|
11586
|
+
columns: resolvedColumns,
|
|
10229
11587
|
rows,
|
|
10230
|
-
rowCount: typeof result.rowCount === 'number' ? result.rowCount :
|
|
11588
|
+
rowCount: typeof result.rowCount === 'number' ? result.rowCount : allRows.length,
|
|
11589
|
+
stats: computeResultStats(resolvedColumns, allRows),
|
|
10231
11590
|
};
|
|
10232
11591
|
}
|
|
10233
|
-
const SIGNAL_STOPWORDS = new Set([
|
|
10234
|
-
'the', 'a', 'an', 'of', 'for', 'to', 'by', 'in', 'on', 'and', 'or', 'is', 'are', 'was', 'were',
|
|
10235
|
-
'what', 'which', 'how', 'show', 'me', 'my', 'our', 'this', 'that', 'top', 'give', 'get', 'list',
|
|
10236
|
-
]);
|
|
10237
|
-
function signalTokens(text) {
|
|
10238
|
-
return new Set(text.toLowerCase().replace(/[^a-z0-9\s]/g, ' ').split(/\s+/)
|
|
10239
|
-
.filter((token) => token.length > 2 && !SIGNAL_STOPWORDS.has(token)));
|
|
10240
|
-
}
|
|
10241
|
-
/** Best token-overlap ratio between the question and any candidate name/description. */
|
|
10242
|
-
function bestOverlap(questionTokens, candidates) {
|
|
10243
|
-
if (questionTokens.size === 0)
|
|
10244
|
-
return 0;
|
|
10245
|
-
let best = 0;
|
|
10246
|
-
for (const candidate of candidates) {
|
|
10247
|
-
const candTokens = signalTokens(candidate);
|
|
10248
|
-
if (candTokens.size === 0)
|
|
10249
|
-
continue;
|
|
10250
|
-
let hits = 0;
|
|
10251
|
-
for (const token of candTokens)
|
|
10252
|
-
if (questionTokens.has(token))
|
|
10253
|
-
hits += 1;
|
|
10254
|
-
if (hits === 0)
|
|
10255
|
-
continue;
|
|
10256
|
-
// Ratio of the candidate's tokens the question covers — rewards a tight match.
|
|
10257
|
-
const ratio = hits / Math.min(candTokens.size, questionTokens.size);
|
|
10258
|
-
if (ratio > best)
|
|
10259
|
-
best = ratio;
|
|
10260
|
-
}
|
|
10261
|
-
return Math.min(1, best);
|
|
10262
|
-
}
|
|
10263
|
-
/**
|
|
10264
|
-
* Cheap, offline retrieval-signal probe. Fills certifiedScore/metricScore/hasRetrieval
|
|
10265
|
-
* from local catalog name/description overlap so the deterministic router can reach
|
|
10266
|
-
* confidence (and skip the LLM) on obvious governed matches. Best-effort — any failure
|
|
10267
|
-
* yields empty signals and the router simply leans on its LLM classification.
|
|
10268
|
-
*/
|
|
10269
|
-
function computeAgentRunSignals(projectRoot, question) {
|
|
10270
|
-
try {
|
|
10271
|
-
const tokens = signalTokens(question);
|
|
10272
|
-
if (tokens.size === 0)
|
|
10273
|
-
return { certifiedScore: 0, metricScore: 0, hasRetrieval: false };
|
|
10274
|
-
const certifiedBlocks = collectPlanBlocks(projectRoot, { certifiedOnly: true });
|
|
10275
|
-
const blockStrings = certifiedBlocks.flatMap((block) => [block.name, block.description ?? ''].filter(Boolean));
|
|
10276
|
-
const metrics = loadSemanticMetrics(projectRoot);
|
|
10277
|
-
const metricStrings = metrics.map((metric) => {
|
|
10278
|
-
const node = metric;
|
|
10279
|
-
return node.name ?? node.label ?? node.id ?? '';
|
|
10280
|
-
}).filter(Boolean);
|
|
10281
|
-
const certifiedScore = bestOverlap(tokens, blockStrings);
|
|
10282
|
-
const metricScore = bestOverlap(tokens, metricStrings);
|
|
10283
|
-
return { certifiedScore, metricScore, hasRetrieval: certifiedScore > 0 || metricScore > 0 };
|
|
10284
|
-
}
|
|
10285
|
-
catch {
|
|
10286
|
-
return { certifiedScore: 0, metricScore: 0, hasRetrieval: false };
|
|
10287
|
-
}
|
|
10288
|
-
}
|
|
10289
11592
|
/** Up to three example questions built from real catalog blocks (falls back to generic asks). */
|
|
10290
11593
|
function buildConversationSuggestions(projectRoot, kind) {
|
|
10291
11594
|
if (kind === 'gratitude')
|
|
@@ -10340,6 +11643,141 @@ function buildConversationSystemPrompt(kind, isGeneralKnowledge, catalogContext,
|
|
|
10340
11643
|
}
|
|
10341
11644
|
return lines.join('\n');
|
|
10342
11645
|
}
|
|
11646
|
+
function renderConversationMemoryForPrompt(context) {
|
|
11647
|
+
const recap = buildConversationContextRecap(context);
|
|
11648
|
+
if (!recap)
|
|
11649
|
+
return undefined;
|
|
11650
|
+
return [
|
|
11651
|
+
'Current conversation context:',
|
|
11652
|
+
recap,
|
|
11653
|
+
'If the user asks what this conversation is about, summarize this context directly. Do not query data or invent values.',
|
|
11654
|
+
].join('\n');
|
|
11655
|
+
}
|
|
11656
|
+
function buildConversationContextRecap(context) {
|
|
11657
|
+
if (!context)
|
|
11658
|
+
return undefined;
|
|
11659
|
+
const turnRecap = buildConversationTurnsRecap(context);
|
|
11660
|
+
if (turnRecap)
|
|
11661
|
+
return turnRecap;
|
|
11662
|
+
const sourceQuestion = agentRunString(context.sourceQuestion);
|
|
11663
|
+
const sourceAnswerSummary = agentRunString(context.sourceAnswerSummary);
|
|
11664
|
+
const columns = agentRunStringArray(context.resultColumns ?? context.outputColumns).slice(0, 8);
|
|
11665
|
+
const values = agentRunRecord(context.resultDimensionValues);
|
|
11666
|
+
const rows = Array.isArray(context.resultRowsSample)
|
|
11667
|
+
? context.resultRowsSample.map(agentRunRecord).filter((row) => Boolean(row)).slice(0, 1)
|
|
11668
|
+
: [];
|
|
11669
|
+
const firstRow = rows[0];
|
|
11670
|
+
const leadParts = buildPriorValueLeadParts(values);
|
|
11671
|
+
const rowFacts = firstRow
|
|
11672
|
+
? columns
|
|
11673
|
+
.slice(0, 5)
|
|
11674
|
+
.map((column) => {
|
|
11675
|
+
const value = firstRow[column];
|
|
11676
|
+
return value === undefined || value === null || typeof value === 'object' ? '' : `${column}=${String(value)}`;
|
|
11677
|
+
})
|
|
11678
|
+
.filter(Boolean)
|
|
11679
|
+
: [];
|
|
11680
|
+
const contextBits = [
|
|
11681
|
+
sourceQuestion ? `the question "${sourceQuestion}"` : '',
|
|
11682
|
+
sourceAnswerSummary ? `the latest answer summary: ${sourceAnswerSummary}` : '',
|
|
11683
|
+
columns.length ? `result columns: ${columns.join(', ')}` : '',
|
|
11684
|
+
leadParts.length ? `notable prior values: ${leadParts.join(', ')}` : '',
|
|
11685
|
+
rowFacts.length ? `first result row: ${rowFacts.join(', ')}` : '',
|
|
11686
|
+
].filter(Boolean);
|
|
11687
|
+
if (contextBits.length === 0)
|
|
11688
|
+
return undefined;
|
|
11689
|
+
return `We were talking about ${contextBits.join('. ')}.`;
|
|
11690
|
+
}
|
|
11691
|
+
function buildConversationTurnsRecap(context) {
|
|
11692
|
+
const turns = Array.isArray(context.turns)
|
|
11693
|
+
? context.turns.map(agentRunRecord).filter((turn) => Boolean(turn)).slice(-3)
|
|
11694
|
+
: [];
|
|
11695
|
+
if (turns.length === 0)
|
|
11696
|
+
return undefined;
|
|
11697
|
+
const activeTurnId = agentRunString(context.activeTurnId);
|
|
11698
|
+
const active = activeTurnId
|
|
11699
|
+
? turns.find((turn) => agentRunString(turn.id) === activeTurnId)
|
|
11700
|
+
: undefined;
|
|
11701
|
+
const selected = active ?? turns[turns.length - 1];
|
|
11702
|
+
if (!selected)
|
|
11703
|
+
return undefined;
|
|
11704
|
+
const result = agentRunRecord(selected.result);
|
|
11705
|
+
const values = agentRunRecord(result?.dimensionValues);
|
|
11706
|
+
const columns = agentRunStringArray(result?.columns).slice(0, 8);
|
|
11707
|
+
const rows = Array.isArray(result?.rowsSample)
|
|
11708
|
+
? result.rowsSample.map(agentRunRecord).filter((row) => Boolean(row)).slice(0, 1)
|
|
11709
|
+
: [];
|
|
11710
|
+
const firstRow = rows[0];
|
|
11711
|
+
const leadParts = buildPriorValueLeadParts(values);
|
|
11712
|
+
const rowFacts = firstRow
|
|
11713
|
+
? columns
|
|
11714
|
+
.slice(0, 5)
|
|
11715
|
+
.map((column) => {
|
|
11716
|
+
const value = firstRow[column];
|
|
11717
|
+
return value === undefined || value === null || typeof value === 'object' ? '' : `${column}=${String(value)}`;
|
|
11718
|
+
})
|
|
11719
|
+
.filter(Boolean)
|
|
11720
|
+
: [];
|
|
11721
|
+
const previous = turns
|
|
11722
|
+
.filter((turn) => turn !== selected)
|
|
11723
|
+
.map((turn) => agentRunString(turn.question))
|
|
11724
|
+
.filter((value) => Boolean(value))
|
|
11725
|
+
.slice(-2);
|
|
11726
|
+
const bits = [
|
|
11727
|
+
agentRunString(selected.question) ? `the latest question "${agentRunString(selected.question)}"` : '',
|
|
11728
|
+
agentRunString(selected.answerSummary) ? `latest answer summary: ${agentRunString(selected.answerSummary)}` : '',
|
|
11729
|
+
columns.length ? `result columns: ${columns.join(', ')}` : '',
|
|
11730
|
+
leadParts.length ? `notable prior values: ${leadParts.join(', ')}` : '',
|
|
11731
|
+
rowFacts.length ? `first result row: ${rowFacts.join(', ')}` : '',
|
|
11732
|
+
previous.length ? `earlier in this thread: ${previous.join(' / ')}` : '',
|
|
11733
|
+
].filter(Boolean);
|
|
11734
|
+
return bits.length > 0 ? `We were talking about ${bits.join('. ')}.` : undefined;
|
|
11735
|
+
}
|
|
11736
|
+
function agentRunStringArray(value) {
|
|
11737
|
+
if (!Array.isArray(value))
|
|
11738
|
+
return [];
|
|
11739
|
+
return Array.from(new Set(value.map(agentRunString).filter((item) => Boolean(item))));
|
|
11740
|
+
}
|
|
11741
|
+
function firstPriorValue(values, keys) {
|
|
11742
|
+
if (!values)
|
|
11743
|
+
return undefined;
|
|
11744
|
+
for (const key of keys) {
|
|
11745
|
+
const list = Array.isArray(values[key]) ? values[key] : [];
|
|
11746
|
+
const first = list.map(agentRunString).find(Boolean);
|
|
11747
|
+
if (first)
|
|
11748
|
+
return first;
|
|
11749
|
+
}
|
|
11750
|
+
return undefined;
|
|
11751
|
+
}
|
|
11752
|
+
/**
|
|
11753
|
+
* Build human-readable "notable prior values" for conversation recap from whatever
|
|
11754
|
+
* dimension columns the prior result actually had. Domain-agnostic: name/label-ish
|
|
11755
|
+
* columns are surfaced first, then any other dimension — no hard-coded jaffle entity
|
|
11756
|
+
* columns — so follow-up questions get meaningful context on ANY repo.
|
|
11757
|
+
*/
|
|
11758
|
+
function buildPriorValueLeadParts(values) {
|
|
11759
|
+
if (!values)
|
|
11760
|
+
return [];
|
|
11761
|
+
const keys = Object.keys(values);
|
|
11762
|
+
const nameLike = keys.filter((key) => /(?:^|_)(?:name|label|title|nm)$/i.test(key));
|
|
11763
|
+
const ordered = [...nameLike, ...keys.filter((key) => !nameLike.includes(key))];
|
|
11764
|
+
const parts = [];
|
|
11765
|
+
const seenLabels = new Set();
|
|
11766
|
+
for (const key of ordered) {
|
|
11767
|
+
if (parts.length >= 3)
|
|
11768
|
+
break;
|
|
11769
|
+
const value = firstPriorValue(values, [key]);
|
|
11770
|
+
if (!value)
|
|
11771
|
+
continue;
|
|
11772
|
+
const label = (key.replace(/_?(?:name|id|key|label|title|nm)$/i, '').replace(/[_-]+/g, ' ').trim()
|
|
11773
|
+
|| key.replace(/[_-]+/g, ' ').trim()).toLowerCase();
|
|
11774
|
+
if (seenLabels.has(label))
|
|
11775
|
+
continue;
|
|
11776
|
+
seenLabels.add(label);
|
|
11777
|
+
parts.push(label ? `${label} "${value}"` : `"${value}"`);
|
|
11778
|
+
}
|
|
11779
|
+
return parts;
|
|
11780
|
+
}
|
|
10343
11781
|
/** Deterministic, warm reply when no AI provider is configured — never governance-speak. */
|
|
10344
11782
|
function buildConversationalFallback(kind, isGeneralKnowledge, question, suggestions) {
|
|
10345
11783
|
const examples = suggestions.length > 0
|
|
@@ -12104,13 +13542,6 @@ function collectSettingsEnvStatus() {
|
|
|
12104
13542
|
},
|
|
12105
13543
|
];
|
|
12106
13544
|
}
|
|
12107
|
-
function isProviderSettingsId(value) {
|
|
12108
|
-
return value === 'anthropic'
|
|
12109
|
-
|| value === 'openai'
|
|
12110
|
-
|| value === 'gemini'
|
|
12111
|
-
|| value === 'ollama'
|
|
12112
|
-
|| value === 'custom-openai';
|
|
12113
|
-
}
|
|
12114
13545
|
function isDeterministicDqlGenerationProvider(value) {
|
|
12115
13546
|
if (typeof value !== 'string')
|
|
12116
13547
|
return false;
|
|
@@ -12131,8 +13562,8 @@ async function testProviderConfig(projectRoot, id, overrides) {
|
|
|
12131
13562
|
const label = providerSettingsLabel(id);
|
|
12132
13563
|
const details = providerConfigDetails(id, config);
|
|
12133
13564
|
const isSubscriptionCli = id === 'claude-code' || id === 'codex';
|
|
12134
|
-
// Subscription
|
|
12135
|
-
// needs no saved config; the others require enabling
|
|
13565
|
+
// Subscription providers are always testable — the connection (browser OAuth, or
|
|
13566
|
+
// an installed+logged-in CLI) needs no saved config; the others require enabling first.
|
|
12136
13567
|
if (!config.enabled && !isSubscriptionCli) {
|
|
12137
13568
|
return { ok: false, message: `${label} is disabled. Enable it (or Save) before testing.` };
|
|
12138
13569
|
}
|
|
@@ -12140,6 +13571,11 @@ async function testProviderConfig(projectRoot, id, overrides) {
|
|
|
12140
13571
|
return testOpenAIProviderConfig(config, label, details);
|
|
12141
13572
|
if (id === 'anthropic')
|
|
12142
13573
|
return testAnthropicProviderConfig(config, label, details);
|
|
13574
|
+
// OAuth-first for subscriptions: when the user signed in via the browser, test the
|
|
13575
|
+
// OAuth-backed provider (no CLI required); otherwise fall back to the CLI passthrough.
|
|
13576
|
+
// Mirrors the runtime selection in dql-agent-provider / createBlockStudioAssistProvider.
|
|
13577
|
+
const claudeOAuth = id === 'claude-code' && claudeOAuthConnected(projectRoot);
|
|
13578
|
+
const codexOAuth = id === 'codex' && codexOAuthConnected(projectRoot);
|
|
12143
13579
|
let provider;
|
|
12144
13580
|
switch (id) {
|
|
12145
13581
|
case 'gemini':
|
|
@@ -12149,10 +13585,14 @@ async function testProviderConfig(projectRoot, id, overrides) {
|
|
|
12149
13585
|
provider = new OllamaProvider({ baseUrl: config.baseUrl, model: config.model });
|
|
12150
13586
|
break;
|
|
12151
13587
|
case 'claude-code':
|
|
12152
|
-
provider =
|
|
13588
|
+
provider = claudeOAuth
|
|
13589
|
+
? new ClaudeOAuthProvider({ projectRoot, model: config.model })
|
|
13590
|
+
: new ClaudeCodeCliProvider({ model: config.model });
|
|
12153
13591
|
break;
|
|
12154
13592
|
case 'codex':
|
|
12155
|
-
provider =
|
|
13593
|
+
provider = codexOAuth
|
|
13594
|
+
? new CodexOAuthProvider({ projectRoot, model: config.model })
|
|
13595
|
+
: new CodexCliProvider({ model: config.model });
|
|
12156
13596
|
break;
|
|
12157
13597
|
case 'custom-openai':
|
|
12158
13598
|
default:
|
|
@@ -12162,10 +13602,11 @@ async function testProviderConfig(projectRoot, id, overrides) {
|
|
|
12162
13602
|
const available = await provider.available().catch(() => false);
|
|
12163
13603
|
if (!available) {
|
|
12164
13604
|
const cli = id === 'claude-code' ? 'claude' : 'codex';
|
|
13605
|
+
const signIn = id === 'claude-code' ? 'Sign in with Claude' : 'Sign in with ChatGPT';
|
|
12165
13606
|
return {
|
|
12166
13607
|
ok: false,
|
|
12167
13608
|
message: isSubscriptionCli
|
|
12168
|
-
? `${label} is not ready.
|
|
13609
|
+
? `${label} is not ready. Click "${signIn}" to sign in with your subscription, or install the \`${cli}\` CLI and run \`${cli} ${id === 'claude-code' ? '/login' : 'login'}\`, then test again.`
|
|
12169
13610
|
: `${label} is not configured or reachable${details}. Check API key, base URL, and local service state.`,
|
|
12170
13611
|
};
|
|
12171
13612
|
}
|
|
@@ -12248,10 +13689,16 @@ function providerSettingsLabel(id) {
|
|
|
12248
13689
|
}
|
|
12249
13690
|
}
|
|
12250
13691
|
function providerConfigDetails(id, config) {
|
|
13692
|
+
// Subscription providers (Claude Code / Codex) authenticate via OAuth/CLI, not an
|
|
13693
|
+
// API key — omit the key clause so we don't report a misleading "API key missing".
|
|
13694
|
+
const isSubscription = id === 'claude-code' || id === 'codex';
|
|
13695
|
+
const authNote = isSubscription
|
|
13696
|
+
? ''
|
|
13697
|
+
: id === 'ollama' ? 'local endpoint' : config.apiKey ? 'API key present' : 'API key missing';
|
|
12251
13698
|
const parts = [
|
|
12252
13699
|
config.model ? `model ${config.model}` : '',
|
|
12253
13700
|
config.baseUrl ? `base URL ${config.baseUrl}` : '',
|
|
12254
|
-
|
|
13701
|
+
authNote,
|
|
12255
13702
|
].filter(Boolean);
|
|
12256
13703
|
return parts.length ? ` (${parts.join(', ')})` : '';
|
|
12257
13704
|
}
|
|
@@ -12273,7 +13720,7 @@ function recordAgentRuntimeSchemaSnapshot(projectRoot, schemaContext, source) {
|
|
|
12273
13720
|
try {
|
|
12274
13721
|
recordRuntimeSchemaSnapshot(projectRoot, {
|
|
12275
13722
|
source,
|
|
12276
|
-
tables: schemaContext.
|
|
13723
|
+
tables: schemaContext.map((table) => ({
|
|
12277
13724
|
relation: table.relation,
|
|
12278
13725
|
schema: table.schema,
|
|
12279
13726
|
name: table.name,
|
|
@@ -12427,7 +13874,9 @@ function buildNotebookResearchReviewChecklist(run) {
|
|
|
12427
13874
|
const questionReady = Boolean(notebookResearchString(run.question));
|
|
12428
13875
|
const hasReviewedSql = Boolean(notebookResearchString(run.reviewedSql));
|
|
12429
13876
|
const hasGeneratedSql = Boolean(notebookResearchString(run.generatedSql));
|
|
13877
|
+
const hasDqlArtifact = Boolean(normalizeDqlArtifactReference(run.dqlArtifact)?.source);
|
|
12430
13878
|
const hasSql = hasReviewedSql || hasGeneratedSql;
|
|
13879
|
+
const hasPromotableSource = hasReviewedSql || hasDqlArtifact;
|
|
12431
13880
|
const preview = notebookResearchPreviewInfo(run.resultPreview);
|
|
12432
13881
|
const previewReady = run.status === 'ready' && preview.hasPreview;
|
|
12433
13882
|
const evidenceReady = notebookResearchEvidenceCount(run.evidence) > 0 || Boolean(run.contextPackId);
|
|
@@ -12446,8 +13895,8 @@ function buildNotebookResearchReviewChecklist(run) {
|
|
|
12446
13895
|
const warnings = [];
|
|
12447
13896
|
if (!questionReady)
|
|
12448
13897
|
blockers.push('Add a business question before review.');
|
|
12449
|
-
if (!hasSql && !reuseRecommended)
|
|
12450
|
-
blockers.push('Add reviewed SQL
|
|
13898
|
+
if (!hasSql && !hasDqlArtifact && !reuseRecommended)
|
|
13899
|
+
blockers.push('Add reviewed SQL, generate SQL, or attach a DQL artifact before promotion.');
|
|
12451
13900
|
if (run.status === 'error')
|
|
12452
13901
|
blockers.push(run.error ? `Fix preview error: ${run.error}` : 'Fix the failed preview before promotion.');
|
|
12453
13902
|
if (!previewReady && run.status !== 'error' && !reuseRecommended)
|
|
@@ -12473,15 +13922,17 @@ function buildNotebookResearchReviewChecklist(run) {
|
|
|
12473
13922
|
},
|
|
12474
13923
|
{
|
|
12475
13924
|
id: 'sql',
|
|
12476
|
-
label: 'Reviewed SQL',
|
|
12477
|
-
status: hasReviewedSql || (reuseRecommended && !hasSql) ? 'passed' : hasGeneratedSql ? 'warning' : 'blocked',
|
|
13925
|
+
label: hasDqlArtifact && !hasReviewedSql ? 'DQL artifact' : 'Reviewed SQL',
|
|
13926
|
+
status: hasReviewedSql || hasDqlArtifact || (reuseRecommended && !hasSql) ? 'passed' : hasGeneratedSql ? 'warning' : 'blocked',
|
|
12478
13927
|
detail: reuseRecommended && !hasSql
|
|
12479
13928
|
? 'Existing certified DQL should be reused; no new SQL is required.'
|
|
12480
|
-
: hasReviewedSql
|
|
12481
|
-
? '
|
|
12482
|
-
:
|
|
12483
|
-
? '
|
|
12484
|
-
:
|
|
13929
|
+
: hasDqlArtifact && !hasReviewedSql
|
|
13930
|
+
? 'Generated DQL artifact is saved for reviewer promotion.'
|
|
13931
|
+
: hasReviewedSql
|
|
13932
|
+
? 'Reviewer SQL is saved.'
|
|
13933
|
+
: hasGeneratedSql
|
|
13934
|
+
? 'Generated SQL exists; review it before promotion.'
|
|
13935
|
+
: 'No SQL is available for preview or DQL generation.',
|
|
12485
13936
|
},
|
|
12486
13937
|
{
|
|
12487
13938
|
id: 'preview',
|
|
@@ -12518,33 +13969,35 @@ function buildNotebookResearchReviewChecklist(run) {
|
|
|
12518
13969
|
{
|
|
12519
13970
|
id: 'parameters',
|
|
12520
13971
|
label: 'Parameter review',
|
|
12521
|
-
status: reuseRecommended && !hasSql ? 'passed' : !hasSql ? 'pending' : dynamicParameters.length > 0 ? 'passed' : 'warning',
|
|
13972
|
+
status: reuseRecommended && !hasSql ? 'passed' : !hasSql && !hasDqlArtifact ? 'pending' : hasDqlArtifact && !hasSql ? 'passed' : dynamicParameters.length > 0 ? 'passed' : 'warning',
|
|
12522
13973
|
detail: reuseRecommended && !hasSql
|
|
12523
13974
|
? 'Use the certified block parameters; no new SQL parameterization is required.'
|
|
12524
|
-
: !hasSql
|
|
12525
|
-
? '
|
|
12526
|
-
:
|
|
12527
|
-
?
|
|
12528
|
-
:
|
|
12529
|
-
? `
|
|
12530
|
-
:
|
|
13975
|
+
: !hasSql && hasDqlArtifact
|
|
13976
|
+
? 'Review parameters declared in the DQL artifact.'
|
|
13977
|
+
: !hasSql
|
|
13978
|
+
? 'Add SQL or a DQL artifact before checking runtime parameters.'
|
|
13979
|
+
: dynamicParameters.length > 0
|
|
13980
|
+
? `Detected ${dynamicParameters.length.toLocaleString()} dynamic parameter${dynamicParameters.length === 1 ? '' : 's'}: ${dynamicParameters.slice(0, 6).map((item) => item.name).join(', ')}.`
|
|
13981
|
+
: staticParameters.length > 0
|
|
13982
|
+
? `Only static or review-required literals were detected: ${staticParameters.slice(0, 6).map((item) => `${item.name} (${item.policy})`).join(', ')}.`
|
|
13983
|
+
: 'No reusable runtime parameters were detected. Certify as static only if the business question is intentionally fixed.',
|
|
12531
13984
|
},
|
|
12532
13985
|
{
|
|
12533
13986
|
id: 'dql_draft',
|
|
12534
13987
|
label: 'DQL draft',
|
|
12535
|
-
status: reuseRecommended && !hasDraft ? 'passed' : hasDraft ? 'passed' :
|
|
13988
|
+
status: reuseRecommended && !hasDraft ? 'passed' : hasDraft ? 'passed' : hasPromotableSource && evidenceReady ? 'pending' : 'blocked',
|
|
12536
13989
|
detail: reuseRecommended && !hasDraft
|
|
12537
13990
|
? 'No new draft is required because an existing DQL block should be reused.'
|
|
12538
13991
|
: hasDraft
|
|
12539
13992
|
? `Draft saved at ${run.draftBlockPath}.`
|
|
12540
|
-
: !
|
|
12541
|
-
? 'Save reviewed SQL before DQL draft creation.'
|
|
13993
|
+
: !hasPromotableSource
|
|
13994
|
+
? 'Save reviewed SQL or review the generated DQL artifact before DQL draft creation.'
|
|
12542
13995
|
: evidenceReady
|
|
12543
|
-
? 'Create a draft after SQL review.'
|
|
13996
|
+
? (hasDqlArtifact && !hasReviewedSql ? 'Create a draft after DQL artifact review.' : 'Create a draft after SQL review.')
|
|
12544
13997
|
: 'Save metadata evidence before creating a reusable DQL draft.',
|
|
12545
13998
|
},
|
|
12546
13999
|
];
|
|
12547
|
-
const readyForDqlDraft = questionReady &&
|
|
14000
|
+
const readyForDqlDraft = questionReady && hasPromotableSource && evidenceReady && run.status !== 'error';
|
|
12548
14001
|
const readyForCertificationReview = readyForDqlDraft && previewReady && hasDraft && !reuseRecommended && !promotionReviewRequired;
|
|
12549
14002
|
return { readyForDqlDraft, readyForCertificationReview, blockers, warnings, items };
|
|
12550
14003
|
}
|
|
@@ -12552,6 +14005,8 @@ function buildNotebookResearchPlan(input) {
|
|
|
12552
14005
|
const reviewedSql = notebookResearchString(input.reviewedSql);
|
|
12553
14006
|
const generatedSql = notebookResearchString(input.generatedSql);
|
|
12554
14007
|
const sql = reviewedSql ?? generatedSql;
|
|
14008
|
+
const dqlArtifact = normalizeDqlArtifactReference(input.dqlArtifact) ?? input.run.dqlArtifact;
|
|
14009
|
+
const hasDqlArtifact = Boolean(dqlArtifact?.source);
|
|
12555
14010
|
const parameterized = sql ? parameterizeSqlForDqlImport(sql) : null;
|
|
12556
14011
|
const evidence = input.evidence && typeof input.evidence === 'object' && !Array.isArray(input.evidence)
|
|
12557
14012
|
? input.evidence
|
|
@@ -12590,6 +14045,7 @@ function buildNotebookResearchPlan(input) {
|
|
|
12590
14045
|
promotion: {
|
|
12591
14046
|
path: notebookResearchPlanPromotionPath({
|
|
12592
14047
|
hasSql: Boolean(sql),
|
|
14048
|
+
hasDqlArtifact,
|
|
12593
14049
|
hasEvidence,
|
|
12594
14050
|
hasPreview,
|
|
12595
14051
|
previewError: input.previewError,
|
|
@@ -12600,6 +14056,7 @@ function buildNotebookResearchPlan(input) {
|
|
|
12600
14056
|
},
|
|
12601
14057
|
reviewFocus: notebookResearchPlanFocus(input.run.intent, {
|
|
12602
14058
|
hasSql: Boolean(sql),
|
|
14059
|
+
hasDqlArtifact,
|
|
12603
14060
|
hasEvidence,
|
|
12604
14061
|
hasPreview,
|
|
12605
14062
|
dynamicParameterCount: parameterized?.parameterPolicy.filter((entry) => entry.policy === 'dynamic').length ?? 0,
|
|
@@ -12612,7 +14069,7 @@ function buildNotebookResearchPlan(input) {
|
|
|
12612
14069
|
function notebookResearchPlanPromotionPath(input) {
|
|
12613
14070
|
if (input.promotionAction === 'reuse_existing')
|
|
12614
14071
|
return 'reuse_existing';
|
|
12615
|
-
if (!input.hasSql)
|
|
14072
|
+
if (!input.hasSql && !input.hasDqlArtifact)
|
|
12616
14073
|
return 'needs_sql';
|
|
12617
14074
|
if (!input.hasEvidence)
|
|
12618
14075
|
return 'review_context';
|
|
@@ -12624,8 +14081,8 @@ function notebookResearchPlanPromotionPath(input) {
|
|
|
12624
14081
|
}
|
|
12625
14082
|
function notebookResearchPlanFocus(intent, input) {
|
|
12626
14083
|
const focus = new Set();
|
|
12627
|
-
if (!input.hasSql)
|
|
12628
|
-
focus.add('Add reviewed SQL or configure AI
|
|
14084
|
+
if (!input.hasSql && !input.hasDqlArtifact)
|
|
14085
|
+
focus.add('Add reviewed SQL, a DQL artifact, or configure AI generation.');
|
|
12629
14086
|
if (!input.hasEvidence)
|
|
12630
14087
|
focus.add('Save metadata context evidence before DQL promotion.');
|
|
12631
14088
|
if (!input.hasPreview)
|
|
@@ -13096,7 +14553,7 @@ function dedupeAgentSchemaColumns(columns) {
|
|
|
13096
14553
|
}
|
|
13097
14554
|
return Array.from(byName.values());
|
|
13098
14555
|
}
|
|
13099
|
-
export function buildAgentSchemaContext(question, rows) {
|
|
14556
|
+
export function buildAgentSchemaContext(question, rows, options = {}) {
|
|
13100
14557
|
const byRelation = new Map();
|
|
13101
14558
|
for (const row of rows) {
|
|
13102
14559
|
if (!row || typeof row !== 'object')
|
|
@@ -13125,16 +14582,82 @@ export function buildAgentSchemaContext(question, rows) {
|
|
|
13125
14582
|
}
|
|
13126
14583
|
const tokens = agentSchemaTokens(question);
|
|
13127
14584
|
const shouldProbeValues = extractAgentValueSearchTerms(question).length > 0;
|
|
14585
|
+
const limit = Math.max(1, Math.trunc(options.limit ?? 12));
|
|
13128
14586
|
return Array.from(byRelation.values())
|
|
13129
14587
|
.map((table) => ({
|
|
13130
14588
|
table,
|
|
13131
14589
|
score: scoreAgentSchemaTable(table, tokens) + (shouldProbeValues ? scoreAgentValueProbeTable(table) : 0),
|
|
13132
14590
|
}))
|
|
13133
|
-
.filter((entry) => entry.score > 0)
|
|
14591
|
+
.filter((entry) => options.includeUnscored || entry.score > 0)
|
|
13134
14592
|
.sort((a, b) => b.score - a.score || a.table.relation.localeCompare(b.table.relation))
|
|
13135
|
-
.slice(0,
|
|
14593
|
+
.slice(0, limit)
|
|
13136
14594
|
.map((entry) => entry.table);
|
|
13137
14595
|
}
|
|
14596
|
+
function mergeAgentSchemaContexts(...contexts) {
|
|
14597
|
+
const byRelation = new Map();
|
|
14598
|
+
for (const table of contexts.flat()) {
|
|
14599
|
+
const key = table.relation.toLowerCase();
|
|
14600
|
+
const existing = byRelation.get(key);
|
|
14601
|
+
if (!existing) {
|
|
14602
|
+
byRelation.set(key, {
|
|
14603
|
+
...table,
|
|
14604
|
+
columns: dedupeAgentSchemaColumns(table.columns).slice(0, 80),
|
|
14605
|
+
});
|
|
14606
|
+
continue;
|
|
14607
|
+
}
|
|
14608
|
+
byRelation.set(key, {
|
|
14609
|
+
...existing,
|
|
14610
|
+
description: existing.description ?? table.description,
|
|
14611
|
+
source: existing.source === table.source ? existing.source : 'catalog and runtime schema',
|
|
14612
|
+
columns: dedupeAgentSchemaColumns([...existing.columns, ...table.columns]).slice(0, 80),
|
|
14613
|
+
});
|
|
14614
|
+
}
|
|
14615
|
+
return Array.from(byRelation.values()).slice(0, 16);
|
|
14616
|
+
}
|
|
14617
|
+
function mergeAgentSchemaSampleValues(snapshot, enriched) {
|
|
14618
|
+
const samples = new Map();
|
|
14619
|
+
for (const table of enriched) {
|
|
14620
|
+
for (const column of table.columns) {
|
|
14621
|
+
if (!column.sampleValues?.length)
|
|
14622
|
+
continue;
|
|
14623
|
+
samples.set(`${table.relation.toLowerCase()}\u0000${column.name.toLowerCase()}`, column.sampleValues);
|
|
14624
|
+
}
|
|
14625
|
+
}
|
|
14626
|
+
if (samples.size === 0)
|
|
14627
|
+
return snapshot;
|
|
14628
|
+
return snapshot.map((table) => ({
|
|
14629
|
+
...table,
|
|
14630
|
+
columns: table.columns.map((column) => {
|
|
14631
|
+
const sampleValues = samples.get(`${table.relation.toLowerCase()}\u0000${column.name.toLowerCase()}`);
|
|
14632
|
+
return sampleValues?.length
|
|
14633
|
+
? { ...column, sampleValues: uniqueStrings([...(column.sampleValues ?? []), ...sampleValues]).slice(0, 8) }
|
|
14634
|
+
: column;
|
|
14635
|
+
}),
|
|
14636
|
+
}));
|
|
14637
|
+
}
|
|
14638
|
+
// Common English plurals that are NOT entity nouns, so a multi-entity heuristic
|
|
14639
|
+
// doesn't count them. Domain-agnostic (no project-specific vocabulary).
|
|
14640
|
+
const NON_ENTITY_PLURALS = new Set([
|
|
14641
|
+
'details', 'values', 'results', 'rows', 'numbers', 'records', 'items', 'things',
|
|
14642
|
+
'totals', 'amounts', 'metrics', 'columns', 'fields', 'names', 'ids',
|
|
14643
|
+
]);
|
|
14644
|
+
/**
|
|
14645
|
+
* Decide whether to run the (advisory, best-effort) runtime value-scan + schema
|
|
14646
|
+
* enrichment for a question. Generic multi-entity / detail intent — no hard-coded
|
|
14647
|
+
* project vocabulary — so it fires for ANY repo's join-shaped questions, not just
|
|
14648
|
+
* jaffle. Over-triggering only costs a few extra bounded probes (all in try/catch).
|
|
14649
|
+
*/
|
|
14650
|
+
function shouldAugmentAgentRuntimeSchema(question) {
|
|
14651
|
+
const lower = question.toLowerCase();
|
|
14652
|
+
const wantsMetric = /\brevenu|\bsales\b|\bamount\b|\bspend\b|\bcost\b|\bcount\b|\btotal\b|\bsum\b|\bavg\b|\baverage\b|\bvalue\b|\bprice\b/.test(lower);
|
|
14653
|
+
const wantsDetail = /\bdetails?\b|\bcomplete\b|\bbreakdown\b|\ball\s+values?\b|\beach\b|\bevery\b/.test(lower);
|
|
14654
|
+
const referencesPriorRows = /\b(?:above|previous|prior|these|those|same)\s+\w+/.test(lower);
|
|
14655
|
+
const explicitJoin = /\bjoin(?:ed|ing)?\b|\bcombine[ds]?\b|\bcross[- ]?reference\b|\balong\s+with\b|\btogether\s+with\b/.test(lower);
|
|
14656
|
+
// Multi-entity intent: a linking word + >= 2 distinct plural content nouns.
|
|
14657
|
+
const nouns = new Set((lower.match(/\b[a-z]{4,}s\b/g) ?? []).filter((word) => !NON_ENTITY_PLURALS.has(word)));
|
|
14658
|
+
const multiEntity = /\b(?:and|with|plus|per|for\s+each|by)\b/.test(lower) && nouns.size >= 2;
|
|
14659
|
+
return explicitJoin || referencesPriorRows || (multiEntity && (wantsMetric || wantsDetail));
|
|
14660
|
+
}
|
|
13138
14661
|
async function enrichAgentSchemaContextWithValueMatches(question, schemaContext, executor, connection) {
|
|
13139
14662
|
const searchTerms = extractAgentValueSearchTerms(question);
|
|
13140
14663
|
if (schemaContext.length === 0 || searchTerms.length === 0)
|
|
@@ -13186,8 +14709,57 @@ function scoreAgentSchemaTable(table, tokens) {
|
|
|
13186
14709
|
score += 3;
|
|
13187
14710
|
}
|
|
13188
14711
|
}
|
|
13189
|
-
|
|
14712
|
+
score += scoreCompositeAgentSchemaTable(table, tokens);
|
|
14713
|
+
// Small tiebreaker toward modeled analytics tables (fct_/dim_/fact_/mart_ …),
|
|
14714
|
+
// a warehouse-modeling convention that generalizes across repos — not a fixed
|
|
14715
|
+
// list of jaffle entity names.
|
|
14716
|
+
if (/^(?:fct|fact|dim|dimension|mart|rpt|report|agg|f|d)[_-]/i.test(table.name)
|
|
14717
|
+
|| /(?:^|[_-])(?:fact|dim|mart|summary|report)(?:$|[_-])/i.test(table.name)) {
|
|
13190
14718
|
score += 1;
|
|
14719
|
+
}
|
|
14720
|
+
return score;
|
|
14721
|
+
}
|
|
14722
|
+
const COMPOSITE_METRIC_TOKENS = new Set([
|
|
14723
|
+
'revenue', 'sales', 'amount', 'spend', 'price', 'cost', 'total', 'sum',
|
|
14724
|
+
'count', 'quantity', 'qty', 'value', 'subtotal', 'order', 'orders',
|
|
14725
|
+
]);
|
|
14726
|
+
const COMPOSITE_STOPWORD_TOKENS = new Set([
|
|
14727
|
+
'the', 'and', 'with', 'for', 'each', 'per', 'all', 'top', 'show', 'list',
|
|
14728
|
+
'give', 'get', 'find', 'plus', 'along', 'together', 'combined', 'combine',
|
|
14729
|
+
'detail', 'details', 'complete', 'breakdown', 'values', 'name', 'names',
|
|
14730
|
+
...COMPOSITE_METRIC_TOKENS,
|
|
14731
|
+
]);
|
|
14732
|
+
/**
|
|
14733
|
+
* Boost the fact/junction table for a multi-entity ("composite") question. Fully
|
|
14734
|
+
* domain-agnostic: the ask is detected from >= 2 distinct entity tokens + a metric,
|
|
14735
|
+
* and the winning table is identified STRUCTURALLY (>= 2 foreign-key-style id columns
|
|
14736
|
+
* plus a numeric measure column) rather than by hard-coded jaffle table names — so it
|
|
14737
|
+
* ranks the right join table for any repo, while jaffle's fct_orders still qualifies.
|
|
14738
|
+
*/
|
|
14739
|
+
function scoreCompositeAgentSchemaTable(table, tokens) {
|
|
14740
|
+
const entityTokens = [...tokens].filter((token) => token.length >= 3 && !COMPOSITE_STOPWORD_TOKENS.has(token));
|
|
14741
|
+
const hasMetric = [...tokens].some((token) => COMPOSITE_METRIC_TOKENS.has(token));
|
|
14742
|
+
const hasCompositeAsk = entityTokens.length >= 2 && hasMetric;
|
|
14743
|
+
const hasPriorRowsAsk = ['above', 'previous', 'prior', 'these', 'those', 'same'].some((token) => tokens.has(token))
|
|
14744
|
+
&& entityTokens.length >= 1;
|
|
14745
|
+
if (!hasCompositeAsk && !hasPriorRowsAsk)
|
|
14746
|
+
return 0;
|
|
14747
|
+
const columns = table.columns.map((column) => column.name.toLowerCase());
|
|
14748
|
+
let score = 0;
|
|
14749
|
+
// Structural fact/junction-table signal: multiple FK-style id columns + a measure.
|
|
14750
|
+
const idColumnCount = columns.filter((name) => /(?:^id$|_id$|_key$|_sk$|_uuid$|_fk$)/.test(name)).length;
|
|
14751
|
+
const hasMeasureColumn = columns.some((name) => /revenue|sales|amount|price|spend|cost|subtotal|total|quantity|\bqty\b|count|value/.test(name));
|
|
14752
|
+
if (idColumnCount >= 2 && hasMeasureColumn)
|
|
14753
|
+
score += 18;
|
|
14754
|
+
else if (idColumnCount >= 2)
|
|
14755
|
+
score += 8;
|
|
14756
|
+
if (hasMeasureColumn)
|
|
14757
|
+
score += 8;
|
|
14758
|
+
// Reward columns whose names overlap the question's actual entity tokens (any domain).
|
|
14759
|
+
for (const token of entityTokens) {
|
|
14760
|
+
if (columns.some((name) => name.includes(token)))
|
|
14761
|
+
score += 4;
|
|
14762
|
+
}
|
|
13191
14763
|
return score;
|
|
13192
14764
|
}
|
|
13193
14765
|
function scoreAgentValueProbeTable(table) {
|