@duckcodeailabs/dql-cli 1.6.30 → 1.6.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/apps-api.d.ts.map +1 -1
- package/dist/apps-api.js +3 -1
- package/dist/apps-api.js.map +1 -1
- package/dist/args.d.ts +18 -0
- package/dist/args.d.ts.map +1 -1
- package/dist/args.js +37 -0
- package/dist/args.js.map +1 -1
- package/dist/assets/dql-notebook/assets/index-B6x1cRJo.js +5698 -0
- package/dist/assets/dql-notebook/index.html +1 -1
- package/dist/commands/agent.d.ts +47 -1
- package/dist/commands/agent.d.ts.map +1 -1
- package/dist/commands/agent.js +635 -27
- package/dist/commands/agent.js.map +1 -1
- package/dist/commands/app.d.ts.map +1 -1
- package/dist/commands/app.js +3 -1
- package/dist/commands/app.js.map +1 -1
- package/dist/commands/diff.d.ts.map +1 -1
- package/dist/commands/diff.js +111 -22
- package/dist/commands/diff.js.map +1 -1
- package/dist/commands/eval-judge.d.ts +37 -0
- package/dist/commands/eval-judge.d.ts.map +1 -0
- package/dist/commands/eval-judge.js +99 -0
- package/dist/commands/eval-judge.js.map +1 -0
- package/dist/commands/eval.d.ts +27 -1
- package/dist/commands/eval.d.ts.map +1 -1
- package/dist/commands/eval.js +143 -6
- package/dist/commands/eval.js.map +1 -1
- package/dist/commands/mcp.js +27 -3
- package/dist/commands/mcp.js.map +1 -1
- package/dist/index.js +6 -0
- package/dist/index.js.map +1 -1
- package/dist/llm/analytics-tools.d.ts.map +1 -1
- package/dist/llm/analytics-tools.js +12 -7
- package/dist/llm/analytics-tools.js.map +1 -1
- package/dist/llm/answer-loop-tools.d.ts +7 -0
- package/dist/llm/answer-loop-tools.d.ts.map +1 -0
- package/dist/llm/answer-loop-tools.js +67 -0
- package/dist/llm/answer-loop-tools.js.map +1 -0
- package/dist/llm/proposal-metadata.d.ts +5 -0
- package/dist/llm/proposal-metadata.d.ts.map +1 -0
- package/dist/llm/proposal-metadata.js +27 -0
- package/dist/llm/proposal-metadata.js.map +1 -0
- package/dist/llm/providers/claude-agent-sdk.d.ts +6 -0
- package/dist/llm/providers/claude-agent-sdk.d.ts.map +1 -1
- package/dist/llm/providers/claude-agent-sdk.js +17 -5
- package/dist/llm/providers/claude-agent-sdk.js.map +1 -1
- package/dist/llm/providers/claude-code.d.ts +7 -1
- package/dist/llm/providers/claude-code.d.ts.map +1 -1
- package/dist/llm/providers/claude-code.js +11 -2
- package/dist/llm/providers/claude-code.js.map +1 -1
- package/dist/llm/providers/dql-agent-provider.d.ts +29 -0
- package/dist/llm/providers/dql-agent-provider.d.ts.map +1 -1
- package/dist/llm/providers/dql-agent-provider.js +777 -32
- package/dist/llm/providers/dql-agent-provider.js.map +1 -1
- package/dist/llm/providers/native-sdk-provider.d.ts.map +1 -1
- package/dist/llm/providers/native-sdk-provider.js +7 -1
- package/dist/llm/providers/native-sdk-provider.js.map +1 -1
- package/dist/llm/tools.d.ts.map +1 -1
- package/dist/llm/tools.js +34 -137
- package/dist/llm/tools.js.map +1 -1
- package/dist/llm/types.d.ts +73 -2
- package/dist/llm/types.d.ts.map +1 -1
- package/dist/local-runtime.d.ts +160 -5
- package/dist/local-runtime.d.ts.map +1 -1
- package/dist/local-runtime.js +1862 -181
- package/dist/local-runtime.js.map +1 -1
- package/dist/package.json +10 -10
- package/dist/providers/oauth/claude-oauth.d.ts +80 -0
- package/dist/providers/oauth/claude-oauth.d.ts.map +1 -0
- package/dist/providers/oauth/claude-oauth.js +352 -0
- package/dist/providers/oauth/claude-oauth.js.map +1 -0
- package/dist/providers/oauth/codex-oauth.d.ts +70 -0
- package/dist/providers/oauth/codex-oauth.d.ts.map +1 -0
- package/dist/providers/oauth/codex-oauth.js +413 -0
- package/dist/providers/oauth/codex-oauth.js.map +1 -0
- package/dist/providers/oauth/oauth-store.d.ts +31 -0
- package/dist/providers/oauth/oauth-store.d.ts.map +1 -0
- package/dist/providers/oauth/oauth-store.js +61 -0
- package/dist/providers/oauth/oauth-store.js.map +1 -0
- package/dist/settings/provider-settings.d.ts +18 -0
- package/dist/settings/provider-settings.d.ts.map +1 -1
- package/dist/settings/provider-settings.js +44 -10
- package/dist/settings/provider-settings.js.map +1 -1
- package/package.json +11 -11
- package/dist/assets/dql-notebook/assets/index-QJKCrLXe.js +0 -5648
package/dist/local-runtime.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { execFileSync, execSync } from 'node:child_process';
|
|
2
|
+
import { createHash } from 'node:crypto';
|
|
2
3
|
import { createServer } from 'node:http';
|
|
3
4
|
import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, watch, writeFileSync } from 'node:fs';
|
|
4
5
|
import { createRequire } from 'node:module';
|
|
@@ -7,17 +8,19 @@ import { dirname, extname, join, normalize, relative, resolve } from 'node:path'
|
|
|
7
8
|
import Anthropic from '@anthropic-ai/sdk';
|
|
8
9
|
import OpenAI from 'openai';
|
|
9
10
|
import { buildExecutionPlan, createWelcomeNotebook, deserializeNotebook, getConnectorFormSchemas, hasSemanticRefs, resolveSemanticRefs, } from '@duckcodeailabs/dql-notebook';
|
|
10
|
-
import { loadSemanticLayerFromDir, resolveSemanticLayerAsync, getDialect, Parser, buildLineageGraph, buildManifest, findAppDocuments, findDashboardsForApp, isBlockIdRef, loadAppDocument, loadDashboardDocument, analyzeImpact, buildTrustChain, detectDomainFlows, getDomainTrustOverview, queryLineage, queryBusiness360, queryCompleteLineagePaths, LineageGraph, canonicalize, canonicalizeNotebook, diffDQL, diffNotebook, writeDomainDeclaration, deleteDomainDeclaration, domainFolderSlug, } from '@duckcodeailabs/dql-core';
|
|
11
|
+
import { loadSemanticLayerFromDir, normalizeDqlArtifactReference, serializeMetricDefinitionToYaml, resolveSemanticLayerAsync, getDialect, Parser, buildLineageGraph, buildManifest, findAppDocuments, findDashboardsForApp, isBlockIdRef, loadAppDocument, loadDashboardDocument, analyzeImpact, buildTrustChain, detectDomainFlows, getDomainTrustOverview, queryLineage, queryBusiness360, queryCompleteLineagePaths, LineageGraph, canonicalize, canonicalizeNotebook, diffDQL, diffNotebook, writeDomainDeclaration, deleteDomainDeclaration, domainFolderSlug, } from '@duckcodeailabs/dql-core';
|
|
11
12
|
import { load as loadYaml } from 'js-yaml';
|
|
12
13
|
import { listBlockTemplates } from './block-templates.js';
|
|
13
14
|
import { getRunner as getLLMRunner } from './llm/index.js';
|
|
14
15
|
import { createDqlAgentProviderRunner } from './llm/providers/dql-agent-provider.js';
|
|
15
16
|
import { listRemoteMcpSettings, saveRemoteMcpSettings } from './llm/mcp-config.js';
|
|
16
|
-
import { ClaudeProvider, GeminiProvider, MemoryStore, OllamaProvider, OpenAIProvider, buildBlockBusinessFingerprint, buildBlockSqlFingerprints, buildLocalContextPack, defaultMemoryPath, ensureDefaultMemoryFiles, ensureMetadataCatalogFresh, propose, proposePlan, recordCorrectionTrace, reviewHint, AgentRunEngine, FileAgentRunStore, defaultAgentRunGates, createLlmAgentRunPlanner, createHybridRouter, synthesizeAnswer, streamOrGenerate, narrateResult, normalizeAnthropicBaseUrl, buildProposePreview, buildFromPrompt, defaultAgentRunStorePath, resolveLocalOwner, resolveProposeConfig, recordQueryRun, recordRuntimeSchemaSnapshot, loadSkills, writeSkill, deleteSkill, reindexProject, defaultKgPath, planApp, planResearch, loadSemanticMetrics, } from '@duckcodeailabs/dql-agent';
|
|
17
|
+
import { ClaudeProvider, ConversationStore, advanceThreadState, buildConversationSnapshot, recallRelevantTurns, GeminiProvider, MemoryStore, OllamaProvider, OpenAIProvider, buildBlockBusinessFingerprint, buildBlockSqlFingerprints, buildLocalContextPack, defaultConversationPath, defaultMemoryPath, ensureDefaultMemoryFiles, ensureMetadataCatalogFresh, propose, proposePlan, recordCorrectionTrace, reviewHint, AgentRunEngine, FileAgentRunStore, defaultAgentRunGates, createLlmAgentRunPlanner, createHybridRouter, computeResultStats, synthesizeAnswer, streamOrGenerate, narrateResult, normalizeAnthropicBaseUrl, buildProposePreview, buildFromPrompt, defaultAgentRunStorePath, resolveLocalOwner, resolveProposeConfig, recordQueryRun, recordRuntimeSchemaSnapshot, loadSkills, writeSkill, deleteSkill, deriveGeneratedDraftSlug, reindexProject, defaultKgPath, planApp, planResearch, loadSemanticMetrics, routeReasoningEffort, routeForCascadeAnswerTier, clampReasoningEffort, bumpReasoningEffort, upsertGeneratedDqlArtifactDraft, } from '@duckcodeailabs/dql-agent';
|
|
17
18
|
import { gatherProposeEnrichment } from './propose-enrich.js';
|
|
18
19
|
import { handleAppsApi, proposeAppAiBuild, recommendVisualization } from './apps-api.js';
|
|
19
|
-
import { getActiveProvider, getEffectiveProviderConfig, listProviderSettings, saveProviderSettings, } from './settings/provider-settings.js';
|
|
20
|
+
import { getActiveProvider, getEffectiveProviderConfig, isProviderSettingsId, listProviderSettings, saveProviderSettings, } from './settings/provider-settings.js';
|
|
20
21
|
import { ClaudeCodeCliProvider, CodexCliProvider } from './providers/subscription-cli.js';
|
|
22
|
+
import { ClaudeOAuthManager, ClaudeOAuthProvider, claudeOAuthConnected, CLAUDE_OAUTH_MODELS, CLAUDE_OAUTH_DEFAULT_MODEL, } from './providers/oauth/claude-oauth.js';
|
|
23
|
+
import { CodexOAuthManager, CodexOAuthProvider, codexOAuthConnected, CODEX_OAUTH_MODELS, CODEX_OAUTH_DEFAULT_MODEL, } from './providers/oauth/codex-oauth.js';
|
|
21
24
|
import { DQLAccessDeniedError, activePersonaAppId, assertAppAccess, loadRuntimeApp, runtimeVariables, } from './governance-runtime.js';
|
|
22
25
|
import { LocalAppStorage, LocalNotebookResearchStorage, defaultLocalAppsDbPath, defaultNotebookResearchDbPath } from '@duckcodeailabs/dql-project';
|
|
23
26
|
import { Certifier, ENTERPRISE_RULES, evaluateInvariants, hasInvariantViolation, } from '@duckcodeailabs/dql-governance';
|
|
@@ -76,6 +79,16 @@ function parseAgentRunRequestedMode(value) {
|
|
|
76
79
|
? value
|
|
77
80
|
: undefined;
|
|
78
81
|
}
|
|
82
|
+
function parseAgentRunReasoningEffort(value) {
|
|
83
|
+
return value === 'low' || value === 'medium' || value === 'high'
|
|
84
|
+
? value
|
|
85
|
+
: undefined;
|
|
86
|
+
}
|
|
87
|
+
function parseAgentRunAnalysisDepth(value) {
|
|
88
|
+
return value === 'quick' || value === 'deep'
|
|
89
|
+
? value
|
|
90
|
+
: undefined;
|
|
91
|
+
}
|
|
79
92
|
function parseAgentRunSelectedObject(value) {
|
|
80
93
|
const record = agentRunRecord(value);
|
|
81
94
|
if (!record)
|
|
@@ -103,7 +116,7 @@ function parseAgentRunHistory(value) {
|
|
|
103
116
|
});
|
|
104
117
|
return history.length > 0 ? history.slice(-20) : undefined;
|
|
105
118
|
}
|
|
106
|
-
function parseAgentRunRequestBody(body) {
|
|
119
|
+
export function parseAgentRunRequestBody(body) {
|
|
107
120
|
const record = agentRunRecord(body);
|
|
108
121
|
if (!record)
|
|
109
122
|
return { error: 'Invalid JSON body.' };
|
|
@@ -126,11 +139,181 @@ function parseAgentRunRequestBody(body) {
|
|
|
126
139
|
signals: signals,
|
|
127
140
|
selectedObject,
|
|
128
141
|
workspaceContext,
|
|
142
|
+
conversationContext: agentRunRecord(record.conversationContext),
|
|
129
143
|
history: parseAgentRunHistory(record.history),
|
|
144
|
+
threadId: agentRunString(record.threadId),
|
|
130
145
|
runId: agentRunString(record.runId),
|
|
146
|
+
reasoningEffort: parseAgentRunReasoningEffort(record.reasoningEffort),
|
|
147
|
+
analysisDepth: parseAgentRunAnalysisDepth(record.analysisDepth) ?? parseAgentRunAnalysisDepth(record.depth),
|
|
131
148
|
},
|
|
132
149
|
};
|
|
133
150
|
}
|
|
151
|
+
export function shouldSynthesizeAgentRunAnswer(governedAnswer) {
|
|
152
|
+
if (governedAnswer.kind === 'no_answer')
|
|
153
|
+
return false;
|
|
154
|
+
if (governedAnswer.kind === 'certified' || governedAnswer.certification === 'certified')
|
|
155
|
+
return false;
|
|
156
|
+
const finalText = (governedAnswer.answer ?? governedAnswer.text ?? '').trim();
|
|
157
|
+
if (governedAnswer.dqlArtifact && finalText)
|
|
158
|
+
return false;
|
|
159
|
+
return true;
|
|
160
|
+
}
|
|
161
|
+
// ── Server-side conversation threads ────────────────────────────────────────
|
|
162
|
+
// When a run carries a threadId, the persisted thread is the authoritative
|
|
163
|
+
// source of prior turns (survives refresh); the client-built context remains
|
|
164
|
+
// the fallback for embedders that never send a threadId.
|
|
165
|
+
async function conversationContextFromThread(store, threadId, clientContext, question) {
|
|
166
|
+
const turns = store.recentTurns(threadId, 8).map((turn) => {
|
|
167
|
+
const contract = turn.contract ?? {};
|
|
168
|
+
const topN = contract.topN;
|
|
169
|
+
const topNValue = typeof topN === 'number'
|
|
170
|
+
? topN
|
|
171
|
+
: topN && typeof topN === 'object' && typeof topN.n === 'number'
|
|
172
|
+
? topN.n
|
|
173
|
+
: undefined;
|
|
174
|
+
return compactConversationRecord({
|
|
175
|
+
id: turn.id,
|
|
176
|
+
question: turn.question,
|
|
177
|
+
answerSummary: turn.answerSummary,
|
|
178
|
+
sourceCertifiedBlock: turn.sourceCertifiedBlock,
|
|
179
|
+
route: turn.route,
|
|
180
|
+
trustLabel: turn.trustLabel,
|
|
181
|
+
certification: turn.certification,
|
|
182
|
+
contextPackId: turn.contextPackId,
|
|
183
|
+
dqlArtifact: turn.dqlArtifact,
|
|
184
|
+
cascade: turn.cascade,
|
|
185
|
+
sourceSql: turn.sql,
|
|
186
|
+
requestedFilters: conversationStringArray(contract.filters),
|
|
187
|
+
requestedDimensions: conversationStringArray(contract.dimensions),
|
|
188
|
+
requestedMeasures: conversationStringArray(contract.measures),
|
|
189
|
+
topN: topNValue,
|
|
190
|
+
result: turn.result
|
|
191
|
+
? compactConversationRecord({
|
|
192
|
+
columns: turn.result.columns,
|
|
193
|
+
rowsSample: turn.result.rowsSample,
|
|
194
|
+
dimensionValues: turn.result.dimensionValues,
|
|
195
|
+
measureColumns: turn.result.measureColumns,
|
|
196
|
+
rowCount: turn.result.rowCount,
|
|
197
|
+
})
|
|
198
|
+
: undefined,
|
|
199
|
+
});
|
|
200
|
+
}).filter((turn) => Boolean(turn));
|
|
201
|
+
const thread = store.getThread(threadId);
|
|
202
|
+
// Bounded structured snapshot (working state + rolling summary + topic relation)
|
|
203
|
+
// for the answer loop's conversation-state prompt section.
|
|
204
|
+
const serverSnapshot = buildConversationSnapshot(store, threadId, { question });
|
|
205
|
+
if (serverSnapshot && question) {
|
|
206
|
+
// Semantic recall over OLDER turns (the recent window is already verbatim).
|
|
207
|
+
serverSnapshot.recalledTurns = await recallRelevantTurns(store, threadId, question, {
|
|
208
|
+
limit: 3,
|
|
209
|
+
excludeTurnIds: serverSnapshot.recentTurns.map((turn) => turn.id),
|
|
210
|
+
});
|
|
211
|
+
}
|
|
212
|
+
return {
|
|
213
|
+
...(clientContext ?? {}),
|
|
214
|
+
conversationStateVersion: 1,
|
|
215
|
+
threadId,
|
|
216
|
+
...(serverSnapshot ? { serverSnapshot } : {}),
|
|
217
|
+
...(thread?.rollingSummary ? { conversationSummary: thread.rollingSummary } : {}),
|
|
218
|
+
...(turns.length > 0 ? { turns, activeTurnId: turns[turns.length - 1].id } : {}),
|
|
219
|
+
};
|
|
220
|
+
}
|
|
221
|
+
/** Best-effort: persist a completed run as a conversation turn (never throws). */
|
|
222
|
+
function recordConversationTurn(store, threadId, run) {
|
|
223
|
+
if (!store || !threadId)
|
|
224
|
+
return;
|
|
225
|
+
try {
|
|
226
|
+
if (!store.getThread(threadId))
|
|
227
|
+
return;
|
|
228
|
+
const turn = store.appendTurn(threadId, conversationTurnInputFromRun(run));
|
|
229
|
+
// Fold the turn into the thread's working state + rolling summary (incremental).
|
|
230
|
+
advanceThreadState(store, threadId, turn);
|
|
231
|
+
}
|
|
232
|
+
catch {
|
|
233
|
+
// Conversation persistence is additive; a failed write must not fail the run.
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
function conversationTurnInputFromRun(run) {
|
|
237
|
+
const artifact = run.artifacts.find((candidate) => candidate.kind === 'answer')
|
|
238
|
+
?? run.artifacts.find((candidate) => candidate.kind === 'research_run')
|
|
239
|
+
?? run.artifacts[0];
|
|
240
|
+
const payload = agentRunRecord(artifact?.payload);
|
|
241
|
+
const result = agentRunRecord(payload?.result);
|
|
242
|
+
const columns = conversationResultColumns(result?.columns);
|
|
243
|
+
const rows = Array.isArray(result?.rows)
|
|
244
|
+
? result.rows.filter((row) => Boolean(row && typeof row === 'object' && !Array.isArray(row))).slice(0, 8)
|
|
245
|
+
: [];
|
|
246
|
+
const rowsSample = rows.map((row) => columns.map((column) => row[column]));
|
|
247
|
+
const contextPack = agentRunRecord(payload?.contextPack);
|
|
248
|
+
const questionPlan = agentRunRecord(contextPack?.questionPlan);
|
|
249
|
+
const requestedShape = agentRunRecord(questionPlan?.requestedShape);
|
|
250
|
+
const rowCountRaw = result?.rowCount;
|
|
251
|
+
const measureColumns = conversationMeasureColumns(columns, requestedShape, rows);
|
|
252
|
+
return {
|
|
253
|
+
question: run.question,
|
|
254
|
+
answerSummary: run.answer ?? run.summary,
|
|
255
|
+
route: run.route,
|
|
256
|
+
trustLabel: agentRunString(payload?.trustLabel) ?? run.trustState,
|
|
257
|
+
certification: agentRunString(payload?.certification),
|
|
258
|
+
sourceCertifiedBlock: agentRunString(payload?.sourceCertifiedBlock)
|
|
259
|
+
?? (artifact?.kind === 'answer' ? agentRunString(artifact.ref) : undefined),
|
|
260
|
+
contextPackId: agentRunString(payload?.contextPackId),
|
|
261
|
+
sql: agentRunString(payload?.proposedSql) ?? agentRunString(payload?.sql),
|
|
262
|
+
dqlArtifact: agentRunRecord(payload?.dqlArtifact),
|
|
263
|
+
cascade: agentRunRecord(payload?.cascade),
|
|
264
|
+
result: columns.length > 0 || rows.length > 0
|
|
265
|
+
? {
|
|
266
|
+
columns,
|
|
267
|
+
rowsSample,
|
|
268
|
+
dimensionValues: conversationDimensionValues(columns, rows),
|
|
269
|
+
measureColumns,
|
|
270
|
+
rowCount: typeof rowCountRaw === 'number' ? rowCountRaw : rows.length || undefined,
|
|
271
|
+
}
|
|
272
|
+
: undefined,
|
|
273
|
+
contract: requestedShape,
|
|
274
|
+
};
|
|
275
|
+
}
|
|
276
|
+
function conversationResultColumns(value) {
|
|
277
|
+
if (!Array.isArray(value))
|
|
278
|
+
return [];
|
|
279
|
+
return value
|
|
280
|
+
.map((column) => typeof column === 'string'
|
|
281
|
+
? column
|
|
282
|
+
: agentRunString(column?.name) ?? '')
|
|
283
|
+
.filter(Boolean)
|
|
284
|
+
.slice(0, 24);
|
|
285
|
+
}
|
|
286
|
+
function conversationMeasureColumns(columns, requestedShape, rows) {
|
|
287
|
+
const requested = conversationStringArray(requestedShape?.measures) ?? [];
|
|
288
|
+
const numericColumns = columns.filter((column) => rows.some((row) => typeof row[column] === 'number' && Number.isFinite(row[column])));
|
|
289
|
+
const metricNamedColumns = columns.filter((column) => /\b(revenue|sales|amount|total|count|average|avg|sum|spend|cost|margin|profit|value|points?|score|quantity|units?|rate|volume)\b/i.test(column.replace(/_/g, ' ')));
|
|
290
|
+
const unique = Array.from(new Set([...requested, ...numericColumns, ...metricNamedColumns]
|
|
291
|
+
.map((value) => value.trim())
|
|
292
|
+
.filter(Boolean)));
|
|
293
|
+
return unique.length > 0 ? unique.slice(0, 24) : undefined;
|
|
294
|
+
}
|
|
295
|
+
/** Distinct string values per low-cardinality column — the deictic-resolution fuel. */
|
|
296
|
+
function conversationDimensionValues(columns, rows) {
|
|
297
|
+
if (columns.length === 0 || rows.length === 0)
|
|
298
|
+
return undefined;
|
|
299
|
+
const out = {};
|
|
300
|
+
for (const column of columns.slice(0, 8)) {
|
|
301
|
+
const values = Array.from(new Set(rows.map((row) => row[column]).filter((value) => typeof value === 'string' && value.trim().length > 0))).slice(0, 24);
|
|
302
|
+
if (values.length > 0 && values.length <= 24)
|
|
303
|
+
out[column] = values;
|
|
304
|
+
}
|
|
305
|
+
return Object.keys(out).length > 0 ? out : undefined;
|
|
306
|
+
}
|
|
307
|
+
function conversationStringArray(value) {
|
|
308
|
+
if (!Array.isArray(value))
|
|
309
|
+
return undefined;
|
|
310
|
+
const values = value.filter((item) => typeof item === 'string' && item.trim().length > 0);
|
|
311
|
+
return values.length > 0 ? values : undefined;
|
|
312
|
+
}
|
|
313
|
+
function compactConversationRecord(record) {
|
|
314
|
+
const entries = Object.entries(record).filter(([, value]) => value !== undefined && value !== null);
|
|
315
|
+
return entries.length > 0 ? Object.fromEntries(entries) : undefined;
|
|
316
|
+
}
|
|
134
317
|
export async function startLocalServer(opts) {
|
|
135
318
|
const { rootDir, executor, connection: rawConnection, preferredPort, projectRoot = process.cwd() } = opts;
|
|
136
319
|
const bindHost = opts.host ?? process.env.DQL_HOST ?? '127.0.0.1';
|
|
@@ -366,7 +549,7 @@ export async function startLocalServer(opts) {
|
|
|
366
549
|
return provider.generate([{ role: 'system', content: system }, { role: 'user', content: user }], { maxTokens: 600, temperature: 0.2, signal });
|
|
367
550
|
},
|
|
368
551
|
});
|
|
369
|
-
async function runGovernedAgentAnswerForRun(request, repair) {
|
|
552
|
+
async function runGovernedAgentAnswerForRun(request, repair, route = 'generated_answer') {
|
|
370
553
|
const governed = resolveGovernedAnswerRunner(projectRoot);
|
|
371
554
|
const resolvedProvider = governed?.provider ?? null;
|
|
372
555
|
const runner = governed?.runner ?? null;
|
|
@@ -376,6 +559,10 @@ export async function startLocalServer(opts) {
|
|
|
376
559
|
let governedAnswer;
|
|
377
560
|
let providerError;
|
|
378
561
|
const isRepair = (repair?.attempt ?? 0) > 0 && Boolean(repair?.repairHint);
|
|
562
|
+
const reasoningEffort = request.reasoningEffort
|
|
563
|
+
? resolveRequestedRunReasoningEffort(projectRoot, resolvedProvider, request.reasoningEffort, isRepair)
|
|
564
|
+
: resolveRunReasoningEffort(projectRoot, resolvedProvider, route, isRepair);
|
|
565
|
+
const analysisDepth = request.analysisDepth ?? (route === 'research' ? 'deep' : undefined);
|
|
379
566
|
const contextEnvelope = {
|
|
380
567
|
mode: 'agent_run',
|
|
381
568
|
selectedObject: request.selectedObject,
|
|
@@ -383,23 +570,37 @@ export async function startLocalServer(opts) {
|
|
|
383
570
|
instruction: [
|
|
384
571
|
'Route through the governed DQL answer loop.',
|
|
385
572
|
'Prefer certified DQL blocks when they exactly cover the question.',
|
|
386
|
-
'Generated
|
|
573
|
+
'Generated DQL artifacts remain review-required; SQL is only the bounded preview/compiled evidence.',
|
|
387
574
|
'If the question needs investigation, return the clearest answer and next review action without certifying generated work.',
|
|
388
575
|
...(isRepair ? [`This is a repair attempt — fix the previous failure: ${repair?.repairHint}`] : []),
|
|
389
576
|
].join(' '),
|
|
390
577
|
};
|
|
391
578
|
const controller = new AbortController();
|
|
579
|
+
// Best-effort active warehouse dialect so Lane-2 semantic compiles emit
|
|
580
|
+
// dialect-correct SQL (e.g. DATE_TRUNC / identifier quoting). Absent when no
|
|
581
|
+
// connection is configured — the compiler then uses its default dialect.
|
|
582
|
+
let semanticDriver;
|
|
583
|
+
try {
|
|
584
|
+
semanticDriver = requireActiveConnection().driver;
|
|
585
|
+
}
|
|
586
|
+
catch {
|
|
587
|
+
semanticDriver = undefined;
|
|
588
|
+
}
|
|
392
589
|
await runner.run({
|
|
393
590
|
provider: resolvedProvider,
|
|
394
591
|
messages: [
|
|
395
592
|
...(request.history ?? []).map((message) => ({ role: message.role, content: message.text })),
|
|
396
593
|
{ role: 'user', content: isRepair ? `${request.question}\n\nFix the previous attempt: ${repair?.repairHint}` : request.question },
|
|
397
594
|
],
|
|
595
|
+
conversationContext: request.conversationContext,
|
|
398
596
|
upstream: {
|
|
399
597
|
cellId: `agent-run:${request.selectedObject?.kind ?? 'workspace'}:${request.selectedObject?.id ?? request.runId ?? 'auto'}`,
|
|
400
598
|
sql: JSON.stringify(contextEnvelope, null, 2),
|
|
401
599
|
},
|
|
600
|
+
reasoningEffort,
|
|
601
|
+
...(analysisDepth ? { analysisDepth } : {}),
|
|
402
602
|
projectRoot,
|
|
603
|
+
...(semanticDriver ? { semanticDriver } : {}),
|
|
403
604
|
executeCertifiedBlock: executeCertifiedBlockForAgent,
|
|
404
605
|
executeGeneratedSql: executeGeneratedSqlForAgent,
|
|
405
606
|
getSchemaContext: getSchemaContextForAgent,
|
|
@@ -416,10 +617,26 @@ export async function startLocalServer(opts) {
|
|
|
416
617
|
}
|
|
417
618
|
return governedAnswer;
|
|
418
619
|
}
|
|
419
|
-
|
|
620
|
+
function resolvedRunRouteFromAnswer(governedAnswer) {
|
|
621
|
+
return routeForCascadeAnswerTier(governedAnswer.route?.tier);
|
|
622
|
+
}
|
|
623
|
+
function groundingGapRepairHint(governedAnswer) {
|
|
624
|
+
const details = governedAnswer.refusalDetails;
|
|
625
|
+
if (!details)
|
|
626
|
+
return governedAnswer.text;
|
|
627
|
+
const offending = details.offending;
|
|
628
|
+
const tokens = [
|
|
629
|
+
details.code ? `code=${details.code}` : undefined,
|
|
630
|
+
offending?.relation ? `relation=${offending.relation}` : undefined,
|
|
631
|
+
offending?.column ? `column=${offending.column}` : undefined,
|
|
632
|
+
details.message ? `message=${details.message}` : undefined,
|
|
633
|
+
].filter((token) => Boolean(token));
|
|
634
|
+
return tokens.length > 0 ? tokens.join('; ') : governedAnswer.text;
|
|
635
|
+
}
|
|
636
|
+
const answerRunExecutor = async ({ request, route, routeDecision, attempt, repairHint, emitAnswerDelta }) => {
|
|
420
637
|
let governedAnswer;
|
|
421
638
|
try {
|
|
422
|
-
governedAnswer = await runGovernedAgentAnswerForRun(request, { attempt, repairHint });
|
|
639
|
+
governedAnswer = await runGovernedAgentAnswerForRun(request, { attempt, repairHint }, route);
|
|
423
640
|
// Surface the approved Hint-Graph corrections that shaped this answer so the
|
|
424
641
|
// UI can show an "applied learnings" chip (memoryContext is already on the answer).
|
|
425
642
|
if (!governedAnswer.appliedHints) {
|
|
@@ -443,14 +660,20 @@ export async function startLocalServer(opts) {
|
|
|
443
660
|
],
|
|
444
661
|
};
|
|
445
662
|
}
|
|
663
|
+
const resolvedRoute = resolvedRunRouteFromAnswer(governedAnswer) ?? route;
|
|
446
664
|
const isCertified = governedAnswer.certification === 'certified' || governedAnswer.kind === 'certified';
|
|
447
|
-
const
|
|
665
|
+
const isGroundingGap = governedAnswer.kind === 'no_answer' && governedAnswer.refusalCode === 'grounding_gap';
|
|
666
|
+
const isProviderError = governedAnswer.kind === 'no_answer' && governedAnswer.refusalCode === 'provider_error';
|
|
667
|
+
const groundingRepairHint = isGroundingGap ? groundingGapRepairHint(governedAnswer) : undefined;
|
|
668
|
+
// A provider outage is a retryable infrastructure failure, not a question the
|
|
669
|
+
// user needs to clarify — surface it as blocked so the UI offers a retry.
|
|
670
|
+
const needsClarification = governedAnswer.kind === 'no_answer' && !isGroundingGap && !isProviderError;
|
|
448
671
|
const sql = governedAnswer.proposedSql ?? governedAnswer.sql;
|
|
449
|
-
|
|
450
|
-
//
|
|
451
|
-
//
|
|
672
|
+
const runnableSql = governedAnswer.kind === 'no_answer' ? undefined : sql;
|
|
673
|
+
// Synthesis is a legacy polish pass. Certified/no-answer paths and lanes
|
|
674
|
+
// that already produced DQL-first final prose keep the fast path.
|
|
452
675
|
let synthesizedAnswer;
|
|
453
|
-
if (
|
|
676
|
+
if (shouldSynthesizeAgentRunAnswer(governedAnswer)) {
|
|
454
677
|
try {
|
|
455
678
|
const provider = await createBlockStudioAssistProvider(projectRoot);
|
|
456
679
|
if (provider) {
|
|
@@ -477,32 +700,55 @@ export async function startLocalServer(opts) {
|
|
|
477
700
|
synthesizedAnswer = undefined;
|
|
478
701
|
}
|
|
479
702
|
}
|
|
480
|
-
const status = needsClarification ? 'needs_clarification' : isCertified ? 'completed' : 'needs_review';
|
|
481
|
-
const trustState = needsClarification ? 'not_applicable' : isCertified ? 'certified' : 'review_required';
|
|
482
|
-
const stopReason = needsClarification ? 'needs_clarification' : isCertified ? 'certified_answer_found' : 'human_review_required';
|
|
703
|
+
const status = isProviderError ? 'blocked' : needsClarification ? 'needs_clarification' : isCertified ? 'completed' : 'needs_review';
|
|
704
|
+
const trustState = isProviderError ? 'blocked' : needsClarification ? 'not_applicable' : isCertified ? 'certified' : 'review_required';
|
|
705
|
+
const stopReason = isProviderError ? 'blocked' : needsClarification ? 'needs_clarification' : isCertified ? 'certified_answer_found' : 'human_review_required';
|
|
483
706
|
const nextActions = needsClarification
|
|
484
707
|
? [{ id: 'clarify', label: 'Clarify question', route: 'generated_answer' }]
|
|
485
708
|
: [
|
|
486
|
-
|
|
709
|
+
{ id: 'create-block', label: governedAnswer.dqlArtifact ? 'Review DQL draft' : 'Create DQL draft', route: 'dql_block_draft', artifactKind: 'dql_block_draft' },
|
|
487
710
|
{ id: 'research-gap', label: 'Research deeper', route: 'research' },
|
|
488
|
-
{ id: '
|
|
711
|
+
...(runnableSql ? [{ id: 'insert-sql', label: 'Insert SQL preview', route: 'sql_cell', artifactKind: 'sql_cell' }] : []),
|
|
489
712
|
];
|
|
490
713
|
return {
|
|
714
|
+
resolvedRoute,
|
|
715
|
+
answerTier: governedAnswer.route?.tier,
|
|
491
716
|
summary: governedAnswer.route?.label ?? (isCertified ? 'Answered from certified DQL context.' : 'Answered with review-required generated analysis.'),
|
|
492
717
|
answer: synthesizedAnswer ?? governedAnswer.answer ?? governedAnswer.text,
|
|
493
718
|
status,
|
|
494
719
|
trustState,
|
|
495
720
|
stopReason,
|
|
496
|
-
artifacts:
|
|
497
|
-
|
|
721
|
+
artifacts: governedAnswer.kind === 'no_answer'
|
|
722
|
+
// A refusal still keeps the DQL draft the answer loop produced (when any),
|
|
723
|
+
// so the "Review DQL draft" next-action isn't a dead link and the user can
|
|
724
|
+
// see the SQL that was about to run. Provider outages carry no draft.
|
|
725
|
+
? (governedAnswer.dqlArtifact && !isProviderError
|
|
726
|
+
? [agentRunArtifact('dql_block_draft', 'DQL draft (review required)', governedAnswer.dqlArtifact, undefined, 'review_required')]
|
|
727
|
+
: [])
|
|
498
728
|
: [agentRunArtifact('answer', isCertified ? 'Certified answer' : 'Review-required answer', governedAnswer, governedAnswer.sourceCertifiedBlock ?? governedAnswer.block?.name, isCertified ? 'certified' : 'review_required')],
|
|
499
729
|
evaluations: [
|
|
500
|
-
agentRunEvaluation('route-decision', 'Route decision', true, 'info', routeDecision?.reason ?? 'Routed request to governed answer.'
|
|
730
|
+
agentRunEvaluation('route-decision', 'Route decision', true, 'info', routeDecision?.reason ?? 'Routed request to governed answer.', {
|
|
731
|
+
plannedRoute: route,
|
|
732
|
+
resolvedRoute,
|
|
733
|
+
aiRoute: governedAnswer.route,
|
|
734
|
+
}),
|
|
501
735
|
agentRunEvaluation('trust-boundary', 'Trust boundary', isCertified, isCertified ? 'info' : 'warning', isCertified
|
|
502
736
|
? 'The answer came from certified DQL context.'
|
|
503
737
|
: needsClarification
|
|
504
738
|
? 'The answer loop needs more context before producing a governed answer.'
|
|
505
739
|
: 'The answer is generated or semantic-layer backed and remains review-required.', governedAnswer.route),
|
|
740
|
+
...(isGroundingGap ? [
|
|
741
|
+
{
|
|
742
|
+
...agentRunEvaluation('grounding-gap', 'Metadata grounding', false, 'warning', 'The answer loop found a metadata grounding gap that can be retried with wider context.', {
|
|
743
|
+
refusalCode: governedAnswer.refusalCode,
|
|
744
|
+
refusalDetails: governedAnswer.refusalDetails,
|
|
745
|
+
validationWarnings: governedAnswer.validationWarnings,
|
|
746
|
+
route: governedAnswer.route,
|
|
747
|
+
}),
|
|
748
|
+
suggestedRepair: groundingRepairHint,
|
|
749
|
+
repairAction: { kind: 'retry', hint: groundingRepairHint },
|
|
750
|
+
},
|
|
751
|
+
] : []),
|
|
506
752
|
...(governedAnswer.executionError ? [
|
|
507
753
|
agentRunEvaluation('execution-error', 'Execution error', false, 'warning', governedAnswer.executionError),
|
|
508
754
|
] : []),
|
|
@@ -525,8 +771,10 @@ export async function startLocalServer(opts) {
|
|
|
525
771
|
const provider = await createBlockStudioAssistProvider(projectRoot);
|
|
526
772
|
if (provider) {
|
|
527
773
|
const system = buildConversationSystemPrompt(kind, isGeneralKnowledge, catalogContext, request.audience ?? 'analyst');
|
|
774
|
+
const conversationMemory = renderConversationMemoryForPrompt(request.conversationContext);
|
|
528
775
|
const messages = [
|
|
529
776
|
{ role: 'system', content: system },
|
|
777
|
+
...(conversationMemory ? [{ role: 'system', content: conversationMemory }] : []),
|
|
530
778
|
...(request.history ?? []).slice(-6).map((message) => ({ role: message.role, content: message.text })),
|
|
531
779
|
{ role: 'user', content: request.question },
|
|
532
780
|
];
|
|
@@ -541,7 +789,8 @@ export async function startLocalServer(opts) {
|
|
|
541
789
|
text = undefined;
|
|
542
790
|
}
|
|
543
791
|
if (!text) {
|
|
544
|
-
text =
|
|
792
|
+
text = buildConversationContextRecap(request.conversationContext)
|
|
793
|
+
?? buildConversationalFallback(kind, isGeneralKnowledge, request.question, suggestions);
|
|
545
794
|
emitAnswerDelta?.(text);
|
|
546
795
|
}
|
|
547
796
|
return {
|
|
@@ -560,7 +809,8 @@ export async function startLocalServer(opts) {
|
|
|
560
809
|
conversation: conversationRunExecutor,
|
|
561
810
|
certified_answer: answerRunExecutor,
|
|
562
811
|
generated_answer: answerRunExecutor,
|
|
563
|
-
research: async (
|
|
812
|
+
research: async (researchContext) => {
|
|
813
|
+
const { runId, request, routeDecision, emit } = researchContext;
|
|
564
814
|
const metrics = loadSemanticMetrics(projectRoot);
|
|
565
815
|
let blocks = collectPlanBlocks(projectRoot, { certifiedOnly: true });
|
|
566
816
|
const usedCertifiedOnly = blocks.length > 0;
|
|
@@ -581,6 +831,13 @@ export async function startLocalServer(opts) {
|
|
|
581
831
|
// When the user explicitly picked research, investigate — don't collapse to one step.
|
|
582
832
|
forceInvestigate: request.requestedMode === 'research',
|
|
583
833
|
});
|
|
834
|
+
// Direct governed answer ("the metric answers this directly"): don't wrap it in
|
|
835
|
+
// a review-required research dossier — run the query through the answer loop and
|
|
836
|
+
// return the executed result table + DQL artifact. A dossier is only for genuine
|
|
837
|
+
// multi-step investigation (plan.done === false) or explicit research mode.
|
|
838
|
+
if (plan.done && !plan.followUp && request.requestedMode !== 'research') {
|
|
839
|
+
return answerRunExecutor(researchContext);
|
|
840
|
+
}
|
|
584
841
|
const needsClarification = Boolean(plan.followUp);
|
|
585
842
|
const notebookPath = agentRunNotebookPath(request, runId);
|
|
586
843
|
const researchIntent = agentRunResearchIntent(request);
|
|
@@ -646,9 +903,15 @@ export async function startLocalServer(opts) {
|
|
|
646
903
|
researchWorkspaceError = formatNotebookResearchStorageError(error);
|
|
647
904
|
}
|
|
648
905
|
}
|
|
649
|
-
const
|
|
650
|
-
|
|
651
|
-
const
|
|
906
|
+
const researchResultPreview = researchRun?.resultPreview;
|
|
907
|
+
const researchResultData = coerceNarrateResultData(researchResultPreview);
|
|
908
|
+
const researchResultRecord = agentRunRecord(researchResultPreview);
|
|
909
|
+
// A query that ran and matched 0 rows STILL executed — treat it as a clean,
|
|
910
|
+
// grounded execution (not "no result"), so an empty answer is surfaced as
|
|
911
|
+
// "0 rows matched" rather than silently downgraded to review-required.
|
|
912
|
+
const researchDidExecute = Boolean(researchResultData) || Boolean(researchResultRecord && Array.isArray(researchResultRecord.rows));
|
|
913
|
+
const researchZeroRows = !researchResultData && researchDidExecute;
|
|
914
|
+
const researchExecutedCleanly = researchDidExecute && !researchWorkspaceError && researchRun?.status !== 'error';
|
|
652
915
|
const narration = !needsClarification && researchResultData
|
|
653
916
|
? await narrateForAgentRun({
|
|
654
917
|
question: request.question,
|
|
@@ -661,18 +924,22 @@ export async function startLocalServer(opts) {
|
|
|
661
924
|
const summary = needsClarification
|
|
662
925
|
? 'Needs clarification before running deeper research.'
|
|
663
926
|
: narration?.summary
|
|
664
|
-
?? (
|
|
665
|
-
? '
|
|
666
|
-
: researchRun?.status === '
|
|
667
|
-
? 'Saved a research dossier
|
|
668
|
-
:
|
|
669
|
-
? '
|
|
670
|
-
:
|
|
671
|
-
? 'Prepared a
|
|
672
|
-
:
|
|
927
|
+
?? (researchZeroRows
|
|
928
|
+
? 'The query executed cleanly against real data and matched 0 rows.'
|
|
929
|
+
: researchRun?.status === 'ready'
|
|
930
|
+
? 'Saved a grounded research dossier with context evidence and next review actions.'
|
|
931
|
+
: researchRun?.status === 'error'
|
|
932
|
+
? 'Saved a research dossier, but the preview needs review before promotion.'
|
|
933
|
+
: researchWorkspaceError
|
|
934
|
+
? 'Prepared a grounded research plan; durable research storage is unavailable in this runtime.'
|
|
935
|
+
: plan.done
|
|
936
|
+
? 'Prepared a direct grounded-answer plan.'
|
|
937
|
+
: 'Prepared a grounded research plan over real DQL assets.');
|
|
673
938
|
return {
|
|
674
939
|
summary,
|
|
675
|
-
answer: plan.followUp?.question ?? narration?.summary
|
|
940
|
+
answer: plan.followUp?.question ?? narration?.summary
|
|
941
|
+
?? (researchZeroRows ? 'The query executed cleanly and matched 0 rows.' : undefined)
|
|
942
|
+
?? researchRun?.summary,
|
|
676
943
|
status: needsClarification ? 'needs_clarification' : 'needs_review',
|
|
677
944
|
trustState: needsClarification ? 'not_applicable' : (researchExecutedCleanly ? 'grounded' : 'review_required'),
|
|
678
945
|
stopReason: needsClarification ? 'needs_clarification' : 'human_review_required',
|
|
@@ -702,15 +969,17 @@ export async function startLocalServer(opts) {
|
|
|
702
969
|
? 'Research workspace storage is unavailable; the plan remains available in this agent run.'
|
|
703
970
|
: 'No durable research record was created because the run needs clarification first.', { notebookPath, researchRunId: researchRun?.id, error: researchWorkspaceError }),
|
|
704
971
|
agentRunEvaluation('result-executed', 'Executed against data', researchExecutedCleanly, researchExecutedCleanly ? 'info' : 'warning', researchExecutedCleanly
|
|
705
|
-
?
|
|
706
|
-
|
|
972
|
+
? (researchZeroRows
|
|
973
|
+
? 'The query executed cleanly against real data and matched 0 rows.'
|
|
974
|
+
: 'The query executed cleanly against real data and returned rows.')
|
|
975
|
+
: 'No executed result was available; the output stays exploratory pending review.', { rowCount: Array.isArray(researchResultRecord?.rows) ? researchResultRecord.rows.length : 0 }),
|
|
707
976
|
],
|
|
708
977
|
nextActions: needsClarification
|
|
709
978
|
? [{ id: 'answer-follow-up', label: 'Answer follow-up', route: 'research' }]
|
|
710
979
|
: [
|
|
711
980
|
...(researchRun?.id ? [{ id: 'open-research', label: 'Open research dossier', artifactKind: 'research_run' }] : []),
|
|
712
|
-
|
|
713
|
-
{ id: '
|
|
981
|
+
{ id: 'create-block', label: 'Review DQL draft', route: 'dql_block_draft', artifactKind: 'dql_block_draft' },
|
|
982
|
+
...(researchRun?.generatedSql || researchRun?.reviewedSql ? [{ id: 'insert-sql', label: 'Insert SQL preview', route: 'sql_cell', artifactKind: 'sql_cell' }] : []),
|
|
714
983
|
],
|
|
715
984
|
};
|
|
716
985
|
},
|
|
@@ -725,12 +994,67 @@ export async function startLocalServer(opts) {
|
|
|
725
994
|
agentRunEvaluation('review-boundary', 'Review boundary', true, 'warning', 'Generated SQL must be reviewed before it becomes certified analytics.'),
|
|
726
995
|
],
|
|
727
996
|
nextActions: [
|
|
728
|
-
{ id: 'insert-sql', label: 'Insert SQL
|
|
729
|
-
{ id: 'create-block', label: '
|
|
997
|
+
{ id: 'insert-sql', label: 'Insert SQL preview', artifactKind: 'sql_cell' },
|
|
998
|
+
{ id: 'create-block', label: 'Review as DQL draft', route: 'dql_block_draft', artifactKind: 'dql_block_draft' },
|
|
730
999
|
],
|
|
731
1000
|
};
|
|
732
1001
|
},
|
|
733
1002
|
dql_block_draft: async ({ request, routeDecision, attempt, repairHint }) => {
|
|
1003
|
+
const contextRecord = agentRunRecord(request.conversationContext);
|
|
1004
|
+
const carriedDqlArtifact = normalizeDqlArtifactReference(contextRecord?.dqlArtifact)
|
|
1005
|
+
?? normalizeDqlArtifactReference(request.workspaceContext?.dqlArtifact);
|
|
1006
|
+
if (carriedDqlArtifact && (attempt ?? 0) === 0) {
|
|
1007
|
+
const sourceQuestion = agentRunString(contextRecord?.sourceQuestion) ?? request.question;
|
|
1008
|
+
const session = await createDqlArtifactGenerationSessionForProject(projectRoot, {
|
|
1009
|
+
question: sourceQuestion,
|
|
1010
|
+
dqlArtifact: carriedDqlArtifact,
|
|
1011
|
+
inputMode: 'upload',
|
|
1012
|
+
inputPath: carriedDqlArtifact.sourcePath,
|
|
1013
|
+
domain: agentRunWorkspaceValue(request, 'domain'),
|
|
1014
|
+
owner: agentRunWorkspaceValue(request, 'owner'),
|
|
1015
|
+
tags: ['agent-run', 'review-required'],
|
|
1016
|
+
contextPackId: agentRunString(contextRecord?.contextPackId),
|
|
1017
|
+
routeIntent: agentRunString(contextRecord?.route),
|
|
1018
|
+
sourceBlock: carriedDqlArtifact.sourcePath,
|
|
1019
|
+
}, semanticLayer);
|
|
1020
|
+
const candidate = session.candidates[0];
|
|
1021
|
+
const validation = candidate?.validation && typeof candidate.validation === 'object'
|
|
1022
|
+
? candidate.validation
|
|
1023
|
+
: undefined;
|
|
1024
|
+
const ready = validation?.valid !== false;
|
|
1025
|
+
const payload = {
|
|
1026
|
+
target: 'block',
|
|
1027
|
+
name: candidate?.name ?? carriedDqlArtifact.name ?? 'DQL block draft',
|
|
1028
|
+
path: candidate?.draftSave?.path ?? candidate?.savedPath,
|
|
1029
|
+
importId: session.id,
|
|
1030
|
+
candidateId: candidate?.id,
|
|
1031
|
+
source: candidate?.dqlSource,
|
|
1032
|
+
dqlSource: candidate?.dqlSource,
|
|
1033
|
+
dqlArtifact: carriedDqlArtifact,
|
|
1034
|
+
certifierVerdict: {
|
|
1035
|
+
ready,
|
|
1036
|
+
diagnostics: validation?.diagnostics ?? [],
|
|
1037
|
+
},
|
|
1038
|
+
generation: session.generation,
|
|
1039
|
+
};
|
|
1040
|
+
return {
|
|
1041
|
+
summary: ready
|
|
1042
|
+
? 'Saved the generated DQL artifact as a review-required block draft.'
|
|
1043
|
+
: 'Saved the generated DQL artifact as a block draft with review blockers or warnings.',
|
|
1044
|
+
artifacts: [agentRunArtifact('dql_block_draft', payload.name, payload, payload.path)],
|
|
1045
|
+
evaluations: [
|
|
1046
|
+
agentRunEvaluation('route-decision', 'Route decision', true, 'info', routeDecision?.reason ?? 'Routed request to DQL block draft generation.'),
|
|
1047
|
+
agentRunEvaluation('dql-artifact-reuse', 'Used generated DQL artifact', true, 'info', 'Persisted the existing DQL artifact without regenerating it from SQL.', { importId: session.id, path: payload.path }),
|
|
1048
|
+
agentRunEvaluation('certification-boundary', 'Certification boundary', ready, 'warning', ready
|
|
1049
|
+
? 'The draft has no automatic validation blockers, but certification still requires human review.'
|
|
1050
|
+
: 'The draft has validation blockers that must be resolved before certification review.', payload),
|
|
1051
|
+
],
|
|
1052
|
+
nextActions: [
|
|
1053
|
+
{ id: 'open-review', label: 'Open review checklist', artifactKind: 'dql_block_draft' },
|
|
1054
|
+
{ id: 'build-app', label: 'Build app from block', route: 'app_build', artifactKind: 'app_draft' },
|
|
1055
|
+
],
|
|
1056
|
+
};
|
|
1057
|
+
}
|
|
734
1058
|
const result = await buildAgentPromptArtifact(request, 'block', { attempt, repairHint });
|
|
735
1059
|
const ready = result.target === 'block' ? result.certifierVerdict.ready : false;
|
|
736
1060
|
return {
|
|
@@ -898,9 +1222,27 @@ export async function startLocalServer(opts) {
|
|
|
898
1222
|
getCatalogContext: () => buildAgentRunCatalogContext(),
|
|
899
1223
|
});
|
|
900
1224
|
const agentRunStore = new FileAgentRunStore({ path: defaultAgentRunStorePath(projectRoot) });
|
|
1225
|
+
// Server-side conversation threads: persisted multi-turn state (survives refresh).
|
|
1226
|
+
// Lazy so a failed native-sqlite load never blocks the runtime; conversation
|
|
1227
|
+
// persistence degrades to the per-request context instead.
|
|
1228
|
+
let conversationStoreInstance;
|
|
1229
|
+
const getConversationStore = () => {
|
|
1230
|
+
if (conversationStoreInstance !== undefined)
|
|
1231
|
+
return conversationStoreInstance;
|
|
1232
|
+
try {
|
|
1233
|
+
conversationStoreInstance = new ConversationStore(defaultConversationPath(projectRoot));
|
|
1234
|
+
}
|
|
1235
|
+
catch {
|
|
1236
|
+
conversationStoreInstance = null;
|
|
1237
|
+
}
|
|
1238
|
+
return conversationStoreInstance;
|
|
1239
|
+
};
|
|
901
1240
|
const agentRunEngine = new AgentRunEngine({
|
|
902
1241
|
store: agentRunStore,
|
|
903
|
-
executors:
|
|
1242
|
+
executors: {
|
|
1243
|
+
...agentRunExecutors,
|
|
1244
|
+
...(opts.agentRunExecutors ?? {}),
|
|
1245
|
+
},
|
|
904
1246
|
gates: defaultAgentRunGates,
|
|
905
1247
|
planner: agentRunPlanner,
|
|
906
1248
|
router: agentRunRouter,
|
|
@@ -1059,25 +1401,42 @@ export async function startLocalServer(opts) {
|
|
|
1059
1401
|
};
|
|
1060
1402
|
};
|
|
1061
1403
|
const getSchemaContextForAgent = async (question) => {
|
|
1404
|
+
const scanRuntimeSchema = async () => {
|
|
1405
|
+
if (!connection)
|
|
1406
|
+
return { ranked: [], snapshot: [] };
|
|
1407
|
+
const result = await executor.executeQuery(`SELECT table_schema, table_name, column_name, data_type
|
|
1408
|
+
FROM information_schema.columns
|
|
1409
|
+
WHERE table_schema NOT IN ('information_schema', 'pg_catalog')
|
|
1410
|
+
ORDER BY table_schema, table_name, ordinal_position
|
|
1411
|
+
LIMIT 2000`, [], runtimeVariables({}), connection);
|
|
1412
|
+
return {
|
|
1413
|
+
ranked: buildAgentSchemaContext(question, result.rows),
|
|
1414
|
+
snapshot: buildAgentSchemaContext(question, result.rows, { includeUnscored: true, limit: 500 }),
|
|
1415
|
+
};
|
|
1416
|
+
};
|
|
1062
1417
|
const catalogContext = await buildAgentSchemaContextFromCatalog(projectRoot, question).catch(() => []);
|
|
1063
1418
|
if (catalogContext.length > 0) {
|
|
1064
1419
|
if (!connection)
|
|
1065
1420
|
return catalogContext;
|
|
1066
|
-
const
|
|
1067
|
-
|
|
1421
|
+
const runtimeScan = shouldAugmentAgentRuntimeSchema(question)
|
|
1422
|
+
? await scanRuntimeSchema().catch(() => undefined)
|
|
1423
|
+
: undefined;
|
|
1424
|
+
const runtimeContext = runtimeScan?.ranked ?? [];
|
|
1425
|
+
const merged = mergeAgentSchemaContexts(catalogContext, runtimeContext);
|
|
1426
|
+
const enriched = await enrichAgentSchemaContextWithValueMatches(question, merged, executor, connection);
|
|
1427
|
+
recordAgentRuntimeSchemaSnapshot(projectRoot, !runtimeScan?.snapshot.length
|
|
1428
|
+
? enriched
|
|
1429
|
+
: mergeAgentSchemaSampleValues(runtimeScan.snapshot, enriched), runtimeContext.length > 0
|
|
1430
|
+
? 'full information_schema runtime scan for composite Ask AI question'
|
|
1431
|
+
: 'catalog enriched runtime schema');
|
|
1068
1432
|
return enriched;
|
|
1069
1433
|
}
|
|
1070
1434
|
if (!connection)
|
|
1071
1435
|
return [];
|
|
1072
1436
|
try {
|
|
1073
|
-
const
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
ORDER BY table_schema, table_name, ordinal_position
|
|
1077
|
-
LIMIT 2000`, [], runtimeVariables({}), connection);
|
|
1078
|
-
const schemaContext = buildAgentSchemaContext(question, result.rows);
|
|
1079
|
-
const enriched = await enrichAgentSchemaContextWithValueMatches(question, schemaContext, executor, connection);
|
|
1080
|
-
recordAgentRuntimeSchemaSnapshot(projectRoot, enriched, 'information_schema runtime scan');
|
|
1437
|
+
const schemaContext = await scanRuntimeSchema();
|
|
1438
|
+
const enriched = await enrichAgentSchemaContextWithValueMatches(question, schemaContext.ranked, executor, connection);
|
|
1439
|
+
recordAgentRuntimeSchemaSnapshot(projectRoot, schemaContext.snapshot, 'full information_schema runtime scan');
|
|
1081
1440
|
return enriched;
|
|
1082
1441
|
}
|
|
1083
1442
|
catch {
|
|
@@ -1094,6 +1453,7 @@ export async function startLocalServer(opts) {
|
|
|
1094
1453
|
let governedAnswer;
|
|
1095
1454
|
let providerError;
|
|
1096
1455
|
const memoOnly = input.mode === 'memo_only';
|
|
1456
|
+
const reasoningEffort = resolveRunReasoningEffort(projectRoot, resolvedProvider, 'research', false);
|
|
1097
1457
|
const contextEnvelope = {
|
|
1098
1458
|
mode: 'app_research',
|
|
1099
1459
|
generationMode: input.mode ?? 'sql_and_memo',
|
|
@@ -1141,6 +1501,8 @@ export async function startLocalServer(opts) {
|
|
|
1141
1501
|
cellId: `app-research:${input.appId}:${input.dashboardId ?? 'app'}`,
|
|
1142
1502
|
sql: JSON.stringify(contextEnvelope, null, 2),
|
|
1143
1503
|
},
|
|
1504
|
+
reasoningEffort,
|
|
1505
|
+
analysisDepth: 'deep',
|
|
1144
1506
|
projectRoot,
|
|
1145
1507
|
executeCertifiedBlock: executeCertifiedBlockForAgent,
|
|
1146
1508
|
executeGeneratedSql: executeGeneratedSqlForAgent,
|
|
@@ -1217,6 +1579,21 @@ export async function startLocalServer(opts) {
|
|
|
1217
1579
|
intents: 0,
|
|
1218
1580
|
notebooks: 0,
|
|
1219
1581
|
},
|
|
1582
|
+
reviewMetrics: {
|
|
1583
|
+
totalReviewCount: 0,
|
|
1584
|
+
openReviewCount: 0,
|
|
1585
|
+
terminalReviewCount: 0,
|
|
1586
|
+
draftCreatedCount: 0,
|
|
1587
|
+
certifiedCount: 0,
|
|
1588
|
+
completedCount: 0,
|
|
1589
|
+
rejectedCount: 0,
|
|
1590
|
+
draftCreationRate: null,
|
|
1591
|
+
certifyConversionRate: null,
|
|
1592
|
+
medianOpenReviewAgeMs: null,
|
|
1593
|
+
medianTimeToDraftMs: null,
|
|
1594
|
+
medianTimeToCertificationMs: null,
|
|
1595
|
+
medianTimeToTerminalMs: null,
|
|
1596
|
+
},
|
|
1220
1597
|
limit: input.limit,
|
|
1221
1598
|
offset: input.offset ?? 0,
|
|
1222
1599
|
});
|
|
@@ -1259,6 +1636,7 @@ export async function startLocalServer(opts) {
|
|
|
1259
1636
|
const context = input.context === undefined ? run.context : input.context;
|
|
1260
1637
|
let generatedSql = notebookResearchString(input.generatedSql) ?? run.generatedSql;
|
|
1261
1638
|
let reviewedSql = notebookResearchString(input.reviewedSql) ?? run.reviewedSql;
|
|
1639
|
+
let dqlArtifact = normalizeDqlArtifactReference(input.dqlArtifact) ?? run.dqlArtifact;
|
|
1262
1640
|
const startedAt = new Date().toISOString();
|
|
1263
1641
|
storage.updateRun(run.id, {
|
|
1264
1642
|
domain,
|
|
@@ -1269,6 +1647,7 @@ export async function startLocalServer(opts) {
|
|
|
1269
1647
|
context,
|
|
1270
1648
|
generatedSql,
|
|
1271
1649
|
reviewedSql,
|
|
1650
|
+
dqlArtifact,
|
|
1272
1651
|
status: 'running',
|
|
1273
1652
|
reviewStatus: 'needs_review',
|
|
1274
1653
|
error: '',
|
|
@@ -1285,8 +1664,8 @@ export async function startLocalServer(opts) {
|
|
|
1285
1664
|
intent,
|
|
1286
1665
|
researchPattern: notebookResearchIntentPattern(intent),
|
|
1287
1666
|
},
|
|
1288
|
-
strictness: '
|
|
1289
|
-
limit:
|
|
1667
|
+
strictness: 'exploratory',
|
|
1668
|
+
limit: 160,
|
|
1290
1669
|
}).catch((error) => {
|
|
1291
1670
|
const message = error instanceof Error ? error.message : String(error);
|
|
1292
1671
|
return {
|
|
@@ -1327,6 +1706,8 @@ export async function startLocalServer(opts) {
|
|
|
1327
1706
|
context,
|
|
1328
1707
|
}, null, 2),
|
|
1329
1708
|
},
|
|
1709
|
+
reasoningEffort: resolveRunReasoningEffort(projectRoot, resolvedProvider, 'research', false),
|
|
1710
|
+
analysisDepth: 'deep',
|
|
1330
1711
|
projectRoot,
|
|
1331
1712
|
executeCertifiedBlock: executeCertifiedBlockForAgent,
|
|
1332
1713
|
executeGeneratedSql: executeGeneratedSqlForAgent,
|
|
@@ -1346,8 +1727,9 @@ export async function startLocalServer(opts) {
|
|
|
1346
1727
|
generationWarnings.push(`AI SQL generation did not return a governed answer. Metadata context was saved for review.${providerError ? ` ${providerError}` : ''}`);
|
|
1347
1728
|
}
|
|
1348
1729
|
else {
|
|
1730
|
+
dqlArtifact = normalizeDqlArtifactReference(governedAnswer.dqlArtifact) ?? dqlArtifact;
|
|
1349
1731
|
generatedSql = notebookResearchString(governedAnswer.proposedSql) ?? notebookResearchString(governedAnswer.sql);
|
|
1350
|
-
if (!generatedSql) {
|
|
1732
|
+
if (!generatedSql && !dqlArtifact) {
|
|
1351
1733
|
generationWarnings.push('AI returned a governed answer without SQL. Metadata context was saved; add reviewed SQL before DQL promotion.');
|
|
1352
1734
|
}
|
|
1353
1735
|
}
|
|
@@ -1409,7 +1791,7 @@ export async function startLocalServer(opts) {
|
|
|
1409
1791
|
const evidence = {
|
|
1410
1792
|
trustStatus: {
|
|
1411
1793
|
label: governedAnswer?.trustLabel
|
|
1412
|
-
?? (reviewedSql ? 'Reviewed notebook SQL' : generatedSql ? 'AI-generated research SQL' : 'Metadata-grounded research plan'),
|
|
1794
|
+
?? (reviewedSql ? 'Reviewed notebook SQL' : generatedSql ? 'AI-generated research SQL' : dqlArtifact ? 'AI-generated DQL artifact' : 'Metadata-grounded research plan'),
|
|
1413
1795
|
reviewRequired: true,
|
|
1414
1796
|
},
|
|
1415
1797
|
contextPackId: contextPack.id,
|
|
@@ -1448,9 +1830,11 @@ export async function startLocalServer(opts) {
|
|
|
1448
1830
|
?? notebookResearchSummary(question, resultPreview, previewError);
|
|
1449
1831
|
const recommendation = previewError
|
|
1450
1832
|
? 'Review the SQL, selected metadata, and connection context before rerunning.'
|
|
1451
|
-
:
|
|
1452
|
-
? 'Review the
|
|
1453
|
-
:
|
|
1833
|
+
: dqlArtifact && !reviewedSql
|
|
1834
|
+
? 'Review the DQL artifact, parameter choices, grain, and evidence before promoting this research into a DQL draft block.'
|
|
1835
|
+
: sqlForPreview
|
|
1836
|
+
? 'Review the SQL, parameter choices, grain, and evidence before promoting this research into a DQL draft block.'
|
|
1837
|
+
: 'Review the selected metadata context, then paste reviewed SQL or configure an AI provider before DQL draft promotion.';
|
|
1454
1838
|
const researchPlan = buildNotebookResearchPlan({
|
|
1455
1839
|
run,
|
|
1456
1840
|
evidence,
|
|
@@ -1458,6 +1842,7 @@ export async function startLocalServer(opts) {
|
|
|
1458
1842
|
previewError,
|
|
1459
1843
|
generatedSql,
|
|
1460
1844
|
reviewedSql,
|
|
1845
|
+
dqlArtifact,
|
|
1461
1846
|
routeDecision,
|
|
1462
1847
|
});
|
|
1463
1848
|
return storage.updateRun(run.id, {
|
|
@@ -1475,6 +1860,7 @@ export async function startLocalServer(opts) {
|
|
|
1475
1860
|
researchPlan,
|
|
1476
1861
|
generatedSql,
|
|
1477
1862
|
reviewedSql,
|
|
1863
|
+
dqlArtifact,
|
|
1478
1864
|
display: display && display.ok ? display.display : undefined,
|
|
1479
1865
|
contextPackId: contextPack.id,
|
|
1480
1866
|
routeDecision,
|
|
@@ -1501,10 +1887,48 @@ export async function startLocalServer(opts) {
|
|
|
1501
1887
|
}
|
|
1502
1888
|
};
|
|
1503
1889
|
const promoteNotebookResearchToDql = async (storage, run, input = {}) => {
|
|
1504
|
-
const
|
|
1505
|
-
|
|
1506
|
-
|
|
1890
|
+
const reviewedSql = notebookResearchString(run.reviewedSql);
|
|
1891
|
+
const dqlArtifact = normalizeDqlArtifactReference(run.dqlArtifact);
|
|
1892
|
+
const sql = reviewedSql ?? notebookResearchString(run.generatedSql);
|
|
1507
1893
|
const sourcePath = `${run.notebookPath}${run.sourceCellId ? `#${run.sourceCellId}` : ''}`;
|
|
1894
|
+
if (!reviewedSql && dqlArtifact) {
|
|
1895
|
+
const session = await createDqlArtifactGenerationSessionForProject(projectRoot, {
|
|
1896
|
+
question: run.question,
|
|
1897
|
+
dqlArtifact,
|
|
1898
|
+
inputMode: 'upload',
|
|
1899
|
+
inputPath: dqlArtifact.sourcePath ?? sourcePath,
|
|
1900
|
+
domain: notebookResearchString(input.domain) ?? run.domain,
|
|
1901
|
+
owner: notebookResearchString(input.owner) ?? run.owner,
|
|
1902
|
+
tags: ['notebook-research', 'review-required', ...(Array.isArray(input.tags) ? input.tags : [])],
|
|
1903
|
+
generatedSql: notebookResearchString(run.generatedSql),
|
|
1904
|
+
contextPackId: run.contextPackId,
|
|
1905
|
+
routeIntent: run.intent,
|
|
1906
|
+
sourceBlock: dqlArtifact.sourcePath,
|
|
1907
|
+
}, semanticLayer);
|
|
1908
|
+
const draftPath = session.candidates.find((candidate) => candidate.draftSave?.path)?.draftSave?.path
|
|
1909
|
+
?? session.candidates.find((candidate) => candidate.savedPath)?.savedPath;
|
|
1910
|
+
const promotion = buildNotebookDqlPromotionSummary(session, draftPath);
|
|
1911
|
+
const updated = storage.markPromoted(run.id, {
|
|
1912
|
+
draftBlockPath: draftPath,
|
|
1913
|
+
dqlImportId: session.id,
|
|
1914
|
+
dqlCandidateIds: session.candidates.map((candidate) => candidate.id),
|
|
1915
|
+
dqlPromotion: promotion,
|
|
1916
|
+
}) ?? run;
|
|
1917
|
+
const planned = storage.updateRun(updated.id, {
|
|
1918
|
+
researchPlan: buildNotebookResearchPlan({
|
|
1919
|
+
run: updated,
|
|
1920
|
+
evidence: updated.evidence,
|
|
1921
|
+
resultPreview: updated.resultPreview,
|
|
1922
|
+
generatedSql: updated.generatedSql,
|
|
1923
|
+
reviewedSql: updated.reviewedSql,
|
|
1924
|
+
dqlArtifact: updated.dqlArtifact,
|
|
1925
|
+
routeDecision: updated.routeDecision,
|
|
1926
|
+
}),
|
|
1927
|
+
}) ?? updated;
|
|
1928
|
+
return { run: planned, session };
|
|
1929
|
+
}
|
|
1930
|
+
if (!sql)
|
|
1931
|
+
throw new Error('Notebook research needs reviewed SQL, generated SQL, or a DQL artifact before DQL draft promotion.');
|
|
1508
1932
|
const session = await createDqlGenerationSessionForProject(projectRoot, {
|
|
1509
1933
|
inputMode: 'upload',
|
|
1510
1934
|
sources: [{ path: sourcePath.endsWith('.sql') ? sourcePath : `${sourcePath}.sql`, content: sql }],
|
|
@@ -1530,6 +1954,7 @@ export async function startLocalServer(opts) {
|
|
|
1530
1954
|
resultPreview: updated.resultPreview,
|
|
1531
1955
|
generatedSql: updated.generatedSql,
|
|
1532
1956
|
reviewedSql: updated.reviewedSql,
|
|
1957
|
+
dqlArtifact: updated.dqlArtifact,
|
|
1533
1958
|
routeDecision: updated.routeDecision,
|
|
1534
1959
|
}),
|
|
1535
1960
|
}) ?? updated;
|
|
@@ -2054,10 +2479,14 @@ export async function startLocalServer(opts) {
|
|
|
2054
2479
|
res.end(serializeJSON({ error: parsed.error ?? 'Invalid agent run request.' }));
|
|
2055
2480
|
return;
|
|
2056
2481
|
}
|
|
2057
|
-
// The
|
|
2058
|
-
//
|
|
2059
|
-
|
|
2060
|
-
|
|
2482
|
+
// The answer-loop cascade now proves certified/semantic/generated tiers
|
|
2483
|
+
// after retrieval and execution. Avoid pre-routing ordinary Ask runs from
|
|
2484
|
+
// token-overlap signals; explicit callers may still provide signals.
|
|
2485
|
+
// Thread-scoped runs: the persisted thread supplies prior turns (server
|
|
2486
|
+
// state wins; the client-built context stays the no-threadId fallback).
|
|
2487
|
+
const conversationStore = parsed.request.threadId ? getConversationStore() : null;
|
|
2488
|
+
if (conversationStore && parsed.request.threadId && conversationStore.getThread(parsed.request.threadId)) {
|
|
2489
|
+
parsed.request.conversationContext = await conversationContextFromThread(conversationStore, parsed.request.threadId, parsed.request.conversationContext, parsed.request.question);
|
|
2061
2490
|
}
|
|
2062
2491
|
const wantsStream = url.searchParams.get('stream') === '1' || url.searchParams.get('stream') === 'true';
|
|
2063
2492
|
if (wantsStream) {
|
|
@@ -2072,11 +2501,13 @@ export async function startLocalServer(opts) {
|
|
|
2072
2501
|
}, (delta) => {
|
|
2073
2502
|
writeAgentRunSse(res, 'agent-run-answer-delta', { runId: parsed.request.runId, delta });
|
|
2074
2503
|
});
|
|
2504
|
+
recordConversationTurn(conversationStore, parsed.request.threadId, run);
|
|
2075
2505
|
writeAgentRunSse(res, 'agent-run-complete', run);
|
|
2076
2506
|
res.end();
|
|
2077
2507
|
return;
|
|
2078
2508
|
}
|
|
2079
2509
|
const run = await agentRunEngine.run(parsed.request);
|
|
2510
|
+
recordConversationTurn(conversationStore, parsed.request.threadId, run);
|
|
2080
2511
|
res.writeHead(201, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2081
2512
|
res.end(serializeJSON({ run }));
|
|
2082
2513
|
}
|
|
@@ -2108,6 +2539,7 @@ export async function startLocalServer(opts) {
|
|
|
2108
2539
|
const notebookPath = (record && agentRunString(record.notebookPath))
|
|
2109
2540
|
?? `notebooks/certification-requests/${Date.now()}.dqlnb`;
|
|
2110
2541
|
const generatedSql = record ? agentRunString(record.generatedSql) : undefined;
|
|
2542
|
+
const dqlArtifact = record ? normalizeDqlArtifactReference(record.dqlArtifact) : undefined;
|
|
2111
2543
|
try {
|
|
2112
2544
|
const storage = openNotebookResearchStorage();
|
|
2113
2545
|
try {
|
|
@@ -2119,9 +2551,11 @@ export async function startLocalServer(opts) {
|
|
|
2119
2551
|
domain: record ? agentRunString(record.domain) : undefined,
|
|
2120
2552
|
owner: record ? agentRunString(record.owner) : undefined,
|
|
2121
2553
|
generatedSql,
|
|
2554
|
+
dqlArtifact,
|
|
2122
2555
|
context: {
|
|
2123
2556
|
surface: 'stakeholder_request_certification',
|
|
2124
2557
|
requestedContext: agentRunRecord(record?.context) ?? null,
|
|
2558
|
+
dqlArtifact: dqlArtifact ?? null,
|
|
2125
2559
|
},
|
|
2126
2560
|
});
|
|
2127
2561
|
res.writeHead(201, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
@@ -2159,6 +2593,23 @@ export async function startLocalServer(opts) {
|
|
|
2159
2593
|
// DRAFT proposals (each with its stored Certifier verdict) so the notebook
|
|
2160
2594
|
// "Get Started" surface can route them into human review. dryRun preview —
|
|
2161
2595
|
// nothing is written or certified by this call.
|
|
2596
|
+
if (req.method === 'GET' && path === '/api/agent-runs/tier-distribution') {
|
|
2597
|
+
try {
|
|
2598
|
+
const store = getConversationStore();
|
|
2599
|
+
const limitParam = Number(url.searchParams.get('limit'));
|
|
2600
|
+
const limit = Number.isFinite(limitParam) && limitParam > 0 ? limitParam : undefined;
|
|
2601
|
+
const distribution = store
|
|
2602
|
+
? store.tierDistribution({ ...(limit ? { limit } : {}) })
|
|
2603
|
+
: { total: 0, byRouteTier: {}, byTerminalLane: {} };
|
|
2604
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2605
|
+
res.end(JSON.stringify(distribution));
|
|
2606
|
+
}
|
|
2607
|
+
catch (error) {
|
|
2608
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2609
|
+
res.end(JSON.stringify({ error: error instanceof Error ? error.message : String(error) }));
|
|
2610
|
+
}
|
|
2611
|
+
return;
|
|
2612
|
+
}
|
|
2162
2613
|
if ((req.method === 'GET' || req.method === 'POST') && path === '/api/propose') {
|
|
2163
2614
|
try {
|
|
2164
2615
|
let owner;
|
|
@@ -2189,6 +2640,69 @@ export async function startLocalServer(opts) {
|
|
|
2189
2640
|
}
|
|
2190
2641
|
return;
|
|
2191
2642
|
}
|
|
2643
|
+
// Composting preview: mine certified block clusters for semantic metric
|
|
2644
|
+
// candidates. Read-only; returns draft YAML + PR body text for review.
|
|
2645
|
+
if ((req.method === 'GET' || req.method === 'POST') && path === '/api/propose/composting') {
|
|
2646
|
+
try {
|
|
2647
|
+
let owner;
|
|
2648
|
+
let limit;
|
|
2649
|
+
let minSupport;
|
|
2650
|
+
if (req.method === 'POST') {
|
|
2651
|
+
const body = await readJSON(req).catch(() => ({}));
|
|
2652
|
+
if (typeof body?.owner === 'string')
|
|
2653
|
+
owner = body.owner;
|
|
2654
|
+
if (typeof body?.limit === 'number' && Number.isFinite(body.limit) && body.limit > 0)
|
|
2655
|
+
limit = body.limit;
|
|
2656
|
+
if (typeof body?.minSupport === 'number' && Number.isFinite(body.minSupport) && body.minSupport > 1)
|
|
2657
|
+
minSupport = body.minSupport;
|
|
2658
|
+
}
|
|
2659
|
+
else {
|
|
2660
|
+
const ownerParam = url.searchParams.get('owner');
|
|
2661
|
+
if (ownerParam)
|
|
2662
|
+
owner = ownerParam;
|
|
2663
|
+
const limitParam = Number(url.searchParams.get('limit'));
|
|
2664
|
+
if (Number.isFinite(limitParam) && limitParam > 0)
|
|
2665
|
+
limit = limitParam;
|
|
2666
|
+
const minSupportParam = Number(url.searchParams.get('minSupport'));
|
|
2667
|
+
if (Number.isFinite(minSupportParam) && minSupportParam > 1)
|
|
2668
|
+
minSupport = minSupportParam;
|
|
2669
|
+
}
|
|
2670
|
+
const changeset = buildSemanticCompostingChangeset(projectRoot, { owner, limit, minSupport });
|
|
2671
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2672
|
+
res.end(serializeJSON(changeset));
|
|
2673
|
+
}
|
|
2674
|
+
catch (error) {
|
|
2675
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2676
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
2677
|
+
}
|
|
2678
|
+
return;
|
|
2679
|
+
}
|
|
2680
|
+
// Composting generation: writes only approved metric candidate ids under
|
|
2681
|
+
// semantic-layer/metrics/_drafts plus a PR-body artifact. Nothing certified.
|
|
2682
|
+
if (req.method === 'POST' && path === '/api/propose/composting/generate') {
|
|
2683
|
+
try {
|
|
2684
|
+
const body = (await readJSON(req).catch(() => ({})));
|
|
2685
|
+
const ids = Array.isArray(body?.ids)
|
|
2686
|
+
? body.ids.filter((id) => typeof id === 'string' && id.trim().length > 0)
|
|
2687
|
+
: [];
|
|
2688
|
+
if (ids.length === 0) {
|
|
2689
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2690
|
+
res.end(serializeJSON({ error: 'Provide non-empty { ids } to generate semantic composting drafts.' }));
|
|
2691
|
+
return;
|
|
2692
|
+
}
|
|
2693
|
+
const owner = typeof body?.owner === 'string' ? body.owner : undefined;
|
|
2694
|
+
const limit = typeof body?.limit === 'number' && Number.isFinite(body.limit) && body.limit > 0 ? body.limit : undefined;
|
|
2695
|
+
const minSupport = typeof body?.minSupport === 'number' && Number.isFinite(body.minSupport) && body.minSupport > 1 ? body.minSupport : undefined;
|
|
2696
|
+
const result = generateSemanticCompostingDrafts(projectRoot, ids, { owner, limit, minSupport });
|
|
2697
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2698
|
+
res.end(serializeJSON(result));
|
|
2699
|
+
}
|
|
2700
|
+
catch (error) {
|
|
2701
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2702
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
2703
|
+
}
|
|
2704
|
+
return;
|
|
2705
|
+
}
|
|
2192
2706
|
// Deterministic PLAN only (classify → plan). Writes NOTHING. Same data the
|
|
2193
2707
|
// readiness endpoint embeds, exposed standalone for the approve gate.
|
|
2194
2708
|
if (req.method === 'POST' && path === '/api/propose/plan') {
|
|
@@ -2612,6 +3126,7 @@ export async function startLocalServer(opts) {
|
|
|
2612
3126
|
apiKey: typeof body.apiKey === 'string' ? body.apiKey : undefined,
|
|
2613
3127
|
baseUrl: typeof body.baseUrl === 'string' ? body.baseUrl : undefined,
|
|
2614
3128
|
model: typeof body.model === 'string' ? body.model : undefined,
|
|
3129
|
+
reasoningEffort: isReasoningEffortSetting(body.reasoningEffort) ? body.reasoningEffort : undefined,
|
|
2615
3130
|
});
|
|
2616
3131
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2617
3132
|
res.end(serializeJSON({ ok: true, providers }));
|
|
@@ -2622,6 +3137,60 @@ export async function startLocalServer(opts) {
|
|
|
2622
3137
|
}
|
|
2623
3138
|
return;
|
|
2624
3139
|
}
|
|
3140
|
+
// ── Subscription OAuth (Claude Pro/Max, ChatGPT Plus/Pro) ──────────────
|
|
3141
|
+
{
|
|
3142
|
+
const authMatch = path.match(/^\/api\/auth\/(claude|codex)\/(start|status|signout)$/);
|
|
3143
|
+
if (authMatch) {
|
|
3144
|
+
const oauthProvider = authMatch[1];
|
|
3145
|
+
const action = authMatch[2];
|
|
3146
|
+
const info = oauthProvider === 'claude'
|
|
3147
|
+
? { models: CLAUDE_OAUTH_MODELS, defaultModel: CLAUDE_OAUTH_DEFAULT_MODEL }
|
|
3148
|
+
: { models: CODEX_OAUTH_MODELS, defaultModel: CODEX_OAUTH_DEFAULT_MODEL };
|
|
3149
|
+
const manager = getOAuthManager(projectRoot, oauthProvider);
|
|
3150
|
+
const statusPayload = () => {
|
|
3151
|
+
const connected = oauthProvider === 'claude' ? claudeOAuthConnected(projectRoot) : codexOAuthConnected(projectRoot);
|
|
3152
|
+
return {
|
|
3153
|
+
provider: oauthProvider,
|
|
3154
|
+
connected,
|
|
3155
|
+
email: connected ? manager.getEmail() : null,
|
|
3156
|
+
models: info.models,
|
|
3157
|
+
defaultModel: info.defaultModel,
|
|
3158
|
+
pending: manager.isPending(),
|
|
3159
|
+
};
|
|
3160
|
+
};
|
|
3161
|
+
try {
|
|
3162
|
+
if (action === 'status' && req.method === 'GET') {
|
|
3163
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3164
|
+
res.end(serializeJSON(statusPayload()));
|
|
3165
|
+
return;
|
|
3166
|
+
}
|
|
3167
|
+
if (action === 'signout' && req.method === 'POST') {
|
|
3168
|
+
manager.signOut();
|
|
3169
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3170
|
+
res.end(serializeJSON(statusPayload()));
|
|
3171
|
+
return;
|
|
3172
|
+
}
|
|
3173
|
+
if (action === 'start' && req.method === 'POST') {
|
|
3174
|
+
// Begin a fresh flow: build the authorize URL and start the loopback
|
|
3175
|
+
// callback server. The client opens the URL; the server captures the
|
|
3176
|
+
// redirect and persists tokens. The pending state lives on the manager,
|
|
3177
|
+
// so a superseded flow settles cleanly. Errors surface on the next poll.
|
|
3178
|
+
const url = manager.startAuthorizationFlow();
|
|
3179
|
+
void manager.waitForCallback().catch(() => { });
|
|
3180
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3181
|
+
res.end(serializeJSON({ url, ...statusPayload() }));
|
|
3182
|
+
return;
|
|
3183
|
+
}
|
|
3184
|
+
res.writeHead(405, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3185
|
+
res.end(serializeJSON({ error: 'Method not allowed' }));
|
|
3186
|
+
}
|
|
3187
|
+
catch (error) {
|
|
3188
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3189
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
3190
|
+
}
|
|
3191
|
+
return;
|
|
3192
|
+
}
|
|
3193
|
+
}
|
|
2625
3194
|
if (req.method === 'POST' && path === '/api/settings/providers/test') {
|
|
2626
3195
|
try {
|
|
2627
3196
|
const body = await readJSON(req);
|
|
@@ -2662,6 +3231,117 @@ export async function startLocalServer(opts) {
|
|
|
2662
3231
|
}
|
|
2663
3232
|
return;
|
|
2664
3233
|
}
|
|
3234
|
+
// ── Conversation threads (server-side session persistence) ────────────
|
|
3235
|
+
if (path === '/api/agent/threads' || path.startsWith('/api/agent/threads/')) {
|
|
3236
|
+
const store = getConversationStore();
|
|
3237
|
+
if (!store) {
|
|
3238
|
+
res.writeHead(503, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3239
|
+
res.end(serializeJSON({ error: 'Conversation store is unavailable.' }));
|
|
3240
|
+
return;
|
|
3241
|
+
}
|
|
3242
|
+
try {
|
|
3243
|
+
if (req.method === 'GET' && path === '/api/agent/threads') {
|
|
3244
|
+
const limit = Number(url.searchParams.get('limit') ?? '50');
|
|
3245
|
+
const includeArchived = url.searchParams.get('archived') === '1';
|
|
3246
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3247
|
+
res.end(serializeJSON({ threads: store.listThreads({ limit: Number.isFinite(limit) ? limit : 50, includeArchived }) }));
|
|
3248
|
+
return;
|
|
3249
|
+
}
|
|
3250
|
+
if (req.method === 'POST' && path === '/api/agent/threads') {
|
|
3251
|
+
const body = await readJSON(req).catch(() => null);
|
|
3252
|
+
const record = agentRunRecord(body) ?? {};
|
|
3253
|
+
const thread = store.createThread({
|
|
3254
|
+
surface: agentRunString(record.surface),
|
|
3255
|
+
title: agentRunString(record.title),
|
|
3256
|
+
notebookPath: agentRunString(record.notebookPath),
|
|
3257
|
+
});
|
|
3258
|
+
res.writeHead(201, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3259
|
+
res.end(serializeJSON({ thread }));
|
|
3260
|
+
return;
|
|
3261
|
+
}
|
|
3262
|
+
// Cross-thread turn search ("search conversations") — must precede :id.
|
|
3263
|
+
if (req.method === 'GET' && path === '/api/agent/threads/search') {
|
|
3264
|
+
const query = url.searchParams.get('q') ?? '';
|
|
3265
|
+
const limit = Number(url.searchParams.get('limit') ?? '10');
|
|
3266
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3267
|
+
res.end(serializeJSON({
|
|
3268
|
+
turns: query.trim()
|
|
3269
|
+
? store.searchTurns({ query, limit: Number.isFinite(limit) ? limit : 10 })
|
|
3270
|
+
: [],
|
|
3271
|
+
}));
|
|
3272
|
+
return;
|
|
3273
|
+
}
|
|
3274
|
+
const threadMatch = path.match(/^\/api\/agent\/threads\/([^/]+)(?:\/(archive|promote))?$/);
|
|
3275
|
+
if (threadMatch) {
|
|
3276
|
+
const threadId = decodeURIComponent(threadMatch[1]);
|
|
3277
|
+
const action = threadMatch[2];
|
|
3278
|
+
const thread = store.getThread(threadId);
|
|
3279
|
+
if (!thread) {
|
|
3280
|
+
res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3281
|
+
res.end(serializeJSON({ error: 'Unknown thread id.' }));
|
|
3282
|
+
return;
|
|
3283
|
+
}
|
|
3284
|
+
if (req.method === 'POST' && action === 'archive') {
|
|
3285
|
+
store.archiveThread(threadId);
|
|
3286
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3287
|
+
res.end(serializeJSON({ ok: true }));
|
|
3288
|
+
return;
|
|
3289
|
+
}
|
|
3290
|
+
// Explicit promotion — the ONLY path from conversation into durable
|
|
3291
|
+
// memory ("remember this"). Deliberate user act, tagged for audit.
|
|
3292
|
+
if (req.method === 'POST' && action === 'promote') {
|
|
3293
|
+
const body = await readJSON(req).catch(() => null);
|
|
3294
|
+
const record = agentRunRecord(body) ?? {};
|
|
3295
|
+
const turnId = agentRunString(record.turnId);
|
|
3296
|
+
const turn = turnId
|
|
3297
|
+
? store.recentTurns(threadId, 200).find((candidate) => candidate.id === turnId)
|
|
3298
|
+
: undefined;
|
|
3299
|
+
if (!turn) {
|
|
3300
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3301
|
+
res.end(serializeJSON({ error: 'turnId is required and must belong to this thread.' }));
|
|
3302
|
+
return;
|
|
3303
|
+
}
|
|
3304
|
+
const scopeRaw = agentRunString(record.scope);
|
|
3305
|
+
const scope = scopeRaw === 'notebook' || scopeRaw === 'project' || scopeRaw === 'user' ? scopeRaw : 'project';
|
|
3306
|
+
const memory = new MemoryStore(defaultMemoryPath(projectRoot));
|
|
3307
|
+
try {
|
|
3308
|
+
const saved = memory.upsert({
|
|
3309
|
+
scope,
|
|
3310
|
+
title: agentRunString(record.title) ?? turn.question.slice(0, 120),
|
|
3311
|
+
content: `Q: ${turn.question}\nA: ${turn.answerSummary ?? turn.answerText ?? '(no answer summary)'}`,
|
|
3312
|
+
tags: ['conversation', threadId],
|
|
3313
|
+
source: 'conversation',
|
|
3314
|
+
confidence: 0.6,
|
|
3315
|
+
importance: 0.5,
|
|
3316
|
+
enabled: true,
|
|
3317
|
+
});
|
|
3318
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3319
|
+
res.end(serializeJSON({ ok: true, memory: saved }));
|
|
3320
|
+
}
|
|
3321
|
+
finally {
|
|
3322
|
+
memory.close();
|
|
3323
|
+
}
|
|
3324
|
+
return;
|
|
3325
|
+
}
|
|
3326
|
+
if (req.method === 'GET' && !action) {
|
|
3327
|
+
const limit = Number(url.searchParams.get('limit') ?? '50');
|
|
3328
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3329
|
+
res.end(serializeJSON({
|
|
3330
|
+
thread,
|
|
3331
|
+
turns: store.recentTurns(threadId, Number.isFinite(limit) ? limit : 50),
|
|
3332
|
+
}));
|
|
3333
|
+
return;
|
|
3334
|
+
}
|
|
3335
|
+
}
|
|
3336
|
+
res.writeHead(405, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3337
|
+
res.end(serializeJSON({ error: 'Method not allowed' }));
|
|
3338
|
+
}
|
|
3339
|
+
catch (error) {
|
|
3340
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3341
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
3342
|
+
}
|
|
3343
|
+
return;
|
|
3344
|
+
}
|
|
2665
3345
|
if (req.method === 'GET' && path === '/api/agent/memory') {
|
|
2666
3346
|
const memory = new MemoryStore(defaultMemoryPath(projectRoot));
|
|
2667
3347
|
try {
|
|
@@ -3185,6 +3865,7 @@ export async function startLocalServer(opts) {
|
|
|
3185
3865
|
const sourceCellId = notebookResearchString(body.sourceCellId) ?? notebookResearchSourceCellId(sourceCell);
|
|
3186
3866
|
const sourceCellName = notebookResearchString(body.sourceCellName) ?? notebookResearchSourceCellName(sourceCell);
|
|
3187
3867
|
const sourceCellFingerprint = notebookResearchString(body.sourceCellFingerprint) ?? notebookResearchSourceCellFingerprint(sourceCell);
|
|
3868
|
+
const dqlArtifact = normalizeDqlArtifactReference(body.dqlArtifact);
|
|
3188
3869
|
if (!notebookPath || !question) {
|
|
3189
3870
|
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3190
3871
|
res.end(serializeJSON({ error: 'notebookPath and question are required.' }));
|
|
@@ -3204,6 +3885,7 @@ export async function startLocalServer(opts) {
|
|
|
3204
3885
|
context: body.context,
|
|
3205
3886
|
generatedSql: notebookResearchString(body.generatedSql),
|
|
3206
3887
|
reviewedSql: notebookResearchString(body.reviewedSql),
|
|
3888
|
+
dqlArtifact,
|
|
3207
3889
|
});
|
|
3208
3890
|
const run = body.run === true
|
|
3209
3891
|
? await runNotebookResearch(storage, created, {
|
|
@@ -3214,6 +3896,7 @@ export async function startLocalServer(opts) {
|
|
|
3214
3896
|
context: body.context,
|
|
3215
3897
|
generatedSql: notebookResearchString(body.generatedSql),
|
|
3216
3898
|
reviewedSql: notebookResearchString(body.reviewedSql),
|
|
3899
|
+
dqlArtifact,
|
|
3217
3900
|
})
|
|
3218
3901
|
: created;
|
|
3219
3902
|
res.writeHead(201, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
@@ -3256,8 +3939,8 @@ export async function startLocalServer(opts) {
|
|
|
3256
3939
|
intent,
|
|
3257
3940
|
researchPattern: notebookResearchIntentPattern(intent),
|
|
3258
3941
|
},
|
|
3259
|
-
strictness: '
|
|
3260
|
-
limit:
|
|
3942
|
+
strictness: 'exploratory',
|
|
3943
|
+
limit: 160,
|
|
3261
3944
|
});
|
|
3262
3945
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3263
3946
|
res.end(serializeJSON(notebookResearchContextPreview(contextPack)));
|
|
@@ -3481,6 +4164,7 @@ export async function startLocalServer(opts) {
|
|
|
3481
4164
|
routeDecision: body.routeDecision,
|
|
3482
4165
|
generatedSql: notebookResearchString(body.generatedSql),
|
|
3483
4166
|
reviewedSql: notebookResearchString(body.reviewedSql),
|
|
4167
|
+
dqlArtifact: normalizeDqlArtifactReference(body.dqlArtifact),
|
|
3484
4168
|
warnings: Array.isArray(body.warnings) ? body.warnings.filter((item) => typeof item === 'string') : undefined,
|
|
3485
4169
|
reviewStatus: notebookResearchReviewStatus(body.reviewStatus),
|
|
3486
4170
|
dqlPromotionAction: notebookResearchPromotionAction(body.dqlPromotionAction),
|
|
@@ -3493,6 +4177,7 @@ export async function startLocalServer(opts) {
|
|
|
3493
4177
|
previewError: updated.error,
|
|
3494
4178
|
generatedSql: updated.generatedSql,
|
|
3495
4179
|
reviewedSql: updated.reviewedSql,
|
|
4180
|
+
dqlArtifact: updated.dqlArtifact,
|
|
3496
4181
|
routeDecision: updated.routeDecision,
|
|
3497
4182
|
}),
|
|
3498
4183
|
}) ?? updated;
|
|
@@ -3512,6 +4197,7 @@ export async function startLocalServer(opts) {
|
|
|
3512
4197
|
context: body.context,
|
|
3513
4198
|
generatedSql: notebookResearchString(body.generatedSql),
|
|
3514
4199
|
reviewedSql: notebookResearchString(body.reviewedSql),
|
|
4200
|
+
dqlArtifact: normalizeDqlArtifactReference(body.dqlArtifact),
|
|
3515
4201
|
});
|
|
3516
4202
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3517
4203
|
res.end(serializeJSON({ run: withNotebookResearchChecklist(updated) }));
|
|
@@ -5676,6 +6362,10 @@ export async function startLocalServer(opts) {
|
|
|
5676
6362
|
conversationContext: conversationContext && typeof conversationContext === 'object' && !Array.isArray(conversationContext)
|
|
5677
6363
|
? conversationContext
|
|
5678
6364
|
: undefined,
|
|
6365
|
+
// Intentionally NOT threading a route effort here: this raw block/chat
|
|
6366
|
+
// authoring endpoint keeps each SDK runner's own tuned default (the
|
|
6367
|
+
// claude-agent-sdk runs `xhigh` for deep authoring). Governed answer /
|
|
6368
|
+
// research paths carry task-adaptive effort; this one shouldn't cap it.
|
|
5679
6369
|
projectRoot,
|
|
5680
6370
|
executeCertifiedBlock: executeCertifiedBlockForAgent,
|
|
5681
6371
|
executeGeneratedSql: executeGeneratedSqlForAgent,
|
|
@@ -6663,16 +7353,75 @@ export function resolveDefaultLLMProvider(projectRoot) {
|
|
|
6663
7353
|
* answer-loop runner — never the MCP `claudeCodeRunner`, which doesn't emit a governed
|
|
6664
7354
|
* answer envelope. Everything else uses the Settings-resolved default runner.
|
|
6665
7355
|
*/
|
|
6666
|
-
function resolveGovernedAnswerRunner(projectRoot) {
|
|
7356
|
+
export function resolveGovernedAnswerRunner(projectRoot) {
|
|
6667
7357
|
const active = getActiveProvider(projectRoot);
|
|
6668
|
-
if (active
|
|
7358
|
+
if (isGovernedAnswerProviderId(active)) {
|
|
6669
7359
|
return { provider: active, runner: createDqlAgentProviderRunner(active) };
|
|
6670
7360
|
}
|
|
6671
7361
|
const resolved = resolveDefaultLLMProvider(projectRoot);
|
|
6672
7362
|
if (!resolved)
|
|
6673
7363
|
return null;
|
|
6674
|
-
|
|
6675
|
-
|
|
7364
|
+
if (isGovernedAnswerProviderId(resolved)) {
|
|
7365
|
+
return { provider: resolved, runner: createDqlAgentProviderRunner(resolved) };
|
|
7366
|
+
}
|
|
7367
|
+
return null;
|
|
7368
|
+
}
|
|
7369
|
+
function isGovernedAnswerProviderId(value) {
|
|
7370
|
+
return value === 'anthropic'
|
|
7371
|
+
|| value === 'openai'
|
|
7372
|
+
|| value === 'gemini'
|
|
7373
|
+
|| value === 'ollama'
|
|
7374
|
+
|| value === 'custom-openai'
|
|
7375
|
+
|| value === 'claude-code'
|
|
7376
|
+
|| value === 'codex';
|
|
7377
|
+
}
|
|
7378
|
+
/** Map a runner provider id to the settings id whose reasoning ceiling applies. */
|
|
7379
|
+
function reasoningSettingsIdFor(provider) {
|
|
7380
|
+
return provider === 'claude-agent-sdk' ? 'anthropic' : provider;
|
|
7381
|
+
}
|
|
7382
|
+
/** Type-guard for the reasoning-effort settings value from an untrusted request body. */
|
|
7383
|
+
function isReasoningEffortSetting(value) {
|
|
7384
|
+
return value === 'auto' || value === 'low' || value === 'medium' || value === 'high';
|
|
7385
|
+
}
|
|
7386
|
+
// Per-project singleton OAuth managers — they hold pending-auth state that must
|
|
7387
|
+
// survive between the `/start` request and the loopback callback. Keyed by
|
|
7388
|
+
// project root. "Pending" is derived from the manager itself (isPending) so a
|
|
7389
|
+
// superseded double-start can't desync a separate flag.
|
|
7390
|
+
const claudeOAuthManagers = new Map();
|
|
7391
|
+
const codexOAuthManagers = new Map();
|
|
7392
|
+
function getOAuthManager(projectRoot, provider) {
|
|
7393
|
+
if (provider === 'claude') {
|
|
7394
|
+
let m = claudeOAuthManagers.get(projectRoot);
|
|
7395
|
+
if (!m) {
|
|
7396
|
+
m = new ClaudeOAuthManager(projectRoot);
|
|
7397
|
+
claudeOAuthManagers.set(projectRoot, m);
|
|
7398
|
+
}
|
|
7399
|
+
return m;
|
|
7400
|
+
}
|
|
7401
|
+
let m = codexOAuthManagers.get(projectRoot);
|
|
7402
|
+
if (!m) {
|
|
7403
|
+
m = new CodexOAuthManager(projectRoot);
|
|
7404
|
+
codexOAuthManagers.set(projectRoot, m);
|
|
7405
|
+
}
|
|
7406
|
+
return m;
|
|
7407
|
+
}
|
|
7408
|
+
/**
|
|
7409
|
+
* Resolve the concrete reasoning effort for a run: the route's task-adaptive
|
|
7410
|
+
* effort (bumped one level on a repair attempt), clamped by the provider's
|
|
7411
|
+
* Settings ceiling (`auto` = no cap). Providers no-op when their model has no
|
|
7412
|
+
* reasoning surface, so returning a concrete level here is always safe.
|
|
7413
|
+
*/
|
|
7414
|
+
function resolveRunReasoningEffort(projectRoot, provider, route, isRepair) {
|
|
7415
|
+
const ceiling = getEffectiveProviderConfig(projectRoot, reasoningSettingsIdFor(provider)).reasoningEffort;
|
|
7416
|
+
let desired = route ? routeReasoningEffort(route) : 'medium';
|
|
7417
|
+
if (isRepair)
|
|
7418
|
+
desired = bumpReasoningEffort(desired);
|
|
7419
|
+
return ceiling ? clampReasoningEffort(desired, ceiling) : desired;
|
|
7420
|
+
}
|
|
7421
|
+
function resolveRequestedRunReasoningEffort(projectRoot, provider, requested, isRepair) {
|
|
7422
|
+
const ceiling = getEffectiveProviderConfig(projectRoot, reasoningSettingsIdFor(provider)).reasoningEffort;
|
|
7423
|
+
const desired = isRepair ? bumpReasoningEffort(requested) : requested;
|
|
7424
|
+
return ceiling ? clampReasoningEffort(desired, ceiling) : desired;
|
|
6676
7425
|
}
|
|
6677
7426
|
function loadAppDashboard(projectRoot, appId, dashboardId) {
|
|
6678
7427
|
for (const p of findAppDocuments(projectRoot)) {
|
|
@@ -6998,6 +7747,11 @@ export function buildProposeReadiness(projectRoot, projectConfig = loadProjectCo
|
|
|
6998
7747
|
if (proposal.certification.errors.length === 0)
|
|
6999
7748
|
readyForReview += 1;
|
|
7000
7749
|
}
|
|
7750
|
+
const reviewableProposals = buildReviewableProposals(projectRoot, manifestPath, projectConfig, summary.proposals, options.owner);
|
|
7751
|
+
const reviewAges = reviewableProposals
|
|
7752
|
+
.map((proposal) => proposal.review.reviewAgeHours)
|
|
7753
|
+
.filter((value) => typeof value === 'number' && Number.isFinite(value));
|
|
7754
|
+
const estimatedReviewMinutes = reviewableProposals.reduce((sum, proposal) => sum + proposal.review.estimatedReviewMinutes, 0);
|
|
7001
7755
|
return {
|
|
7002
7756
|
ready: true,
|
|
7003
7757
|
summary: {
|
|
@@ -7012,11 +7766,595 @@ export function buildProposeReadiness(projectRoot, projectConfig = loadProjectCo
|
|
|
7012
7766
|
readyForReview,
|
|
7013
7767
|
blockingTotal,
|
|
7014
7768
|
warningTotal,
|
|
7769
|
+
reviewTelemetry: {
|
|
7770
|
+
existingDrafts: reviewableProposals.filter((proposal) => proposal.review.draftExists).length,
|
|
7771
|
+
medianReviewAgeHours: median(reviewAges),
|
|
7772
|
+
readyForReviewRate: ratio(readyForReview, summary.proposalsRanked),
|
|
7773
|
+
estimatedReviewMinutes,
|
|
7774
|
+
},
|
|
7015
7775
|
},
|
|
7016
7776
|
plan,
|
|
7017
|
-
proposals:
|
|
7777
|
+
proposals: reviewableProposals,
|
|
7018
7778
|
};
|
|
7019
7779
|
}
|
|
7780
|
+
function buildReviewableProposals(projectRoot, dbtManifestPath, projectConfig, proposals, owner) {
|
|
7781
|
+
const manifest = safeBuildProjectManifest(projectRoot);
|
|
7782
|
+
return proposals.map((proposal, index) => {
|
|
7783
|
+
const preview = buildProposePreview(projectRoot, dbtManifestPath, proposal.slug, {
|
|
7784
|
+
config: projectConfig.propose,
|
|
7785
|
+
owner,
|
|
7786
|
+
});
|
|
7787
|
+
const draftPath = proposal.path ?? resolveProposeDraftPath(projectRoot, proposal.domain, proposal.slug);
|
|
7788
|
+
const draftFile = join(projectRoot, draftPath);
|
|
7789
|
+
const draftExists = existsSync(draftFile);
|
|
7790
|
+
const stats = draftExists ? statSync(draftFile) : undefined;
|
|
7791
|
+
const firstSeenAt = stats ? stableFsTime(stats.birthtimeMs, stats.ctimeMs) : undefined;
|
|
7792
|
+
const lastUpdatedAt = stats ? stableFsTime(stats.mtimeMs, stats.ctimeMs) : undefined;
|
|
7793
|
+
const reviewAgeHours = firstSeenAt ? hoursSince(firstSeenAt) : undefined;
|
|
7794
|
+
const blockingCount = proposal.certification.errors.length;
|
|
7795
|
+
const warningCount = proposal.certification.warnings.length;
|
|
7796
|
+
const outputs = preview?.outputs ?? proposal.inference.declaredOutputs;
|
|
7797
|
+
const status = blockingCount === 0
|
|
7798
|
+
? 'ready_for_review'
|
|
7799
|
+
: draftExists
|
|
7800
|
+
? 'draft_exists'
|
|
7801
|
+
: 'new';
|
|
7802
|
+
return {
|
|
7803
|
+
...proposal,
|
|
7804
|
+
review: {
|
|
7805
|
+
queueRank: index + 1,
|
|
7806
|
+
status,
|
|
7807
|
+
priorityScore: proposal.ranking.score,
|
|
7808
|
+
blockingCount,
|
|
7809
|
+
warningCount,
|
|
7810
|
+
estimatedReviewMinutes: estimateProposalReviewMinutes(blockingCount, warningCount, Boolean(preview?.sqlPreview)),
|
|
7811
|
+
draftPath,
|
|
7812
|
+
draftExists,
|
|
7813
|
+
firstSeenAt,
|
|
7814
|
+
lastUpdatedAt,
|
|
7815
|
+
reviewAgeHours,
|
|
7816
|
+
certifyCommand: `dql certify --from-draft ${quoteCliArg(draftPath)} --owner you@example.com`,
|
|
7817
|
+
payload: {
|
|
7818
|
+
question: preview?.examples?.[0] ?? proposal.inference.examples[0]?.question ?? `Review ${proposal.slug}`,
|
|
7819
|
+
model: proposal.model,
|
|
7820
|
+
domain: proposal.domain,
|
|
7821
|
+
sqlPreview: preview?.sqlPreview,
|
|
7822
|
+
outputs,
|
|
7823
|
+
grain: preview?.grain ?? proposal.inference.grain,
|
|
7824
|
+
pattern: preview?.pattern ?? proposal.inference.pattern,
|
|
7825
|
+
evidence: proposal.evidence,
|
|
7826
|
+
resultSample: { status: 'not_run', rows: [] },
|
|
7827
|
+
nearestCertifiedBlock: findNearestCertifiedBlock(manifest, proposal, outputs),
|
|
7828
|
+
},
|
|
7829
|
+
},
|
|
7830
|
+
};
|
|
7831
|
+
});
|
|
7832
|
+
}
|
|
7833
|
+
function safeBuildProjectManifest(projectRoot) {
|
|
7834
|
+
try {
|
|
7835
|
+
return buildManifest({ projectRoot });
|
|
7836
|
+
}
|
|
7837
|
+
catch {
|
|
7838
|
+
return { blocks: {} };
|
|
7839
|
+
}
|
|
7840
|
+
}
|
|
7841
|
+
function resolveProposeDraftPath(projectRoot, domain, slug) {
|
|
7842
|
+
const safeDomain = domain
|
|
7843
|
+
.trim()
|
|
7844
|
+
.toLowerCase()
|
|
7845
|
+
.replace(/[^a-z0-9/_-]+/g, '-')
|
|
7846
|
+
.replace(/^[-/]+|[-/]+$/g, '');
|
|
7847
|
+
return safeDomain && existsSync(join(projectRoot, 'domains', safeDomain))
|
|
7848
|
+
? `domains/${safeDomain}/blocks/_drafts/${slug}.dql`
|
|
7849
|
+
: `blocks/_drafts/${slug}.dql`;
|
|
7850
|
+
}
|
|
7851
|
+
function findNearestCertifiedBlock(manifest, proposal, outputs) {
|
|
7852
|
+
const outputSet = new Set(outputs.map((output) => output.toLowerCase()));
|
|
7853
|
+
let best;
|
|
7854
|
+
for (const block of Object.values(manifest.blocks ?? {})) {
|
|
7855
|
+
if (String(block.status ?? '').toLowerCase() !== 'certified')
|
|
7856
|
+
continue;
|
|
7857
|
+
const blockOutputs = (block.declaredOutputs ?? []).map((output) => output.toLowerCase());
|
|
7858
|
+
const sharedOutputs = blockOutputs.filter((output) => outputSet.has(output));
|
|
7859
|
+
const sameDomain = block.domain === proposal.domain;
|
|
7860
|
+
const overlapScore = sharedOutputs.length * 10 + (sameDomain ? 5 : 0);
|
|
7861
|
+
if (overlapScore <= 0)
|
|
7862
|
+
continue;
|
|
7863
|
+
if (!best || overlapScore > best.overlapScore) {
|
|
7864
|
+
best = {
|
|
7865
|
+
name: block.name,
|
|
7866
|
+
path: block.filePath,
|
|
7867
|
+
domain: block.domain,
|
|
7868
|
+
overlapScore,
|
|
7869
|
+
sharedOutputs,
|
|
7870
|
+
};
|
|
7871
|
+
}
|
|
7872
|
+
}
|
|
7873
|
+
return best;
|
|
7874
|
+
}
|
|
7875
|
+
function estimateProposalReviewMinutes(blockingCount, warningCount, hasSqlPreview) {
|
|
7876
|
+
const base = hasSqlPreview ? 12 : 18;
|
|
7877
|
+
return base + blockingCount * 8 + warningCount * 4;
|
|
7878
|
+
}
|
|
7879
|
+
function stableFsTime(primaryMs, fallbackMs) {
|
|
7880
|
+
const ms = Number.isFinite(primaryMs) && primaryMs > 0 ? primaryMs : fallbackMs;
|
|
7881
|
+
return Number.isFinite(ms) && ms > 0 ? new Date(ms).toISOString() : undefined;
|
|
7882
|
+
}
|
|
7883
|
+
function hoursSince(iso) {
|
|
7884
|
+
const ts = Date.parse(iso);
|
|
7885
|
+
if (!Number.isFinite(ts))
|
|
7886
|
+
return 0;
|
|
7887
|
+
return Number(Math.max(0, (Date.now() - ts) / 3_600_000).toFixed(2));
|
|
7888
|
+
}
|
|
7889
|
+
function median(values) {
|
|
7890
|
+
if (values.length === 0)
|
|
7891
|
+
return null;
|
|
7892
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
7893
|
+
const mid = Math.floor(sorted.length / 2);
|
|
7894
|
+
const value = sorted.length % 2 === 1
|
|
7895
|
+
? sorted[mid]
|
|
7896
|
+
: (sorted[mid - 1] + sorted[mid]) / 2;
|
|
7897
|
+
return Number(value.toFixed(2));
|
|
7898
|
+
}
|
|
7899
|
+
function ratio(numerator, denominator) {
|
|
7900
|
+
if (denominator === 0)
|
|
7901
|
+
return null;
|
|
7902
|
+
return Number((numerator / denominator).toFixed(4));
|
|
7903
|
+
}
|
|
7904
|
+
function quoteCliArg(value) {
|
|
7905
|
+
if (/^[A-Za-z0-9_./:-]+$/.test(value))
|
|
7906
|
+
return value;
|
|
7907
|
+
return `'${value.replace(/'/g, "'\\''")}'`;
|
|
7908
|
+
}
|
|
7909
|
+
/**
|
|
7910
|
+
* Mine certified block clusters for recurring metric definitions and render a
|
|
7911
|
+
* reviewable semantic-layer changeset. This is read-only: callers must confirm
|
|
7912
|
+
* candidate ids through generateSemanticCompostingDrafts before files are written.
|
|
7913
|
+
*/
|
|
7914
|
+
export function buildSemanticCompostingChangeset(projectRoot, options = {}) {
|
|
7915
|
+
let manifest;
|
|
7916
|
+
try {
|
|
7917
|
+
manifest = buildManifest({ projectRoot });
|
|
7918
|
+
}
|
|
7919
|
+
catch (error) {
|
|
7920
|
+
return {
|
|
7921
|
+
ready: false,
|
|
7922
|
+
reason: error instanceof Error ? error.message : String(error),
|
|
7923
|
+
summary: {
|
|
7924
|
+
certifiedBlocksScanned: 0,
|
|
7925
|
+
eligibleMetricClusters: 0,
|
|
7926
|
+
candidatesRanked: 0,
|
|
7927
|
+
existingDrafts: 0,
|
|
7928
|
+
donorBlocksUsed: 0,
|
|
7929
|
+
minSupport: Math.max(2, Math.floor(options.minSupport ?? 2)),
|
|
7930
|
+
},
|
|
7931
|
+
candidates: [],
|
|
7932
|
+
prBody: '',
|
|
7933
|
+
};
|
|
7934
|
+
}
|
|
7935
|
+
const minSupport = Math.max(2, Math.floor(options.minSupport ?? 2));
|
|
7936
|
+
const limit = Number.isFinite(options.limit) && Number(options.limit) > 0 ? Math.floor(Number(options.limit)) : 10;
|
|
7937
|
+
const owner = options.owner?.trim() || resolveLocalOwner(projectRoot, { persist: false });
|
|
7938
|
+
const existingMetricNames = new Set(Object.entries(manifest.metrics ?? {})
|
|
7939
|
+
.filter(([, metric]) => !isSemanticDraftMetricPath(metric.filePath))
|
|
7940
|
+
.map(([name]) => name.toLowerCase()));
|
|
7941
|
+
const certifiedBlocks = Object.values(manifest.blocks ?? {})
|
|
7942
|
+
.filter((block) => String(block.status ?? '').toLowerCase() === 'certified');
|
|
7943
|
+
const groups = new Map();
|
|
7944
|
+
for (const block of certifiedBlocks) {
|
|
7945
|
+
const table = singleTableDependency(block);
|
|
7946
|
+
if (!table)
|
|
7947
|
+
continue;
|
|
7948
|
+
const where = extractCompostingWhereClause(block.sql);
|
|
7949
|
+
const normalizedFilter = where ? normalizeCompostingSql(where) : undefined;
|
|
7950
|
+
const metrics = extractCompostingAggregateMetrics(block.sql);
|
|
7951
|
+
if (metrics.length === 0)
|
|
7952
|
+
continue;
|
|
7953
|
+
const donor = {
|
|
7954
|
+
name: block.name,
|
|
7955
|
+
path: block.filePath,
|
|
7956
|
+
domain: block.domain,
|
|
7957
|
+
outputs: blockOutputNames(block),
|
|
7958
|
+
filters: where ? splitCompostingFilters(where) : [],
|
|
7959
|
+
};
|
|
7960
|
+
for (const metric of metrics) {
|
|
7961
|
+
const normalizedExpression = normalizeCompostingSql(metric.expression);
|
|
7962
|
+
const domain = block.domain?.trim() || 'misc';
|
|
7963
|
+
const key = [
|
|
7964
|
+
domain.toLowerCase(),
|
|
7965
|
+
table.toLowerCase(),
|
|
7966
|
+
metric.type,
|
|
7967
|
+
normalizedExpression,
|
|
7968
|
+
normalizedFilter ?? '',
|
|
7969
|
+
].join('\0');
|
|
7970
|
+
const existing = groups.get(key);
|
|
7971
|
+
if (existing) {
|
|
7972
|
+
existing.aliases.push(metric.alias ?? '');
|
|
7973
|
+
existing.donors.set(block.filePath, donor);
|
|
7974
|
+
}
|
|
7975
|
+
else {
|
|
7976
|
+
groups.set(key, {
|
|
7977
|
+
domain,
|
|
7978
|
+
table,
|
|
7979
|
+
type: metric.type,
|
|
7980
|
+
expression: metric.expression,
|
|
7981
|
+
normalizedExpression,
|
|
7982
|
+
filter: where,
|
|
7983
|
+
normalizedFilter,
|
|
7984
|
+
aliases: [metric.alias ?? ''],
|
|
7985
|
+
donors: new Map([[block.filePath, donor]]),
|
|
7986
|
+
});
|
|
7987
|
+
}
|
|
7988
|
+
}
|
|
7989
|
+
}
|
|
7990
|
+
const usedNames = new Set(existingMetricNames);
|
|
7991
|
+
const candidates = Array.from(groups.values())
|
|
7992
|
+
.filter((group) => group.donors.size >= minSupport)
|
|
7993
|
+
.sort((a, b) => b.donors.size - a.donors.size || a.normalizedExpression.localeCompare(b.normalizedExpression))
|
|
7994
|
+
.map((group) => buildCompostingMetricCandidate(projectRoot, group, owner, usedNames))
|
|
7995
|
+
.filter((candidate) => Boolean(candidate))
|
|
7996
|
+
.slice(0, limit);
|
|
7997
|
+
const donorBlocksUsed = new Set(candidates.flatMap((candidate) => candidate.donorBlocks.map((donor) => donor.path))).size;
|
|
7998
|
+
return {
|
|
7999
|
+
ready: true,
|
|
8000
|
+
summary: {
|
|
8001
|
+
certifiedBlocksScanned: certifiedBlocks.length,
|
|
8002
|
+
eligibleMetricClusters: Array.from(groups.values()).filter((group) => group.donors.size >= minSupport).length,
|
|
8003
|
+
candidatesRanked: candidates.length,
|
|
8004
|
+
existingDrafts: candidates.filter((candidate) => candidate.draftExists).length,
|
|
8005
|
+
donorBlocksUsed,
|
|
8006
|
+
minSupport,
|
|
8007
|
+
},
|
|
8008
|
+
candidates,
|
|
8009
|
+
prBody: renderCompostingPrBody(candidates),
|
|
8010
|
+
};
|
|
8011
|
+
}
|
|
8012
|
+
export function generateSemanticCompostingDrafts(projectRoot, candidateIds, options = {}) {
|
|
8013
|
+
const changeset = buildSemanticCompostingChangeset(projectRoot, options);
|
|
8014
|
+
if (!changeset.ready) {
|
|
8015
|
+
return {
|
|
8016
|
+
ready: false,
|
|
8017
|
+
reason: changeset.reason,
|
|
8018
|
+
draftsWritten: 0,
|
|
8019
|
+
draftsSkipped: 0,
|
|
8020
|
+
candidates: [],
|
|
8021
|
+
paths: [],
|
|
8022
|
+
prBody: '',
|
|
8023
|
+
};
|
|
8024
|
+
}
|
|
8025
|
+
const wanted = candidateIds && candidateIds.length > 0 ? new Set(candidateIds) : undefined;
|
|
8026
|
+
const selected = wanted ? changeset.candidates.filter((candidate) => wanted.has(candidate.id)) : changeset.candidates;
|
|
8027
|
+
const paths = [];
|
|
8028
|
+
let draftsWritten = 0;
|
|
8029
|
+
let draftsSkipped = 0;
|
|
8030
|
+
for (const candidate of selected) {
|
|
8031
|
+
const absolutePath = join(projectRoot, candidate.draftPath);
|
|
8032
|
+
if (existsSync(absolutePath)) {
|
|
8033
|
+
draftsSkipped += 1;
|
|
8034
|
+
}
|
|
8035
|
+
else {
|
|
8036
|
+
mkdirSync(dirname(absolutePath), { recursive: true });
|
|
8037
|
+
writeFileSync(absolutePath, candidate.yaml, 'utf-8');
|
|
8038
|
+
draftsWritten += 1;
|
|
8039
|
+
}
|
|
8040
|
+
paths.push(candidate.draftPath);
|
|
8041
|
+
}
|
|
8042
|
+
let prBodyPath;
|
|
8043
|
+
const prBody = renderCompostingPrBody(selected);
|
|
8044
|
+
if (selected.length > 0) {
|
|
8045
|
+
prBodyPath = 'semantic-layer/metrics/_drafts/PR_BODY.md';
|
|
8046
|
+
const absolutePrBodyPath = join(projectRoot, prBodyPath);
|
|
8047
|
+
mkdirSync(dirname(absolutePrBodyPath), { recursive: true });
|
|
8048
|
+
writeFileSync(absolutePrBodyPath, prBody, 'utf-8');
|
|
8049
|
+
}
|
|
8050
|
+
return {
|
|
8051
|
+
ready: true,
|
|
8052
|
+
draftsWritten,
|
|
8053
|
+
draftsSkipped,
|
|
8054
|
+
candidates: selected,
|
|
8055
|
+
paths,
|
|
8056
|
+
prBody,
|
|
8057
|
+
prBodyPath,
|
|
8058
|
+
};
|
|
8059
|
+
}
|
|
8060
|
+
function buildCompostingMetricCandidate(projectRoot, group, owner, usedNames) {
|
|
8061
|
+
const baseName = preferredCompostingMetricName(group);
|
|
8062
|
+
const name = dedupeCompostingName(baseName, usedNames);
|
|
8063
|
+
if (!name)
|
|
8064
|
+
return undefined;
|
|
8065
|
+
usedNames.add(name.toLowerCase());
|
|
8066
|
+
const label = titleFromSlug(name);
|
|
8067
|
+
const donors = Array.from(group.donors.values()).sort((a, b) => a.path.localeCompare(b.path));
|
|
8068
|
+
const support = donors.length;
|
|
8069
|
+
const domainSlug = slugForPath(group.domain);
|
|
8070
|
+
const draftPath = `semantic-layer/metrics/_drafts/${domainSlug}/${name}.yaml`;
|
|
8071
|
+
const recurringFilters = group.filter ? splitCompostingFilters(group.filter) : [];
|
|
8072
|
+
const metric = {
|
|
8073
|
+
name,
|
|
8074
|
+
label,
|
|
8075
|
+
description: `Draft metric composted from ${support} certified DQL blocks. Review the expression, filter, grain, and owner before certification.`,
|
|
8076
|
+
domain: group.domain,
|
|
8077
|
+
status: 'draft',
|
|
8078
|
+
sql: group.expression,
|
|
8079
|
+
type: group.type,
|
|
8080
|
+
table: group.table,
|
|
8081
|
+
filter: group.filter,
|
|
8082
|
+
tags: ['composted', 'human-review'],
|
|
8083
|
+
owner,
|
|
8084
|
+
source: {
|
|
8085
|
+
provider: 'dql',
|
|
8086
|
+
objectType: 'composted_metric',
|
|
8087
|
+
objectId: `composted:${name}`,
|
|
8088
|
+
objectName: label,
|
|
8089
|
+
importedAt: new Date().toISOString(),
|
|
8090
|
+
extra: {
|
|
8091
|
+
support,
|
|
8092
|
+
donorBlocks: donors.map((donor) => donor.name),
|
|
8093
|
+
donorPaths: donors.map((donor) => donor.path),
|
|
8094
|
+
normalizedExpression: group.normalizedExpression,
|
|
8095
|
+
normalizedFilter: group.normalizedFilter,
|
|
8096
|
+
},
|
|
8097
|
+
},
|
|
8098
|
+
};
|
|
8099
|
+
return {
|
|
8100
|
+
id: `metric:${name}:${stableShortHash(`${group.domain}|${group.table}|${group.normalizedExpression}|${group.normalizedFilter ?? ''}`)}`,
|
|
8101
|
+
kind: 'metric',
|
|
8102
|
+
name,
|
|
8103
|
+
label,
|
|
8104
|
+
description: metric.description,
|
|
8105
|
+
domain: group.domain,
|
|
8106
|
+
table: group.table,
|
|
8107
|
+
type: group.type,
|
|
8108
|
+
sql: group.expression,
|
|
8109
|
+
filter: group.filter,
|
|
8110
|
+
status: 'draft',
|
|
8111
|
+
support,
|
|
8112
|
+
donorBlocks: donors,
|
|
8113
|
+
draftPath,
|
|
8114
|
+
draftExists: existsSync(join(projectRoot, draftPath)),
|
|
8115
|
+
yaml: serializeMetricDefinitionToYaml(metric),
|
|
8116
|
+
provenance: {
|
|
8117
|
+
normalizedExpression: group.normalizedExpression,
|
|
8118
|
+
normalizedFilter: group.normalizedFilter,
|
|
8119
|
+
recurringFilters,
|
|
8120
|
+
},
|
|
8121
|
+
review: {
|
|
8122
|
+
priorityScore: support * 10 + recurringFilters.length * 2,
|
|
8123
|
+
rationale: `Recurring ${group.type} metric found in ${support} certified blocks${group.filter ? ' with the same filter' : ''}.`,
|
|
8124
|
+
},
|
|
8125
|
+
};
|
|
8126
|
+
}
|
|
8127
|
+
function singleTableDependency(block) {
|
|
8128
|
+
const dependencies = Array.from(new Set((block.tableDependencies ?? []).filter((table) => table && !table.startsWith('ref:'))));
|
|
8129
|
+
if (dependencies.length === 1)
|
|
8130
|
+
return dependencies[0];
|
|
8131
|
+
const rawRefs = Array.from(new Set((block.rawTableRefs ?? []).filter(Boolean)));
|
|
8132
|
+
return rawRefs.length === 1 ? rawRefs[0] : undefined;
|
|
8133
|
+
}
|
|
8134
|
+
function blockOutputNames(block) {
|
|
8135
|
+
if (Array.isArray(block.declaredOutputs) && block.declaredOutputs.length > 0)
|
|
8136
|
+
return block.declaredOutputs;
|
|
8137
|
+
if (Array.isArray(block.outputContract) && block.outputContract.length > 0)
|
|
8138
|
+
return block.outputContract.map((output) => output.name);
|
|
8139
|
+
if (Array.isArray(block.outputs) && block.outputs.length > 0)
|
|
8140
|
+
return block.outputs.map((output) => output.name);
|
|
8141
|
+
return [];
|
|
8142
|
+
}
|
|
8143
|
+
function isSemanticDraftMetricPath(filePath) {
|
|
8144
|
+
return Boolean(filePath?.replaceAll('\\', '/').includes('semantic-layer/metrics/_drafts/'));
|
|
8145
|
+
}
|
|
8146
|
+
function extractCompostingAggregateMetrics(sql) {
|
|
8147
|
+
const select = extractCompostingSelectClause(sql);
|
|
8148
|
+
if (!select)
|
|
8149
|
+
return [];
|
|
8150
|
+
return splitCompostingTopLevel(select, ',').flatMap((item) => {
|
|
8151
|
+
const parsed = parseCompostingSelectItem(item);
|
|
8152
|
+
if (!parsed)
|
|
8153
|
+
return [];
|
|
8154
|
+
return [parsed];
|
|
8155
|
+
});
|
|
8156
|
+
}
|
|
8157
|
+
function extractCompostingSelectClause(sql) {
|
|
8158
|
+
const source = stripSqlTerminator(sql);
|
|
8159
|
+
const selectMatch = /\bselect\b/i.exec(source);
|
|
8160
|
+
if (!selectMatch)
|
|
8161
|
+
return undefined;
|
|
8162
|
+
const fromIndex = findTopLevelKeyword(source, 'from', selectMatch.index + selectMatch[0].length);
|
|
8163
|
+
if (fromIndex < 0)
|
|
8164
|
+
return undefined;
|
|
8165
|
+
return source.slice(selectMatch.index + selectMatch[0].length, fromIndex).trim();
|
|
8166
|
+
}
|
|
8167
|
+
function parseCompostingSelectItem(item) {
|
|
8168
|
+
const { expression, alias } = splitCompostingAlias(item.trim());
|
|
8169
|
+
const normalizedExpression = expression.trim();
|
|
8170
|
+
const distinctCount = normalizedExpression.match(/^count\s*\(\s*distinct\s+[\s\S]+\)$/i);
|
|
8171
|
+
if (distinctCount) {
|
|
8172
|
+
return { expression: normalizedExpression, alias, type: 'count_distinct' };
|
|
8173
|
+
}
|
|
8174
|
+
const count = normalizedExpression.match(/^count\s*\([\s\S]*\)$/i);
|
|
8175
|
+
if (count)
|
|
8176
|
+
return { expression: normalizedExpression, alias, type: 'count' };
|
|
8177
|
+
const aggregate = normalizedExpression.match(/^(sum|avg|min|max)\s*\([\s\S]+\)$/i);
|
|
8178
|
+
if (!aggregate)
|
|
8179
|
+
return undefined;
|
|
8180
|
+
return {
|
|
8181
|
+
expression: normalizedExpression,
|
|
8182
|
+
alias,
|
|
8183
|
+
type: aggregate[1].toLowerCase(),
|
|
8184
|
+
};
|
|
8185
|
+
}
|
|
8186
|
+
function splitCompostingAlias(item) {
|
|
8187
|
+
const asMatch = item.match(/^([\s\S]+?)\s+as\s+("[^"]+"|`[^`]+`|\[[^\]]+\]|[A-Za-z_][A-Za-z0-9_$]*)$/i);
|
|
8188
|
+
if (asMatch)
|
|
8189
|
+
return { expression: asMatch[1].trim(), alias: stripIdentifierQuotes(asMatch[2]) };
|
|
8190
|
+
return { expression: item.trim() };
|
|
8191
|
+
}
|
|
8192
|
+
function extractCompostingWhereClause(sql) {
|
|
8193
|
+
const source = stripSqlTerminator(sql);
|
|
8194
|
+
const whereIndex = findTopLevelKeyword(source, 'where', 0);
|
|
8195
|
+
if (whereIndex < 0)
|
|
8196
|
+
return undefined;
|
|
8197
|
+
const start = whereIndex + 'where'.length;
|
|
8198
|
+
const endCandidates = ['group by', 'having', 'order by', 'limit', 'qualify']
|
|
8199
|
+
.map((keyword) => findTopLevelKeyword(source, keyword, start))
|
|
8200
|
+
.filter((index) => index >= 0);
|
|
8201
|
+
const end = endCandidates.length > 0 ? Math.min(...endCandidates) : source.length;
|
|
8202
|
+
const where = source.slice(start, end).trim();
|
|
8203
|
+
return where || undefined;
|
|
8204
|
+
}
|
|
8205
|
+
function splitCompostingFilters(where) {
|
|
8206
|
+
return splitCompostingTopLevel(where, 'and')
|
|
8207
|
+
.map((part) => part.trim())
|
|
8208
|
+
.filter(Boolean);
|
|
8209
|
+
}
|
|
8210
|
+
function splitCompostingTopLevel(input, delimiter) {
|
|
8211
|
+
const out = [];
|
|
8212
|
+
let start = 0;
|
|
8213
|
+
let depth = 0;
|
|
8214
|
+
let quote;
|
|
8215
|
+
for (let i = 0; i < input.length; i += 1) {
|
|
8216
|
+
const char = input[i];
|
|
8217
|
+
if (quote) {
|
|
8218
|
+
if (char === quote && input[i - 1] !== '\\')
|
|
8219
|
+
quote = undefined;
|
|
8220
|
+
continue;
|
|
8221
|
+
}
|
|
8222
|
+
if (char === '"' || char === "'" || char === '`') {
|
|
8223
|
+
quote = char;
|
|
8224
|
+
continue;
|
|
8225
|
+
}
|
|
8226
|
+
if (char === '(')
|
|
8227
|
+
depth += 1;
|
|
8228
|
+
else if (char === ')')
|
|
8229
|
+
depth = Math.max(0, depth - 1);
|
|
8230
|
+
if (depth !== 0)
|
|
8231
|
+
continue;
|
|
8232
|
+
if (delimiter === ',' && char === ',') {
|
|
8233
|
+
out.push(input.slice(start, i).trim());
|
|
8234
|
+
start = i + 1;
|
|
8235
|
+
}
|
|
8236
|
+
else if (delimiter === 'and'
|
|
8237
|
+
&& input.slice(i, i + 3).toLowerCase() === 'and'
|
|
8238
|
+
&& !/[A-Za-z0-9_]/.test(input[i - 1] ?? '')
|
|
8239
|
+
&& !/[A-Za-z0-9_]/.test(input[i + 3] ?? '')) {
|
|
8240
|
+
out.push(input.slice(start, i).trim());
|
|
8241
|
+
start = i + 3;
|
|
8242
|
+
i += 2;
|
|
8243
|
+
}
|
|
8244
|
+
}
|
|
8245
|
+
out.push(input.slice(start).trim());
|
|
8246
|
+
return out.filter(Boolean);
|
|
8247
|
+
}
|
|
8248
|
+
function findTopLevelKeyword(input, keyword, fromIndex) {
|
|
8249
|
+
const lower = input.toLowerCase();
|
|
8250
|
+
const target = keyword.toLowerCase();
|
|
8251
|
+
let depth = 0;
|
|
8252
|
+
let quote;
|
|
8253
|
+
for (let i = Math.max(0, fromIndex); i <= input.length - target.length; i += 1) {
|
|
8254
|
+
const char = input[i];
|
|
8255
|
+
if (quote) {
|
|
8256
|
+
if (char === quote && input[i - 1] !== '\\')
|
|
8257
|
+
quote = undefined;
|
|
8258
|
+
continue;
|
|
8259
|
+
}
|
|
8260
|
+
if (char === '"' || char === "'" || char === '`') {
|
|
8261
|
+
quote = char;
|
|
8262
|
+
continue;
|
|
8263
|
+
}
|
|
8264
|
+
if (char === '(') {
|
|
8265
|
+
depth += 1;
|
|
8266
|
+
continue;
|
|
8267
|
+
}
|
|
8268
|
+
if (char === ')') {
|
|
8269
|
+
depth = Math.max(0, depth - 1);
|
|
8270
|
+
continue;
|
|
8271
|
+
}
|
|
8272
|
+
if (depth !== 0)
|
|
8273
|
+
continue;
|
|
8274
|
+
if (lower.slice(i, i + target.length) === target
|
|
8275
|
+
&& !/[A-Za-z0-9_]/.test(input[i - 1] ?? '')
|
|
8276
|
+
&& !/[A-Za-z0-9_]/.test(input[i + target.length] ?? '')) {
|
|
8277
|
+
return i;
|
|
8278
|
+
}
|
|
8279
|
+
}
|
|
8280
|
+
return -1;
|
|
8281
|
+
}
|
|
8282
|
+
function preferredCompostingMetricName(group) {
|
|
8283
|
+
const aliases = group.aliases.map((alias) => slugForIdentifier(alias)).filter(Boolean);
|
|
8284
|
+
const counts = new Map();
|
|
8285
|
+
for (const alias of aliases)
|
|
8286
|
+
counts.set(alias, (counts.get(alias) ?? 0) + 1);
|
|
8287
|
+
const preferredAlias = Array.from(counts.entries()).sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))[0]?.[0];
|
|
8288
|
+
if (preferredAlias)
|
|
8289
|
+
return preferredAlias;
|
|
8290
|
+
const expressionName = slugForIdentifier(group.expression.replace(/\b(sum|avg|min|max|count|distinct)\b/gi, ''));
|
|
8291
|
+
return expressionName ? `${expressionName}_${group.type}` : `${group.type}_metric`;
|
|
8292
|
+
}
|
|
8293
|
+
function dedupeCompostingName(baseName, usedNames) {
|
|
8294
|
+
const base = slugForIdentifier(baseName).slice(0, 72).replace(/_+$/g, '');
|
|
8295
|
+
if (!base)
|
|
8296
|
+
return undefined;
|
|
8297
|
+
if (!usedNames.has(base.toLowerCase()))
|
|
8298
|
+
return base;
|
|
8299
|
+
let index = 2;
|
|
8300
|
+
while (usedNames.has(`${base}_${index}`.toLowerCase()))
|
|
8301
|
+
index += 1;
|
|
8302
|
+
return `${base}_${index}`;
|
|
8303
|
+
}
|
|
8304
|
+
function normalizeCompostingSql(value) {
|
|
8305
|
+
return value
|
|
8306
|
+
.replace(/["`]/g, '')
|
|
8307
|
+
.replace(/\s+/g, ' ')
|
|
8308
|
+
.trim()
|
|
8309
|
+
.toLowerCase();
|
|
8310
|
+
}
|
|
8311
|
+
function slugForIdentifier(value) {
|
|
8312
|
+
return value
|
|
8313
|
+
.trim()
|
|
8314
|
+
.replace(/["`[\]]/g, '')
|
|
8315
|
+
.replace(/([a-z0-9])([A-Z])/g, '$1_$2')
|
|
8316
|
+
.toLowerCase()
|
|
8317
|
+
.replace(/[^a-z0-9]+/g, '_')
|
|
8318
|
+
.replace(/^_+|_+$/g, '');
|
|
8319
|
+
}
|
|
8320
|
+
function slugForPath(value) {
|
|
8321
|
+
return slugForIdentifier(value) || 'misc';
|
|
8322
|
+
}
|
|
8323
|
+
function stripIdentifierQuotes(value) {
|
|
8324
|
+
return value.replace(/^["`\[]|["`\]]$/g, '').trim();
|
|
8325
|
+
}
|
|
8326
|
+
function titleFromSlug(value) {
|
|
8327
|
+
return value.split('_').filter(Boolean).map((part) => part.charAt(0).toUpperCase() + part.slice(1)).join(' ');
|
|
8328
|
+
}
|
|
8329
|
+
function stableShortHash(value) {
|
|
8330
|
+
return createHash('sha1').update(value).digest('hex').slice(0, 10);
|
|
8331
|
+
}
|
|
8332
|
+
function renderCompostingPrBody(candidates) {
|
|
8333
|
+
const lines = [
|
|
8334
|
+
'# Semantic Composting Changeset',
|
|
8335
|
+
'',
|
|
8336
|
+
'This changeset proposes draft semantic metrics mined from recurring logic in certified DQL blocks.',
|
|
8337
|
+
'Review each metric expression, filter, owner, grain compatibility, and downstream impact before marking it certified.',
|
|
8338
|
+
'',
|
|
8339
|
+
];
|
|
8340
|
+
if (candidates.length === 0) {
|
|
8341
|
+
lines.push('No recurring certified-block metric clusters met the current support threshold.');
|
|
8342
|
+
return `${lines.join('\n')}\n`;
|
|
8343
|
+
}
|
|
8344
|
+
for (const candidate of candidates) {
|
|
8345
|
+
lines.push(`## ${candidate.label}`);
|
|
8346
|
+
lines.push('');
|
|
8347
|
+
lines.push(`- Draft path: \`${candidate.draftPath}\``);
|
|
8348
|
+
lines.push(`- Domain/table: \`${candidate.domain}\` / \`${candidate.table}\``);
|
|
8349
|
+
lines.push(`- Expression: \`${candidate.sql}\``);
|
|
8350
|
+
if (candidate.filter)
|
|
8351
|
+
lines.push(`- Filter: \`${candidate.filter}\``);
|
|
8352
|
+
lines.push(`- Support: ${candidate.support} certified block(s)`);
|
|
8353
|
+
lines.push(`- Donors: ${candidate.donorBlocks.map((donor) => `\`${donor.path}\``).join(', ')}`);
|
|
8354
|
+
lines.push('');
|
|
8355
|
+
}
|
|
8356
|
+
return `${lines.join('\n')}\n`;
|
|
8357
|
+
}
|
|
7020
8358
|
export async function generateProposeDrafts(projectRoot, slugs, projectConfig = loadProjectConfig(projectRoot), options = {}) {
|
|
7021
8359
|
const manifestPath = resolveDbtManifestPath(projectRoot, projectConfig);
|
|
7022
8360
|
if (!manifestPath) {
|
|
@@ -8150,6 +9488,20 @@ function parseBlockStudioArrayField(source, key) {
|
|
|
8150
9488
|
function parseBlockStudioStringField(source, key) {
|
|
8151
9489
|
return source.match(new RegExp(`\\b${key}\\s*=\\s*"([^"]*)"`, 'i'))?.[1] ?? undefined;
|
|
8152
9490
|
}
|
|
9491
|
+
function parseSemanticRequestedFilters(source) {
|
|
9492
|
+
return parseBlockStudioArrayField(source, 'requested_filters')
|
|
9493
|
+
.map((entry) => {
|
|
9494
|
+
const match = entry.match(/^([^=<>!]+)\s*=\s*(.+)$/);
|
|
9495
|
+
if (!match)
|
|
9496
|
+
return undefined;
|
|
9497
|
+
const dimension = match[1]?.trim();
|
|
9498
|
+
const value = match[2]?.trim();
|
|
9499
|
+
if (!dimension || !value)
|
|
9500
|
+
return undefined;
|
|
9501
|
+
return { dimension, operator: 'equals', values: [value] };
|
|
9502
|
+
})
|
|
9503
|
+
.filter((filter) => Boolean(filter));
|
|
9504
|
+
}
|
|
8153
9505
|
function parseSemanticBlockConfig(source) {
|
|
8154
9506
|
const blockType = (parseBlockStudioStringField(source, 'type') ?? 'custom').toLowerCase() === 'semantic'
|
|
8155
9507
|
? 'semantic'
|
|
@@ -8157,6 +9509,7 @@ function parseSemanticBlockConfig(source) {
|
|
|
8157
9509
|
const metric = parseBlockStudioStringField(source, 'metric');
|
|
8158
9510
|
const metrics = parseBlockStudioArrayField(source, 'metrics');
|
|
8159
9511
|
const dimensions = parseBlockStudioArrayField(source, 'dimensions');
|
|
9512
|
+
const filters = parseSemanticRequestedFilters(source);
|
|
8160
9513
|
const timeDimension = parseBlockStudioStringField(source, 'time_dimension');
|
|
8161
9514
|
const granularity = parseBlockStudioStringField(source, 'granularity');
|
|
8162
9515
|
const limitMatch = source.match(/\blimit\s*=\s*(\d+)/i);
|
|
@@ -8165,6 +9518,7 @@ function parseSemanticBlockConfig(source) {
|
|
|
8165
9518
|
metric,
|
|
8166
9519
|
metrics,
|
|
8167
9520
|
dimensions,
|
|
9521
|
+
filters,
|
|
8168
9522
|
timeDimension,
|
|
8169
9523
|
granularity,
|
|
8170
9524
|
limit: limitMatch ? Number.parseInt(limitMatch[1], 10) : undefined,
|
|
@@ -8310,6 +9664,7 @@ function composeSemanticBlockSql(source, semanticLayer, options) {
|
|
|
8310
9664
|
? composeRuntimeSemanticQuery({
|
|
8311
9665
|
metrics,
|
|
8312
9666
|
dimensions: config.dimensions,
|
|
9667
|
+
filters: config.filters,
|
|
8313
9668
|
timeDimension: config.timeDimension && config.granularity
|
|
8314
9669
|
? { name: config.timeDimension, granularity: config.granularity }
|
|
8315
9670
|
: undefined,
|
|
@@ -8324,6 +9679,7 @@ function composeSemanticBlockSql(source, semanticLayer, options) {
|
|
|
8324
9679
|
: semanticLayer.composeQuery({
|
|
8325
9680
|
metrics,
|
|
8326
9681
|
dimensions: config.dimensions,
|
|
9682
|
+
filters: config.filters,
|
|
8327
9683
|
timeDimension: config.timeDimension && config.granularity
|
|
8328
9684
|
? { name: config.timeDimension, granularity: config.granularity }
|
|
8329
9685
|
: undefined,
|
|
@@ -8490,6 +9846,101 @@ export function validateBlockStudioSource(source, semanticLayer) {
|
|
|
8490
9846
|
executableSql,
|
|
8491
9847
|
};
|
|
8492
9848
|
}
|
|
9849
|
+
export async function createDqlArtifactGenerationSessionForProject(projectRoot, options, semanticLayer) {
|
|
9850
|
+
const dqlArtifact = normalizeDqlArtifactReference(options.dqlArtifact);
|
|
9851
|
+
if (!dqlArtifact)
|
|
9852
|
+
throw new Error('A DQL artifact with source is required for DQL draft promotion.');
|
|
9853
|
+
const question = notebookResearchString(options.question) ?? dqlArtifact.name ?? 'Generated DQL artifact';
|
|
9854
|
+
const slug = deriveGeneratedDraftSlug(dqlArtifact.name ?? question);
|
|
9855
|
+
const domain = notebookResearchString(options.domain) ?? 'misc';
|
|
9856
|
+
const owner = notebookResearchString(options.owner) ?? '';
|
|
9857
|
+
const tags = Array.from(new Set(['notebook-research', 'review-required', 'dql-artifact', ...(Array.isArray(options.tags) ? options.tags : [])]
|
|
9858
|
+
.map((tag) => notebookResearchString(tag))
|
|
9859
|
+
.filter((tag) => Boolean(tag))));
|
|
9860
|
+
const proposedContractId = `${domain}.Unknown.${slug}`;
|
|
9861
|
+
const draft = upsertGeneratedDqlArtifactDraft(projectRoot, {
|
|
9862
|
+
slug,
|
|
9863
|
+
question,
|
|
9864
|
+
proposedContractId,
|
|
9865
|
+
owner,
|
|
9866
|
+
proposedDomain: domain,
|
|
9867
|
+
dqlArtifact,
|
|
9868
|
+
sourceQuestion: question,
|
|
9869
|
+
sourceBlock: options.sourceBlock,
|
|
9870
|
+
contextPackId: options.contextPackId,
|
|
9871
|
+
routeIntent: options.routeIntent,
|
|
9872
|
+
outputs: Array.from(new Set([...(dqlArtifact.dimensions ?? []), ...(dqlArtifact.metrics ?? [])])),
|
|
9873
|
+
});
|
|
9874
|
+
const now = new Date().toISOString();
|
|
9875
|
+
const uniqueSuffix = createHash('sha1').update(`${now}:${draft.path}:${dqlArtifact.source}`).digest('hex').slice(0, 12);
|
|
9876
|
+
const importId = `imp_dql_artifact_${uniqueSuffix}`;
|
|
9877
|
+
const candidateId = `cand_${slug.replace(/[^a-z0-9_]+/g, '_').replace(/^_+|_+$/g, '') || 'dql_artifact'}_${uniqueSuffix.slice(0, 6)}`;
|
|
9878
|
+
const sourcePath = dqlArtifact.sourcePath ?? options.inputPath ?? draft.path;
|
|
9879
|
+
const savedSource = readFileSync(join(projectRoot, draft.path), 'utf-8');
|
|
9880
|
+
const candidate = {
|
|
9881
|
+
id: candidateId,
|
|
9882
|
+
sourceKind: 'raw-sql-file',
|
|
9883
|
+
sourcePath,
|
|
9884
|
+
name: dqlArtifact.name ?? slug,
|
|
9885
|
+
domain,
|
|
9886
|
+
description: question,
|
|
9887
|
+
owner,
|
|
9888
|
+
tags,
|
|
9889
|
+
grain: dqlArtifact.timeDimension ? `${dqlArtifact.timeDimension.name}.${dqlArtifact.timeDimension.granularity}` : undefined,
|
|
9890
|
+
outputs: Array.from(new Set([...(dqlArtifact.dimensions ?? []), ...(dqlArtifact.metrics ?? [])])),
|
|
9891
|
+
dimensions: dqlArtifact.dimensions,
|
|
9892
|
+
allowedFilters: dqlArtifact.filters?.map((filter) => filter.dimension),
|
|
9893
|
+
parameterPolicy: dqlArtifact.filters?.map((filter) => ({ name: filter.dimension, policy: 'dynamic' })),
|
|
9894
|
+
sql: notebookResearchString(options.generatedSql) ?? '',
|
|
9895
|
+
dqlSource: savedSource,
|
|
9896
|
+
validation: validateBlockStudioSource(savedSource, semanticLayer),
|
|
9897
|
+
preview: null,
|
|
9898
|
+
lineage: {
|
|
9899
|
+
sourceTables: [],
|
|
9900
|
+
parameters: [],
|
|
9901
|
+
warnings: [],
|
|
9902
|
+
statementIndex: 1,
|
|
9903
|
+
totalStatements: 1,
|
|
9904
|
+
},
|
|
9905
|
+
confidence: 0.95,
|
|
9906
|
+
splitStrategy: 'manual',
|
|
9907
|
+
warnings: [],
|
|
9908
|
+
conversionNotes: ['Persisted the DQL artifact returned by the governed answer; no SQL-to-DQL regeneration was required.'],
|
|
9909
|
+
aiAssistance: [],
|
|
9910
|
+
reviewStatus: 'draft',
|
|
9911
|
+
savedPath: draft.path,
|
|
9912
|
+
generationMode: 'deterministic',
|
|
9913
|
+
generationProvider: 'dql-artifact',
|
|
9914
|
+
llmContext: `Persisted generated DQL artifact for: ${question}`,
|
|
9915
|
+
evidence: [],
|
|
9916
|
+
draftSave: { status: 'saved', path: draft.path, savedAt: now },
|
|
9917
|
+
recommendedAction: 'create_new',
|
|
9918
|
+
similarityMatches: [],
|
|
9919
|
+
};
|
|
9920
|
+
const session = {
|
|
9921
|
+
id: importId,
|
|
9922
|
+
sourceKind: 'raw-sql-file',
|
|
9923
|
+
inputPath: sourcePath,
|
|
9924
|
+
inputMode: options.inputMode ?? 'upload',
|
|
9925
|
+
sourceFiles: [sourcePath],
|
|
9926
|
+
createdAt: now,
|
|
9927
|
+
updatedAt: now,
|
|
9928
|
+
defaults: { domain, owner, tags },
|
|
9929
|
+
candidateIds: [candidate.id],
|
|
9930
|
+
mode: 'ai-import',
|
|
9931
|
+
generation: {
|
|
9932
|
+
provider: 'dql-artifact',
|
|
9933
|
+
aiEnabled: false,
|
|
9934
|
+
contextObjectCount: 0,
|
|
9935
|
+
createdDrafts: 1,
|
|
9936
|
+
warnings: [],
|
|
9937
|
+
},
|
|
9938
|
+
candidates: [candidate],
|
|
9939
|
+
};
|
|
9940
|
+
writeBlockStudioImportSession(projectRoot, session);
|
|
9941
|
+
await refreshLocalMetadataCatalog(projectRoot);
|
|
9942
|
+
return session;
|
|
9943
|
+
}
|
|
8493
9944
|
export async function createDqlGenerationSessionForProject(projectRoot, options, semanticLayer) {
|
|
8494
9945
|
const deterministicOnly = isDeterministicDqlGenerationProvider(options.provider);
|
|
8495
9946
|
const requestedProvider = !deterministicOnly && isProviderSettingsId(options.provider) ? options.provider : undefined;
|
|
@@ -9488,7 +10939,13 @@ function inferDqlGenerationPattern(sql) {
|
|
|
9488
10939
|
return 'trend';
|
|
9489
10940
|
if (groupFields.length === 1 && /_id$|_key$/i.test(groupFields[0]))
|
|
9490
10941
|
return 'entity_rollup';
|
|
9491
|
-
|
|
10942
|
+
// Entity profile = a non-aggregated, un-grouped row lookup filtered by an
|
|
10943
|
+
// identifier (…WHERE <something>_id = / IN …). Structural + domain-agnostic —
|
|
10944
|
+
// no hard-coded entity vocabulary.
|
|
10945
|
+
const hasAggregate = /\b(sum|count|avg|min|max|median|percentile|rank)\s*\(/i.test(sql);
|
|
10946
|
+
const hasGroupBy = /\bgroup\s+by\b/i.test(sql);
|
|
10947
|
+
const filtersByIdentifier = /\bwhere\b[\s\S]*?\b[A-Za-z_][A-Za-z0-9_]*(?:_id|_key|id)\b\s*(?:=|in\b)/i.test(sql);
|
|
10948
|
+
if (!hasAggregate && !hasGroupBy && filtersByIdentifier) {
|
|
9492
10949
|
return 'entity_profile';
|
|
9493
10950
|
}
|
|
9494
10951
|
return 'custom';
|
|
@@ -10086,10 +11543,15 @@ async function createBlockStudioAssistProvider(projectRoot, requestedProvider) {
|
|
|
10086
11543
|
provider = new OpenAIProvider({ apiKey: config.apiKey, baseUrl: config.baseUrl, model: config.model, allowNoApiKey: true });
|
|
10087
11544
|
break;
|
|
10088
11545
|
case 'claude-code':
|
|
10089
|
-
|
|
11546
|
+
// OAuth-first: use the browser-login token when connected; fall back to the CLI passthrough.
|
|
11547
|
+
provider = claudeOAuthConnected(projectRoot)
|
|
11548
|
+
? new ClaudeOAuthProvider({ projectRoot, model: config.model })
|
|
11549
|
+
: new ClaudeCodeCliProvider({ model: config.model });
|
|
10090
11550
|
break;
|
|
10091
11551
|
case 'codex':
|
|
10092
|
-
provider =
|
|
11552
|
+
provider = codexOAuthConnected(projectRoot)
|
|
11553
|
+
? new CodexOAuthProvider({ projectRoot, model: config.model })
|
|
11554
|
+
: new CodexCliProvider({ model: config.model });
|
|
10093
11555
|
break;
|
|
10094
11556
|
default:
|
|
10095
11557
|
return null;
|
|
@@ -10103,8 +11565,7 @@ function agentResultToSynthesisPreview(result) {
|
|
|
10103
11565
|
const columns = Array.isArray(result.columns)
|
|
10104
11566
|
? result.columns.map((col) => typeof col === 'string' ? col : col?.name ?? String(col))
|
|
10105
11567
|
: [];
|
|
10106
|
-
const
|
|
10107
|
-
const rows = rawRows.map((row) => {
|
|
11568
|
+
const normalizeRow = (row) => {
|
|
10108
11569
|
if (row && typeof row === 'object' && !Array.isArray(row))
|
|
10109
11570
|
return row;
|
|
10110
11571
|
// Array-shaped rows → zip with column names.
|
|
@@ -10114,69 +11575,20 @@ function agentResultToSynthesisPreview(result) {
|
|
|
10114
11575
|
return obj;
|
|
10115
11576
|
}
|
|
10116
11577
|
return { value: row };
|
|
10117
|
-
}
|
|
11578
|
+
};
|
|
11579
|
+
// Normalize ALL rows so verified statistics cover the full result — the
|
|
11580
|
+
// narrator sees only a 20-row sample, and sample-derived aggregate claims
|
|
11581
|
+
// ("spend ranges from X to Y") were the last hallucination surface.
|
|
11582
|
+
const allRows = Array.isArray(result.rows) ? result.rows.map(normalizeRow) : [];
|
|
11583
|
+
const rows = allRows.slice(0, 20);
|
|
11584
|
+
const resolvedColumns = columns.length > 0 ? columns : Object.keys(rows[0] ?? {});
|
|
10118
11585
|
return {
|
|
10119
|
-
columns:
|
|
11586
|
+
columns: resolvedColumns,
|
|
10120
11587
|
rows,
|
|
10121
|
-
rowCount: typeof result.rowCount === 'number' ? result.rowCount :
|
|
11588
|
+
rowCount: typeof result.rowCount === 'number' ? result.rowCount : allRows.length,
|
|
11589
|
+
stats: computeResultStats(resolvedColumns, allRows),
|
|
10122
11590
|
};
|
|
10123
11591
|
}
|
|
10124
|
-
const SIGNAL_STOPWORDS = new Set([
|
|
10125
|
-
'the', 'a', 'an', 'of', 'for', 'to', 'by', 'in', 'on', 'and', 'or', 'is', 'are', 'was', 'were',
|
|
10126
|
-
'what', 'which', 'how', 'show', 'me', 'my', 'our', 'this', 'that', 'top', 'give', 'get', 'list',
|
|
10127
|
-
]);
|
|
10128
|
-
function signalTokens(text) {
|
|
10129
|
-
return new Set(text.toLowerCase().replace(/[^a-z0-9\s]/g, ' ').split(/\s+/)
|
|
10130
|
-
.filter((token) => token.length > 2 && !SIGNAL_STOPWORDS.has(token)));
|
|
10131
|
-
}
|
|
10132
|
-
/** Best token-overlap ratio between the question and any candidate name/description. */
|
|
10133
|
-
function bestOverlap(questionTokens, candidates) {
|
|
10134
|
-
if (questionTokens.size === 0)
|
|
10135
|
-
return 0;
|
|
10136
|
-
let best = 0;
|
|
10137
|
-
for (const candidate of candidates) {
|
|
10138
|
-
const candTokens = signalTokens(candidate);
|
|
10139
|
-
if (candTokens.size === 0)
|
|
10140
|
-
continue;
|
|
10141
|
-
let hits = 0;
|
|
10142
|
-
for (const token of candTokens)
|
|
10143
|
-
if (questionTokens.has(token))
|
|
10144
|
-
hits += 1;
|
|
10145
|
-
if (hits === 0)
|
|
10146
|
-
continue;
|
|
10147
|
-
// Ratio of the candidate's tokens the question covers — rewards a tight match.
|
|
10148
|
-
const ratio = hits / Math.min(candTokens.size, questionTokens.size);
|
|
10149
|
-
if (ratio > best)
|
|
10150
|
-
best = ratio;
|
|
10151
|
-
}
|
|
10152
|
-
return Math.min(1, best);
|
|
10153
|
-
}
|
|
10154
|
-
/**
|
|
10155
|
-
* Cheap, offline retrieval-signal probe. Fills certifiedScore/metricScore/hasRetrieval
|
|
10156
|
-
* from local catalog name/description overlap so the deterministic router can reach
|
|
10157
|
-
* confidence (and skip the LLM) on obvious governed matches. Best-effort — any failure
|
|
10158
|
-
* yields empty signals and the router simply leans on its LLM classification.
|
|
10159
|
-
*/
|
|
10160
|
-
function computeAgentRunSignals(projectRoot, question) {
|
|
10161
|
-
try {
|
|
10162
|
-
const tokens = signalTokens(question);
|
|
10163
|
-
if (tokens.size === 0)
|
|
10164
|
-
return { certifiedScore: 0, metricScore: 0, hasRetrieval: false };
|
|
10165
|
-
const certifiedBlocks = collectPlanBlocks(projectRoot, { certifiedOnly: true });
|
|
10166
|
-
const blockStrings = certifiedBlocks.flatMap((block) => [block.name, block.description ?? ''].filter(Boolean));
|
|
10167
|
-
const metrics = loadSemanticMetrics(projectRoot);
|
|
10168
|
-
const metricStrings = metrics.map((metric) => {
|
|
10169
|
-
const node = metric;
|
|
10170
|
-
return node.name ?? node.label ?? node.id ?? '';
|
|
10171
|
-
}).filter(Boolean);
|
|
10172
|
-
const certifiedScore = bestOverlap(tokens, blockStrings);
|
|
10173
|
-
const metricScore = bestOverlap(tokens, metricStrings);
|
|
10174
|
-
return { certifiedScore, metricScore, hasRetrieval: certifiedScore > 0 || metricScore > 0 };
|
|
10175
|
-
}
|
|
10176
|
-
catch {
|
|
10177
|
-
return { certifiedScore: 0, metricScore: 0, hasRetrieval: false };
|
|
10178
|
-
}
|
|
10179
|
-
}
|
|
10180
11592
|
/** Up to three example questions built from real catalog blocks (falls back to generic asks). */
|
|
10181
11593
|
function buildConversationSuggestions(projectRoot, kind) {
|
|
10182
11594
|
if (kind === 'gratitude')
|
|
@@ -10231,6 +11643,141 @@ function buildConversationSystemPrompt(kind, isGeneralKnowledge, catalogContext,
|
|
|
10231
11643
|
}
|
|
10232
11644
|
return lines.join('\n');
|
|
10233
11645
|
}
|
|
11646
|
+
function renderConversationMemoryForPrompt(context) {
|
|
11647
|
+
const recap = buildConversationContextRecap(context);
|
|
11648
|
+
if (!recap)
|
|
11649
|
+
return undefined;
|
|
11650
|
+
return [
|
|
11651
|
+
'Current conversation context:',
|
|
11652
|
+
recap,
|
|
11653
|
+
'If the user asks what this conversation is about, summarize this context directly. Do not query data or invent values.',
|
|
11654
|
+
].join('\n');
|
|
11655
|
+
}
|
|
11656
|
+
function buildConversationContextRecap(context) {
|
|
11657
|
+
if (!context)
|
|
11658
|
+
return undefined;
|
|
11659
|
+
const turnRecap = buildConversationTurnsRecap(context);
|
|
11660
|
+
if (turnRecap)
|
|
11661
|
+
return turnRecap;
|
|
11662
|
+
const sourceQuestion = agentRunString(context.sourceQuestion);
|
|
11663
|
+
const sourceAnswerSummary = agentRunString(context.sourceAnswerSummary);
|
|
11664
|
+
const columns = agentRunStringArray(context.resultColumns ?? context.outputColumns).slice(0, 8);
|
|
11665
|
+
const values = agentRunRecord(context.resultDimensionValues);
|
|
11666
|
+
const rows = Array.isArray(context.resultRowsSample)
|
|
11667
|
+
? context.resultRowsSample.map(agentRunRecord).filter((row) => Boolean(row)).slice(0, 1)
|
|
11668
|
+
: [];
|
|
11669
|
+
const firstRow = rows[0];
|
|
11670
|
+
const leadParts = buildPriorValueLeadParts(values);
|
|
11671
|
+
const rowFacts = firstRow
|
|
11672
|
+
? columns
|
|
11673
|
+
.slice(0, 5)
|
|
11674
|
+
.map((column) => {
|
|
11675
|
+
const value = firstRow[column];
|
|
11676
|
+
return value === undefined || value === null || typeof value === 'object' ? '' : `${column}=${String(value)}`;
|
|
11677
|
+
})
|
|
11678
|
+
.filter(Boolean)
|
|
11679
|
+
: [];
|
|
11680
|
+
const contextBits = [
|
|
11681
|
+
sourceQuestion ? `the question "${sourceQuestion}"` : '',
|
|
11682
|
+
sourceAnswerSummary ? `the latest answer summary: ${sourceAnswerSummary}` : '',
|
|
11683
|
+
columns.length ? `result columns: ${columns.join(', ')}` : '',
|
|
11684
|
+
leadParts.length ? `notable prior values: ${leadParts.join(', ')}` : '',
|
|
11685
|
+
rowFacts.length ? `first result row: ${rowFacts.join(', ')}` : '',
|
|
11686
|
+
].filter(Boolean);
|
|
11687
|
+
if (contextBits.length === 0)
|
|
11688
|
+
return undefined;
|
|
11689
|
+
return `We were talking about ${contextBits.join('. ')}.`;
|
|
11690
|
+
}
|
|
11691
|
+
function buildConversationTurnsRecap(context) {
|
|
11692
|
+
const turns = Array.isArray(context.turns)
|
|
11693
|
+
? context.turns.map(agentRunRecord).filter((turn) => Boolean(turn)).slice(-3)
|
|
11694
|
+
: [];
|
|
11695
|
+
if (turns.length === 0)
|
|
11696
|
+
return undefined;
|
|
11697
|
+
const activeTurnId = agentRunString(context.activeTurnId);
|
|
11698
|
+
const active = activeTurnId
|
|
11699
|
+
? turns.find((turn) => agentRunString(turn.id) === activeTurnId)
|
|
11700
|
+
: undefined;
|
|
11701
|
+
const selected = active ?? turns[turns.length - 1];
|
|
11702
|
+
if (!selected)
|
|
11703
|
+
return undefined;
|
|
11704
|
+
const result = agentRunRecord(selected.result);
|
|
11705
|
+
const values = agentRunRecord(result?.dimensionValues);
|
|
11706
|
+
const columns = agentRunStringArray(result?.columns).slice(0, 8);
|
|
11707
|
+
const rows = Array.isArray(result?.rowsSample)
|
|
11708
|
+
? result.rowsSample.map(agentRunRecord).filter((row) => Boolean(row)).slice(0, 1)
|
|
11709
|
+
: [];
|
|
11710
|
+
const firstRow = rows[0];
|
|
11711
|
+
const leadParts = buildPriorValueLeadParts(values);
|
|
11712
|
+
const rowFacts = firstRow
|
|
11713
|
+
? columns
|
|
11714
|
+
.slice(0, 5)
|
|
11715
|
+
.map((column) => {
|
|
11716
|
+
const value = firstRow[column];
|
|
11717
|
+
return value === undefined || value === null || typeof value === 'object' ? '' : `${column}=${String(value)}`;
|
|
11718
|
+
})
|
|
11719
|
+
.filter(Boolean)
|
|
11720
|
+
: [];
|
|
11721
|
+
const previous = turns
|
|
11722
|
+
.filter((turn) => turn !== selected)
|
|
11723
|
+
.map((turn) => agentRunString(turn.question))
|
|
11724
|
+
.filter((value) => Boolean(value))
|
|
11725
|
+
.slice(-2);
|
|
11726
|
+
const bits = [
|
|
11727
|
+
agentRunString(selected.question) ? `the latest question "${agentRunString(selected.question)}"` : '',
|
|
11728
|
+
agentRunString(selected.answerSummary) ? `latest answer summary: ${agentRunString(selected.answerSummary)}` : '',
|
|
11729
|
+
columns.length ? `result columns: ${columns.join(', ')}` : '',
|
|
11730
|
+
leadParts.length ? `notable prior values: ${leadParts.join(', ')}` : '',
|
|
11731
|
+
rowFacts.length ? `first result row: ${rowFacts.join(', ')}` : '',
|
|
11732
|
+
previous.length ? `earlier in this thread: ${previous.join(' / ')}` : '',
|
|
11733
|
+
].filter(Boolean);
|
|
11734
|
+
return bits.length > 0 ? `We were talking about ${bits.join('. ')}.` : undefined;
|
|
11735
|
+
}
|
|
11736
|
+
function agentRunStringArray(value) {
|
|
11737
|
+
if (!Array.isArray(value))
|
|
11738
|
+
return [];
|
|
11739
|
+
return Array.from(new Set(value.map(agentRunString).filter((item) => Boolean(item))));
|
|
11740
|
+
}
|
|
11741
|
+
function firstPriorValue(values, keys) {
|
|
11742
|
+
if (!values)
|
|
11743
|
+
return undefined;
|
|
11744
|
+
for (const key of keys) {
|
|
11745
|
+
const list = Array.isArray(values[key]) ? values[key] : [];
|
|
11746
|
+
const first = list.map(agentRunString).find(Boolean);
|
|
11747
|
+
if (first)
|
|
11748
|
+
return first;
|
|
11749
|
+
}
|
|
11750
|
+
return undefined;
|
|
11751
|
+
}
|
|
11752
|
+
/**
|
|
11753
|
+
* Build human-readable "notable prior values" for conversation recap from whatever
|
|
11754
|
+
* dimension columns the prior result actually had. Domain-agnostic: name/label-ish
|
|
11755
|
+
* columns are surfaced first, then any other dimension — no hard-coded jaffle entity
|
|
11756
|
+
* columns — so follow-up questions get meaningful context on ANY repo.
|
|
11757
|
+
*/
|
|
11758
|
+
function buildPriorValueLeadParts(values) {
|
|
11759
|
+
if (!values)
|
|
11760
|
+
return [];
|
|
11761
|
+
const keys = Object.keys(values);
|
|
11762
|
+
const nameLike = keys.filter((key) => /(?:^|_)(?:name|label|title|nm)$/i.test(key));
|
|
11763
|
+
const ordered = [...nameLike, ...keys.filter((key) => !nameLike.includes(key))];
|
|
11764
|
+
const parts = [];
|
|
11765
|
+
const seenLabels = new Set();
|
|
11766
|
+
for (const key of ordered) {
|
|
11767
|
+
if (parts.length >= 3)
|
|
11768
|
+
break;
|
|
11769
|
+
const value = firstPriorValue(values, [key]);
|
|
11770
|
+
if (!value)
|
|
11771
|
+
continue;
|
|
11772
|
+
const label = (key.replace(/_?(?:name|id|key|label|title|nm)$/i, '').replace(/[_-]+/g, ' ').trim()
|
|
11773
|
+
|| key.replace(/[_-]+/g, ' ').trim()).toLowerCase();
|
|
11774
|
+
if (seenLabels.has(label))
|
|
11775
|
+
continue;
|
|
11776
|
+
seenLabels.add(label);
|
|
11777
|
+
parts.push(label ? `${label} "${value}"` : `"${value}"`);
|
|
11778
|
+
}
|
|
11779
|
+
return parts;
|
|
11780
|
+
}
|
|
10234
11781
|
/** Deterministic, warm reply when no AI provider is configured — never governance-speak. */
|
|
10235
11782
|
function buildConversationalFallback(kind, isGeneralKnowledge, question, suggestions) {
|
|
10236
11783
|
const examples = suggestions.length > 0
|
|
@@ -11995,13 +13542,6 @@ function collectSettingsEnvStatus() {
|
|
|
11995
13542
|
},
|
|
11996
13543
|
];
|
|
11997
13544
|
}
|
|
11998
|
-
function isProviderSettingsId(value) {
|
|
11999
|
-
return value === 'anthropic'
|
|
12000
|
-
|| value === 'openai'
|
|
12001
|
-
|| value === 'gemini'
|
|
12002
|
-
|| value === 'ollama'
|
|
12003
|
-
|| value === 'custom-openai';
|
|
12004
|
-
}
|
|
12005
13545
|
function isDeterministicDqlGenerationProvider(value) {
|
|
12006
13546
|
if (typeof value !== 'string')
|
|
12007
13547
|
return false;
|
|
@@ -12022,8 +13562,8 @@ async function testProviderConfig(projectRoot, id, overrides) {
|
|
|
12022
13562
|
const label = providerSettingsLabel(id);
|
|
12023
13563
|
const details = providerConfigDetails(id, config);
|
|
12024
13564
|
const isSubscriptionCli = id === 'claude-code' || id === 'codex';
|
|
12025
|
-
// Subscription
|
|
12026
|
-
// needs no saved config; the others require enabling
|
|
13565
|
+
// Subscription providers are always testable — the connection (browser OAuth, or
|
|
13566
|
+
// an installed+logged-in CLI) needs no saved config; the others require enabling first.
|
|
12027
13567
|
if (!config.enabled && !isSubscriptionCli) {
|
|
12028
13568
|
return { ok: false, message: `${label} is disabled. Enable it (or Save) before testing.` };
|
|
12029
13569
|
}
|
|
@@ -12031,6 +13571,11 @@ async function testProviderConfig(projectRoot, id, overrides) {
|
|
|
12031
13571
|
return testOpenAIProviderConfig(config, label, details);
|
|
12032
13572
|
if (id === 'anthropic')
|
|
12033
13573
|
return testAnthropicProviderConfig(config, label, details);
|
|
13574
|
+
// OAuth-first for subscriptions: when the user signed in via the browser, test the
|
|
13575
|
+
// OAuth-backed provider (no CLI required); otherwise fall back to the CLI passthrough.
|
|
13576
|
+
// Mirrors the runtime selection in dql-agent-provider / createBlockStudioAssistProvider.
|
|
13577
|
+
const claudeOAuth = id === 'claude-code' && claudeOAuthConnected(projectRoot);
|
|
13578
|
+
const codexOAuth = id === 'codex' && codexOAuthConnected(projectRoot);
|
|
12034
13579
|
let provider;
|
|
12035
13580
|
switch (id) {
|
|
12036
13581
|
case 'gemini':
|
|
@@ -12040,10 +13585,14 @@ async function testProviderConfig(projectRoot, id, overrides) {
|
|
|
12040
13585
|
provider = new OllamaProvider({ baseUrl: config.baseUrl, model: config.model });
|
|
12041
13586
|
break;
|
|
12042
13587
|
case 'claude-code':
|
|
12043
|
-
provider =
|
|
13588
|
+
provider = claudeOAuth
|
|
13589
|
+
? new ClaudeOAuthProvider({ projectRoot, model: config.model })
|
|
13590
|
+
: new ClaudeCodeCliProvider({ model: config.model });
|
|
12044
13591
|
break;
|
|
12045
13592
|
case 'codex':
|
|
12046
|
-
provider =
|
|
13593
|
+
provider = codexOAuth
|
|
13594
|
+
? new CodexOAuthProvider({ projectRoot, model: config.model })
|
|
13595
|
+
: new CodexCliProvider({ model: config.model });
|
|
12047
13596
|
break;
|
|
12048
13597
|
case 'custom-openai':
|
|
12049
13598
|
default:
|
|
@@ -12053,10 +13602,11 @@ async function testProviderConfig(projectRoot, id, overrides) {
|
|
|
12053
13602
|
const available = await provider.available().catch(() => false);
|
|
12054
13603
|
if (!available) {
|
|
12055
13604
|
const cli = id === 'claude-code' ? 'claude' : 'codex';
|
|
13605
|
+
const signIn = id === 'claude-code' ? 'Sign in with Claude' : 'Sign in with ChatGPT';
|
|
12056
13606
|
return {
|
|
12057
13607
|
ok: false,
|
|
12058
13608
|
message: isSubscriptionCli
|
|
12059
|
-
? `${label} is not ready.
|
|
13609
|
+
? `${label} is not ready. Click "${signIn}" to sign in with your subscription, or install the \`${cli}\` CLI and run \`${cli} ${id === 'claude-code' ? '/login' : 'login'}\`, then test again.`
|
|
12060
13610
|
: `${label} is not configured or reachable${details}. Check API key, base URL, and local service state.`,
|
|
12061
13611
|
};
|
|
12062
13612
|
}
|
|
@@ -12139,10 +13689,16 @@ function providerSettingsLabel(id) {
|
|
|
12139
13689
|
}
|
|
12140
13690
|
}
|
|
12141
13691
|
function providerConfigDetails(id, config) {
|
|
13692
|
+
// Subscription providers (Claude Code / Codex) authenticate via OAuth/CLI, not an
|
|
13693
|
+
// API key — omit the key clause so we don't report a misleading "API key missing".
|
|
13694
|
+
const isSubscription = id === 'claude-code' || id === 'codex';
|
|
13695
|
+
const authNote = isSubscription
|
|
13696
|
+
? ''
|
|
13697
|
+
: id === 'ollama' ? 'local endpoint' : config.apiKey ? 'API key present' : 'API key missing';
|
|
12142
13698
|
const parts = [
|
|
12143
13699
|
config.model ? `model ${config.model}` : '',
|
|
12144
13700
|
config.baseUrl ? `base URL ${config.baseUrl}` : '',
|
|
12145
|
-
|
|
13701
|
+
authNote,
|
|
12146
13702
|
].filter(Boolean);
|
|
12147
13703
|
return parts.length ? ` (${parts.join(', ')})` : '';
|
|
12148
13704
|
}
|
|
@@ -12164,7 +13720,7 @@ function recordAgentRuntimeSchemaSnapshot(projectRoot, schemaContext, source) {
|
|
|
12164
13720
|
try {
|
|
12165
13721
|
recordRuntimeSchemaSnapshot(projectRoot, {
|
|
12166
13722
|
source,
|
|
12167
|
-
tables: schemaContext.
|
|
13723
|
+
tables: schemaContext.map((table) => ({
|
|
12168
13724
|
relation: table.relation,
|
|
12169
13725
|
schema: table.schema,
|
|
12170
13726
|
name: table.name,
|
|
@@ -12318,7 +13874,9 @@ function buildNotebookResearchReviewChecklist(run) {
|
|
|
12318
13874
|
const questionReady = Boolean(notebookResearchString(run.question));
|
|
12319
13875
|
const hasReviewedSql = Boolean(notebookResearchString(run.reviewedSql));
|
|
12320
13876
|
const hasGeneratedSql = Boolean(notebookResearchString(run.generatedSql));
|
|
13877
|
+
const hasDqlArtifact = Boolean(normalizeDqlArtifactReference(run.dqlArtifact)?.source);
|
|
12321
13878
|
const hasSql = hasReviewedSql || hasGeneratedSql;
|
|
13879
|
+
const hasPromotableSource = hasReviewedSql || hasDqlArtifact;
|
|
12322
13880
|
const preview = notebookResearchPreviewInfo(run.resultPreview);
|
|
12323
13881
|
const previewReady = run.status === 'ready' && preview.hasPreview;
|
|
12324
13882
|
const evidenceReady = notebookResearchEvidenceCount(run.evidence) > 0 || Boolean(run.contextPackId);
|
|
@@ -12337,8 +13895,8 @@ function buildNotebookResearchReviewChecklist(run) {
|
|
|
12337
13895
|
const warnings = [];
|
|
12338
13896
|
if (!questionReady)
|
|
12339
13897
|
blockers.push('Add a business question before review.');
|
|
12340
|
-
if (!hasSql && !reuseRecommended)
|
|
12341
|
-
blockers.push('Add reviewed SQL
|
|
13898
|
+
if (!hasSql && !hasDqlArtifact && !reuseRecommended)
|
|
13899
|
+
blockers.push('Add reviewed SQL, generate SQL, or attach a DQL artifact before promotion.');
|
|
12342
13900
|
if (run.status === 'error')
|
|
12343
13901
|
blockers.push(run.error ? `Fix preview error: ${run.error}` : 'Fix the failed preview before promotion.');
|
|
12344
13902
|
if (!previewReady && run.status !== 'error' && !reuseRecommended)
|
|
@@ -12364,15 +13922,17 @@ function buildNotebookResearchReviewChecklist(run) {
|
|
|
12364
13922
|
},
|
|
12365
13923
|
{
|
|
12366
13924
|
id: 'sql',
|
|
12367
|
-
label: 'Reviewed SQL',
|
|
12368
|
-
status: hasReviewedSql || (reuseRecommended && !hasSql) ? 'passed' : hasGeneratedSql ? 'warning' : 'blocked',
|
|
13925
|
+
label: hasDqlArtifact && !hasReviewedSql ? 'DQL artifact' : 'Reviewed SQL',
|
|
13926
|
+
status: hasReviewedSql || hasDqlArtifact || (reuseRecommended && !hasSql) ? 'passed' : hasGeneratedSql ? 'warning' : 'blocked',
|
|
12369
13927
|
detail: reuseRecommended && !hasSql
|
|
12370
13928
|
? 'Existing certified DQL should be reused; no new SQL is required.'
|
|
12371
|
-
: hasReviewedSql
|
|
12372
|
-
? '
|
|
12373
|
-
:
|
|
12374
|
-
? '
|
|
12375
|
-
:
|
|
13929
|
+
: hasDqlArtifact && !hasReviewedSql
|
|
13930
|
+
? 'Generated DQL artifact is saved for reviewer promotion.'
|
|
13931
|
+
: hasReviewedSql
|
|
13932
|
+
? 'Reviewer SQL is saved.'
|
|
13933
|
+
: hasGeneratedSql
|
|
13934
|
+
? 'Generated SQL exists; review it before promotion.'
|
|
13935
|
+
: 'No SQL is available for preview or DQL generation.',
|
|
12376
13936
|
},
|
|
12377
13937
|
{
|
|
12378
13938
|
id: 'preview',
|
|
@@ -12409,33 +13969,35 @@ function buildNotebookResearchReviewChecklist(run) {
|
|
|
12409
13969
|
{
|
|
12410
13970
|
id: 'parameters',
|
|
12411
13971
|
label: 'Parameter review',
|
|
12412
|
-
status: reuseRecommended && !hasSql ? 'passed' : !hasSql ? 'pending' : dynamicParameters.length > 0 ? 'passed' : 'warning',
|
|
13972
|
+
status: reuseRecommended && !hasSql ? 'passed' : !hasSql && !hasDqlArtifact ? 'pending' : hasDqlArtifact && !hasSql ? 'passed' : dynamicParameters.length > 0 ? 'passed' : 'warning',
|
|
12413
13973
|
detail: reuseRecommended && !hasSql
|
|
12414
13974
|
? 'Use the certified block parameters; no new SQL parameterization is required.'
|
|
12415
|
-
: !hasSql
|
|
12416
|
-
? '
|
|
12417
|
-
:
|
|
12418
|
-
?
|
|
12419
|
-
:
|
|
12420
|
-
? `
|
|
12421
|
-
:
|
|
13975
|
+
: !hasSql && hasDqlArtifact
|
|
13976
|
+
? 'Review parameters declared in the DQL artifact.'
|
|
13977
|
+
: !hasSql
|
|
13978
|
+
? 'Add SQL or a DQL artifact before checking runtime parameters.'
|
|
13979
|
+
: dynamicParameters.length > 0
|
|
13980
|
+
? `Detected ${dynamicParameters.length.toLocaleString()} dynamic parameter${dynamicParameters.length === 1 ? '' : 's'}: ${dynamicParameters.slice(0, 6).map((item) => item.name).join(', ')}.`
|
|
13981
|
+
: staticParameters.length > 0
|
|
13982
|
+
? `Only static or review-required literals were detected: ${staticParameters.slice(0, 6).map((item) => `${item.name} (${item.policy})`).join(', ')}.`
|
|
13983
|
+
: 'No reusable runtime parameters were detected. Certify as static only if the business question is intentionally fixed.',
|
|
12422
13984
|
},
|
|
12423
13985
|
{
|
|
12424
13986
|
id: 'dql_draft',
|
|
12425
13987
|
label: 'DQL draft',
|
|
12426
|
-
status: reuseRecommended && !hasDraft ? 'passed' : hasDraft ? 'passed' :
|
|
13988
|
+
status: reuseRecommended && !hasDraft ? 'passed' : hasDraft ? 'passed' : hasPromotableSource && evidenceReady ? 'pending' : 'blocked',
|
|
12427
13989
|
detail: reuseRecommended && !hasDraft
|
|
12428
13990
|
? 'No new draft is required because an existing DQL block should be reused.'
|
|
12429
13991
|
: hasDraft
|
|
12430
13992
|
? `Draft saved at ${run.draftBlockPath}.`
|
|
12431
|
-
: !
|
|
12432
|
-
? 'Save reviewed SQL before DQL draft creation.'
|
|
13993
|
+
: !hasPromotableSource
|
|
13994
|
+
? 'Save reviewed SQL or review the generated DQL artifact before DQL draft creation.'
|
|
12433
13995
|
: evidenceReady
|
|
12434
|
-
? 'Create a draft after SQL review.'
|
|
13996
|
+
? (hasDqlArtifact && !hasReviewedSql ? 'Create a draft after DQL artifact review.' : 'Create a draft after SQL review.')
|
|
12435
13997
|
: 'Save metadata evidence before creating a reusable DQL draft.',
|
|
12436
13998
|
},
|
|
12437
13999
|
];
|
|
12438
|
-
const readyForDqlDraft = questionReady &&
|
|
14000
|
+
const readyForDqlDraft = questionReady && hasPromotableSource && evidenceReady && run.status !== 'error';
|
|
12439
14001
|
const readyForCertificationReview = readyForDqlDraft && previewReady && hasDraft && !reuseRecommended && !promotionReviewRequired;
|
|
12440
14002
|
return { readyForDqlDraft, readyForCertificationReview, blockers, warnings, items };
|
|
12441
14003
|
}
|
|
@@ -12443,6 +14005,8 @@ function buildNotebookResearchPlan(input) {
|
|
|
12443
14005
|
const reviewedSql = notebookResearchString(input.reviewedSql);
|
|
12444
14006
|
const generatedSql = notebookResearchString(input.generatedSql);
|
|
12445
14007
|
const sql = reviewedSql ?? generatedSql;
|
|
14008
|
+
const dqlArtifact = normalizeDqlArtifactReference(input.dqlArtifact) ?? input.run.dqlArtifact;
|
|
14009
|
+
const hasDqlArtifact = Boolean(dqlArtifact?.source);
|
|
12446
14010
|
const parameterized = sql ? parameterizeSqlForDqlImport(sql) : null;
|
|
12447
14011
|
const evidence = input.evidence && typeof input.evidence === 'object' && !Array.isArray(input.evidence)
|
|
12448
14012
|
? input.evidence
|
|
@@ -12481,6 +14045,7 @@ function buildNotebookResearchPlan(input) {
|
|
|
12481
14045
|
promotion: {
|
|
12482
14046
|
path: notebookResearchPlanPromotionPath({
|
|
12483
14047
|
hasSql: Boolean(sql),
|
|
14048
|
+
hasDqlArtifact,
|
|
12484
14049
|
hasEvidence,
|
|
12485
14050
|
hasPreview,
|
|
12486
14051
|
previewError: input.previewError,
|
|
@@ -12491,6 +14056,7 @@ function buildNotebookResearchPlan(input) {
|
|
|
12491
14056
|
},
|
|
12492
14057
|
reviewFocus: notebookResearchPlanFocus(input.run.intent, {
|
|
12493
14058
|
hasSql: Boolean(sql),
|
|
14059
|
+
hasDqlArtifact,
|
|
12494
14060
|
hasEvidence,
|
|
12495
14061
|
hasPreview,
|
|
12496
14062
|
dynamicParameterCount: parameterized?.parameterPolicy.filter((entry) => entry.policy === 'dynamic').length ?? 0,
|
|
@@ -12503,7 +14069,7 @@ function buildNotebookResearchPlan(input) {
|
|
|
12503
14069
|
function notebookResearchPlanPromotionPath(input) {
|
|
12504
14070
|
if (input.promotionAction === 'reuse_existing')
|
|
12505
14071
|
return 'reuse_existing';
|
|
12506
|
-
if (!input.hasSql)
|
|
14072
|
+
if (!input.hasSql && !input.hasDqlArtifact)
|
|
12507
14073
|
return 'needs_sql';
|
|
12508
14074
|
if (!input.hasEvidence)
|
|
12509
14075
|
return 'review_context';
|
|
@@ -12515,8 +14081,8 @@ function notebookResearchPlanPromotionPath(input) {
|
|
|
12515
14081
|
}
|
|
12516
14082
|
function notebookResearchPlanFocus(intent, input) {
|
|
12517
14083
|
const focus = new Set();
|
|
12518
|
-
if (!input.hasSql)
|
|
12519
|
-
focus.add('Add reviewed SQL or configure AI
|
|
14084
|
+
if (!input.hasSql && !input.hasDqlArtifact)
|
|
14085
|
+
focus.add('Add reviewed SQL, a DQL artifact, or configure AI generation.');
|
|
12520
14086
|
if (!input.hasEvidence)
|
|
12521
14087
|
focus.add('Save metadata context evidence before DQL promotion.');
|
|
12522
14088
|
if (!input.hasPreview)
|
|
@@ -12987,7 +14553,7 @@ function dedupeAgentSchemaColumns(columns) {
|
|
|
12987
14553
|
}
|
|
12988
14554
|
return Array.from(byName.values());
|
|
12989
14555
|
}
|
|
12990
|
-
export function buildAgentSchemaContext(question, rows) {
|
|
14556
|
+
export function buildAgentSchemaContext(question, rows, options = {}) {
|
|
12991
14557
|
const byRelation = new Map();
|
|
12992
14558
|
for (const row of rows) {
|
|
12993
14559
|
if (!row || typeof row !== 'object')
|
|
@@ -13016,16 +14582,82 @@ export function buildAgentSchemaContext(question, rows) {
|
|
|
13016
14582
|
}
|
|
13017
14583
|
const tokens = agentSchemaTokens(question);
|
|
13018
14584
|
const shouldProbeValues = extractAgentValueSearchTerms(question).length > 0;
|
|
14585
|
+
const limit = Math.max(1, Math.trunc(options.limit ?? 12));
|
|
13019
14586
|
return Array.from(byRelation.values())
|
|
13020
14587
|
.map((table) => ({
|
|
13021
14588
|
table,
|
|
13022
14589
|
score: scoreAgentSchemaTable(table, tokens) + (shouldProbeValues ? scoreAgentValueProbeTable(table) : 0),
|
|
13023
14590
|
}))
|
|
13024
|
-
.filter((entry) => entry.score > 0)
|
|
14591
|
+
.filter((entry) => options.includeUnscored || entry.score > 0)
|
|
13025
14592
|
.sort((a, b) => b.score - a.score || a.table.relation.localeCompare(b.table.relation))
|
|
13026
|
-
.slice(0,
|
|
14593
|
+
.slice(0, limit)
|
|
13027
14594
|
.map((entry) => entry.table);
|
|
13028
14595
|
}
|
|
14596
|
+
function mergeAgentSchemaContexts(...contexts) {
|
|
14597
|
+
const byRelation = new Map();
|
|
14598
|
+
for (const table of contexts.flat()) {
|
|
14599
|
+
const key = table.relation.toLowerCase();
|
|
14600
|
+
const existing = byRelation.get(key);
|
|
14601
|
+
if (!existing) {
|
|
14602
|
+
byRelation.set(key, {
|
|
14603
|
+
...table,
|
|
14604
|
+
columns: dedupeAgentSchemaColumns(table.columns).slice(0, 80),
|
|
14605
|
+
});
|
|
14606
|
+
continue;
|
|
14607
|
+
}
|
|
14608
|
+
byRelation.set(key, {
|
|
14609
|
+
...existing,
|
|
14610
|
+
description: existing.description ?? table.description,
|
|
14611
|
+
source: existing.source === table.source ? existing.source : 'catalog and runtime schema',
|
|
14612
|
+
columns: dedupeAgentSchemaColumns([...existing.columns, ...table.columns]).slice(0, 80),
|
|
14613
|
+
});
|
|
14614
|
+
}
|
|
14615
|
+
return Array.from(byRelation.values()).slice(0, 16);
|
|
14616
|
+
}
|
|
14617
|
+
function mergeAgentSchemaSampleValues(snapshot, enriched) {
|
|
14618
|
+
const samples = new Map();
|
|
14619
|
+
for (const table of enriched) {
|
|
14620
|
+
for (const column of table.columns) {
|
|
14621
|
+
if (!column.sampleValues?.length)
|
|
14622
|
+
continue;
|
|
14623
|
+
samples.set(`${table.relation.toLowerCase()}\u0000${column.name.toLowerCase()}`, column.sampleValues);
|
|
14624
|
+
}
|
|
14625
|
+
}
|
|
14626
|
+
if (samples.size === 0)
|
|
14627
|
+
return snapshot;
|
|
14628
|
+
return snapshot.map((table) => ({
|
|
14629
|
+
...table,
|
|
14630
|
+
columns: table.columns.map((column) => {
|
|
14631
|
+
const sampleValues = samples.get(`${table.relation.toLowerCase()}\u0000${column.name.toLowerCase()}`);
|
|
14632
|
+
return sampleValues?.length
|
|
14633
|
+
? { ...column, sampleValues: uniqueStrings([...(column.sampleValues ?? []), ...sampleValues]).slice(0, 8) }
|
|
14634
|
+
: column;
|
|
14635
|
+
}),
|
|
14636
|
+
}));
|
|
14637
|
+
}
|
|
14638
|
+
// Common English plurals that are NOT entity nouns, so a multi-entity heuristic
|
|
14639
|
+
// doesn't count them. Domain-agnostic (no project-specific vocabulary).
|
|
14640
|
+
const NON_ENTITY_PLURALS = new Set([
|
|
14641
|
+
'details', 'values', 'results', 'rows', 'numbers', 'records', 'items', 'things',
|
|
14642
|
+
'totals', 'amounts', 'metrics', 'columns', 'fields', 'names', 'ids',
|
|
14643
|
+
]);
|
|
14644
|
+
/**
|
|
14645
|
+
* Decide whether to run the (advisory, best-effort) runtime value-scan + schema
|
|
14646
|
+
* enrichment for a question. Generic multi-entity / detail intent — no hard-coded
|
|
14647
|
+
* project vocabulary — so it fires for ANY repo's join-shaped questions, not just
|
|
14648
|
+
* jaffle. Over-triggering only costs a few extra bounded probes (all in try/catch).
|
|
14649
|
+
*/
|
|
14650
|
+
function shouldAugmentAgentRuntimeSchema(question) {
|
|
14651
|
+
const lower = question.toLowerCase();
|
|
14652
|
+
const wantsMetric = /\brevenu|\bsales\b|\bamount\b|\bspend\b|\bcost\b|\bcount\b|\btotal\b|\bsum\b|\bavg\b|\baverage\b|\bvalue\b|\bprice\b/.test(lower);
|
|
14653
|
+
const wantsDetail = /\bdetails?\b|\bcomplete\b|\bbreakdown\b|\ball\s+values?\b|\beach\b|\bevery\b/.test(lower);
|
|
14654
|
+
const referencesPriorRows = /\b(?:above|previous|prior|these|those|same)\s+\w+/.test(lower);
|
|
14655
|
+
const explicitJoin = /\bjoin(?:ed|ing)?\b|\bcombine[ds]?\b|\bcross[- ]?reference\b|\balong\s+with\b|\btogether\s+with\b/.test(lower);
|
|
14656
|
+
// Multi-entity intent: a linking word + >= 2 distinct plural content nouns.
|
|
14657
|
+
const nouns = new Set((lower.match(/\b[a-z]{4,}s\b/g) ?? []).filter((word) => !NON_ENTITY_PLURALS.has(word)));
|
|
14658
|
+
const multiEntity = /\b(?:and|with|plus|per|for\s+each|by)\b/.test(lower) && nouns.size >= 2;
|
|
14659
|
+
return explicitJoin || referencesPriorRows || (multiEntity && (wantsMetric || wantsDetail));
|
|
14660
|
+
}
|
|
13029
14661
|
async function enrichAgentSchemaContextWithValueMatches(question, schemaContext, executor, connection) {
|
|
13030
14662
|
const searchTerms = extractAgentValueSearchTerms(question);
|
|
13031
14663
|
if (schemaContext.length === 0 || searchTerms.length === 0)
|
|
@@ -13077,8 +14709,57 @@ function scoreAgentSchemaTable(table, tokens) {
|
|
|
13077
14709
|
score += 3;
|
|
13078
14710
|
}
|
|
13079
14711
|
}
|
|
13080
|
-
|
|
14712
|
+
score += scoreCompositeAgentSchemaTable(table, tokens);
|
|
14713
|
+
// Small tiebreaker toward modeled analytics tables (fct_/dim_/fact_/mart_ …),
|
|
14714
|
+
// a warehouse-modeling convention that generalizes across repos — not a fixed
|
|
14715
|
+
// list of jaffle entity names.
|
|
14716
|
+
if (/^(?:fct|fact|dim|dimension|mart|rpt|report|agg|f|d)[_-]/i.test(table.name)
|
|
14717
|
+
|| /(?:^|[_-])(?:fact|dim|mart|summary|report)(?:$|[_-])/i.test(table.name)) {
|
|
13081
14718
|
score += 1;
|
|
14719
|
+
}
|
|
14720
|
+
return score;
|
|
14721
|
+
}
|
|
14722
|
+
const COMPOSITE_METRIC_TOKENS = new Set([
|
|
14723
|
+
'revenue', 'sales', 'amount', 'spend', 'price', 'cost', 'total', 'sum',
|
|
14724
|
+
'count', 'quantity', 'qty', 'value', 'subtotal', 'order', 'orders',
|
|
14725
|
+
]);
|
|
14726
|
+
const COMPOSITE_STOPWORD_TOKENS = new Set([
|
|
14727
|
+
'the', 'and', 'with', 'for', 'each', 'per', 'all', 'top', 'show', 'list',
|
|
14728
|
+
'give', 'get', 'find', 'plus', 'along', 'together', 'combined', 'combine',
|
|
14729
|
+
'detail', 'details', 'complete', 'breakdown', 'values', 'name', 'names',
|
|
14730
|
+
...COMPOSITE_METRIC_TOKENS,
|
|
14731
|
+
]);
|
|
14732
|
+
/**
|
|
14733
|
+
* Boost the fact/junction table for a multi-entity ("composite") question. Fully
|
|
14734
|
+
* domain-agnostic: the ask is detected from >= 2 distinct entity tokens + a metric,
|
|
14735
|
+
* and the winning table is identified STRUCTURALLY (>= 2 foreign-key-style id columns
|
|
14736
|
+
* plus a numeric measure column) rather than by hard-coded jaffle table names — so it
|
|
14737
|
+
* ranks the right join table for any repo, while jaffle's fct_orders still qualifies.
|
|
14738
|
+
*/
|
|
14739
|
+
function scoreCompositeAgentSchemaTable(table, tokens) {
|
|
14740
|
+
const entityTokens = [...tokens].filter((token) => token.length >= 3 && !COMPOSITE_STOPWORD_TOKENS.has(token));
|
|
14741
|
+
const hasMetric = [...tokens].some((token) => COMPOSITE_METRIC_TOKENS.has(token));
|
|
14742
|
+
const hasCompositeAsk = entityTokens.length >= 2 && hasMetric;
|
|
14743
|
+
const hasPriorRowsAsk = ['above', 'previous', 'prior', 'these', 'those', 'same'].some((token) => tokens.has(token))
|
|
14744
|
+
&& entityTokens.length >= 1;
|
|
14745
|
+
if (!hasCompositeAsk && !hasPriorRowsAsk)
|
|
14746
|
+
return 0;
|
|
14747
|
+
const columns = table.columns.map((column) => column.name.toLowerCase());
|
|
14748
|
+
let score = 0;
|
|
14749
|
+
// Structural fact/junction-table signal: multiple FK-style id columns + a measure.
|
|
14750
|
+
const idColumnCount = columns.filter((name) => /(?:^id$|_id$|_key$|_sk$|_uuid$|_fk$)/.test(name)).length;
|
|
14751
|
+
const hasMeasureColumn = columns.some((name) => /revenue|sales|amount|price|spend|cost|subtotal|total|quantity|\bqty\b|count|value/.test(name));
|
|
14752
|
+
if (idColumnCount >= 2 && hasMeasureColumn)
|
|
14753
|
+
score += 18;
|
|
14754
|
+
else if (idColumnCount >= 2)
|
|
14755
|
+
score += 8;
|
|
14756
|
+
if (hasMeasureColumn)
|
|
14757
|
+
score += 8;
|
|
14758
|
+
// Reward columns whose names overlap the question's actual entity tokens (any domain).
|
|
14759
|
+
for (const token of entityTokens) {
|
|
14760
|
+
if (columns.some((name) => name.includes(token)))
|
|
14761
|
+
score += 4;
|
|
14762
|
+
}
|
|
13082
14763
|
return score;
|
|
13083
14764
|
}
|
|
13084
14765
|
function scoreAgentValueProbeTable(table) {
|