@duckcodeailabs/dql-agent 1.6.33 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-run-engine.d.ts +39 -8
- package/dist/agent-run-engine.d.ts.map +1 -1
- package/dist/agent-run-engine.js +169 -10
- package/dist/agent-run-engine.js.map +1 -1
- package/dist/agent-run-gates.d.ts.map +1 -1
- package/dist/agent-run-gates.js +21 -3
- package/dist/agent-run-gates.js.map +1 -1
- package/dist/agent-run-planner.d.ts.map +1 -1
- package/dist/agent-run-planner.js +14 -3
- package/dist/agent-run-planner.js.map +1 -1
- package/dist/agentic/answer-contract.d.ts +37 -0
- package/dist/agentic/answer-contract.d.ts.map +1 -0
- package/dist/agentic/answer-contract.js +47 -0
- package/dist/agentic/answer-contract.js.map +1 -0
- package/dist/agentic/tool-loop.d.ts +48 -0
- package/dist/agentic/tool-loop.d.ts.map +1 -0
- package/dist/agentic/tool-loop.js +181 -0
- package/dist/agentic/tool-loop.js.map +1 -0
- package/dist/agentic/toolset.d.ts +40 -0
- package/dist/agentic/toolset.d.ts.map +1 -0
- package/dist/agentic/toolset.js +265 -0
- package/dist/agentic/toolset.js.map +1 -0
- package/dist/answer-loop.d.ts +82 -6
- package/dist/answer-loop.d.ts.map +1 -1
- package/dist/answer-loop.js +799 -112
- package/dist/answer-loop.js.map +1 -1
- package/dist/app-builder.d.ts.map +1 -1
- package/dist/app-builder.js +18 -10
- package/dist/app-builder.js.map +1 -1
- package/dist/cascade/budgets.d.ts +20 -2
- package/dist/cascade/budgets.d.ts.map +1 -1
- package/dist/cascade/budgets.js +83 -45
- package/dist/cascade/budgets.js.map +1 -1
- package/dist/cascade/route-policy.d.ts +1 -1
- package/dist/cascade/route-policy.d.ts.map +1 -1
- package/dist/cascade/route-policy.js +6 -0
- package/dist/cascade/route-policy.js.map +1 -1
- package/dist/domain-context.d.ts +30 -0
- package/dist/domain-context.d.ts.map +1 -0
- package/dist/domain-context.js +68 -0
- package/dist/domain-context.js.map +1 -0
- package/dist/embeddings/provider.d.ts +67 -4
- package/dist/embeddings/provider.d.ts.map +1 -1
- package/dist/embeddings/provider.js +167 -6
- package/dist/embeddings/provider.js.map +1 -1
- package/dist/hints/correction-eval.d.ts +30 -0
- package/dist/hints/correction-eval.d.ts.map +1 -0
- package/dist/hints/correction-eval.js +63 -0
- package/dist/hints/correction-eval.js.map +1 -0
- package/dist/hints/git-store.d.ts +39 -3
- package/dist/hints/git-store.d.ts.map +1 -1
- package/dist/hints/git-store.js +172 -2
- package/dist/hints/git-store.js.map +1 -1
- package/dist/hints/staleness.d.ts +30 -0
- package/dist/hints/staleness.d.ts.map +1 -0
- package/dist/hints/staleness.js +38 -0
- package/dist/hints/staleness.js.map +1 -0
- package/dist/hints/store.d.ts.map +1 -1
- package/dist/hints/store.js +3 -32
- package/dist/hints/store.js.map +1 -1
- package/dist/hints/types.d.ts +41 -0
- package/dist/hints/types.d.ts.map +1 -1
- package/dist/hints/types.js +1 -0
- package/dist/hints/types.js.map +1 -1
- package/dist/index.d.ts +46 -10
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +180 -9
- package/dist/index.js.map +1 -1
- package/dist/intent-controller.d.ts.map +1 -1
- package/dist/intent-controller.js +15 -3
- package/dist/intent-controller.js.map +1 -1
- package/dist/kg/build.d.ts.map +1 -1
- package/dist/kg/build.js +196 -4
- package/dist/kg/build.js.map +1 -1
- package/dist/kg/sqlite-fts.d.ts +13 -0
- package/dist/kg/sqlite-fts.d.ts.map +1 -1
- package/dist/kg/sqlite-fts.js +82 -49
- package/dist/kg/sqlite-fts.js.map +1 -1
- package/dist/kg/types.d.ts +9 -2
- package/dist/kg/types.d.ts.map +1 -1
- package/dist/memory/fts-query.d.ts +37 -4
- package/dist/memory/fts-query.d.ts.map +1 -1
- package/dist/memory/fts-query.js +53 -12
- package/dist/memory/fts-query.js.map +1 -1
- package/dist/metadata/analysis-planner.d.ts.map +1 -1
- package/dist/metadata/analysis-planner.js +179 -28
- package/dist/metadata/analysis-planner.js.map +1 -1
- package/dist/metadata/analytical-policy.d.ts +51 -0
- package/dist/metadata/analytical-policy.d.ts.map +1 -0
- package/dist/metadata/analytical-policy.js +309 -0
- package/dist/metadata/analytical-policy.js.map +1 -0
- package/dist/metadata/block-adapt.d.ts +37 -0
- package/dist/metadata/block-adapt.d.ts.map +1 -0
- package/dist/metadata/block-adapt.js +89 -0
- package/dist/metadata/block-adapt.js.map +1 -0
- package/dist/metadata/block-fit.js +37 -4
- package/dist/metadata/block-fit.js.map +1 -1
- package/dist/metadata/catalog.d.ts +85 -1
- package/dist/metadata/catalog.d.ts.map +1 -1
- package/dist/metadata/catalog.js +674 -92
- package/dist/metadata/catalog.js.map +1 -1
- package/dist/metadata/dbt-first-safety.d.ts +19 -0
- package/dist/metadata/dbt-first-safety.d.ts.map +1 -0
- package/dist/metadata/dbt-first-safety.js +17 -0
- package/dist/metadata/dbt-first-safety.js.map +1 -0
- package/dist/metadata/example-match.d.ts +30 -0
- package/dist/metadata/example-match.d.ts.map +1 -0
- package/dist/metadata/example-match.js +69 -0
- package/dist/metadata/example-match.js.map +1 -0
- package/dist/metadata/grain-ledger.d.ts +51 -0
- package/dist/metadata/grain-ledger.d.ts.map +1 -0
- package/dist/metadata/grain-ledger.js +174 -0
- package/dist/metadata/grain-ledger.js.map +1 -0
- package/dist/metadata/metric-match.d.ts +21 -1
- package/dist/metadata/metric-match.d.ts.map +1 -1
- package/dist/metadata/metric-match.js +157 -21
- package/dist/metadata/metric-match.js.map +1 -1
- package/dist/metadata/sql-context-validation.d.ts.map +1 -1
- package/dist/metadata/sql-context-validation.js +42 -2
- package/dist/metadata/sql-context-validation.js.map +1 -1
- package/dist/metadata/sql-grounding.d.ts +4 -2
- package/dist/metadata/sql-grounding.d.ts.map +1 -1
- package/dist/metadata/sql-grounding.js +29 -4
- package/dist/metadata/sql-grounding.js.map +1 -1
- package/dist/metadata/sql-retrieval.d.ts +5 -2
- package/dist/metadata/sql-retrieval.d.ts.map +1 -1
- package/dist/metadata/sql-retrieval.js +28 -6
- package/dist/metadata/sql-retrieval.js.map +1 -1
- package/dist/propose/build-from-prompt.d.ts +34 -0
- package/dist/propose/build-from-prompt.d.ts.map +1 -1
- package/dist/propose/build-from-prompt.js +205 -19
- package/dist/propose/build-from-prompt.js.map +1 -1
- package/dist/propose/failure-analysis.d.ts +48 -0
- package/dist/propose/failure-analysis.d.ts.map +1 -0
- package/dist/propose/failure-analysis.js +71 -0
- package/dist/propose/failure-analysis.js.map +1 -0
- package/dist/propose/join-mining.d.ts +18 -0
- package/dist/propose/join-mining.d.ts.map +1 -0
- package/dist/propose/join-mining.js +74 -0
- package/dist/propose/join-mining.js.map +1 -0
- package/dist/providers/claude.d.ts.map +1 -1
- package/dist/providers/claude.js +36 -1
- package/dist/providers/claude.js.map +1 -1
- package/dist/providers/index.d.ts +2 -2
- package/dist/providers/index.d.ts.map +1 -1
- package/dist/providers/index.js +1 -1
- package/dist/providers/index.js.map +1 -1
- package/dist/providers/openai.d.ts.map +1 -1
- package/dist/providers/openai.js +34 -1
- package/dist/providers/openai.js.map +1 -1
- package/dist/providers/reasoning-effort.d.ts +27 -0
- package/dist/providers/reasoning-effort.d.ts.map +1 -1
- package/dist/providers/reasoning-effort.js +32 -0
- package/dist/providers/reasoning-effort.js.map +1 -1
- package/dist/providers/types.d.ts +2 -1
- package/dist/providers/types.d.ts.map +1 -1
- package/dist/research-loop.d.ts.map +1 -1
- package/dist/research-loop.js +9 -2
- package/dist/research-loop.js.map +1 -1
- package/dist/semantic-bridge/compose.d.ts.map +1 -1
- package/dist/semantic-bridge/compose.js +316 -25
- package/dist/semantic-bridge/compose.js.map +1 -1
- package/dist/skills/bootstrap.d.ts +57 -0
- package/dist/skills/bootstrap.d.ts.map +1 -0
- package/dist/skills/bootstrap.js +260 -0
- package/dist/skills/bootstrap.js.map +1 -0
- package/dist/skills/defaults.d.ts +2 -2
- package/dist/skills/defaults.js +2 -2
- package/dist/skills/domain-skills.d.ts +24 -0
- package/dist/skills/domain-skills.d.ts.map +1 -0
- package/dist/skills/domain-skills.js +76 -0
- package/dist/skills/domain-skills.js.map +1 -0
- package/dist/skills/loader.d.ts +56 -8
- package/dist/skills/loader.d.ts.map +1 -1
- package/dist/skills/loader.js +257 -41
- package/dist/skills/loader.js.map +1 -1
- package/dist/synthesize.d.ts.map +1 -1
- package/dist/synthesize.js +152 -16
- package/dist/synthesize.js.map +1 -1
- package/dist/tools/registry.d.ts +1 -1
- package/dist/tools/registry.d.ts.map +1 -1
- package/dist/tools/registry.js +54 -12
- package/dist/tools/registry.js.map +1 -1
- package/package.json +4 -4
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/** Compatibility wrapper around the unified manifest-v3 analytical policy service. */
|
|
2
|
+
import type { DQLManifest } from '@duckcodeailabs/dql-core';
|
|
3
|
+
import type { DomainContextEnvelope } from '../domain-context.js';
|
|
4
|
+
import { type AnalyticalPolicyCode } from './analytical-policy.js';
|
|
5
|
+
export type DbtFirstJoinSafetyCode = AnalyticalPolicyCode;
|
|
6
|
+
export interface DbtFirstJoinSafetyDecision {
|
|
7
|
+
safe: boolean;
|
|
8
|
+
code?: DbtFirstJoinSafetyCode;
|
|
9
|
+
message?: string;
|
|
10
|
+
entities: string[];
|
|
11
|
+
relationshipIds: string[];
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Validate generated SQL against exact certified path and join-key proof.
|
|
15
|
+
* Kept under the historical name so existing Ask AI integrations remain API
|
|
16
|
+
* compatible while MCP, notebooks, and Apps adopt the same policy service.
|
|
17
|
+
*/
|
|
18
|
+
export declare function evaluateDbtFirstGeneratedSql(sql: string, manifest: DQLManifest, purpose?: string, domainContext?: DomainContextEnvelope): DbtFirstJoinSafetyDecision;
|
|
19
|
+
//# sourceMappingURL=dbt-first-safety.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"dbt-first-safety.d.ts","sourceRoot":"","sources":["../../src/metadata/dbt-first-safety.ts"],"names":[],"mappings":"AAAA,sFAAsF;AACtF,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAC5D,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAClE,OAAO,EAAyB,KAAK,oBAAoB,EAAE,MAAM,wBAAwB,CAAC;AAE1F,MAAM,MAAM,sBAAsB,GAAG,oBAAoB,CAAC;AAE1D,MAAM,WAAW,0BAA0B;IACzC,IAAI,EAAE,OAAO,CAAC;IACd,IAAI,CAAC,EAAE,sBAAsB,CAAC;IAC9B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,eAAe,EAAE,MAAM,EAAE,CAAC;CAC3B;AAED;;;;GAIG;AACH,wBAAgB,4BAA4B,CAC1C,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,WAAW,EACrB,OAAO,CAAC,EAAE,MAAM,EAChB,aAAa,CAAC,EAAE,qBAAqB,GACpC,0BAA0B,CAS5B"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { validateAnalyticalSql } from './analytical-policy.js';
|
|
2
|
+
/**
|
|
3
|
+
* Validate generated SQL against exact certified path and join-key proof.
|
|
4
|
+
* Kept under the historical name so existing Ask AI integrations remain API
|
|
5
|
+
* compatible while MCP, notebooks, and Apps adopt the same policy service.
|
|
6
|
+
*/
|
|
7
|
+
export function evaluateDbtFirstGeneratedSql(sql, manifest, purpose, domainContext) {
|
|
8
|
+
const decision = validateAnalyticalSql(sql, manifest, 'duckdb', purpose, domainContext);
|
|
9
|
+
return {
|
|
10
|
+
safe: decision.safe,
|
|
11
|
+
code: decision.code,
|
|
12
|
+
message: decision.message,
|
|
13
|
+
entities: decision.entities,
|
|
14
|
+
relationshipIds: decision.relationshipIds,
|
|
15
|
+
};
|
|
16
|
+
}
|
|
17
|
+
//# sourceMappingURL=dbt-first-safety.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"dbt-first-safety.js","sourceRoot":"","sources":["../../src/metadata/dbt-first-safety.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,qBAAqB,EAA6B,MAAM,wBAAwB,CAAC;AAY1F;;;;GAIG;AACH,MAAM,UAAU,4BAA4B,CAC1C,GAAW,EACX,QAAqB,EACrB,OAAgB,EAChB,aAAqC;IAErC,MAAM,QAAQ,GAAG,qBAAqB,CAAC,GAAG,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,aAAa,CAAC,CAAC;IACxF,OAAO;QACL,IAAI,EAAE,QAAQ,CAAC,IAAI;QACnB,IAAI,EAAE,QAAQ,CAAC,IAAI;QACnB,OAAO,EAAE,QAAQ,CAAC,OAAO;QACzB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;QAC3B,eAAe,EAAE,QAAQ,CAAC,eAAe;KAC1C,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Paraphrase-tolerant certified-example matching (W2.1).
|
|
3
|
+
*
|
|
4
|
+
* `hasExactExampleQuestion` matches a question to a block's `examples[].question`
|
|
5
|
+
* by normalized string equality, so a paraphrase ("which 5 products earn the most"
|
|
6
|
+
* vs the certified "top 5 products by revenue") misses Tier 1 and falls through to
|
|
7
|
+
* generation. This module adds a semantic (embedding cosine) match. It is used only
|
|
8
|
+
* to PROMOTE a block to the certified candidate — the promoted block still runs the
|
|
9
|
+
* full shape/grain fit + direction check, so a paraphrase never bypasses grain the
|
|
10
|
+
* way a user naming the block directly does. Degrades safely on the default hashed
|
|
11
|
+
* embedder (behaves like fuzzy token overlap); lights up with a real provider.
|
|
12
|
+
*/
|
|
13
|
+
import { type EmbeddingProvider } from '../embeddings/provider.js';
|
|
14
|
+
import type { MetadataObject } from './catalog.js';
|
|
15
|
+
/** Default cosine threshold for a paraphrase match over the resolved provider. */
|
|
16
|
+
export declare const DEFAULT_PARAPHRASE_THRESHOLD = 0.82;
|
|
17
|
+
export interface ParaphraseMatchOptions {
|
|
18
|
+
threshold?: number;
|
|
19
|
+
}
|
|
20
|
+
/** The highest cosine between the question and any of the block's example questions. */
|
|
21
|
+
export declare function bestExampleParaphrase(question: string, block: MetadataObject, provider: EmbeddingProvider): Promise<{
|
|
22
|
+
cosine: number;
|
|
23
|
+
example?: string;
|
|
24
|
+
}>;
|
|
25
|
+
/**
|
|
26
|
+
* True when the question paraphrases one of the block's certified example
|
|
27
|
+
* questions (cosine ≥ threshold) AND the ranking direction is compatible.
|
|
28
|
+
*/
|
|
29
|
+
export declare function matchExampleParaphrase(question: string, block: MetadataObject, provider: EmbeddingProvider, options?: ParaphraseMatchOptions): Promise<boolean>;
|
|
30
|
+
//# sourceMappingURL=example-match.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"example-match.d.ts","sourceRoot":"","sources":["../../src/metadata/example-match.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AACH,OAAO,EAAoB,KAAK,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AACrF,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAEnD,kFAAkF;AAClF,eAAO,MAAM,4BAA4B,OAAO,CAAC;AAEjD,MAAM,WAAW,sBAAsB;IACrC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AA0BD,wFAAwF;AACxF,wBAAsB,qBAAqB,CACzC,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,cAAc,EACrB,QAAQ,EAAE,iBAAiB,GAC1B,OAAO,CAAC;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,CAe/C;AAED;;;GAGG;AACH,wBAAsB,sBAAsB,CAC1C,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,cAAc,EACrB,QAAQ,EAAE,iBAAiB,EAC3B,OAAO,GAAE,sBAA2B,GACnC,OAAO,CAAC,OAAO,CAAC,CAKlB"}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Paraphrase-tolerant certified-example matching (W2.1).
|
|
3
|
+
*
|
|
4
|
+
* `hasExactExampleQuestion` matches a question to a block's `examples[].question`
|
|
5
|
+
* by normalized string equality, so a paraphrase ("which 5 products earn the most"
|
|
6
|
+
* vs the certified "top 5 products by revenue") misses Tier 1 and falls through to
|
|
7
|
+
* generation. This module adds a semantic (embedding cosine) match. It is used only
|
|
8
|
+
* to PROMOTE a block to the certified candidate — the promoted block still runs the
|
|
9
|
+
* full shape/grain fit + direction check, so a paraphrase never bypasses grain the
|
|
10
|
+
* way a user naming the block directly does. Degrades safely on the default hashed
|
|
11
|
+
* embedder (behaves like fuzzy token overlap); lights up with a real provider.
|
|
12
|
+
*/
|
|
13
|
+
import { cosineSimilarity } from '../embeddings/provider.js';
|
|
14
|
+
/** Default cosine threshold for a paraphrase match over the resolved provider. */
|
|
15
|
+
export const DEFAULT_PARAPHRASE_THRESHOLD = 0.82;
|
|
16
|
+
function exampleQuestions(block) {
|
|
17
|
+
const examples = Array.isArray(block.payload?.examples) ? block.payload.examples : [];
|
|
18
|
+
return examples
|
|
19
|
+
.map((example) => (example && typeof example === 'object' ? example.question : undefined))
|
|
20
|
+
.filter((question) => typeof question === 'string' && question.trim().length > 0);
|
|
21
|
+
}
|
|
22
|
+
/** A "top" question must not match a "bottom" example (and vice-versa). */
|
|
23
|
+
function rankingDirection(value) {
|
|
24
|
+
const lower = value.toLowerCase();
|
|
25
|
+
const bottom = /\b(bottom|worst|lowest|least|fewest|minimum|min|smallest)\b/.test(lower);
|
|
26
|
+
const top = /\b(top|best|highest|most|maximum|max|largest|leading|leaders?)\b/.test(lower);
|
|
27
|
+
if (bottom && !top)
|
|
28
|
+
return 'bottom';
|
|
29
|
+
if (top && !bottom)
|
|
30
|
+
return 'top';
|
|
31
|
+
return undefined;
|
|
32
|
+
}
|
|
33
|
+
function directionCompatible(question, example) {
|
|
34
|
+
const q = rankingDirection(question);
|
|
35
|
+
if (!q)
|
|
36
|
+
return true;
|
|
37
|
+
const e = rankingDirection(example);
|
|
38
|
+
return !e || e === q;
|
|
39
|
+
}
|
|
40
|
+
/** The highest cosine between the question and any of the block's example questions. */
|
|
41
|
+
export async function bestExampleParaphrase(question, block, provider) {
|
|
42
|
+
const examples = exampleQuestions(block);
|
|
43
|
+
if (examples.length === 0)
|
|
44
|
+
return { cosine: 0 };
|
|
45
|
+
const vectors = await provider.embed([question, ...examples]);
|
|
46
|
+
const questionVector = vectors[0];
|
|
47
|
+
let best = 0;
|
|
48
|
+
let bestExample;
|
|
49
|
+
for (let i = 0; i < examples.length; i++) {
|
|
50
|
+
const cosine = cosineSimilarity(questionVector, vectors[i + 1]);
|
|
51
|
+
if (cosine > best) {
|
|
52
|
+
best = cosine;
|
|
53
|
+
bestExample = examples[i];
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
return { cosine: best, example: bestExample };
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* True when the question paraphrases one of the block's certified example
|
|
60
|
+
* questions (cosine ≥ threshold) AND the ranking direction is compatible.
|
|
61
|
+
*/
|
|
62
|
+
export async function matchExampleParaphrase(question, block, provider, options = {}) {
|
|
63
|
+
const threshold = options.threshold ?? DEFAULT_PARAPHRASE_THRESHOLD;
|
|
64
|
+
const { cosine, example } = await bestExampleParaphrase(question, block, provider);
|
|
65
|
+
if (cosine < threshold)
|
|
66
|
+
return false;
|
|
67
|
+
return example ? directionCompatible(question, example) : true;
|
|
68
|
+
}
|
|
69
|
+
//# sourceMappingURL=example-match.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"example-match.js","sourceRoot":"","sources":["../../src/metadata/example-match.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AACH,OAAO,EAAE,gBAAgB,EAA0B,MAAM,2BAA2B,CAAC;AAGrF,kFAAkF;AAClF,MAAM,CAAC,MAAM,4BAA4B,GAAG,IAAI,CAAC;AAMjD,SAAS,gBAAgB,CAAC,KAAqB;IAC7C,MAAM,QAAQ,GAAc,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,OAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;IAClG,OAAO,QAAQ;SACZ,GAAG,CAAC,CAAC,OAAgB,EAAE,EAAE,CAAC,CAAC,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAE,OAAkC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;SAC9H,MAAM,CAAC,CAAC,QAAiB,EAAsB,EAAE,CAAC,OAAO,QAAQ,KAAK,QAAQ,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACnH,CAAC;AAED,2EAA2E;AAC3E,SAAS,gBAAgB,CAAC,KAAa;IACrC,MAAM,KAAK,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC;IAClC,MAAM,MAAM,GAAG,6DAA6D,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACzF,MAAM,GAAG,GAAG,kEAAkE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC3F,IAAI,MAAM,IAAI,CAAC,GAAG;QAAE,OAAO,QAAQ,CAAC;IACpC,IAAI,GAAG,IAAI,CAAC,MAAM;QAAE,OAAO,KAAK,CAAC;IACjC,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,mBAAmB,CAAC,QAAgB,EAAE,OAAe;IAC5D,MAAM,CAAC,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IACrC,IAAI,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IACpB,MAAM,CAAC,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;IACpC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AACvB,CAAC;AAED,wFAAwF;AACxF,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,QAAgB,EAChB,KAAqB,EACrB,QAA2B;IAE3B,MAAM,QAAQ,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC;IACzC,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;IAChD,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAC,QAAQ,EAAE,GAAG,QAAQ,CAAC,CAAC,CAAC;IAC9D,MAAM,cAAc,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IAClC,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,WAA+B,CAAC;IACpC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACzC,MAAM,MAAM,GAAG,gBAAgB,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAChE,IAAI,MAAM,GAAG,IAAI,EAAE,CAAC;YAClB,IAAI,GAAG,MAAM,CAAC;YACd,WAAW,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;AAChD,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,QAAgB,EAChB,KAAqB,EACrB,QAA2B,EAC3B,UAAkC,EAAE;IAEpC,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,4BAA4B,CAAC;IACpE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,qBAAqB,CAAC,QAAQ,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC;IACnF,IAAI,MAAM,GAAG,SAAS;QAAE,OAAO,KAAK,CAAC;IACrC,OAAO,OAAO,CAAC,CAAC,CAAC,mBAAmB,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACjE,CAAC"}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Grain ledger + static fan-out detection (W1.3).
|
|
3
|
+
*
|
|
4
|
+
* The class-B (wrong-number) failure mode is a VALID query over real columns that
|
|
5
|
+
* returns an inflated total because an additive aggregate is computed after a
|
|
6
|
+
* one-to-many join — every row of the aggregated relation is multiplied by the
|
|
7
|
+
* number of matching rows on the "many" side. `validateSql` proves identifiers
|
|
8
|
+
* exist; it says nothing about join grain. This module adds a deterministic,
|
|
9
|
+
* zero-LLM check: given per-relation unique keys (the ledger) and the parsed join
|
|
10
|
+
* graph, flag an additive aggregate over relation R when R is joined to a relation
|
|
11
|
+
* S on a column that is NOT a unique key of S (so S contributes multiple rows per
|
|
12
|
+
* R row). It is intentionally conservative — no key data for a relation ⇒ no flag —
|
|
13
|
+
* so it never demotes a correct answer; it is wired warn-only.
|
|
14
|
+
*/
|
|
15
|
+
import { type SqlReferenceAnalysis } from '@duckcodeailabs/dql-core';
|
|
16
|
+
import type { MetadataObject } from './catalog.js';
|
|
17
|
+
export interface RelationGrain {
|
|
18
|
+
/** Normalized relation tail token (e.g. `fct_orders`). */
|
|
19
|
+
relation: string;
|
|
20
|
+
/** Columns that ALONE uniquely identify a row (single-column keys only). */
|
|
21
|
+
uniqueKeys: Set<string>;
|
|
22
|
+
source: string;
|
|
23
|
+
}
|
|
24
|
+
export type GrainLedger = Map<string, RelationGrain>;
|
|
25
|
+
export interface FanoutRisk {
|
|
26
|
+
aggregate: string;
|
|
27
|
+
aggregatedRelation: string;
|
|
28
|
+
fanoutRelation: string;
|
|
29
|
+
joinColumn: string;
|
|
30
|
+
message: string;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Build a grain ledger from catalog objects. v0 sources single-column unique keys
|
|
34
|
+
* from DataLex entity primary-key fields and any dbt/catalog object that carries a
|
|
35
|
+
* `primaryKey` / `uniqueColumns` payload (the hook dbt unique-test extraction fills
|
|
36
|
+
* in W5.3). Composite keys are intentionally skipped — a single column of a
|
|
37
|
+
* composite key is not unique on its own, so treating it as one would miss fan-out.
|
|
38
|
+
*/
|
|
39
|
+
export declare function buildGrainLedger(objects: MetadataObject[]): GrainLedger;
|
|
40
|
+
/**
|
|
41
|
+
* Detect additive aggregates that risk fan-out double-counting given the ledger.
|
|
42
|
+
* An aggregate over relation R is flagged when R is joined to a relation S on a
|
|
43
|
+
* column that is NOT a known unique key of S (S can contribute many rows per R row).
|
|
44
|
+
*/
|
|
45
|
+
export declare function detectFanoutRisks(analysis: SqlReferenceAnalysis, ledger: GrainLedger): FanoutRisk[];
|
|
46
|
+
/**
|
|
47
|
+
* Convenience wrapper: parse the SQL, build the ledger from the objects, and return
|
|
48
|
+
* fan-out warning strings. Best-effort — any parse failure yields no warnings.
|
|
49
|
+
*/
|
|
50
|
+
export declare function fanoutWarningsForSql(sql: string, objects: MetadataObject[], dialect?: string): string[];
|
|
51
|
+
//# sourceMappingURL=grain-ledger.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"grain-ledger.d.ts","sourceRoot":"","sources":["../../src/metadata/grain-ledger.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AACH,OAAO,EAAwB,KAAK,oBAAoB,EAAE,MAAM,0BAA0B,CAAC;AAC3F,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAEnD,MAAM,WAAW,aAAa;IAC5B,0DAA0D;IAC1D,QAAQ,EAAE,MAAM,CAAC;IACjB,4EAA4E;IAC5E,UAAU,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IACxB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,MAAM,WAAW,GAAG,GAAG,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;AAErD,MAAM,WAAW,UAAU;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;CACjB;AA2BD;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,cAAc,EAAE,GAAG,WAAW,CA0CvE;AAQD;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,oBAAoB,EAAE,MAAM,EAAE,WAAW,GAAG,UAAU,EAAE,CA4CnG;AAED;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,cAAc,EAAE,EAAE,OAAO,SAAW,GAAG,MAAM,EAAE,CAUzG"}
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Grain ledger + static fan-out detection (W1.3).
|
|
3
|
+
*
|
|
4
|
+
* The class-B (wrong-number) failure mode is a VALID query over real columns that
|
|
5
|
+
* returns an inflated total because an additive aggregate is computed after a
|
|
6
|
+
* one-to-many join — every row of the aggregated relation is multiplied by the
|
|
7
|
+
* number of matching rows on the "many" side. `validateSql` proves identifiers
|
|
8
|
+
* exist; it says nothing about join grain. This module adds a deterministic,
|
|
9
|
+
* zero-LLM check: given per-relation unique keys (the ledger) and the parsed join
|
|
10
|
+
* graph, flag an additive aggregate over relation R when R is joined to a relation
|
|
11
|
+
* S on a column that is NOT a unique key of S (so S contributes multiple rows per
|
|
12
|
+
* R row). It is intentionally conservative — no key data for a relation ⇒ no flag —
|
|
13
|
+
* so it never demotes a correct answer; it is wired warn-only.
|
|
14
|
+
*/
|
|
15
|
+
import { analyzeSqlReferences } from '@duckcodeailabs/dql-core';
|
|
16
|
+
/** Additive aggregates whose value changes when rows are duplicated. MIN/MAX are safe. */
|
|
17
|
+
const ADDITIVE_AGGREGATES = new Set(['SUM', 'AVG', 'COUNT']);
|
|
18
|
+
function normalizeRelationTail(relation) {
|
|
19
|
+
if (!relation)
|
|
20
|
+
return undefined;
|
|
21
|
+
const tail = relation.split('.').at(-1) ?? relation;
|
|
22
|
+
const cleaned = tail.replace(/["`]/g, '').trim().toLowerCase();
|
|
23
|
+
return cleaned || undefined;
|
|
24
|
+
}
|
|
25
|
+
function normalizeColumn(column) {
|
|
26
|
+
return column.replace(/["`]/g, '').trim().toLowerCase();
|
|
27
|
+
}
|
|
28
|
+
function addUniqueKey(ledger, relation, column, source) {
|
|
29
|
+
const key = normalizeRelationTail(relation);
|
|
30
|
+
if (!key)
|
|
31
|
+
return;
|
|
32
|
+
const existing = ledger.get(key);
|
|
33
|
+
if (existing) {
|
|
34
|
+
existing.uniqueKeys.add(normalizeColumn(column));
|
|
35
|
+
return;
|
|
36
|
+
}
|
|
37
|
+
ledger.set(key, { relation: key, uniqueKeys: new Set([normalizeColumn(column)]), source });
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Build a grain ledger from catalog objects. v0 sources single-column unique keys
|
|
41
|
+
* from DataLex entity primary-key fields and any dbt/catalog object that carries a
|
|
42
|
+
* `primaryKey` / `uniqueColumns` payload (the hook dbt unique-test extraction fills
|
|
43
|
+
* in W5.3). Composite keys are intentionally skipped — a single column of a
|
|
44
|
+
* composite key is not unique on its own, so treating it as one would miss fan-out.
|
|
45
|
+
*/
|
|
46
|
+
export function buildGrainLedger(objects) {
|
|
47
|
+
const ledger = new Map();
|
|
48
|
+
for (const object of objects) {
|
|
49
|
+
if (object.objectType === 'datalex_entity') {
|
|
50
|
+
const binding = object.payload?.binding;
|
|
51
|
+
const relation = binding?.ref ?? object.fullName ?? object.name;
|
|
52
|
+
const fields = Array.isArray(object.payload?.fields) ? object.payload.fields : [];
|
|
53
|
+
const primaryKeys = fields.filter((field) => field?.primaryKey === true && typeof field.name === 'string');
|
|
54
|
+
// Only a lone primary key column is a single-column unique key.
|
|
55
|
+
if (primaryKeys.length === 1) {
|
|
56
|
+
addUniqueKey(ledger, relation, String(primaryKeys[0].name), 'datalex primary key');
|
|
57
|
+
}
|
|
58
|
+
// Entity-level candidate/business keys (W5.2). Conservatively use only entries
|
|
59
|
+
// that are a SINGLE simple column token — a composite key (e.g. "a, b") is not
|
|
60
|
+
// a single-column unique key, and over-claiming uniqueness would MISS a real
|
|
61
|
+
// fan-out (a silent wrong number), so we skip anything non-atomic.
|
|
62
|
+
for (const keyList of [object.payload?.candidateKeys, object.payload?.businessKeys]) {
|
|
63
|
+
if (!Array.isArray(keyList))
|
|
64
|
+
continue;
|
|
65
|
+
for (const key of keyList) {
|
|
66
|
+
if (typeof key === 'string' && /^[A-Za-z_][\w]*$/.test(key.trim())) {
|
|
67
|
+
addUniqueKey(ledger, relation, key.trim(), 'datalex candidate/business key');
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
continue;
|
|
72
|
+
}
|
|
73
|
+
// Generic hook: an object may declare its own key columns (dbt constraints /
|
|
74
|
+
// unique tests once extracted, or explicit metadata). uniqueColumns lists
|
|
75
|
+
// single-column unique keys; primaryKey is a single-column PK.
|
|
76
|
+
const uniqueColumns = object.payload?.uniqueColumns;
|
|
77
|
+
if (Array.isArray(uniqueColumns)) {
|
|
78
|
+
const relation = metadataRelationName(object);
|
|
79
|
+
for (const column of uniqueColumns) {
|
|
80
|
+
if (typeof column === 'string')
|
|
81
|
+
addUniqueKey(ledger, relation, column, 'declared unique column');
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
const primaryKey = object.payload?.primaryKey;
|
|
85
|
+
if (typeof primaryKey === 'string') {
|
|
86
|
+
addUniqueKey(ledger, metadataRelationName(object), primaryKey, 'declared primary key');
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
return ledger;
|
|
90
|
+
}
|
|
91
|
+
function metadataRelationName(object) {
|
|
92
|
+
const relation = object.payload?.relation;
|
|
93
|
+
if (typeof relation === 'string' && relation.trim())
|
|
94
|
+
return relation;
|
|
95
|
+
return object.fullName ?? object.name;
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* Detect additive aggregates that risk fan-out double-counting given the ledger.
|
|
99
|
+
* An aggregate over relation R is flagged when R is joined to a relation S on a
|
|
100
|
+
* column that is NOT a known unique key of S (S can contribute many rows per R row).
|
|
101
|
+
*/
|
|
102
|
+
export function detectFanoutRisks(analysis, ledger) {
|
|
103
|
+
if (!analysis.parsed || analysis.joins.length === 0)
|
|
104
|
+
return [];
|
|
105
|
+
const risks = [];
|
|
106
|
+
const seen = new Set();
|
|
107
|
+
for (const aggregate of analysis.aggregates) {
|
|
108
|
+
if (aggregate.distinct)
|
|
109
|
+
continue;
|
|
110
|
+
if (!ADDITIVE_AGGREGATES.has(aggregate.func))
|
|
111
|
+
continue;
|
|
112
|
+
const aggregatedRelation = normalizeRelationTail(aggregate.relation);
|
|
113
|
+
if (!aggregatedRelation)
|
|
114
|
+
continue; // COUNT(*) / unattributed — out of scope for v0
|
|
115
|
+
for (const join of analysis.joins) {
|
|
116
|
+
const left = normalizeRelationTail(join.leftRelation);
|
|
117
|
+
const right = normalizeRelationTail(join.rightRelation);
|
|
118
|
+
let counterpart;
|
|
119
|
+
let counterpartColumn;
|
|
120
|
+
if (left === aggregatedRelation) {
|
|
121
|
+
counterpart = right;
|
|
122
|
+
counterpartColumn = normalizeColumn(join.rightColumn);
|
|
123
|
+
}
|
|
124
|
+
else if (right === aggregatedRelation) {
|
|
125
|
+
counterpart = left;
|
|
126
|
+
counterpartColumn = normalizeColumn(join.leftColumn);
|
|
127
|
+
}
|
|
128
|
+
else {
|
|
129
|
+
continue;
|
|
130
|
+
}
|
|
131
|
+
if (!counterpart || !counterpartColumn)
|
|
132
|
+
continue;
|
|
133
|
+
const counterpartGrain = ledger.get(counterpart);
|
|
134
|
+
if (!counterpartGrain)
|
|
135
|
+
continue; // unknown counterpart grain → conservative, no flag
|
|
136
|
+
if (counterpartGrain.uniqueKeys.has(counterpartColumn))
|
|
137
|
+
continue; // N:1 — safe
|
|
138
|
+
const aggregateLabel = `${aggregate.func}(${aggregate.column ?? '*'})`;
|
|
139
|
+
const dedupeKey = `${aggregateLabel}|${aggregatedRelation}|${counterpart}|${counterpartColumn}`;
|
|
140
|
+
if (seen.has(dedupeKey))
|
|
141
|
+
continue;
|
|
142
|
+
seen.add(dedupeKey);
|
|
143
|
+
risks.push({
|
|
144
|
+
aggregate: aggregateLabel,
|
|
145
|
+
aggregatedRelation,
|
|
146
|
+
fanoutRelation: counterpart,
|
|
147
|
+
joinColumn: counterpartColumn,
|
|
148
|
+
message: `Possible fan-out: ${aggregateLabel} aggregates ${aggregatedRelation}, which is joined one-to-many to ` +
|
|
149
|
+
`${counterpart} on ${counterpartColumn} (not a unique key of ${counterpart}). This can double-count. ` +
|
|
150
|
+
`Pre-aggregate ${aggregatedRelation} in a CTE before joining, or use COUNT(DISTINCT ...).`,
|
|
151
|
+
});
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
return risks;
|
|
155
|
+
}
|
|
156
|
+
/**
|
|
157
|
+
* Convenience wrapper: parse the SQL, build the ledger from the objects, and return
|
|
158
|
+
* fan-out warning strings. Best-effort — any parse failure yields no warnings.
|
|
159
|
+
*/
|
|
160
|
+
export function fanoutWarningsForSql(sql, objects, dialect = 'duckdb') {
|
|
161
|
+
try {
|
|
162
|
+
const analysis = analyzeSqlReferences(sql, dialect);
|
|
163
|
+
if (!analysis.parsed)
|
|
164
|
+
return [];
|
|
165
|
+
const ledger = buildGrainLedger(objects);
|
|
166
|
+
if (ledger.size === 0)
|
|
167
|
+
return [];
|
|
168
|
+
return detectFanoutRisks(analysis, ledger).map((risk) => risk.message);
|
|
169
|
+
}
|
|
170
|
+
catch {
|
|
171
|
+
return [];
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
//# sourceMappingURL=grain-ledger.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"grain-ledger.js","sourceRoot":"","sources":["../../src/metadata/grain-ledger.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AACH,OAAO,EAAE,oBAAoB,EAA6B,MAAM,0BAA0B,CAAC;AAqB3F,0FAA0F;AAC1F,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC;AAE7D,SAAS,qBAAqB,CAAC,QAA4B;IACzD,IAAI,CAAC,QAAQ;QAAE,OAAO,SAAS,CAAC;IAChC,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC;IACpD,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC/D,OAAO,OAAO,IAAI,SAAS,CAAC;AAC9B,CAAC;AAED,SAAS,eAAe,CAAC,MAAc;IACrC,OAAO,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;AAC1D,CAAC;AAED,SAAS,YAAY,CAAC,MAAmB,EAAE,QAA4B,EAAE,MAAc,EAAE,MAAc;IACrG,MAAM,GAAG,GAAG,qBAAqB,CAAC,QAAQ,CAAC,CAAC;IAC5C,IAAI,CAAC,GAAG;QAAE,OAAO;IACjB,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACjC,IAAI,QAAQ,EAAE,CAAC;QACb,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC;QACjD,OAAO;IACT,CAAC;IACD,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,UAAU,EAAE,IAAI,GAAG,CAAC,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;AAC7F,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAAyB;IACxD,MAAM,MAAM,GAAgB,IAAI,GAAG,EAAE,CAAC;IACtC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,MAAM,CAAC,UAAU,KAAK,gBAAgB,EAAE,CAAC;YAC3C,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,EAAE,OAAuC,CAAC;YACxE,MAAM,QAAQ,GAAG,OAAO,EAAE,GAAG,IAAI,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,IAAI,CAAC;YAChE,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,OAAQ,CAAC,MAAwC,CAAC,CAAC,CAAC,EAAE,CAAC;YACrH,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,EAAE,UAAU,KAAK,IAAI,IAAI,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC;YAC3G,gEAAgE;YAChE,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC7B,YAAY,CAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,qBAAqB,CAAC,CAAC;YACrF,CAAC;YACD,+EAA+E;YAC/E,+EAA+E;YAC/E,6EAA6E;YAC7E,mEAAmE;YACnE,KAAK,MAAM,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,aAAa,EAAE,MAAM,CAAC,OAAO,EAAE,YAAY,CAAC,EAAE,CAAC;gBACpF,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;oBAAE,SAAS;gBACtC,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;oBAC1B,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,kBAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;wBACnE,YAAY,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,IAAI,EAAE,EAAE,gCAAgC,CAAC,CAAC;oBAC/E,CAAC;gBACH,CAAC;YACH,CAAC;YACD,SAAS;QACX,CAAC;QACD,6EAA6E;QAC7E,0EAA0E;QAC1E,+DAA+D;QAC/D,MAAM,aAAa,GAAG,MAAM,CAAC,OAAO,EAAE,aAAa,CAAC;QACpD,IAAI,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE,CAAC;YACjC,MAAM,QAAQ,GAAG,oBAAoB,CAAC,MAAM,CAAC,CAAC;YAC9C,KAAK,MAAM,MAAM,IAAI,aAAa,EAAE,CAAC;gBACnC,IAAI,OAAO,MAAM,KAAK,QAAQ;oBAAE,YAAY,CAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,wBAAwB,CAAC,CAAC;YACnG,CAAC;QACH,CAAC;QACD,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,EAAE,UAAU,CAAC;QAC9C,IAAI,OAAO,UAAU,KAAK,QAAQ,EAAE,CAAC;YACnC,YAAY,CAAC,MAAM,EAAE,oBAAoB,CAAC,MAAM,CAAC,EAAE,UAAU,EAAE,sBAAsB,CAAC,CAAC;QACzF,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,oBAAoB,CAAC,MAAsB;IAClD,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC;IAC1C,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,QAAQ,CAAC,IAAI,EAAE;QAAE,OAAO,QAAQ,CAAC;IACrE,OAAO,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,IAAI,CAAC;AACxC,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,iBAAiB,CAAC,QAA8B,EAAE,MAAmB;IACnF,IAAI,CAAC,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAC/D,MAAM,KAAK,GAAiB,EAAE,CAAC;IAC/B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,SAAS,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC;QAC5C,IAAI,SAAS,CAAC,QAAQ;YAAE,SAAS;QACjC,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC;YAAE,SAAS;QACvD,MAAM,kBAAkB,GAAG,qBAAqB,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;QACrE,IAAI,CAAC,kBAAkB;YAAE,SAAS,CAAC,gDAAgD;QACnF,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;YAClC,MAAM,IAAI,GAAG,qBAAqB,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YACtD,MAAM,KAAK,GAAG,qBAAqB,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;YACxD,IAAI,WAA+B,CAAC;YACpC,IAAI,iBAAqC,CAAC;YAC1C,IAAI,IAAI,KAAK,kBAAkB,EAAE,CAAC;gBAChC,WAAW,GAAG,KAAK,CAAC;gBACpB,iBAAiB,GAAG,eAAe,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YACxD,CAAC;iBAAM,IAAI,KAAK,KAAK,kBAAkB,EAAE,CAAC;gBACxC,WAAW,GAAG,IAAI,CAAC;gBACnB,iBAAiB,GAAG,eAAe,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YACvD,CAAC;iBAAM,CAAC;gBACN,SAAS;YACX,CAAC;YACD,IAAI,CAAC,WAAW,IAAI,CAAC,iBAAiB;gBAAE,SAAS;YACjD,MAAM,gBAAgB,GAAG,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;YACjD,IAAI,CAAC,gBAAgB;gBAAE,SAAS,CAAC,oDAAoD;YACrF,IAAI,gBAAgB,CAAC,UAAU,CAAC,GAAG,CAAC,iBAAiB,CAAC;gBAAE,SAAS,CAAC,aAAa;YAC/E,MAAM,cAAc,GAAG,GAAG,SAAS,CAAC,IAAI,IAAI,SAAS,CAAC,MAAM,IAAI,GAAG,GAAG,CAAC;YACvE,MAAM,SAAS,GAAG,GAAG,cAAc,IAAI,kBAAkB,IAAI,WAAW,IAAI,iBAAiB,EAAE,CAAC;YAChG,IAAI,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC;gBAAE,SAAS;YAClC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YACpB,KAAK,CAAC,IAAI,CAAC;gBACT,SAAS,EAAE,cAAc;gBACzB,kBAAkB;gBAClB,cAAc,EAAE,WAAW;gBAC3B,UAAU,EAAE,iBAAiB;gBAC7B,OAAO,EACL,qBAAqB,cAAc,eAAe,kBAAkB,mCAAmC;oBACvG,GAAG,WAAW,OAAO,iBAAiB,yBAAyB,WAAW,4BAA4B;oBACtG,iBAAiB,kBAAkB,uDAAuD;aAC7F,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CAAC,GAAW,EAAE,OAAyB,EAAE,OAAO,GAAG,QAAQ;IAC7F,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,oBAAoB,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACpD,IAAI,CAAC,QAAQ,CAAC,MAAM;YAAE,OAAO,EAAE,CAAC;QAChC,MAAM,MAAM,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;QACzC,IAAI,MAAM,CAAC,IAAI,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QACjC,OAAO,iBAAiB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACzE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC"}
|
|
@@ -28,6 +28,15 @@
|
|
|
28
28
|
import { type EmbeddingProvider } from '../embeddings/provider.js';
|
|
29
29
|
import type { KGNode } from '../kg/types.js';
|
|
30
30
|
import type { SemanticLayer } from '@duckcodeailabs/dql-core';
|
|
31
|
+
/**
|
|
32
|
+
* Build the measure-family map for a specific project by merging the seed
|
|
33
|
+
* synonyms with a family for each distinctive token in the metric pool's names.
|
|
34
|
+
* A metric named `tax_amount` contributes the `tax` family; `avg_discount_rate`
|
|
35
|
+
* contributes `discount`. Generic tokens (`amount`, `rate`, `total`) never become
|
|
36
|
+
* families — they'd match everything. Seed families win when a token already
|
|
37
|
+
* belongs to one (so `sales` stays in `revenue`, not its own family).
|
|
38
|
+
*/
|
|
39
|
+
export declare function buildMeasureFamilies(metrics: KGNode[]): Record<string, string[]>;
|
|
31
40
|
/** The measure families a metric KG node resolves to (revenue, orders, ...). */
|
|
32
41
|
export declare function metricFamilies(metric: KGNode): Set<string>;
|
|
33
42
|
export interface MetricMatch {
|
|
@@ -36,16 +45,27 @@ export interface MetricMatch {
|
|
|
36
45
|
score: number;
|
|
37
46
|
/** The shared measure family that justified the match, if any. */
|
|
38
47
|
family?: string;
|
|
48
|
+
/** Why this candidate was safe to advance into semantic compilation. */
|
|
49
|
+
basis?: 'name' | 'family' | 'description' | 'embedding';
|
|
39
50
|
}
|
|
40
51
|
export interface MatchSemanticMetricOptions {
|
|
41
52
|
/**
|
|
42
53
|
* Confidence threshold. A match below this is treated as "no confident metric"
|
|
43
|
-
* so ad-hoc questions still fall through to generated SQL. Default 0.
|
|
54
|
+
* so ad-hoc questions still fall through to generated SQL. Default 0.3.
|
|
44
55
|
*/
|
|
45
56
|
threshold?: number;
|
|
46
57
|
/** Vector-similarity weight; defaults to a small conservative blend. */
|
|
47
58
|
alpha?: number;
|
|
48
59
|
provider?: EmbeddingProvider;
|
|
60
|
+
/**
|
|
61
|
+
* The planner's isolated MEASURE terms for the question (from
|
|
62
|
+
* `requestedShape.measures` / `metricTerms`). When supplied, the measure/family
|
|
63
|
+
* signal is scored against these instead of every content word in the question,
|
|
64
|
+
* so a group-by dimension ("... by product") can't be mistaken for a measure and
|
|
65
|
+
* pull in a dimension-named metric like `product_count`. Falls back to all
|
|
66
|
+
* content tokens when empty, preserving the old behavior for bare callers.
|
|
67
|
+
*/
|
|
68
|
+
measureTerms?: string[];
|
|
49
69
|
}
|
|
50
70
|
export declare const DEFAULT_METRIC_MATCH_EMBEDDING_ALPHA = 0.18;
|
|
51
71
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"metric-match.d.ts","sourceRoot":"","sources":["../../src/metadata/metric-match.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH,OAAO,
|
|
1
|
+
{"version":3,"file":"metric-match.d.ts","sourceRoot":"","sources":["../../src/metadata/metric-match.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH,OAAO,EAA8D,KAAK,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC/H,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAsD9D;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAehF;AAiFD,gFAAgF;AAChF,wBAAgB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,CAE1D;AAED,MAAM,WAAW,WAAW;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,mCAAmC;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,kEAAkE;IAClE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,wEAAwE;IACxE,KAAK,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,aAAa,GAAG,WAAW,CAAC;CACzD;AAED,MAAM,WAAW,0BAA0B;IACzC;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,wEAAwE;IACxE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,iBAAiB,CAAC;IAC7B;;;;;;;OAOG;IACH,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;CACzB;AAED,eAAO,MAAM,oCAAoC,OAAO,CAAC;AAEzD;;;;;;;;;;;;;GAaG;AACH,wBAAsB,mBAAmB,CACvC,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,MAAM,EAAE,EACjB,OAAO,GAAE,0BAA+B,GACvC,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,CA2H7B;AAOD;;;;;GAKG;AACH,wBAAgB,qBAAqB,CACnC,MAAM,EAAE,MAAM,EACd,aAAa,CAAC,EAAE,aAAa,GAC5B;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GAAG,SAAS,CAa7C;AAkCD;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,MAAM,EAAE,aAAa,CAAC,EAAE,aAAa,GAAG,MAAM,GAAG,SAAS,CAKrG;AAED;;;;;;;;GAQG;AACH,wBAAgB,wBAAwB,CACtC,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,MAAM,EAAE,EACd,aAAa,CAAC,EAAE,aAAa,GAC5B;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAAG,SAAS,CAG7C;AAED,wFAAwF;AACxF,wBAAgB,+BAA+B,CAC7C,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,MAAM,EAAE,EACd,aAAa,CAAC,EAAE,aAAa,GAC5B;IAAE,GAAG,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAAG,SAAS,CAWtE;AAED,MAAM,WAAW,yBAAyB;IACxC,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,EAAE,CAAC;CACtB;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,2BAA2B,CAAC,KAAK,EAAE;IACjD,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,cAAc,EAAE;QACd,UAAU,EAAE,MAAM,EAAE,CAAC;QACrB,QAAQ,EAAE,MAAM,EAAE,CAAC;QACnB,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,IAAI,CAAC,EAAE;YAAE,CAAC,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,SAAS,GAAG,WAAW,CAAA;SAAE,CAAC;QACrD,gBAAgB,CAAC,EAAE,KAAK,GAAG,QAAQ,CAAC;KACrC,CAAC;IACF,YAAY,EAAE,KAAK,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,KAAK,CAAC;YAAE,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC,CAAC;IAC3F,aAAa,CAAC,EAAE,aAAa,CAAC;CAC/B,GAAG,yBAAyB,GAAG,SAAS,CA+CxC"}
|