@memberjunction/db-auto-doc 6.1.4 → 6.2.0-edge.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/DBAutoDocAPI.d.ts +23 -0
- package/dist/api/DBAutoDocAPI.d.ts.map +1 -1
- package/dist/api/DBAutoDocAPI.js +20 -4
- package/dist/api/DBAutoDocAPI.js.map +1 -1
- package/dist/commands/export.js +4 -4
- package/dist/commands/generate-queries.js +11 -11
- package/dist/core/AnalysisEngine.d.ts +37 -0
- package/dist/core/AnalysisEngine.d.ts.map +1 -1
- package/dist/core/AnalysisEngine.js +60 -16
- package/dist/core/AnalysisEngine.js.map +1 -1
- package/dist/core/AnalysisOrchestrator.d.ts +2 -0
- package/dist/core/AnalysisOrchestrator.d.ts.map +1 -1
- package/dist/core/AnalysisOrchestrator.js +33 -29
- package/dist/core/AnalysisOrchestrator.js.map +1 -1
- package/dist/core/BackpropagationEngine.d.ts +7 -0
- package/dist/core/BackpropagationEngine.d.ts.map +1 -1
- package/dist/core/BackpropagationEngine.js +10 -2
- package/dist/core/BackpropagationEngine.js.map +1 -1
- package/dist/core/ConvergenceDetector.d.ts +4 -0
- package/dist/core/ConvergenceDetector.d.ts.map +1 -1
- package/dist/core/ConvergenceDetector.js +10 -2
- package/dist/core/ConvergenceDetector.js.map +1 -1
- package/dist/core/GuardrailsManager.d.ts +16 -0
- package/dist/core/GuardrailsManager.d.ts.map +1 -1
- package/dist/core/GuardrailsManager.js +40 -8
- package/dist/core/GuardrailsManager.js.map +1 -1
- package/dist/database/Database.d.ts +25 -0
- package/dist/database/Database.d.ts.map +1 -1
- package/dist/database/Database.js +46 -10
- package/dist/database/Database.js.map +1 -1
- package/dist/database/TopologicalSorter.d.ts +4 -0
- package/dist/database/TopologicalSorter.d.ts.map +1 -1
- package/dist/database/TopologicalSorter.js +10 -2
- package/dist/database/TopologicalSorter.js.map +1 -1
- package/dist/database/index.d.ts +1 -1
- package/dist/database/index.d.ts.map +1 -1
- package/dist/database/index.js +1 -1
- package/dist/database/index.js.map +1 -1
- package/dist/discovery/BridgeViewSQLGenerator.d.ts +28 -14
- package/dist/discovery/BridgeViewSQLGenerator.d.ts.map +1 -1
- package/dist/discovery/BridgeViewSQLGenerator.js +62 -31
- package/dist/discovery/BridgeViewSQLGenerator.js.map +1 -1
- package/dist/discovery/ColumnClusterer.d.ts +11 -5
- package/dist/discovery/ColumnClusterer.d.ts.map +1 -1
- package/dist/discovery/ColumnClusterer.js +21 -9
- package/dist/discovery/ColumnClusterer.js.map +1 -1
- package/dist/discovery/ColumnNormalizer.d.ts +28 -16
- package/dist/discovery/ColumnNormalizer.d.ts.map +1 -1
- package/dist/discovery/ColumnNormalizer.js +35 -27
- package/dist/discovery/ColumnNormalizer.js.map +1 -1
- package/dist/discovery/ColumnStatsCache.d.ts +37 -0
- package/dist/discovery/ColumnStatsCache.d.ts.map +1 -1
- package/dist/discovery/ColumnStatsCache.js +87 -23
- package/dist/discovery/ColumnStatsCache.js.map +1 -1
- package/dist/discovery/Composer.d.ts +14 -4
- package/dist/discovery/Composer.d.ts.map +1 -1
- package/dist/discovery/Composer.js +28 -20
- package/dist/discovery/Composer.js.map +1 -1
- package/dist/discovery/DiscoveryEngine.d.ts +16 -8
- package/dist/discovery/DiscoveryEngine.d.ts.map +1 -1
- package/dist/discovery/DiscoveryEngine.js +43 -27
- package/dist/discovery/DiscoveryEngine.js.map +1 -1
- package/dist/discovery/DiscoveryTriggerAnalyzer.d.ts +20 -0
- package/dist/discovery/DiscoveryTriggerAnalyzer.d.ts.map +1 -1
- package/dist/discovery/DiscoveryTriggerAnalyzer.js +27 -7
- package/dist/discovery/DiscoveryTriggerAnalyzer.js.map +1 -1
- package/dist/discovery/EmbeddingProvider.d.ts +6 -4
- package/dist/discovery/EmbeddingProvider.d.ts.map +1 -1
- package/dist/discovery/EmbeddingProvider.js +7 -3
- package/dist/discovery/EmbeddingProvider.js.map +1 -1
- package/dist/discovery/EnumCandidateGate.d.ts +15 -11
- package/dist/discovery/EnumCandidateGate.d.ts.map +1 -1
- package/dist/discovery/EnumCandidateGate.js +21 -13
- package/dist/discovery/EnumCandidateGate.js.map +1 -1
- package/dist/discovery/FKDetector.d.ts +4 -0
- package/dist/discovery/FKDetector.d.ts.map +1 -1
- package/dist/discovery/FKDetector.js +10 -2
- package/dist/discovery/FKDetector.js.map +1 -1
- package/dist/discovery/FKGraphWalker.d.ts +27 -18
- package/dist/discovery/FKGraphWalker.d.ts.map +1 -1
- package/dist/discovery/FKGraphWalker.js +30 -26
- package/dist/discovery/FKGraphWalker.js.map +1 -1
- package/dist/discovery/LLMDiscoveryValidator.d.ts +2 -0
- package/dist/discovery/LLMDiscoveryValidator.d.ts.map +1 -1
- package/dist/discovery/LLMDiscoveryValidator.js +28 -24
- package/dist/discovery/LLMDiscoveryValidator.js.map +1 -1
- package/dist/discovery/LLMSanityChecker.d.ts +5 -3
- package/dist/discovery/LLMSanityChecker.d.ts.map +1 -1
- package/dist/discovery/LLMSanityChecker.js +16 -12
- package/dist/discovery/LLMSanityChecker.js.map +1 -1
- package/dist/discovery/OrganicKeyDetector.d.ts +12 -4
- package/dist/discovery/OrganicKeyDetector.d.ts.map +1 -1
- package/dist/discovery/OrganicKeyDetector.js +36 -27
- package/dist/discovery/OrganicKeyDetector.js.map +1 -1
- package/dist/discovery/OrganicKeyTranslator.d.ts +9 -0
- package/dist/discovery/OrganicKeyTranslator.d.ts.map +1 -1
- package/dist/discovery/OrganicKeyTranslator.js +15 -7
- package/dist/discovery/OrganicKeyTranslator.js.map +1 -1
- package/dist/discovery/PKDetector.d.ts +4 -0
- package/dist/discovery/PKDetector.d.ts.map +1 -1
- package/dist/discovery/PKDetector.js +10 -2
- package/dist/discovery/PKDetector.js.map +1 -1
- package/dist/discovery/SemanticPhase.d.ts +3 -1
- package/dist/discovery/SemanticPhase.d.ts.map +1 -1
- package/dist/discovery/SemanticPhase.js +55 -51
- package/dist/discovery/SemanticPhase.js.map +1 -1
- package/dist/discovery/StructuralPhase.d.ts +9 -3
- package/dist/discovery/StructuralPhase.d.ts.map +1 -1
- package/dist/discovery/StructuralPhase.js +13 -6
- package/dist/discovery/StructuralPhase.js.map +1 -1
- package/dist/discovery/TransitiveBridgeDetector.d.ts +21 -15
- package/dist/discovery/TransitiveBridgeDetector.d.ts.map +1 -1
- package/dist/discovery/TransitiveBridgeDetector.js +62 -51
- package/dist/discovery/TransitiveBridgeDetector.js.map +1 -1
- package/dist/drivers/BaseAutoDocDriver.d.ts +20 -8
- package/dist/drivers/BaseAutoDocDriver.d.ts.map +1 -1
- package/dist/drivers/BaseAutoDocDriver.js +10 -2
- package/dist/drivers/BaseAutoDocDriver.js.map +1 -1
- package/dist/generators/AdditionalSchemaInfoGenerator.d.ts +9 -7
- package/dist/generators/AdditionalSchemaInfoGenerator.d.ts.map +1 -1
- package/dist/generators/AdditionalSchemaInfoGenerator.js +13 -9
- package/dist/generators/AdditionalSchemaInfoGenerator.js.map +1 -1
- package/dist/generators/CSVGenerator.d.ts +2 -0
- package/dist/generators/CSVGenerator.d.ts.map +1 -1
- package/dist/generators/CSVGenerator.js +5 -1
- package/dist/generators/CSVGenerator.js.map +1 -1
- package/dist/generators/HTMLGenerator.d.ts +2 -0
- package/dist/generators/HTMLGenerator.d.ts.map +1 -1
- package/dist/generators/HTMLGenerator.js +5 -1
- package/dist/generators/HTMLGenerator.js.map +1 -1
- package/dist/generators/MarkdownGenerator.d.ts +2 -0
- package/dist/generators/MarkdownGenerator.d.ts.map +1 -1
- package/dist/generators/MarkdownGenerator.js +5 -1
- package/dist/generators/MarkdownGenerator.js.map +1 -1
- package/dist/generators/MermaidGenerator.d.ts +4 -0
- package/dist/generators/MermaidGenerator.d.ts.map +1 -1
- package/dist/generators/MermaidGenerator.js +11 -3
- package/dist/generators/MermaidGenerator.js.map +1 -1
- package/dist/generators/ReportGenerator.d.ts +2 -0
- package/dist/generators/ReportGenerator.d.ts.map +1 -1
- package/dist/generators/ReportGenerator.js +5 -1
- package/dist/generators/ReportGenerator.js.map +1 -1
- package/dist/generators/SQLGenerator.d.ts +2 -0
- package/dist/generators/SQLGenerator.d.ts.map +1 -1
- package/dist/generators/SQLGenerator.js +5 -1
- package/dist/generators/SQLGenerator.js.map +1 -1
- package/dist/generators/SampleQueryGenerator.d.ts +2 -0
- package/dist/generators/SampleQueryGenerator.d.ts.map +1 -1
- package/dist/generators/SampleQueryGenerator.js +22 -18
- package/dist/generators/SampleQueryGenerator.js.map +1 -1
- package/dist/prompts/PromptEngine.d.ts +26 -0
- package/dist/prompts/PromptEngine.d.ts.map +1 -1
- package/dist/prompts/PromptEngine.js +22 -6
- package/dist/prompts/PromptEngine.js.map +1 -1
- package/dist/prompts/PromptFileLoader.d.ts +9 -1
- package/dist/prompts/PromptFileLoader.d.ts.map +1 -1
- package/dist/prompts/PromptFileLoader.js +19 -3
- package/dist/prompts/PromptFileLoader.js.map +1 -1
- package/dist/state/IterationTracker.d.ts +31 -0
- package/dist/state/IterationTracker.d.ts.map +1 -1
- package/dist/state/IterationTracker.js +66 -14
- package/dist/state/IterationTracker.js.map +1 -1
- package/dist/state/StateManager.d.ts +35 -0
- package/dist/state/StateManager.d.ts.map +1 -1
- package/dist/state/StateManager.js +66 -14
- package/dist/state/StateManager.js.map +1 -1
- package/dist/state/StateValidator.d.ts +4 -0
- package/dist/state/StateValidator.d.ts.map +1 -1
- package/dist/state/StateValidator.js +12 -4
- package/dist/state/StateValidator.js.map +1 -1
- package/dist/types/discovery.d.ts +14 -14
- package/dist/types/organic-keys.d.ts +16 -12
- package/dist/types/organic-keys.d.ts.map +1 -1
- package/dist/types/organic-keys.js +15 -7
- package/dist/types/organic-keys.js.map +1 -1
- package/dist/types/sample-queries.d.ts +13 -13
- package/dist/utils/config-loader.d.ts +6 -0
- package/dist/utils/config-loader.d.ts.map +1 -1
- package/dist/utils/config-loader.js +15 -3
- package/dist/utils/config-loader.js.map +1 -1
- package/dist/utils/ensureArray.d.ts +2 -0
- package/dist/utils/ensureArray.d.ts.map +1 -1
- package/dist/utils/ensureArray.js +5 -1
- package/dist/utils/ensureArray.js.map +1 -1
- package/dist/utils/json.d.ts +6 -0
- package/dist/utils/json.d.ts.map +1 -1
- package/dist/utils/json.js +20 -8
- package/dist/utils/json.js.map +1 -1
- package/dist/utils/llm-factory.d.ts +6 -0
- package/dist/utils/llm-factory.d.ts.map +1 -1
- package/dist/utils/llm-factory.js +15 -3
- package/dist/utils/llm-factory.js.map +1 -1
- package/package.json +5 -5
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/database/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,kBAAkB,EAAE,YAAY,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/database/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,kBAAkB,EAAE,YAAY,EAAE,WAAW,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC1G,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC"}
|
|
@@ -1,13 +1,20 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* BridgeViewSQLGenerator — Pattern 3 SQL emission.
|
|
3
3
|
*
|
|
4
|
-
* Takes a BridgePath (from FKGraphWalker) and produces the
|
|
5
|
-
*
|
|
6
|
-
*
|
|
4
|
+
* Takes a BridgePath (from FKGraphWalker) and produces the SELECT body of the
|
|
5
|
+
* bridge view that PR #2193's CodeGen ingest (`processOrganicKeyConfig`) will
|
|
6
|
+
* wrap in the platform's create-or-replace DDL and EXECUTE when materializing
|
|
7
|
+
* the organic-key configuration.
|
|
7
8
|
*
|
|
8
|
-
*
|
|
9
|
+
* CodeGen executes the body verbatim, so it is emitted in the analyzed
|
|
10
|
+
* database's dialect: identifiers are quoted per `provider` (brackets on SQL
|
|
11
|
+
* Server, double quotes on PostgreSQL/Oracle, backticks on MySQL). On
|
|
12
|
+
* PostgreSQL that quoting is load-bearing — an unquoted mixed-case name folds
|
|
13
|
+
* to lower case and no longer matches the catalog.
|
|
14
|
+
*
|
|
15
|
+
* Output SQL shape (SQL Server shown) — minimal valid bridge view per the
|
|
16
|
+
* PR #2193 docs example; the view header is CodeGen's, not part of the body:
|
|
9
17
|
*
|
|
10
|
-
* CREATE OR ALTER VIEW [schema].[viewName] AS
|
|
11
18
|
* SELECT
|
|
12
19
|
* [hub].[hubKeyField] AS [hubKeyField],
|
|
13
20
|
* [spoke].[spokePK] AS [spokeOutputAlias]
|
|
@@ -31,22 +38,25 @@
|
|
|
31
38
|
* walker doesn't know it.
|
|
32
39
|
*/
|
|
33
40
|
import { BridgePath } from './FKGraphWalker.js';
|
|
41
|
+
import { DatabaseConfig } from '../types/config.js';
|
|
42
|
+
/** Database platform the bridge view body is written for. */
|
|
43
|
+
export type BridgeViewProvider = NonNullable<DatabaseConfig['provider']>;
|
|
34
44
|
/** Bundle returned for each bridge path. */
|
|
35
45
|
export interface GeneratedBridgeView {
|
|
36
46
|
/** View name suitable for the PR #2193 TransitiveView.Name field. */
|
|
37
|
-
|
|
47
|
+
ViewName: string;
|
|
38
48
|
/** Default schema (caller may override). */
|
|
39
49
|
schemaName: string;
|
|
40
50
|
/** The bare SELECT body for the bridge view — exactly what TransitiveView.SQL should carry.
|
|
41
|
-
*
|
|
42
|
-
*
|
|
43
|
-
|
|
51
|
+
* CodeGen wraps it in the platform's create-or-replace view DDL, so the header must NOT be
|
|
52
|
+
* included here. Written in the dialect of the generator's `provider`. */
|
|
53
|
+
Sql: string;
|
|
44
54
|
/** The hub-key field projected by the view (matches TransitiveMatchFieldNames[0]). */
|
|
45
|
-
|
|
55
|
+
HubKeyField: string;
|
|
46
56
|
/** The output column projected for the spoke (matches TransitiveOutputFieldName). */
|
|
47
|
-
|
|
57
|
+
SpokeOutputField: string;
|
|
48
58
|
/** The spoke PK column the form-panel join joins ON (matches RelatedEntityJoinFieldName). */
|
|
49
|
-
|
|
59
|
+
SpokeJoinField: string;
|
|
50
60
|
}
|
|
51
61
|
export interface BridgeViewSQLGeneratorOptions {
|
|
52
62
|
/**
|
|
@@ -59,9 +69,13 @@ export interface BridgeViewSQLGeneratorOptions {
|
|
|
59
69
|
* The generator lowercases the result and replaces non-alphanumeric chars
|
|
60
70
|
* with underscores, since the view must be a valid SQL identifier.
|
|
61
71
|
*/
|
|
62
|
-
|
|
72
|
+
ViewNamePattern?: string;
|
|
63
73
|
/** Override the view's schema. Defaults to the SPOKE's schema (where it'll be most natural to find). */
|
|
64
|
-
|
|
74
|
+
ViewSchema?: string;
|
|
75
|
+
/** Platform whose identifier quoting the body uses. Default `'sqlserver'`, matching `DatabaseConfig.provider`. */
|
|
76
|
+
provider?: BridgeViewProvider;
|
|
65
77
|
}
|
|
78
|
+
export declare function GenerateBridgeView(path: BridgePath, spokePKColumn: string, opts?: BridgeViewSQLGeneratorOptions): GeneratedBridgeView;
|
|
79
|
+
/** @deprecated Use {@link GenerateBridgeView}. */
|
|
66
80
|
export declare function generateBridgeView(path: BridgePath, spokePKColumn: string, opts?: BridgeViewSQLGeneratorOptions): GeneratedBridgeView;
|
|
67
81
|
//# sourceMappingURL=BridgeViewSQLGenerator.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"BridgeViewSQLGenerator.d.ts","sourceRoot":"","sources":["../../src/discovery/BridgeViewSQLGenerator.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"BridgeViewSQLGenerator.d.ts","sourceRoot":"","sources":["../../src/discovery/BridgeViewSQLGenerator.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAChD,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAEpD,6DAA6D;AAC7D,MAAM,MAAM,kBAAkB,GAAG,WAAW,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC,CAAC;AAEzE,4CAA4C;AAC5C,MAAM,WAAW,mBAAmB;IAChC,qEAAqE;IACrE,QAAQ,EAAE,MAAM,CAAC;IACjB,4CAA4C;IAC5C,UAAU,EAAE,MAAM,CAAC;IACnB;;+EAE2E;IAC3E,GAAG,EAAE,MAAM,CAAC;IACZ,sFAAsF;IACtF,WAAW,EAAE,MAAM,CAAC;IACpB,qFAAqF;IACrF,gBAAgB,EAAE,MAAM,CAAC;IACzB,6FAA6F;IAC7F,cAAc,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,6BAA6B;IAC1C;;;;;;;;;OASG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,wGAAwG;IACxG,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,kHAAkH;IAClH,QAAQ,CAAC,EAAE,kBAAkB,CAAC;CACjC;AAQD,wBAAgB,kBAAkB,CAC9B,IAAI,EAAE,UAAU,EAChB,aAAa,EAAE,MAAM,EACrB,IAAI,GAAE,6BAAkC,GACzC,mBAAmB,CA0DrB;AAED,kDAAkD;AAClD,wBAAgB,kBAAkB,CAC9B,IAAI,EAAE,UAAU,EAChB,aAAa,EAAE,MAAM,EACrB,IAAI,GAAE,6BAAkC,GACzC,mBAAmB,CAErB"}
|
|
@@ -1,13 +1,20 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* BridgeViewSQLGenerator — Pattern 3 SQL emission.
|
|
3
3
|
*
|
|
4
|
-
* Takes a BridgePath (from FKGraphWalker) and produces the
|
|
5
|
-
*
|
|
6
|
-
*
|
|
4
|
+
* Takes a BridgePath (from FKGraphWalker) and produces the SELECT body of the
|
|
5
|
+
* bridge view that PR #2193's CodeGen ingest (`processOrganicKeyConfig`) will
|
|
6
|
+
* wrap in the platform's create-or-replace DDL and EXECUTE when materializing
|
|
7
|
+
* the organic-key configuration.
|
|
7
8
|
*
|
|
8
|
-
*
|
|
9
|
+
* CodeGen executes the body verbatim, so it is emitted in the analyzed
|
|
10
|
+
* database's dialect: identifiers are quoted per `provider` (brackets on SQL
|
|
11
|
+
* Server, double quotes on PostgreSQL/Oracle, backticks on MySQL). On
|
|
12
|
+
* PostgreSQL that quoting is load-bearing — an unquoted mixed-case name folds
|
|
13
|
+
* to lower case and no longer matches the catalog.
|
|
14
|
+
*
|
|
15
|
+
* Output SQL shape (SQL Server shown) — minimal valid bridge view per the
|
|
16
|
+
* PR #2193 docs example; the view header is CodeGen's, not part of the body:
|
|
9
17
|
*
|
|
10
|
-
* CREATE OR ALTER VIEW [schema].[viewName] AS
|
|
11
18
|
* SELECT
|
|
12
19
|
* [hub].[hubKeyField] AS [hubKeyField],
|
|
13
20
|
* [spoke].[spokePK] AS [spokeOutputAlias]
|
|
@@ -31,43 +38,47 @@
|
|
|
31
38
|
* walker doesn't know it.
|
|
32
39
|
*/
|
|
33
40
|
const DEFAULTS = {
|
|
34
|
-
|
|
35
|
-
|
|
41
|
+
ViewNamePattern: 'vw{spoke}_{key}_bridge',
|
|
42
|
+
ViewSchema: '',
|
|
43
|
+
provider: 'sqlserver',
|
|
36
44
|
};
|
|
37
|
-
export function
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
const
|
|
41
|
-
const
|
|
45
|
+
export function GenerateBridgeView(path, spokePKColumn, opts = {}) {
|
|
46
|
+
// `?? DEFAULTS.provider`: callers thread an optional provider through as `{ provider }`, and a
|
|
47
|
+
// spread of an explicit `undefined` would overwrite the default.
|
|
48
|
+
const o = { ...DEFAULTS, ...opts, provider: opts.provider ?? DEFAULTS.provider };
|
|
49
|
+
const qident = identifierQuoter(o.provider);
|
|
50
|
+
const viewName = buildViewName(o.ViewNamePattern, path);
|
|
51
|
+
const schemaName = o.ViewSchema || path.SpokeSchema;
|
|
52
|
+
const spokeOutputField = `${path.SpokeTable}_${spokePKColumn}`;
|
|
42
53
|
// ─── Build alias map ────────────────────────────────────────────────────
|
|
43
54
|
// hop[0].fromTable = spoke (alias "spoke")
|
|
44
55
|
// hop[i].toTable for 0<i<len-1 = intermediate (alias t1, t2, ...)
|
|
45
56
|
// hop[last].toTable = hub (alias "hub")
|
|
46
57
|
const aliases = [];
|
|
47
58
|
aliases.push('spoke'); // index 0 = spoke
|
|
48
|
-
for (let i = 1; i < path.
|
|
59
|
+
for (let i = 1; i < path.Hops.length; i++) {
|
|
49
60
|
aliases.push(`t${i}`);
|
|
50
61
|
}
|
|
51
62
|
aliases.push('hub'); // final = hub
|
|
52
63
|
// ─── JOIN clauses ───────────────────────────────────────────────────────
|
|
53
|
-
const fromClause = `FROM ${qident(path.
|
|
64
|
+
const fromClause = `FROM ${qident(path.SpokeSchema)}.${qident(path.SpokeTable)} ${aliases[0]}`;
|
|
54
65
|
const joinClauses = [];
|
|
55
|
-
for (let i = 0; i < path.
|
|
56
|
-
const hop = path.
|
|
66
|
+
for (let i = 0; i < path.Hops.length; i++) {
|
|
67
|
+
const hop = path.Hops[i];
|
|
57
68
|
const fromAlias = aliases[i];
|
|
58
69
|
const toAlias = aliases[i + 1];
|
|
59
70
|
joinClauses.push(`INNER JOIN ${qident(hop.toSchema)}.${qident(hop.toTable)} ${toAlias} ON ${fromAlias}.${qident(hop.fromColumn)} = ${toAlias}.${qident(hop.toColumn)}`);
|
|
60
71
|
}
|
|
61
72
|
// ─── SELECT list ────────────────────────────────────────────────────────
|
|
62
73
|
const selectList = [
|
|
63
|
-
`hub.${qident(path.
|
|
74
|
+
`hub.${qident(path.HubKeyField)} AS ${qident(path.HubKeyField)}`,
|
|
64
75
|
`spoke.${qident(spokePKColumn)} AS ${qident(spokeOutputField)}`,
|
|
65
76
|
];
|
|
66
77
|
// ─── Final SQL — body only ──────────────────────────────────────────────
|
|
67
|
-
//
|
|
68
|
-
//
|
|
69
|
-
//
|
|
70
|
-
//
|
|
78
|
+
// CodeGen's processOrganicKeyConfig() wraps whatever's in TransitiveView.SQL
|
|
79
|
+
// in the platform's create-or-replace view DDL. So we emit ONLY the SELECT
|
|
80
|
+
// body — never a CREATE VIEW header — otherwise the database rejects the
|
|
81
|
+
// double-prefixed statement.
|
|
71
82
|
const sql = [
|
|
72
83
|
`SELECT`,
|
|
73
84
|
' ' + selectList.join(',\n '),
|
|
@@ -75,25 +86,45 @@ export function generateBridgeView(path, spokePKColumn, opts = {}) {
|
|
|
75
86
|
...joinClauses,
|
|
76
87
|
].join('\n');
|
|
77
88
|
return {
|
|
78
|
-
viewName,
|
|
89
|
+
ViewName: viewName,
|
|
79
90
|
schemaName,
|
|
80
|
-
sql,
|
|
81
|
-
|
|
82
|
-
spokeOutputField,
|
|
83
|
-
|
|
91
|
+
Sql: sql,
|
|
92
|
+
HubKeyField: path.HubKeyField,
|
|
93
|
+
SpokeOutputField: spokeOutputField,
|
|
94
|
+
SpokeJoinField: spokePKColumn,
|
|
84
95
|
};
|
|
85
96
|
}
|
|
97
|
+
/** @deprecated Use {@link GenerateBridgeView}. */
|
|
98
|
+
export function generateBridgeView(path, spokePKColumn, opts = {}) {
|
|
99
|
+
return GenerateBridgeView(path, spokePKColumn, opts);
|
|
100
|
+
}
|
|
86
101
|
// ─── Helpers ────────────────────────────────────────────────────────────────
|
|
87
102
|
function buildViewName(pattern, path) {
|
|
103
|
+
// Function replacements: a string replacement expands `$&`, `$\`` and `$1` in the DATA, so a
|
|
104
|
+
// table name containing `$` would splice the pattern's own text into the view name.
|
|
88
105
|
const raw = pattern
|
|
89
|
-
.replace(/\{hub\}/g, path.
|
|
90
|
-
.replace(/\{spoke\}/g, path.
|
|
91
|
-
.replace(/\{key\}/g, path.
|
|
106
|
+
.replace(/\{hub\}/g, () => path.HubTable)
|
|
107
|
+
.replace(/\{spoke\}/g, () => path.SpokeTable)
|
|
108
|
+
.replace(/\{key\}/g, () => path.HubKeyField);
|
|
92
109
|
// Sanitize: SQL identifier — alphanumeric + underscore only, max 128 chars.
|
|
93
110
|
const cleaned = raw.replace(/[^A-Za-z0-9_]/g, '_').slice(0, 128);
|
|
94
111
|
return cleaned;
|
|
95
112
|
}
|
|
96
|
-
|
|
97
|
-
|
|
113
|
+
/**
|
|
114
|
+
* Returns the platform's identifier quoter, doubling the closing delimiter inside a name.
|
|
115
|
+
* Exhaustive on purpose (no `default`): a provider added to `DatabaseConfig['provider']` fails to
|
|
116
|
+
* compile here instead of silently getting SQL Server brackets, and the caller's default is the only
|
|
117
|
+
* place a missing provider becomes `'sqlserver'`.
|
|
118
|
+
*/
|
|
119
|
+
function identifierQuoter(provider) {
|
|
120
|
+
switch (provider) {
|
|
121
|
+
case 'postgresql':
|
|
122
|
+
case 'oracle':
|
|
123
|
+
return (name) => `"${name.replace(/"/g, '""')}"`;
|
|
124
|
+
case 'mysql':
|
|
125
|
+
return (name) => `\`${name.replace(/`/g, '``')}\``;
|
|
126
|
+
case 'sqlserver':
|
|
127
|
+
return (name) => `[${name.replace(/]/g, ']]')}]`;
|
|
128
|
+
}
|
|
98
129
|
}
|
|
99
130
|
//# sourceMappingURL=BridgeViewSQLGenerator.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"BridgeViewSQLGenerator.js","sourceRoot":"","sources":["../../src/discovery/BridgeViewSQLGenerator.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"BridgeViewSQLGenerator.js","sourceRoot":"","sources":["../../src/discovery/BridgeViewSQLGenerator.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AA4CH,MAAM,QAAQ,GAA4C;IACtD,eAAe,EAAE,wBAAwB;IACzC,UAAU,EAAE,EAAE;IACd,QAAQ,EAAE,WAAW;CACxB,CAAC;AAEF,MAAM,UAAU,kBAAkB,CAC9B,IAAgB,EAChB,aAAqB,EACrB,OAAsC,EAAE;IAExC,+FAA+F;IAC/F,iEAAiE;IACjE,MAAM,CAAC,GAAG,EAAE,GAAG,QAAQ,EAAE,GAAG,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,IAAI,QAAQ,CAAC,QAAQ,EAAE,CAAC;IACjF,MAAM,MAAM,GAAG,gBAAgB,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IAC5C,MAAM,QAAQ,GAAG,aAAa,CAAC,CAAC,CAAC,eAAe,EAAE,IAAI,CAAC,CAAC;IACxD,MAAM,UAAU,GAAG,CAAC,CAAC,UAAU,IAAI,IAAI,CAAC,WAAW,CAAC;IACpD,MAAM,gBAAgB,GAAG,GAAG,IAAI,CAAC,UAAU,IAAI,aAAa,EAAE,CAAC;IAE/D,2EAA2E;IAC3E,6CAA6C;IAC7C,kEAAkE;IAClE,2CAA2C;IAC3C,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,kBAAkB;IACzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC1B,CAAC;IACD,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,cAAc;IAEnC,2EAA2E;IAC3E,MAAM,UAAU,GAAG,QAAQ,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;IAC/F,MAAM,WAAW,GAAa,EAAE,CAAC;IACjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACzB,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAC7B,MAAM,OAAO,GAAG,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC/B,WAAW,CAAC,IAAI,CACZ,cAAc,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,OAAO,OAAO,SAAS,IAAI,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CACxJ,CAAC;IACN,CAAC;IAED,2EAA2E;IAC3E,MAAM,UAAU,GAAG;QACf,OAAO,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE;QAChE,SAAS,MAAM,CAAC,aAAa,CAAC,OAAO,MAAM,CAAC,gBAAgB,CAAC,EAAE;KAClE,CAAC;IAEF,2EAA2E;IAC3E,6EAA6E;IAC7E,2EAA2E;IAC3E,yEAAyE;IACzE,6BAA6B;IAC7B,MAAM,GAAG,GAAG;QACR,QAAQ;QACR,MAAM,GAAG,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC;QACnC,UAAU;QACV,GAAG,WAAW;KACjB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAEb,OAAO;QACH,QAAQ,EAAE,QAAQ;QAClB,UAAU;QACV,GAAG,EAAE,GAAG;QACR,WAAW,EAAE,IAAI,CAAC,WAAW;QAC7B,gBAAgB,EAAE,gBAAgB;QAClC,cAAc,EAAE,aAAa;KAChC,CAAC;AACN,CAAC;AAED,kDAAkD;AAClD,MAAM,UAAU,kBAAkB,CAC9B,IAAgB,EAChB,aAAqB,EACrB,OAAsC,EAAE;IAExC,OAAO,kBAAkB,CAAC,IAAI,EAAE,aAAa,EAAE,IAAI,CAAC,CAAC;AACzD,CAAC;AAED,+EAA+E;AAE/E,SAAS,aAAa,CAAC,OAAe,EAAE,IAAgB;IACpD,6FAA6F;IAC7F,oFAAoF;IACpF,MAAM,GAAG,GAAG,OAAO;SACd,OAAO,CAAC,UAAU,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC;SACxC,OAAO,CAAC,YAAY,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC;SAC5C,OAAO,CAAC,UAAU,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACjD,4EAA4E;IAC5E,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC,gBAAgB,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACjE,OAAO,OAAO,CAAC;AACnB,CAAC;AAED;;;;;GAKG;AACH,SAAS,gBAAgB,CAAC,QAA4B;IAClD,QAAQ,QAAQ,EAAE,CAAC;QACf,KAAK,YAAY,CAAC;QAClB,KAAK,QAAQ;YACT,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC;QACrD,KAAK,OAAO;YACR,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC;QACvD,KAAK,WAAW;YACZ,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC;IACzD,CAAC;AACL,CAAC"}
|
|
@@ -46,18 +46,24 @@ export interface ClustererOptions {
|
|
|
46
46
|
* When omitted, the clusterer auto-calibrates from the actual distance
|
|
47
47
|
* distribution (threshold = pN of pairwise distances, default p5).
|
|
48
48
|
*/
|
|
49
|
-
|
|
49
|
+
MergeThreshold?: number;
|
|
50
50
|
/** Percentile (0–100) of the pairwise distance distribution for auto-calibration. Default 5. */
|
|
51
|
-
|
|
51
|
+
MergeThresholdPercentile?: number;
|
|
52
52
|
/** Minimum cluster size to report. */
|
|
53
|
-
|
|
53
|
+
MinClusterSize: number;
|
|
54
54
|
/** Minimum distinct tables a cluster must span. */
|
|
55
|
-
|
|
55
|
+
MinDistinctTables: number;
|
|
56
56
|
}
|
|
57
57
|
export declare class ColumnClusterer {
|
|
58
58
|
private readonly opts;
|
|
59
|
-
|
|
59
|
+
LastResolvedThreshold: number;
|
|
60
|
+
/** @deprecated Use {@link LastResolvedThreshold}. */
|
|
61
|
+
get lastResolvedThreshold(): number;
|
|
62
|
+
/** @deprecated Use {@link LastResolvedThreshold}. */
|
|
63
|
+
set lastResolvedThreshold(value: number);
|
|
60
64
|
constructor(opts: ClustererOptions);
|
|
65
|
+
Cluster(columns: ClustererInputColumn[]): RawCluster[];
|
|
66
|
+
/** @deprecated Use {@link Cluster}. */
|
|
61
67
|
cluster(columns: ClustererInputColumn[]): RawCluster[];
|
|
62
68
|
}
|
|
63
69
|
//# sourceMappingURL=ColumnClusterer.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ColumnClusterer.d.ts","sourceRoot":"","sources":["../../src/discovery/ColumnClusterer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,uBAAuB,EAAE,MAAM,0BAA0B,CAAC;AAEnE,8EAA8E;AAC9E,MAAM,WAAW,oBAAoB;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,YAAY,CAAC;IACxB,gBAAgB,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;IACpE,YAAY,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED,uFAAuF;AACvF,MAAM,WAAW,UAAU;IACvB,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,OAAO,EAAE,uBAAuB,EAAE,CAAC;IACnC,oFAAoF;IACpF,gBAAgB,EAAE,MAAM,CAAC;CAC5B;AAED,MAAM,WAAW,gBAAgB;IAC7B;;;;OAIG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,gGAAgG;IAChG,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,sCAAsC;IACtC,cAAc,EAAE,MAAM,CAAC;IACvB,mDAAmD;IACnD,iBAAiB,EAAE,MAAM,CAAC;CAC7B;AAED,qBAAa,eAAe;
|
|
1
|
+
{"version":3,"file":"ColumnClusterer.d.ts","sourceRoot":"","sources":["../../src/discovery/ColumnClusterer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,uBAAuB,EAAE,MAAM,0BAA0B,CAAC;AAEnE,8EAA8E;AAC9E,MAAM,WAAW,oBAAoB;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,YAAY,CAAC;IACxB,gBAAgB,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;IACpE,YAAY,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED,uFAAuF;AACvF,MAAM,WAAW,UAAU;IACvB,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,OAAO,EAAE,uBAAuB,EAAE,CAAC;IACnC,oFAAoF;IACpF,gBAAgB,EAAE,MAAM,CAAC;CAC5B;AAED,MAAM,WAAW,gBAAgB;IAC7B;;;;OAIG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,gGAAgG;IAChG,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,sCAAsC;IACtC,cAAc,EAAE,MAAM,CAAC;IACvB,mDAAmD;IACnD,iBAAiB,EAAE,MAAM,CAAC;CAC7B;AAED,qBAAa,eAAe;IAYZ,OAAO,CAAC,QAAQ,CAAC,IAAI;IAX1B,qBAAqB,SAAK;IAEjC,qDAAqD;IACrD,IAAW,qBAAqB,WAE/B;IACD,qDAAqD;IACrD,IAAW,qBAAqB,CAAC,KAAK,QAAA,EAErC;gBAE4B,IAAI,EAAE,gBAAgB;IAE5C,OAAO,CAAC,OAAO,EAAE,oBAAoB,EAAE,GAAG,UAAU,EAAE;IAqF7D,uCAAuC;IAChC,OAAO,CAAC,OAAO,EAAE,oBAAoB,EAAE,GAAG,UAAU,EAAE;CAGhE"}
|
|
@@ -19,20 +19,28 @@
|
|
|
19
19
|
* which matches the actual "cluster = one concept" question.
|
|
20
20
|
*/
|
|
21
21
|
export class ColumnClusterer {
|
|
22
|
+
/** @deprecated Use {@link LastResolvedThreshold}. */
|
|
23
|
+
get lastResolvedThreshold() {
|
|
24
|
+
return this.LastResolvedThreshold;
|
|
25
|
+
}
|
|
26
|
+
/** @deprecated Use {@link LastResolvedThreshold}. */
|
|
27
|
+
set lastResolvedThreshold(value) {
|
|
28
|
+
this.LastResolvedThreshold = value;
|
|
29
|
+
}
|
|
22
30
|
constructor(opts) {
|
|
23
31
|
this.opts = opts;
|
|
24
|
-
this.
|
|
32
|
+
this.LastResolvedThreshold = 0;
|
|
25
33
|
}
|
|
26
|
-
|
|
34
|
+
Cluster(columns) {
|
|
27
35
|
const n = columns.length;
|
|
28
|
-
if (n < this.opts.
|
|
36
|
+
if (n < this.opts.MinClusterSize)
|
|
29
37
|
return [];
|
|
30
38
|
// Step 1 — pairwise cosine-distance matrix.
|
|
31
39
|
const distance = computePairwiseDistance(columns);
|
|
32
|
-
const resolvedThreshold = this.opts.
|
|
33
|
-
? this.opts.
|
|
34
|
-
: computeThresholdFromDistribution(distance, this.opts.
|
|
35
|
-
this.
|
|
40
|
+
const resolvedThreshold = this.opts.MergeThreshold !== undefined
|
|
41
|
+
? this.opts.MergeThreshold
|
|
42
|
+
: computeThresholdFromDistribution(distance, this.opts.MergeThresholdPercentile ?? 5);
|
|
43
|
+
this.LastResolvedThreshold = resolvedThreshold;
|
|
36
44
|
// Step 2 — agglomerative merge with average-linkage.
|
|
37
45
|
const clusters = new Map();
|
|
38
46
|
for (let i = 0; i < n; i++)
|
|
@@ -85,10 +93,10 @@ export class ColumnClusterer {
|
|
|
85
93
|
// Step 3 — filter + emit.
|
|
86
94
|
const out = [];
|
|
87
95
|
for (const memberIdxs of clusters.values()) {
|
|
88
|
-
if (memberIdxs.length < this.opts.
|
|
96
|
+
if (memberIdxs.length < this.opts.MinClusterSize)
|
|
89
97
|
continue;
|
|
90
98
|
const tableSet = new Set(memberIdxs.map((i) => `${columns[i].schema}.${columns[i].table}`));
|
|
91
|
-
if (tableSet.size < this.opts.
|
|
99
|
+
if (tableSet.size < this.opts.MinDistinctTables)
|
|
92
100
|
continue;
|
|
93
101
|
const members = memberIdxs.map((i) => ({
|
|
94
102
|
schema: columns[i].schema,
|
|
@@ -107,6 +115,10 @@ export class ColumnClusterer {
|
|
|
107
115
|
out.sort((a, b) => b.members.length - a.members.length);
|
|
108
116
|
return out;
|
|
109
117
|
}
|
|
118
|
+
/** @deprecated Use {@link Cluster}. */
|
|
119
|
+
cluster(columns) {
|
|
120
|
+
return this.Cluster(columns);
|
|
121
|
+
}
|
|
110
122
|
}
|
|
111
123
|
// ─── Helpers ────────────────────────────────────────────────────────────────
|
|
112
124
|
function computePairwiseDistance(columns) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ColumnClusterer.js","sourceRoot":"","sources":["../../src/discovery/ColumnClusterer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAsCH,MAAM,OAAO,eAAe;IAGxB,YAA6B,IAAsB;QAAtB,SAAI,GAAJ,IAAI,CAAkB;
|
|
1
|
+
{"version":3,"file":"ColumnClusterer.js","sourceRoot":"","sources":["../../src/discovery/ColumnClusterer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAsCH,MAAM,OAAO,eAAe;IAGxB,qDAAqD;IACrD,IAAW,qBAAqB;QAC5B,OAAO,IAAI,CAAC,qBAAqB,CAAC;IACtC,CAAC;IACD,qDAAqD;IACrD,IAAW,qBAAqB,CAAC,KAAK;QAClC,IAAI,CAAC,qBAAqB,GAAG,KAAK,CAAC;IACvC,CAAC;IAED,YAA6B,IAAsB;QAAtB,SAAI,GAAJ,IAAI,CAAkB;QAX5C,0BAAqB,GAAG,CAAC,CAAC;IAWqB,CAAC;IAEhD,OAAO,CAAC,OAA+B;QAC1C,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;QACzB,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc;YAAE,OAAO,EAAE,CAAC;QAE5C,4CAA4C;QAC5C,MAAM,QAAQ,GAAG,uBAAuB,CAAC,OAAO,CAAC,CAAC;QAClD,MAAM,iBAAiB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,KAAK,SAAS;YAC5D,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc;YAC1B,CAAC,CAAC,gCAAgC,CAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,wBAAwB,IAAI,CAAC,CAAC,CAAC;QAC1F,IAAI,CAAC,qBAAqB,GAAG,iBAAiB,CAAC;QAE/C,qDAAqD;QACrD,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAoB,CAAC;QAC7C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE;YAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAEjD,MAAM,OAAO,GAAG,IAAI,GAAG,EAA+B,CAAC;QACvD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YACzB,MAAM,GAAG,GAAG,IAAI,GAAG,EAAkB,CAAC;YACtC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBACzB,IAAI,CAAC,KAAK,CAAC;oBAAE,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YAC3D,CAAC;YACD,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QACxB,CAAC;QAED,OAAO,QAAQ,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YACvB,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC;YACf,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC;YACf,IAAI,KAAK,GAAG,MAAM,CAAC,iBAAiB,CAAC;YACrC,KAAK,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,OAAO,EAAE,CAAC;gBAC9B,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;oBACxB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,EAAE,CAAC;wBACrB,KAAK,GAAG,CAAC,CAAC;wBACV,KAAK,GAAG,CAAC,CAAC;wBACV,KAAK,GAAG,CAAC,CAAC;oBACd,CAAC;gBACL,CAAC;YACL,CAAC;YACD,IAAI,KAAK,GAAG,iBAAiB,IAAI,KAAK,GAAG,CAAC;gBAAE,MAAM;YAElD,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAE,CAAC;YACtC,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAE,CAAC;YACtC,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC;YAC9B,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC;YAC9B,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAE,CAAC;YACjC,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAE,CAAC;YACjC,KAAK,MAAM,CAAC,IAAI,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;gBAC9B,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK,KAAK;oBAAE,SAAS;gBACzC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC;gBACzB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC;gBACzB,MAAM,IAAI,GAAG,CAAC,KAAK,GAAG,GAAG,GAAG,KAAK,GAAG,GAAG,CAAC,GAAG,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC;gBAC3D,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;gBAClB,OAAO,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;gBACjC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAClC,CAAC;YACD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACnB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAEtB,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;YAC/C,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC3B,CAAC;QAED,0BAA0B;QAC1B,MAAM,GAAG,GAAiB,EAAE,CAAC;QAC7B,KAAK,MAAM,UAAU,IAAI,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;YACzC,IAAI,UAAU,CAAC,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc;gBAAE,SAAS;YAC3D,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;YAC5F,IAAI,QAAQ,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,iBAAiB;gBAAE,SAAS;YAC1D,MAAM,OAAO,GAA8B,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC9D,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM;gBACzB,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK;gBACvB,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM;gBACzB,gBAAgB,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,gBAAgB;gBAC/C,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,IAAI,IAAI;gBACrC,YAAY,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,YAAY;aAC1C,CAAC,CAAC,CAAC;YACJ,GAAG,CAAC,IAAI,CAAC;gBACL,aAAa,EAAE,UAAU;gBACzB,OAAO;gBACP,gBAAgB,EAAE,gBAAgB,CAAC,UAAU,EAAE,QAAQ,EAAE,CAAC,CAAC;aAC9D,CAAC,CAAC;QACP,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACxD,OAAO,GAAG,CAAC;IACf,CAAC;IAED,uCAAuC;IAChC,OAAO,CAAC,OAA+B;QAC1C,OAAO,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IACjC,CAAC;CACJ;AAED,+EAA+E;AAE/E,SAAS,uBAAuB,CAAC,OAA+B;IAC5D,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;IACzB,MAAM,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAC9B,MAAM,GAAG,GAAG,IAAI,YAAY,CAAC,GAAG,CAAC,CAAC;IAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACzB,MAAM,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAChC,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7B,MAAM,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YAChC,IAAI,CAAC,GAAG,CAAC,CAAC;YACV,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;YAC3C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE;gBAAE,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;YACjD,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAChB,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5C,CAAC;IACL,CAAC;IACD,OAAO,GAAG,CAAC;AACf,CAAC;AAED,SAAS,SAAS,CAAC,CAAS,EAAE,CAAS,EAAE,CAAS;IAC9C,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;AACnD,CAAC;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,SAAS,gCAAgC,CAAC,SAAuB,EAAE,UAAkB;IACjF,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACrC,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC3D,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;IAExB,2EAA2E;IAC3E,2EAA2E;IAC3E,qEAAqE;IACrE,EAAE;IACF,yEAAyE;IACzE,4EAA4E;IAC5E,yEAAyE;IACzE,qCAAqC;IACrC,MAAM,aAAa,GAAG,IAAI,CAAC;IAC3B,MAAM,kBAAkB,GAAG,IAAI,CAAC;IAChC,MAAM,KAAK,GAAG,IAAI,CAAC;IAEnB,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7B,IAAI,MAAM,CAAC,CAAC,CAAC,GAAG,aAAa;YAAE,MAAM;QACrC,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QACtC,IAAI,GAAG,GAAG,OAAO,EAAE,CAAC;YAChB,OAAO,GAAG,GAAG,CAAC;YACd,YAAY,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAC7B,CAAC;IACL,CAAC;IAED,IAAI,OAAO,IAAI,kBAAkB,EAAE,CAAC;QAChC,OAAO,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,GAAG,GAAG,CAAC,CAAC;IACzD,CAAC;IAED,+EAA+E;IAC/E,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC,CAAC;IACjD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC;IACtC,OAAO,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;AACzD,CAAC;AAED,SAAS,UAAU,CAAC,MAAoB,EAAE,CAAS,EAAE,CAAS,EAAE,CAAS;IACrE,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACtB,IAAI,CAAC,GAAG,CAAC;QAAE,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC3B,OAAO,MAAM,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;AACtC,CAAC;AAED,SAAS,gBAAgB,CAAC,UAAoB,EAAE,MAAoB,EAAE,CAAS;IAC3E,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACzC,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7C,MAAM,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;YAC9D,IAAI,CAAC,GAAG,GAAG;gBAAE,GAAG,GAAG,CAAC,CAAC;QACzB,CAAC;IACL,CAAC;IACD,OAAO,GAAG,CAAC;AACf,CAAC"}
|
|
@@ -31,15 +31,15 @@ import { OrganicKeyNormalizationStrategy } from '../types/organic-keys.js';
|
|
|
31
31
|
export interface NormalizerInputColumn {
|
|
32
32
|
schema: string;
|
|
33
33
|
table: string;
|
|
34
|
-
|
|
34
|
+
Column: string;
|
|
35
35
|
dataType: string;
|
|
36
36
|
/** Original LLM-generated description from DBAutoDoc's prior analysis pass. */
|
|
37
|
-
|
|
37
|
+
OriginalDescription: string;
|
|
38
38
|
/** Sample values from the column's actual data. */
|
|
39
|
-
|
|
39
|
+
SampleValues: string[];
|
|
40
40
|
/** Whether the column participates in any FK (declared or detected). */
|
|
41
|
-
|
|
42
|
-
|
|
41
|
+
ParticipatesInFK: boolean;
|
|
42
|
+
FkTarget?: {
|
|
43
43
|
schema: string;
|
|
44
44
|
table: string;
|
|
45
45
|
column: string;
|
|
@@ -49,25 +49,25 @@ export interface NormalizerInputColumn {
|
|
|
49
49
|
/** All columns of one table, batched for a single LLM call. */
|
|
50
50
|
export interface TableNormalizationInput {
|
|
51
51
|
schema: string;
|
|
52
|
-
|
|
52
|
+
SchemaDescription?: string;
|
|
53
53
|
table: string;
|
|
54
|
-
|
|
54
|
+
TableDescription?: string;
|
|
55
55
|
columns: NormalizerInputColumn[];
|
|
56
56
|
}
|
|
57
57
|
/** One column's normalized output. */
|
|
58
58
|
export interface NormalizedColumn extends NormalizerInputColumn {
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
59
|
+
ConceptName: string;
|
|
60
|
+
NormalizationStrategy: OrganicKeyNormalizationStrategy;
|
|
61
|
+
CustomNormalizationExpression?: string;
|
|
62
|
+
NormalizedDescription: string;
|
|
63
|
+
IsUsefulOrganicKey: boolean;
|
|
64
64
|
confidence: number;
|
|
65
65
|
reasoning: string;
|
|
66
66
|
}
|
|
67
67
|
/** Aggregate result of normalizing many tables. */
|
|
68
68
|
export interface NormalizationBatchResult {
|
|
69
69
|
/** Useful organic-key columns surviving normalization (filtered to isUsefulOrganicKey=true). */
|
|
70
|
-
|
|
70
|
+
Normalized: NormalizedColumn[];
|
|
71
71
|
/** Columns the normalizer marked isUsefulOrganicKey=false (audit, free-form, etc.). */
|
|
72
72
|
rejected: number;
|
|
73
73
|
/** Tables where the LLM call failed entirely. */
|
|
@@ -80,17 +80,27 @@ export interface NormalizationBatchResult {
|
|
|
80
80
|
}
|
|
81
81
|
export interface NormalizerOptions {
|
|
82
82
|
/** Concurrency for per-table LLM calls. Default 8. */
|
|
83
|
-
|
|
83
|
+
Concurrency?: number;
|
|
84
84
|
/** Max retries per table on transient failures. Default 2. */
|
|
85
|
-
|
|
85
|
+
MaxRetries?: number;
|
|
86
86
|
/** Progress callback (done count, total count). */
|
|
87
|
-
|
|
87
|
+
OnProgress?: (done: number, total: number) => void;
|
|
88
88
|
}
|
|
89
89
|
export declare class TableNormalizer {
|
|
90
90
|
private readonly aiConfig;
|
|
91
91
|
private readonly llm;
|
|
92
92
|
constructor(aiConfig: AIConfig);
|
|
93
93
|
/** Normalize one table — one LLM call returning per-column entries. */
|
|
94
|
+
NormalizeTable(input: TableNormalizationInput, maxRetries?: number): Promise<{
|
|
95
|
+
normalized: NormalizedColumn[];
|
|
96
|
+
tokens: {
|
|
97
|
+
total: number;
|
|
98
|
+
input: number;
|
|
99
|
+
output: number;
|
|
100
|
+
};
|
|
101
|
+
errorMessage?: string;
|
|
102
|
+
}>;
|
|
103
|
+
/** @deprecated Use {@link NormalizeTable}. */
|
|
94
104
|
normalizeTable(input: TableNormalizationInput, maxRetries?: number): Promise<{
|
|
95
105
|
normalized: NormalizedColumn[];
|
|
96
106
|
tokens: {
|
|
@@ -101,6 +111,8 @@ export declare class TableNormalizer {
|
|
|
101
111
|
errorMessage?: string;
|
|
102
112
|
}>;
|
|
103
113
|
/** Batch normalize many tables with bounded concurrency. */
|
|
114
|
+
NormalizeAll(tables: TableNormalizationInput[], opts?: NormalizerOptions): Promise<NormalizationBatchResult>;
|
|
115
|
+
/** @deprecated Use {@link NormalizeAll}. */
|
|
104
116
|
normalizeAll(tables: TableNormalizationInput[], opts?: NormalizerOptions): Promise<NormalizationBatchResult>;
|
|
105
117
|
}
|
|
106
118
|
//# sourceMappingURL=ColumnNormalizer.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ColumnNormalizer.d.ts","sourceRoot":"","sources":["../../src/discovery/ColumnNormalizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAIH,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,+BAA+B,EAAE,MAAM,0BAA0B,CAAC;AAG3E,4CAA4C;AAC5C,MAAM,WAAW,qBAAqB;IAClC,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,+EAA+E;IAC/E,mBAAmB,EAAE,MAAM,CAAC;IAC5B,mDAAmD;IACnD,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,wEAAwE;IACxE,gBAAgB,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;IACpE,YAAY,EAAE,OAAO,CAAC;CACzB;AAED,+DAA+D;AAC/D,MAAM,WAAW,uBAAuB;IACpC,MAAM,EAAE,MAAM,CAAC;IACf,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,OAAO,EAAE,qBAAqB,EAAE,CAAC;CACpC;AAED,sCAAsC;AACtC,MAAM,WAAW,gBAAiB,SAAQ,qBAAqB;IAC3D,WAAW,EAAE,MAAM,CAAC;IACpB,qBAAqB,EAAE,+BAA+B,CAAC;IACvD,6BAA6B,CAAC,EAAE,MAAM,CAAC;IACvC,qBAAqB,EAAE,MAAM,CAAC;IAC9B,kBAAkB,EAAE,OAAO,CAAC;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,mDAAmD;AACnD,MAAM,WAAW,wBAAwB;IACrC,gGAAgG;IAChG,UAAU,EAAE,gBAAgB,EAAE,CAAC;IAC/B,uFAAuF;IACvF,QAAQ,EAAE,MAAM,CAAC;IACjB,iDAAiD;IACjD,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;CAC5D;AAED,MAAM,WAAW,iBAAiB;IAC9B,sDAAsD;IACtD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,8DAA8D;IAC9D,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,mDAAmD;IACnD,UAAU,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;CACtD;AAED,qBAAa,eAAe;IAGZ,OAAO,CAAC,QAAQ,CAAC,QAAQ;IAFrC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAU;gBAED,QAAQ,EAAE,QAAQ;IAI/C,uEAAuE;IAC1D,cAAc,CACvB,KAAK,EAAE,uBAAuB,EAC9B,UAAU,SAAI,GACf,OAAO,CAAC;QACP,UAAU,EAAE,gBAAgB,EAAE,CAAC;QAC/B,MAAM,EAAE;YAAE,KAAK,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,CAAC;YAAC,MAAM,EAAE,MAAM,CAAA;SAAE,CAAC;QACzD,YAAY,CAAC,EAAE,MAAM,CAAC;KACzB,CAAC;IA6EF,4DAA4D;IAC/C,YAAY,CACrB,MAAM,EAAE,uBAAuB,EAAE,EACjC,IAAI,GAAE,iBAAsB,GAC7B,OAAO,CAAC,wBAAwB,CAAC;
|
|
1
|
+
{"version":3,"file":"ColumnNormalizer.d.ts","sourceRoot":"","sources":["../../src/discovery/ColumnNormalizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAIH,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,+BAA+B,EAAE,MAAM,0BAA0B,CAAC;AAG3E,4CAA4C;AAC5C,MAAM,WAAW,qBAAqB;IAClC,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,+EAA+E;IAC/E,mBAAmB,EAAE,MAAM,CAAC;IAC5B,mDAAmD;IACnD,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,wEAAwE;IACxE,gBAAgB,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;IACpE,YAAY,EAAE,OAAO,CAAC;CACzB;AAED,+DAA+D;AAC/D,MAAM,WAAW,uBAAuB;IACpC,MAAM,EAAE,MAAM,CAAC;IACf,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,OAAO,EAAE,qBAAqB,EAAE,CAAC;CACpC;AAED,sCAAsC;AACtC,MAAM,WAAW,gBAAiB,SAAQ,qBAAqB;IAC3D,WAAW,EAAE,MAAM,CAAC;IACpB,qBAAqB,EAAE,+BAA+B,CAAC;IACvD,6BAA6B,CAAC,EAAE,MAAM,CAAC;IACvC,qBAAqB,EAAE,MAAM,CAAC;IAC9B,kBAAkB,EAAE,OAAO,CAAC;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,mDAAmD;AACnD,MAAM,WAAW,wBAAwB;IACrC,gGAAgG;IAChG,UAAU,EAAE,gBAAgB,EAAE,CAAC;IAC/B,uFAAuF;IACvF,QAAQ,EAAE,MAAM,CAAC;IACjB,iDAAiD;IACjD,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;CAC5D;AAED,MAAM,WAAW,iBAAiB;IAC9B,sDAAsD;IACtD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,8DAA8D;IAC9D,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,mDAAmD;IACnD,UAAU,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;CACtD;AAED,qBAAa,eAAe;IAGZ,OAAO,CAAC,QAAQ,CAAC,QAAQ;IAFrC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAU;gBAED,QAAQ,EAAE,QAAQ;IAI/C,uEAAuE;IAC1D,cAAc,CACvB,KAAK,EAAE,uBAAuB,EAC9B,UAAU,SAAI,GACf,OAAO,CAAC;QACP,UAAU,EAAE,gBAAgB,EAAE,CAAC;QAC/B,MAAM,EAAE;YAAE,KAAK,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,CAAC;YAAC,MAAM,EAAE,MAAM,CAAA;SAAE,CAAC;QACzD,YAAY,CAAC,EAAE,MAAM,CAAC;KACzB,CAAC;IA6EF,8CAA8C;IACjC,cAAc,CACvB,KAAK,EAAE,uBAAuB,EAC9B,UAAU,SAAI,GACf,OAAO,CAAC;QACP,UAAU,EAAE,gBAAgB,EAAE,CAAC;QAC/B,MAAM,EAAE;YAAE,KAAK,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,CAAC;YAAC,MAAM,EAAE,MAAM,CAAA;SAAE,CAAC;QACzD,YAAY,CAAC,EAAE,MAAM,CAAC;KACzB,CAAC;IAIF,4DAA4D;IAC/C,YAAY,CACrB,MAAM,EAAE,uBAAuB,EAAE,EACjC,IAAI,GAAE,iBAAsB,GAC7B,OAAO,CAAC,wBAAwB,CAAC;IAsCpC,4CAA4C;IAC/B,YAAY,CACrB,MAAM,EAAE,uBAAuB,EAAE,EACjC,IAAI,GAAE,iBAAsB,GAC7B,OAAO,CAAC,wBAAwB,CAAC;CAGvC"}
|
|
@@ -25,15 +25,15 @@
|
|
|
25
25
|
* normalizedDescription so the embedding step naturally clusters them. The
|
|
26
26
|
* prompt enforces this via a canonical concept-name list.
|
|
27
27
|
*/
|
|
28
|
-
import {
|
|
29
|
-
import {
|
|
28
|
+
import { CreateLLMInstance } from '../utils/llm-factory.js';
|
|
29
|
+
import { CleanAndParseJSON } from '../utils/json.js';
|
|
30
30
|
export class TableNormalizer {
|
|
31
31
|
constructor(aiConfig) {
|
|
32
32
|
this.aiConfig = aiConfig;
|
|
33
|
-
this.llm =
|
|
33
|
+
this.llm = CreateLLMInstance(aiConfig.provider, aiConfig.apiKey);
|
|
34
34
|
}
|
|
35
35
|
/** Normalize one table — one LLM call returning per-column entries. */
|
|
36
|
-
async
|
|
36
|
+
async NormalizeTable(input, maxRetries = 2) {
|
|
37
37
|
if (input.columns.length === 0) {
|
|
38
38
|
return { normalized: [], tokens: { total: 0, input: 0, output: 0 } };
|
|
39
39
|
}
|
|
@@ -72,7 +72,7 @@ export class TableNormalizer {
|
|
|
72
72
|
};
|
|
73
73
|
let parsed = null;
|
|
74
74
|
try {
|
|
75
|
-
parsed =
|
|
75
|
+
parsed = CleanAndParseJSON(content);
|
|
76
76
|
}
|
|
77
77
|
catch (err) {
|
|
78
78
|
lastError = `JSON parse threw: ${err.message}. Content prefix: ${content.slice(0, 200)}`;
|
|
@@ -87,7 +87,7 @@ export class TableNormalizer {
|
|
|
87
87
|
return { normalized: [], tokens: cumTokens, errorMessage: lastError };
|
|
88
88
|
}
|
|
89
89
|
// Match the LLM's response entries back to input columns by name.
|
|
90
|
-
const byName = new Map(input.columns.map((c) => [c.
|
|
90
|
+
const byName = new Map(input.columns.map((c) => [c.Column.toLowerCase(), c]));
|
|
91
91
|
const normalized = [];
|
|
92
92
|
for (const entry of parsed.columns) {
|
|
93
93
|
if (!entry || typeof entry.column !== 'string')
|
|
@@ -97,11 +97,11 @@ export class TableNormalizer {
|
|
|
97
97
|
continue; // LLM hallucinated a column name; skip
|
|
98
98
|
normalized.push({
|
|
99
99
|
...inputCol,
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
100
|
+
ConceptName: entry.conceptName ?? '',
|
|
101
|
+
NormalizationStrategy: entry.normalizationStrategy ?? 'LowerCaseTrim',
|
|
102
|
+
CustomNormalizationExpression: sanitizePlaceholder(entry.customNormalizationExpression),
|
|
103
|
+
NormalizedDescription: entry.normalizedDescription ?? '',
|
|
104
|
+
IsUsefulOrganicKey: !!entry.isUsefulOrganicKey,
|
|
105
105
|
confidence: clamp01(entry.confidence),
|
|
106
106
|
reasoning: entry.reasoning ?? '',
|
|
107
107
|
});
|
|
@@ -110,10 +110,14 @@ export class TableNormalizer {
|
|
|
110
110
|
}
|
|
111
111
|
return { normalized: [], tokens: cumTokens, errorMessage: lastError || 'unknown failure after retries' };
|
|
112
112
|
}
|
|
113
|
+
/** @deprecated Use {@link NormalizeTable}. */
|
|
114
|
+
async normalizeTable(input, maxRetries = 2) {
|
|
115
|
+
return this.NormalizeTable(input, maxRetries);
|
|
116
|
+
}
|
|
113
117
|
/** Batch normalize many tables with bounded concurrency. */
|
|
114
|
-
async
|
|
115
|
-
const concurrency = Math.max(1, opts.
|
|
116
|
-
const maxRetries = Math.max(0, opts.
|
|
118
|
+
async NormalizeAll(tables, opts = {}) {
|
|
119
|
+
const concurrency = Math.max(1, opts.Concurrency ?? 8);
|
|
120
|
+
const maxRetries = Math.max(0, opts.MaxRetries ?? 2);
|
|
117
121
|
const allNormalized = [];
|
|
118
122
|
let rejected = 0;
|
|
119
123
|
let errors = 0;
|
|
@@ -127,7 +131,7 @@ export class TableNormalizer {
|
|
|
127
131
|
const idx = cursor++;
|
|
128
132
|
if (idx >= tables.length)
|
|
129
133
|
return;
|
|
130
|
-
const r = await this.
|
|
134
|
+
const r = await this.NormalizeTable(tables[idx], maxRetries);
|
|
131
135
|
total += r.tokens.total;
|
|
132
136
|
input += r.tokens.input;
|
|
133
137
|
output += r.tokens.output;
|
|
@@ -136,18 +140,22 @@ export class TableNormalizer {
|
|
|
136
140
|
}
|
|
137
141
|
else {
|
|
138
142
|
for (const n of r.normalized) {
|
|
139
|
-
if (n.
|
|
143
|
+
if (n.IsUsefulOrganicKey)
|
|
140
144
|
allNormalized.push(n);
|
|
141
145
|
else
|
|
142
146
|
rejected++;
|
|
143
147
|
}
|
|
144
148
|
}
|
|
145
149
|
completed++;
|
|
146
|
-
opts.
|
|
150
|
+
opts.OnProgress?.(completed, tables.length);
|
|
147
151
|
}
|
|
148
152
|
});
|
|
149
153
|
await Promise.all(runners);
|
|
150
|
-
return {
|
|
154
|
+
return { Normalized: allNormalized, rejected, errors, tokens: { total, input, output } };
|
|
155
|
+
}
|
|
156
|
+
/** @deprecated Use {@link NormalizeAll}. */
|
|
157
|
+
async normalizeAll(tables, opts = {}) {
|
|
158
|
+
return this.NormalizeAll(tables, opts);
|
|
151
159
|
}
|
|
152
160
|
}
|
|
153
161
|
// ─── Prompt ─────────────────────────────────────────────────────────────────
|
|
@@ -337,18 +345,18 @@ function sanitizePlaceholder(expr) {
|
|
|
337
345
|
function buildUserPrompt(input) {
|
|
338
346
|
const lines = [];
|
|
339
347
|
lines.push(`Table: ${input.schema}.${input.table}`);
|
|
340
|
-
if (input.
|
|
341
|
-
lines.push(`Schema purpose: ${truncate(input.
|
|
342
|
-
if (input.
|
|
343
|
-
lines.push(`Table purpose: ${truncate(input.
|
|
348
|
+
if (input.SchemaDescription)
|
|
349
|
+
lines.push(`Schema purpose: ${truncate(input.SchemaDescription, 240)}`);
|
|
350
|
+
if (input.TableDescription)
|
|
351
|
+
lines.push(`Table purpose: ${truncate(input.TableDescription, 240)}`);
|
|
344
352
|
lines.push('');
|
|
345
353
|
lines.push(`Columns (${input.columns.length}):`);
|
|
346
354
|
for (const c of input.columns) {
|
|
347
|
-
lines.push(` - ${c.
|
|
348
|
-
if (c.
|
|
349
|
-
lines.push(` description: ${truncate(c.
|
|
350
|
-
if (c.
|
|
351
|
-
const samples = c.
|
|
355
|
+
lines.push(` - ${c.Column} [${c.dataType}]${c.isPrimaryKey ? ' PK' : ''}${c.ParticipatesInFK ? ` FK${c.FkTarget ? `→${c.FkTarget.schema}.${c.FkTarget.table}.${c.FkTarget.column}` : ''}` : ''}`);
|
|
356
|
+
if (c.OriginalDescription)
|
|
357
|
+
lines.push(` description: ${truncate(c.OriginalDescription, 240)}`);
|
|
358
|
+
if (c.SampleValues && c.SampleValues.length > 0) {
|
|
359
|
+
const samples = c.SampleValues
|
|
352
360
|
.slice(0, 5)
|
|
353
361
|
.map((v) => JSON.stringify(truncate(String(v), 80)))
|
|
354
362
|
.join(', ');
|