@squadbase/vite-server 0.1.17-dev.a9ddcfa → 0.1.17-dev.e4768d1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/index.js +8941 -3700
- package/dist/connectors/airtable-oauth.js +87 -23
- package/dist/connectors/airtable.js +130 -23
- package/dist/connectors/amplitude.js +47 -23
- package/dist/connectors/anthropic.js +41 -17
- package/dist/connectors/asana.js +76 -25
- package/dist/connectors/attio.js +69 -28
- package/dist/connectors/aws-billing.js +47 -23
- package/dist/connectors/azure-sql.js +322 -25
- package/dist/connectors/backlog-api-key.js +79 -30
- package/dist/connectors/clickup.js +89 -25
- package/dist/connectors/cosmosdb.js +51 -27
- package/dist/connectors/customerio.js +47 -23
- package/dist/connectors/dbt.js +725 -40
- package/dist/connectors/freshdesk.js +121 -23
- package/dist/connectors/freshsales.js +47 -23
- package/dist/connectors/freshservice.js +47 -23
- package/dist/connectors/gamma.js +54 -30
- package/dist/connectors/gemini.js +41 -17
- package/dist/connectors/github.js +51 -27
- package/dist/connectors/gmail-oauth.js +42 -23
- package/dist/connectors/gmail.js +43 -19
- package/dist/connectors/google-ads.js +47 -23
- package/dist/connectors/google-analytics-oauth.js +191 -40
- package/dist/connectors/google-analytics.js +529 -111
- package/dist/connectors/google-audit-log.js +43 -19
- package/dist/connectors/google-calendar-oauth.js +100 -30
- package/dist/connectors/google-calendar.js +100 -26
- package/dist/connectors/google-docs.js +42 -23
- package/dist/connectors/google-drive.js +64 -23
- package/dist/connectors/google-search-console-oauth.js +165 -32
- package/dist/connectors/google-sheets.js +142 -199
- package/dist/connectors/google-slides.js +42 -23
- package/dist/connectors/grafana.js +84 -25
- package/dist/connectors/hackernews.d.ts +5 -0
- package/dist/connectors/hackernews.js +909 -0
- package/dist/connectors/hubspot-oauth.js +73 -22
- package/dist/connectors/hubspot.js +64 -24
- package/dist/connectors/influxdb.js +47 -23
- package/dist/connectors/intercom-oauth.js +104 -25
- package/dist/connectors/intercom.js +51 -27
- package/dist/connectors/jdbc.js +76 -25
- package/dist/connectors/jira-api-key.js +107 -26
- package/dist/connectors/kintone-api-token.js +105 -33
- package/dist/connectors/kintone.js +182 -26
- package/dist/connectors/linear.js +93 -27
- package/dist/connectors/linkedin-ads.js +80 -29
- package/dist/connectors/mailchimp-oauth.js +45 -21
- package/dist/connectors/mailchimp.js +45 -21
- package/dist/connectors/meta-ads-oauth.js +72 -29
- package/dist/connectors/meta-ads.js +74 -29
- package/dist/connectors/mixpanel.js +47 -23
- package/dist/connectors/monday.js +48 -24
- package/dist/connectors/mongodb.js +47 -23
- package/dist/connectors/notion-oauth.js +92 -24
- package/dist/connectors/notion.js +144 -26
- package/dist/connectors/openai.js +41 -17
- package/dist/connectors/oracle.js +316 -26
- package/dist/connectors/outlook-oauth.js +53 -34
- package/dist/connectors/powerbi-oauth.js +45 -26
- package/dist/connectors/salesforce.js +150 -24
- package/dist/connectors/semrush.js +45 -21
- package/dist/connectors/sentry.js +75 -25
- package/dist/connectors/shopify-oauth.js +75 -23
- package/dist/connectors/shopify.js +47 -23
- package/dist/connectors/sqlserver.js +322 -25
- package/dist/connectors/stripe-api-key.js +105 -30
- package/dist/connectors/stripe-oauth.js +102 -32
- package/dist/connectors/supabase.js +315 -21
- package/dist/connectors/tableau.js +54 -30
- package/dist/connectors/tiktok-ads.js +76 -31
- package/dist/connectors/wix-store.js +47 -23
- package/dist/connectors/x.d.ts +5 -0
- package/dist/connectors/x.js +951 -0
- package/dist/connectors/zendesk-oauth.js +87 -25
- package/dist/connectors/zendesk.js +51 -27
- package/dist/index.js +9005 -3709
- package/dist/main.js +9005 -3709
- package/dist/vite-plugin.js +8939 -3698
- package/package.json +12 -4
package/dist/connectors/dbt.js
CHANGED
|
@@ -1,6 +1,11 @@
|
|
|
1
1
|
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
2
|
-
var __esm = (fn, res) => function __init() {
|
|
3
|
-
|
|
2
|
+
var __esm = (fn, res, err) => function __init() {
|
|
3
|
+
if (err) throw err[0];
|
|
4
|
+
try {
|
|
5
|
+
return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
|
|
6
|
+
} catch (e) {
|
|
7
|
+
throw err = [e], e;
|
|
8
|
+
}
|
|
4
9
|
};
|
|
5
10
|
|
|
6
11
|
// ../connectors/src/parameter-definition.ts
|
|
@@ -398,6 +403,15 @@ var ConnectorPlugin = class _ConnectorPlugin {
|
|
|
398
403
|
* `runSetupFlow` from `setup-flow.ts`.
|
|
399
404
|
*/
|
|
400
405
|
setup;
|
|
406
|
+
/**
|
|
407
|
+
* 統一 introspection。SQL DB の `information_schema` / `SHOW` / `DESCRIBE` や SaaS の
|
|
408
|
+
* メタデータ API といった取得経路の差を吸収し、tables / columns / PK / FK を共通型
|
|
409
|
+
* {@link IntrospectedSchema} へ正規化して返す。取得経路は {@link IntrospectContext} で
|
|
410
|
+
* 抽象化され、SQL DB は `ctx.query`、SaaS は `ctx.proxyFetch` + `ctx.params` を使う。
|
|
411
|
+
* `ctx.profile === true` のときは L2 プロファイル(行数・NULL 率等)も best-effort で埋める。
|
|
412
|
+
* coding-agent の `DataContextIndex` がデータ地図を構築する際に使う。
|
|
413
|
+
*/
|
|
414
|
+
introspect;
|
|
401
415
|
/**
|
|
402
416
|
* Opt-out of the default "verify before save" behavior on connection
|
|
403
417
|
* creation. The backend invokes `checkConnection` synchronously while
|
|
@@ -412,6 +426,14 @@ var ConnectorPlugin = class _ConnectorPlugin {
|
|
|
412
426
|
* the default verify-on-create behavior without opt-in.
|
|
413
427
|
*/
|
|
414
428
|
skipConnectionCheckOnCreate;
|
|
429
|
+
/**
|
|
430
|
+
* 決定論的な「立ち上げ survey」レシピ(ルールベース prewarm 基盤)。実装した connector は、LLM を使わずに
|
|
431
|
+
* read-only で自分を survey する probe 列(テーブル/列メタデータ → 小サンプル)を宣言する。coding-agent の
|
|
432
|
+
* recon driver がこれを読み、connector ツールの `execute` を直接呼んで結果を seed にする。未実装の connector は
|
|
433
|
+
* driver 側で safely skip される(その connection は listConnections のみ seed)。標準 `information_schema` を
|
|
434
|
+
* 持つ SQL connector は `makeStandardSqlSurveyProbes` を使える。**probe は必ず read-only**。
|
|
435
|
+
*/
|
|
436
|
+
surveyProbes;
|
|
415
437
|
constructor(config) {
|
|
416
438
|
this.slug = config.slug;
|
|
417
439
|
this.authType = config.authType;
|
|
@@ -429,7 +451,9 @@ var ConnectorPlugin = class _ConnectorPlugin {
|
|
|
429
451
|
this.query = config.query;
|
|
430
452
|
this.checkConnection = config.checkConnection;
|
|
431
453
|
this.setup = config.setup;
|
|
454
|
+
this.introspect = config.introspect;
|
|
432
455
|
this.skipConnectionCheckOnCreate = config.skipConnectionCheckOnCreate;
|
|
456
|
+
this.surveyProbes = config.surveyProbes;
|
|
433
457
|
}
|
|
434
458
|
get connectorKey() {
|
|
435
459
|
return _ConnectorPlugin.deriveKey(this.slug, this.authType);
|
|
@@ -437,7 +461,7 @@ var ConnectorPlugin = class _ConnectorPlugin {
|
|
|
437
461
|
/**
|
|
438
462
|
* Create tools for connections that belong to this connector.
|
|
439
463
|
* Filters connections by connectorKey internally.
|
|
440
|
-
* Returns tools keyed as
|
|
464
|
+
* Returns tools keyed as `connector_${connectorKey}_${toolName}`.
|
|
441
465
|
*/
|
|
442
466
|
createTools(connections, config, opts) {
|
|
443
467
|
const myConnections = connections.filter(
|
|
@@ -447,7 +471,7 @@ var ConnectorPlugin = class _ConnectorPlugin {
|
|
|
447
471
|
for (const t of Object.values(this.tools)) {
|
|
448
472
|
const tool = t.createTool(myConnections, config);
|
|
449
473
|
const originalToModelOutput = tool.toModelOutput;
|
|
450
|
-
result[
|
|
474
|
+
result[`connector_${this.connectorKey}_${t.name}`] = {
|
|
451
475
|
...tool,
|
|
452
476
|
toModelOutput: async (options) => {
|
|
453
477
|
if (!originalToModelOutput) {
|
|
@@ -467,25 +491,25 @@ var ConnectorPlugin = class _ConnectorPlugin {
|
|
|
467
491
|
if (authType) return `${slug}-${authType}`;
|
|
468
492
|
const LEGACY_NULL_AUTH_TYPE_MAP = {
|
|
469
493
|
// user-password
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
494
|
+
postgresql: "user-password",
|
|
495
|
+
mysql: "user-password",
|
|
496
|
+
clickhouse: "user-password",
|
|
497
|
+
kintone: "user-password",
|
|
474
498
|
"squadbase-db": "user-password",
|
|
475
499
|
// service-account
|
|
476
|
-
|
|
477
|
-
|
|
500
|
+
snowflake: "service-account",
|
|
501
|
+
bigquery: "service-account",
|
|
478
502
|
"google-analytics": "service-account",
|
|
479
503
|
"google-calendar": "service-account",
|
|
480
504
|
"aws-athena": "service-account",
|
|
481
|
-
|
|
505
|
+
redshift: "service-account",
|
|
482
506
|
// api-key
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
507
|
+
databricks: "api-key",
|
|
508
|
+
dbt: "api-key",
|
|
509
|
+
airtable: "api-key",
|
|
510
|
+
openai: "api-key",
|
|
511
|
+
gemini: "api-key",
|
|
512
|
+
anthropic: "api-key",
|
|
489
513
|
"wix-store": "api-key"
|
|
490
514
|
};
|
|
491
515
|
const fallbackAuthType = LEGACY_NULL_AUTH_TYPE_MAP[slug];
|
|
@@ -577,11 +601,11 @@ var AUTH_TYPES = {
|
|
|
577
601
|
// ../connectors/src/connectors/dbt/setup.ts
|
|
578
602
|
var dbtOnboarding = new ConnectorOnboarding({
|
|
579
603
|
dataOverviewInstructions: {
|
|
580
|
-
en: `1. Call
|
|
581
|
-
2. For key models, call
|
|
604
|
+
en: `1. Call connector_dbt_request to list models using the GraphQL query for models (first: 100)
|
|
605
|
+
2. For key models, call connector_dbt_request with the model column information query to get column names and types
|
|
582
606
|
3. Explore sources and lineage as needed to understand the data structure`,
|
|
583
|
-
ja: `1.
|
|
584
|
-
2. \u4E3B\u8981\u30E2\u30C7\u30EB\u306B\u3064\u3044\u3066
|
|
607
|
+
ja: `1. connector_dbt_request \u3067\u30E2\u30C7\u30EB\u4E00\u89A7\u53D6\u5F97\u7528\u306EGraphQL\u30AF\u30A8\u30EA\u3092\u5B9F\u884C\uFF08first: 100\uFF09
|
|
608
|
+
2. \u4E3B\u8981\u30E2\u30C7\u30EB\u306B\u3064\u3044\u3066 connector_dbt_request \u3067\u30AB\u30E9\u30E0\u60C5\u5831\u53D6\u5F97\u7528\u306E\u30AF\u30A8\u30EA\u3092\u5B9F\u884C\u3057\u3001\u30AB\u30E9\u30E0\u540D\u3068\u578B\u3092\u53D6\u5F97
|
|
585
609
|
3. \u5FC5\u8981\u306B\u5FDC\u3058\u3066\u30BD\u30FC\u30B9\u3084\u30EA\u30CD\u30FC\u30B8\u3092\u63A2\u7D22\u3057\u3001\u30C7\u30FC\u30BF\u69CB\u9020\u3092\u628A\u63E1`
|
|
586
610
|
}
|
|
587
611
|
});
|
|
@@ -609,10 +633,54 @@ function adminApiFetch(params, path2, init) {
|
|
|
609
633
|
if (!headers.has("Accept")) headers.set("Accept", "application/json");
|
|
610
634
|
return fetch(`${adminApiBase(host)}${trimmedPath}`, { ...init, headers });
|
|
611
635
|
}
|
|
636
|
+
function discoveryEndpoint(host) {
|
|
637
|
+
if (host.includes("emea")) return "https://metadata.emea.dbt.com/graphql";
|
|
638
|
+
if (host.includes(".au.")) return "https://metadata.au.dbt.com/graphql";
|
|
639
|
+
return "https://metadata.cloud.getdbt.com/graphql";
|
|
640
|
+
}
|
|
641
|
+
async function discoveryQuery(params, query, variables) {
|
|
642
|
+
const host = params[parameters.host.slug];
|
|
643
|
+
const token = params[parameters.token.slug];
|
|
644
|
+
const prodEnvId = params[parameters.prodEnvId.slug];
|
|
645
|
+
if (!host || !token || !prodEnvId) return null;
|
|
646
|
+
try {
|
|
647
|
+
const res = await fetch(discoveryEndpoint(host), {
|
|
648
|
+
method: "POST",
|
|
649
|
+
headers: {
|
|
650
|
+
"Content-Type": "application/json",
|
|
651
|
+
Authorization: `Bearer ${token}`,
|
|
652
|
+
Accept: "application/json"
|
|
653
|
+
},
|
|
654
|
+
body: JSON.stringify({
|
|
655
|
+
query,
|
|
656
|
+
variables: { environmentId: Number(prodEnvId), ...variables }
|
|
657
|
+
})
|
|
658
|
+
});
|
|
659
|
+
if (!res.ok) return null;
|
|
660
|
+
const json = await res.json();
|
|
661
|
+
if (json.errors && json.errors.length > 0) return null;
|
|
662
|
+
return json.data ?? null;
|
|
663
|
+
} catch {
|
|
664
|
+
return null;
|
|
665
|
+
}
|
|
666
|
+
}
|
|
612
667
|
|
|
613
668
|
// ../connectors/src/connectors/dbt/setup-flow.ts
|
|
614
669
|
var ALL_PROJECTS = "__ALL_PROJECTS__";
|
|
615
670
|
var DBT_SETUP_MAX_PROJECTS = 10;
|
|
671
|
+
var DISCOVERY_FETCH_LIMIT = 500;
|
|
672
|
+
var MART_TIER1_DETAIL = 20;
|
|
673
|
+
var MART_TIER2_NAMES = 80;
|
|
674
|
+
var INTERMEDIATE_TIER1_DETAIL = 10;
|
|
675
|
+
var INTERMEDIATE_TIER2_NAMES = 40;
|
|
676
|
+
var OTHER_TIER1_DETAIL = 10;
|
|
677
|
+
var OTHER_TIER2_NAMES = 30;
|
|
678
|
+
var SOURCE_ENUMERATION_THRESHOLD = 25;
|
|
679
|
+
var MAX_EXPOSURE_DETAIL = 25;
|
|
680
|
+
var MAX_METRIC_DETAIL = 25;
|
|
681
|
+
var MAX_COLUMNS_PER_MODEL = 12;
|
|
682
|
+
var DESCRIPTION_MAX_PRIMARY = 180;
|
|
683
|
+
var DESCRIPTION_MAX_SECONDARY = 120;
|
|
616
684
|
async function listAllProjects(params) {
|
|
617
685
|
const accountId = params[parameters.accountId.slug];
|
|
618
686
|
if (!accountId) {
|
|
@@ -642,11 +710,556 @@ async function listAllProjects(params) {
|
|
|
642
710
|
return all;
|
|
643
711
|
}
|
|
644
712
|
function repoLabel(p) {
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
return
|
|
713
|
+
return p.repository?.remote_url ?? p.repository?.name ?? "-";
|
|
714
|
+
}
|
|
715
|
+
async function fetchRecentRunCount(params, projectId) {
|
|
716
|
+
const accountId = params[parameters.accountId.slug];
|
|
717
|
+
if (!accountId) return null;
|
|
718
|
+
try {
|
|
719
|
+
const path2 = `/api/v2/accounts/${encodeURIComponent(accountId)}/runs/?project_id=${encodeURIComponent(projectId)}&limit=1&order_by=-id`;
|
|
720
|
+
const res = await adminApiFetch(params, path2);
|
|
721
|
+
if (!res.ok) return null;
|
|
722
|
+
const data = await res.json();
|
|
723
|
+
const total = data.extra?.pagination?.total_count;
|
|
724
|
+
return typeof total === "number" ? total : null;
|
|
725
|
+
} catch {
|
|
726
|
+
return null;
|
|
727
|
+
}
|
|
728
|
+
}
|
|
729
|
+
var APPLIED_MODELS_QUERY = `query($environmentId: BigInt!, $first: Int!) {
|
|
730
|
+
environment(id: $environmentId) {
|
|
731
|
+
applied {
|
|
732
|
+
models(first: $first) {
|
|
733
|
+
totalCount
|
|
734
|
+
edges {
|
|
735
|
+
node {
|
|
736
|
+
uniqueId name description materializedType
|
|
737
|
+
database schema alias tags filePath packageName
|
|
738
|
+
catalog { columns { name type description } }
|
|
739
|
+
}
|
|
740
|
+
}
|
|
741
|
+
}
|
|
742
|
+
}
|
|
743
|
+
}
|
|
744
|
+
}`;
|
|
745
|
+
var DEFINITION_MODELS_QUERY = `query($environmentId: BigInt!, $first: Int!) {
|
|
746
|
+
environment(id: $environmentId) {
|
|
747
|
+
definition {
|
|
748
|
+
models(first: $first) {
|
|
749
|
+
totalCount
|
|
750
|
+
edges {
|
|
751
|
+
node {
|
|
752
|
+
uniqueId name description materializedType
|
|
753
|
+
database schema alias tags filePath packageName
|
|
754
|
+
}
|
|
755
|
+
}
|
|
756
|
+
}
|
|
757
|
+
}
|
|
758
|
+
}
|
|
759
|
+
}`;
|
|
760
|
+
var SOURCES_QUERY = `query($environmentId: BigInt!, $first: Int!) {
|
|
761
|
+
environment(id: $environmentId) {
|
|
762
|
+
applied {
|
|
763
|
+
sources(first: $first) {
|
|
764
|
+
totalCount
|
|
765
|
+
edges {
|
|
766
|
+
node {
|
|
767
|
+
uniqueId name description database schema identifier
|
|
768
|
+
sourceName freshness { freshnessStatus }
|
|
769
|
+
}
|
|
770
|
+
}
|
|
771
|
+
}
|
|
772
|
+
}
|
|
773
|
+
}
|
|
774
|
+
}`;
|
|
775
|
+
var EXPOSURES_QUERY = `query($environmentId: BigInt!, $first: Int!) {
|
|
776
|
+
environment(id: $environmentId) {
|
|
777
|
+
applied {
|
|
778
|
+
exposures(first: $first) {
|
|
779
|
+
totalCount
|
|
780
|
+
edges {
|
|
781
|
+
node {
|
|
782
|
+
uniqueId name exposureType maturity label description url
|
|
783
|
+
ownerName ownerEmail tags
|
|
784
|
+
parents { uniqueId name }
|
|
785
|
+
}
|
|
786
|
+
}
|
|
787
|
+
}
|
|
788
|
+
}
|
|
789
|
+
}
|
|
790
|
+
}`;
|
|
791
|
+
var METRICS_QUERY = `query($environmentId: BigInt!, $first: Int!) {
|
|
792
|
+
environment(id: $environmentId) {
|
|
793
|
+
definition {
|
|
794
|
+
metrics(first: $first) {
|
|
795
|
+
totalCount
|
|
796
|
+
edges {
|
|
797
|
+
node {
|
|
798
|
+
uniqueId name label description type filter formula
|
|
799
|
+
}
|
|
800
|
+
}
|
|
801
|
+
}
|
|
802
|
+
}
|
|
803
|
+
}
|
|
804
|
+
}`;
|
|
805
|
+
async function fetchModels(params) {
|
|
806
|
+
const applied = await discoveryQuery(
|
|
807
|
+
params,
|
|
808
|
+
APPLIED_MODELS_QUERY,
|
|
809
|
+
{ first: DISCOVERY_FETCH_LIMIT }
|
|
810
|
+
);
|
|
811
|
+
const appliedNodes = applied?.environment?.applied?.models?.edges?.map((e) => e.node) ?? [];
|
|
812
|
+
if (appliedNodes.length > 0) {
|
|
813
|
+
return { models: appliedNodes, state: "applied" };
|
|
814
|
+
}
|
|
815
|
+
const definition = await discoveryQuery(
|
|
816
|
+
params,
|
|
817
|
+
DEFINITION_MODELS_QUERY,
|
|
818
|
+
{ first: DISCOVERY_FETCH_LIMIT }
|
|
819
|
+
);
|
|
820
|
+
const definitionNodes = definition?.environment?.definition?.models?.edges?.map((e) => e.node) ?? [];
|
|
821
|
+
if (definitionNodes.length > 0) {
|
|
822
|
+
return { models: definitionNodes, state: "definition" };
|
|
823
|
+
}
|
|
824
|
+
return { models: [], state: "empty" };
|
|
825
|
+
}
|
|
826
|
+
async function fetchSources(params) {
|
|
827
|
+
const data = await discoveryQuery(params, SOURCES_QUERY, { first: DISCOVERY_FETCH_LIMIT });
|
|
828
|
+
return data?.environment?.applied?.sources?.edges?.map((e) => e.node) ?? [];
|
|
829
|
+
}
|
|
830
|
+
async function fetchExposures(params) {
|
|
831
|
+
const data = await discoveryQuery(params, EXPOSURES_QUERY, { first: DISCOVERY_FETCH_LIMIT });
|
|
832
|
+
return data?.environment?.applied?.exposures?.edges?.map((e) => e.node) ?? [];
|
|
833
|
+
}
|
|
834
|
+
async function fetchMetrics(params) {
|
|
835
|
+
const data = await discoveryQuery(params, METRICS_QUERY, { first: DISCOVERY_FETCH_LIMIT });
|
|
836
|
+
return data?.environment?.definition?.metrics?.edges?.map((e) => e.node) ?? [];
|
|
837
|
+
}
|
|
838
|
+
var MART_TAG_SET = /* @__PURE__ */ new Set([
|
|
839
|
+
"mart",
|
|
840
|
+
"marts",
|
|
841
|
+
"gold",
|
|
842
|
+
"final",
|
|
843
|
+
"reporting",
|
|
844
|
+
"presentation",
|
|
845
|
+
"bi",
|
|
846
|
+
"analytics",
|
|
847
|
+
"business",
|
|
848
|
+
"datamart",
|
|
849
|
+
"dm",
|
|
850
|
+
"dwh",
|
|
851
|
+
"exposed",
|
|
852
|
+
"kpi",
|
|
853
|
+
"fact",
|
|
854
|
+
"dimension"
|
|
855
|
+
]);
|
|
856
|
+
var INTERMEDIATE_TAG_SET = /* @__PURE__ */ new Set([
|
|
857
|
+
"intermediate",
|
|
858
|
+
"int",
|
|
859
|
+
"silver",
|
|
860
|
+
"transform",
|
|
861
|
+
"transformed",
|
|
862
|
+
"derived",
|
|
863
|
+
"refined",
|
|
864
|
+
"prep",
|
|
865
|
+
"normalized"
|
|
866
|
+
]);
|
|
867
|
+
var STAGING_TAG_SET = /* @__PURE__ */ new Set([
|
|
868
|
+
"staging",
|
|
869
|
+
"stg",
|
|
870
|
+
"bronze",
|
|
871
|
+
"source",
|
|
872
|
+
"sources",
|
|
873
|
+
"raw",
|
|
874
|
+
"base",
|
|
875
|
+
"ingest",
|
|
876
|
+
"ingested",
|
|
877
|
+
"ods",
|
|
878
|
+
"landing"
|
|
879
|
+
]);
|
|
880
|
+
var MART_PATH_RE = /\/(marts?|gold|presentation|bi|reporting|analytics|datamart|dwh|dm|exposed|final|serve|serving)\//;
|
|
881
|
+
var INT_PATH_RE = /\/(intermediate|silver|transform|transformed|derived|refined|prep|preprocess|normalized)\//;
|
|
882
|
+
var STG_PATH_RE = /\/(staging|bronze|sources?|raw|ingest|ods|landing|base)\//;
|
|
883
|
+
var MART_NAME_RE = /^(fct|fact|dim|dimension|agg|mart|rpt|report|kpi|metric|cube|wide|obt|gold|datamart|dm|dwh|presentation|bi|final)_/;
|
|
884
|
+
var INT_NAME_RE = /^(int|intermediate|silver|derived|transform|refined|prep)_/;
|
|
885
|
+
var STG_NAME_RE = /^(stg|staging|src|source|raw|base|bronze|ingest|ods|landing)_/;
|
|
886
|
+
var MART_SCHEMA_RE = /^(mart|marts|analytics|reporting|presentation|datamart|dwh|dm|gold|bi)$|_(mart|marts|gold|datamart|dm|dwh|reporting|presentation)$/;
|
|
887
|
+
var INT_SCHEMA_RE = /^(int|intermediate|silver)$|_(intermediate|silver)$/;
|
|
888
|
+
var STG_SCHEMA_RE = /^(stg|staging|raw|sources?|bronze|base|ods|ingest|landing)$|_(staging|stg|raw|bronze|base|ods|landing)$/;
|
|
889
|
+
var LAYER_FOLDER_TOKENS = /* @__PURE__ */ new Set([
|
|
890
|
+
// standard
|
|
891
|
+
"marts",
|
|
892
|
+
"mart",
|
|
893
|
+
"intermediate",
|
|
894
|
+
"staging",
|
|
895
|
+
"sources",
|
|
896
|
+
"source",
|
|
897
|
+
"base",
|
|
898
|
+
// medallion
|
|
899
|
+
"bronze",
|
|
900
|
+
"silver",
|
|
901
|
+
"gold",
|
|
902
|
+
// dbt Labs newer
|
|
903
|
+
"presentation",
|
|
904
|
+
"transform",
|
|
905
|
+
// DW vocab
|
|
906
|
+
"reporting",
|
|
907
|
+
"analytics",
|
|
908
|
+
"datamart",
|
|
909
|
+
"dwh",
|
|
910
|
+
"dm",
|
|
911
|
+
"ods",
|
|
912
|
+
"raw",
|
|
913
|
+
"ingest",
|
|
914
|
+
"landing",
|
|
915
|
+
"final",
|
|
916
|
+
"exposed"
|
|
917
|
+
]);
|
|
918
|
+
function martLikelihoodScore(m) {
|
|
919
|
+
const mat = (m.materializedType ?? "").toLowerCase();
|
|
920
|
+
if (mat === "incremental") return 3;
|
|
921
|
+
if (mat === "table") return 3;
|
|
922
|
+
if (mat === "materialized_view") return 2;
|
|
923
|
+
if (mat === "view") return 1;
|
|
924
|
+
if (mat === "ephemeral") return 0;
|
|
925
|
+
if (mat === "seed") return 0;
|
|
926
|
+
return 1;
|
|
927
|
+
}
|
|
928
|
+
function layerModels(models, exposures) {
|
|
929
|
+
const layerOf = /* @__PURE__ */ new Map();
|
|
930
|
+
const byId = /* @__PURE__ */ new Map();
|
|
931
|
+
for (const m of models) {
|
|
932
|
+
const initial = classifyModel(m);
|
|
933
|
+
if (m.uniqueId) {
|
|
934
|
+
layerOf.set(m.uniqueId, initial);
|
|
935
|
+
byId.set(m.uniqueId, m);
|
|
936
|
+
}
|
|
937
|
+
}
|
|
938
|
+
for (const e of exposures) {
|
|
939
|
+
for (const p of e.parents ?? []) {
|
|
940
|
+
if (!p.uniqueId) continue;
|
|
941
|
+
const current = layerOf.get(p.uniqueId);
|
|
942
|
+
if (current === "other" || current === "intermediate") {
|
|
943
|
+
layerOf.set(p.uniqueId, "mart");
|
|
944
|
+
}
|
|
945
|
+
}
|
|
946
|
+
}
|
|
947
|
+
const out = {
|
|
948
|
+
mart: [],
|
|
949
|
+
intermediate: [],
|
|
950
|
+
staging: [],
|
|
951
|
+
other: []
|
|
952
|
+
};
|
|
953
|
+
for (const m of models) {
|
|
954
|
+
const layer = m.uniqueId && layerOf.get(m.uniqueId) || classifyModel(m);
|
|
955
|
+
out[layer].push(m);
|
|
956
|
+
}
|
|
957
|
+
return out;
|
|
958
|
+
}
|
|
959
|
+
function classifyByTags(tags) {
|
|
960
|
+
for (const raw of tags) {
|
|
961
|
+
const t = raw.toLowerCase();
|
|
962
|
+
if (MART_TAG_SET.has(t)) return "mart";
|
|
963
|
+
if (INTERMEDIATE_TAG_SET.has(t)) return "intermediate";
|
|
964
|
+
if (STAGING_TAG_SET.has(t)) return "staging";
|
|
965
|
+
}
|
|
966
|
+
return null;
|
|
967
|
+
}
|
|
968
|
+
function classifyModel(m) {
|
|
969
|
+
const fromTags = classifyByTags(m.tags ?? []);
|
|
970
|
+
if (fromTags) return fromTags;
|
|
971
|
+
const path2 = (m.filePath ?? "").toLowerCase();
|
|
972
|
+
if (path2) {
|
|
973
|
+
if (MART_PATH_RE.test(path2)) return "mart";
|
|
974
|
+
if (INT_PATH_RE.test(path2)) return "intermediate";
|
|
975
|
+
if (STG_PATH_RE.test(path2)) return "staging";
|
|
976
|
+
}
|
|
977
|
+
const name = (m.name ?? "").toLowerCase();
|
|
978
|
+
if (MART_NAME_RE.test(name)) return "mart";
|
|
979
|
+
if (INT_NAME_RE.test(name)) return "intermediate";
|
|
980
|
+
if (STG_NAME_RE.test(name)) return "staging";
|
|
981
|
+
const schema = (m.schema ?? "").toLowerCase();
|
|
982
|
+
if (schema) {
|
|
983
|
+
if (MART_SCHEMA_RE.test(schema)) return "mart";
|
|
984
|
+
if (INT_SCHEMA_RE.test(schema)) return "intermediate";
|
|
985
|
+
if (STG_SCHEMA_RE.test(schema)) return "staging";
|
|
986
|
+
}
|
|
987
|
+
return "other";
|
|
988
|
+
}
|
|
989
|
+
function modelGroupFolder(m) {
|
|
990
|
+
const fp = (m.filePath ?? "").toLowerCase();
|
|
991
|
+
const cleaned = fp.replace(/^models\//, "");
|
|
992
|
+
const segments = cleaned ? cleaned.split("/") : [];
|
|
993
|
+
while (segments.length > 0 && LAYER_FOLDER_TOKENS.has(segments[0])) {
|
|
994
|
+
segments.shift();
|
|
995
|
+
}
|
|
996
|
+
if (segments.length >= 2) return segments[0];
|
|
997
|
+
if (m.schema) return m.schema.toLowerCase();
|
|
998
|
+
const namePrefix = (m.name ?? "").split("_")[0];
|
|
999
|
+
if (namePrefix && namePrefix !== m.name) return namePrefix;
|
|
1000
|
+
return "(ungrouped)";
|
|
1001
|
+
}
|
|
1002
|
+
function escapePipe(s) {
|
|
1003
|
+
return s.replace(/\|/g, "\\|");
|
|
1004
|
+
}
|
|
1005
|
+
function summarizeDescription(d, max = 200) {
|
|
1006
|
+
if (!d) return null;
|
|
1007
|
+
const collapsed = d.replace(/\s+/g, " ").trim();
|
|
1008
|
+
if (!collapsed) return null;
|
|
1009
|
+
return collapsed.length > max ? `${collapsed.slice(0, max - 1)}\u2026` : collapsed;
|
|
1010
|
+
}
|
|
1011
|
+
function countBy(items, key) {
|
|
1012
|
+
const m = /* @__PURE__ */ new Map();
|
|
1013
|
+
for (const it of items) {
|
|
1014
|
+
const k = key(it);
|
|
1015
|
+
m.set(k, (m.get(k) ?? 0) + 1);
|
|
1016
|
+
}
|
|
1017
|
+
return m;
|
|
1018
|
+
}
|
|
1019
|
+
function sortedEntries(m) {
|
|
1020
|
+
return Array.from(m.entries()).sort(
|
|
1021
|
+
(a, b) => b[1] - a[1] || a[0].localeCompare(b[0])
|
|
1022
|
+
);
|
|
1023
|
+
}
|
|
1024
|
+
function modelIdentifier(m) {
|
|
1025
|
+
const parts = [m.schema, m.alias ?? m.name].filter(Boolean);
|
|
1026
|
+
return parts.join(".") || m.name || m.uniqueId || "(unknown)";
|
|
1027
|
+
}
|
|
1028
|
+
function renderInventoryOverview(sections, models, modelState, sources, exposures, metrics, layered) {
|
|
1029
|
+
sections.push("### Inventory overview", "");
|
|
1030
|
+
if (models.length === 0) {
|
|
1031
|
+
sections.push(
|
|
1032
|
+
"_The pinned production environment has no models surfaced by dbt's Discovery API. Trigger a successful run (or verify the environment ID) before generating dashboards._",
|
|
1033
|
+
""
|
|
1034
|
+
);
|
|
1035
|
+
} else {
|
|
1036
|
+
const materializations = countBy(
|
|
1037
|
+
models,
|
|
1038
|
+
(m) => (m.materializedType ?? "unknown").toLowerCase()
|
|
1039
|
+
);
|
|
1040
|
+
const layerOrder = [
|
|
1041
|
+
"mart",
|
|
1042
|
+
"intermediate",
|
|
1043
|
+
"staging",
|
|
1044
|
+
"other"
|
|
1045
|
+
];
|
|
1046
|
+
const layerSummary = layered ? layerOrder.map((l) => [l, layered[l].length]).filter(([, n]) => n > 0).map(([l, n]) => `${l} ${n}`).join(" \xB7 ") : "";
|
|
1047
|
+
const matSummary = sortedEntries(materializations).map(([k, n]) => `${k} ${n}`).join(" \xB7 ");
|
|
1048
|
+
const stateNote = modelState === "applied" ? "from latest successful prod run (columns available)" : "from project definition (no prod run yet \u2014 columns unavailable)";
|
|
1049
|
+
sections.push(`- Models: **${models.length}** _(${stateNote})_`);
|
|
1050
|
+
if (layerSummary) sections.push(` - By layer: ${layerSummary}`);
|
|
1051
|
+
if (matSummary) sections.push(` - By materialization: ${matSummary}`);
|
|
1052
|
+
}
|
|
1053
|
+
if (sources.length > 0) {
|
|
1054
|
+
const bySource = countBy(sources, (s) => s.sourceName ?? "(unnamed)");
|
|
1055
|
+
const summary = sortedEntries(bySource).map(([k, n]) => `${k} ${n}`).join(" \xB7 ");
|
|
1056
|
+
sections.push(`- Sources: **${sources.length}** \u2014 ${summary}`);
|
|
1057
|
+
} else {
|
|
1058
|
+
sections.push(`- Sources: 0`);
|
|
1059
|
+
}
|
|
1060
|
+
sections.push(
|
|
1061
|
+
`- Exposures: **${exposures.length}**${exposures.length === 0 ? " _(no BI dashboards declared in dbt yet)_" : ""}`
|
|
1062
|
+
);
|
|
1063
|
+
sections.push(
|
|
1064
|
+
`- Metrics: **${metrics.length}**${metrics.length === 0 ? " _(no semantic-layer metrics defined \u2014 aggregations need to be derived from models)_" : ""}`
|
|
1065
|
+
);
|
|
1066
|
+
sections.push("");
|
|
1067
|
+
}
|
|
1068
|
+
function renderModelLayerTiered(sections, layerLabel, blurb, models, tier1, tier2, opts) {
|
|
1069
|
+
if (models.length === 0) return;
|
|
1070
|
+
sections.push(`### ${layerLabel} (${models.length} models)`, "", blurb, "");
|
|
1071
|
+
const renderDetailed = (m) => {
|
|
1072
|
+
const id = modelIdentifier(m);
|
|
1073
|
+
const mat = m.materializedType ? ` _(${m.materializedType})_` : "";
|
|
1074
|
+
const desc = summarizeDescription(m.description, DESCRIPTION_MAX_PRIMARY);
|
|
1075
|
+
sections.push(`- \`${id}\`${mat}${desc ? ` \u2014 ${desc}` : ""}`);
|
|
1076
|
+
if (opts.showColumns) {
|
|
1077
|
+
const cols = m.catalog?.columns ?? [];
|
|
1078
|
+
if (cols.length > 0) {
|
|
1079
|
+
const rendered = cols.slice(0, MAX_COLUMNS_PER_MODEL).map((c) => c.type ? `${c.name} (${c.type})` : `${c.name}`).join(", ");
|
|
1080
|
+
const more = cols.length > MAX_COLUMNS_PER_MODEL ? ` (+${cols.length - MAX_COLUMNS_PER_MODEL} more)` : "";
|
|
1081
|
+
sections.push(` - Columns: ${rendered}${more}`);
|
|
1082
|
+
}
|
|
1083
|
+
}
|
|
1084
|
+
};
|
|
1085
|
+
const detailedSelected = /* @__PURE__ */ new Set();
|
|
1086
|
+
if (opts.group) {
|
|
1087
|
+
const buckets = /* @__PURE__ */ new Map();
|
|
1088
|
+
for (const m of models) {
|
|
1089
|
+
const k = modelGroupFolder(m);
|
|
1090
|
+
if (!buckets.has(k)) buckets.set(k, []);
|
|
1091
|
+
buckets.get(k).push(m);
|
|
1092
|
+
}
|
|
1093
|
+
const sortedBuckets = Array.from(buckets.entries()).sort(
|
|
1094
|
+
(a, b) => b[1].length - a[1].length || a[0].localeCompare(b[0])
|
|
1095
|
+
);
|
|
1096
|
+
const useHeadings = sortedBuckets.length > 1;
|
|
1097
|
+
const perFolderCap = Math.max(
|
|
1098
|
+
2,
|
|
1099
|
+
Math.ceil(tier1 / Math.max(1, sortedBuckets.length))
|
|
1100
|
+
);
|
|
1101
|
+
let budget = tier1;
|
|
1102
|
+
for (const [folder, bucket] of sortedBuckets) {
|
|
1103
|
+
if (useHeadings) {
|
|
1104
|
+
sections.push(`#### ${folder} (${bucket.length})`);
|
|
1105
|
+
}
|
|
1106
|
+
const take = Math.min(bucket.length, perFolderCap, budget);
|
|
1107
|
+
const detailedHere = bucket.slice(0, take);
|
|
1108
|
+
for (const m of detailedHere) {
|
|
1109
|
+
renderDetailed(m);
|
|
1110
|
+
detailedSelected.add(m);
|
|
1111
|
+
}
|
|
1112
|
+
budget -= detailedHere.length;
|
|
1113
|
+
if (bucket.length > detailedHere.length && useHeadings) {
|
|
1114
|
+
sections.push(
|
|
1115
|
+
`- _\u2026and ${bucket.length - detailedHere.length} more in this folder._`
|
|
1116
|
+
);
|
|
1117
|
+
}
|
|
1118
|
+
sections.push("");
|
|
1119
|
+
}
|
|
1120
|
+
} else {
|
|
1121
|
+
const detailed = models.slice(0, tier1);
|
|
1122
|
+
for (const m of detailed) {
|
|
1123
|
+
renderDetailed(m);
|
|
1124
|
+
detailedSelected.add(m);
|
|
1125
|
+
}
|
|
1126
|
+
if (detailed.length > 0) sections.push("");
|
|
1127
|
+
}
|
|
1128
|
+
const remaining = models.filter((m) => !detailedSelected.has(m));
|
|
1129
|
+
const named = remaining.slice(0, tier2);
|
|
1130
|
+
const overflow = remaining.length - named.length;
|
|
1131
|
+
if (named.length > 0) {
|
|
1132
|
+
sections.push(
|
|
1133
|
+
`_Also available (name only \u2014 ask via the dbt request tool for details):_`
|
|
1134
|
+
);
|
|
1135
|
+
sections.push(named.map((m) => `\`${m.name}\``).join(", "), "");
|
|
1136
|
+
}
|
|
1137
|
+
if (overflow > 0) {
|
|
1138
|
+
sections.push(
|
|
1139
|
+
`_\u2026and ${overflow} more ${layerLabel.toLowerCase()} models not surfaced here. The dbt request tool can enumerate them on demand._`,
|
|
1140
|
+
""
|
|
1141
|
+
);
|
|
1142
|
+
}
|
|
1143
|
+
}
|
|
1144
|
+
function renderStagingLayer(sections, models) {
|
|
1145
|
+
if (models.length === 0) return;
|
|
1146
|
+
sections.push(
|
|
1147
|
+
`### Staging layer (${models.length} models)`,
|
|
1148
|
+
"",
|
|
1149
|
+
"_Source-aligned cleanups (one staging model \u2248 one raw table). Almost never the right pick for dashboards \u2014 use mart or intermediate instead. Listed by group count only to keep this summary compact; the dbt request tool can enumerate individual names if needed._",
|
|
1150
|
+
""
|
|
1151
|
+
);
|
|
1152
|
+
const groups = /* @__PURE__ */ new Map();
|
|
1153
|
+
for (const m of models) {
|
|
1154
|
+
const k = modelGroupFolder(m);
|
|
1155
|
+
if (!groups.has(k)) groups.set(k, []);
|
|
1156
|
+
groups.get(k).push(m);
|
|
1157
|
+
}
|
|
1158
|
+
const entries = Array.from(groups.entries()).sort(
|
|
1159
|
+
(a, b) => b[1].length - a[1].length || a[0].localeCompare(b[0])
|
|
1160
|
+
);
|
|
1161
|
+
for (const [folder, bucket] of entries) {
|
|
1162
|
+
sections.push(`- **${folder}**: ${bucket.length}`);
|
|
1163
|
+
}
|
|
1164
|
+
sections.push("");
|
|
1165
|
+
}
|
|
1166
|
+
function renderExposures(sections, exposures) {
|
|
1167
|
+
if (exposures.length === 0) return;
|
|
1168
|
+
sections.push(
|
|
1169
|
+
`### Exposures (existing dashboards / downstream apps) \u2014 ${exposures.length}`,
|
|
1170
|
+
"",
|
|
1171
|
+
"_Already declared by the team. Treat them as ground truth for what stakeholders care about; avoid proposing exact duplicates._",
|
|
1172
|
+
""
|
|
1173
|
+
);
|
|
1174
|
+
for (const e of exposures.slice(0, MAX_EXPOSURE_DETAIL)) {
|
|
1175
|
+
const title = e.label || e.name || e.uniqueId || "(unknown)";
|
|
1176
|
+
const meta = [
|
|
1177
|
+
e.exposureType ? `type: ${e.exposureType}` : null,
|
|
1178
|
+
e.maturity ? `maturity: ${e.maturity}` : null
|
|
1179
|
+
].filter(Boolean).join(", ");
|
|
1180
|
+
sections.push(`- **${title}**${meta ? ` (${meta})` : ""}`);
|
|
1181
|
+
const desc = summarizeDescription(e.description, DESCRIPTION_MAX_SECONDARY);
|
|
1182
|
+
if (desc) sections.push(` - ${desc}`);
|
|
1183
|
+
if (e.url) sections.push(` - URL: ${e.url}`);
|
|
1184
|
+
const parents = (e.parents ?? []).map((p) => p.name).filter((n) => !!n).slice(0, 6);
|
|
1185
|
+
if (parents.length > 0) {
|
|
1186
|
+
sections.push(` - Built on: ${parents.join(", ")}`);
|
|
1187
|
+
}
|
|
1188
|
+
}
|
|
1189
|
+
if (exposures.length > MAX_EXPOSURE_DETAIL) {
|
|
1190
|
+
sections.push(
|
|
1191
|
+
`- _\u2026and ${exposures.length - MAX_EXPOSURE_DETAIL} more \u2014 fetch via the dbt request tool._`
|
|
1192
|
+
);
|
|
1193
|
+
}
|
|
1194
|
+
sections.push("");
|
|
1195
|
+
}
|
|
1196
|
+
function renderMetrics(sections, metrics) {
|
|
1197
|
+
if (metrics.length === 0) return;
|
|
1198
|
+
sections.push(
|
|
1199
|
+
`### Metrics \u2014 ${metrics.length}`,
|
|
1200
|
+
"",
|
|
1201
|
+
"_Semantic-layer definitions vetted by the team. Wire them straight into KPI tiles instead of re-deriving the same aggregation in a chart._",
|
|
1202
|
+
""
|
|
1203
|
+
);
|
|
1204
|
+
for (const m of metrics.slice(0, MAX_METRIC_DETAIL)) {
|
|
1205
|
+
const title = m.label || m.name || m.uniqueId || "(unknown)";
|
|
1206
|
+
const meta = m.type ? ` _(type: ${m.type})_` : "";
|
|
1207
|
+
sections.push(`- **${title}**${meta}`);
|
|
1208
|
+
const desc = summarizeDescription(m.description, DESCRIPTION_MAX_SECONDARY);
|
|
1209
|
+
if (desc) sections.push(` - ${desc}`);
|
|
1210
|
+
if (m.formula) sections.push(` - Formula: \`${m.formula}\``);
|
|
1211
|
+
if (m.filter) sections.push(` - Filter: \`${m.filter}\``);
|
|
1212
|
+
}
|
|
1213
|
+
if (metrics.length > MAX_METRIC_DETAIL) {
|
|
1214
|
+
sections.push(
|
|
1215
|
+
`- _\u2026and ${metrics.length - MAX_METRIC_DETAIL} more \u2014 fetch via the dbt request tool._`
|
|
1216
|
+
);
|
|
1217
|
+
}
|
|
1218
|
+
sections.push("");
|
|
1219
|
+
}
|
|
1220
|
+
function renderSources(sections, sources) {
|
|
1221
|
+
if (sources.length === 0) return;
|
|
1222
|
+
sections.push(
|
|
1223
|
+
`### Sources (${sources.length})`,
|
|
1224
|
+
"",
|
|
1225
|
+
"_Upstream raw tables feeding the staging layer. Reference them for lineage / freshness, not for direct dashboard reads._",
|
|
1226
|
+
""
|
|
1227
|
+
);
|
|
1228
|
+
const grouped = /* @__PURE__ */ new Map();
|
|
1229
|
+
for (const s of sources) {
|
|
1230
|
+
const k = s.sourceName ?? "(unnamed)";
|
|
1231
|
+
if (!grouped.has(k)) grouped.set(k, []);
|
|
1232
|
+
grouped.get(k).push(s);
|
|
1233
|
+
}
|
|
1234
|
+
const entries = Array.from(grouped.entries()).sort(
|
|
1235
|
+
(a, b) => b[1].length - a[1].length || a[0].localeCompare(b[0])
|
|
1236
|
+
);
|
|
1237
|
+
const enumerateNames = sources.length <= SOURCE_ENUMERATION_THRESHOLD;
|
|
1238
|
+
for (const [src, bucket] of entries) {
|
|
1239
|
+
const warehouse = bucket[0]?.database && bucket[0]?.schema ? ` \u2014 \`${bucket[0].database}.${bucket[0].schema}\`` : "";
|
|
1240
|
+
if (enumerateNames) {
|
|
1241
|
+
const names = bucket.map((s) => `\`${s.name}\``).join(", ");
|
|
1242
|
+
sections.push(`- **${src}** (${bucket.length})${warehouse}: ${names}`);
|
|
1243
|
+
} else {
|
|
1244
|
+
sections.push(`- **${src}** (${bucket.length})${warehouse}`);
|
|
1245
|
+
}
|
|
1246
|
+
}
|
|
1247
|
+
if (!enumerateNames) {
|
|
1248
|
+
sections.push(
|
|
1249
|
+
"",
|
|
1250
|
+
`_Per-source table names omitted because the total exceeds ${SOURCE_ENUMERATION_THRESHOLD}. Use the dbt request tool to enumerate individual sources._`
|
|
1251
|
+
);
|
|
1252
|
+
}
|
|
1253
|
+
const freshnessIssues = sources.filter(
|
|
1254
|
+
(s) => s.freshness?.freshnessStatus && !["Pass", "Unconfigured"].includes(s.freshness.freshnessStatus)
|
|
1255
|
+
);
|
|
1256
|
+
if (freshnessIssues.length > 0) {
|
|
1257
|
+
sections.push(
|
|
1258
|
+
"",
|
|
1259
|
+
`> \u26A0\uFE0F ${freshnessIssues.length} source(s) failing freshness \u2014 downstream metrics may be stale.`
|
|
1260
|
+
);
|
|
1261
|
+
}
|
|
1262
|
+
sections.push("");
|
|
650
1263
|
}
|
|
651
1264
|
var dbtSetupFlow = {
|
|
652
1265
|
initialState: () => ({}),
|
|
@@ -678,7 +1291,7 @@ var dbtSetupFlow = {
|
|
|
678
1291
|
throw new Error("dbt setup: incomplete state on finalize");
|
|
679
1292
|
}
|
|
680
1293
|
const allProjects = await listAllProjects(rt.params);
|
|
681
|
-
const
|
|
1294
|
+
const projectById = new Map(allProjects.map((p) => [String(p.id), p]));
|
|
682
1295
|
const targetIds = await resolveSetupSelection({
|
|
683
1296
|
selected: state.projects,
|
|
684
1297
|
allSentinel: ALL_PROJECTS,
|
|
@@ -689,25 +1302,97 @@ var dbtSetupFlow = {
|
|
|
689
1302
|
const accountId = rt.params[parameters.accountId.slug];
|
|
690
1303
|
const prodEnvId = rt.params[parameters.prodEnvId.slug];
|
|
691
1304
|
if (accountId) sections.push(`- Account: \`${accountId}\``);
|
|
692
|
-
if (prodEnvId) sections.push(`-
|
|
693
|
-
sections.push(
|
|
694
|
-
|
|
1305
|
+
if (prodEnvId) sections.push(`- Production environment: \`${prodEnvId}\``);
|
|
1306
|
+
sections.push(
|
|
1307
|
+
"",
|
|
1308
|
+
"_The downstream coding agent should use this summary as the canonical map of dbt assets for dashboard / app proposals. Reach for the mart layer first; fall back to intermediate models for custom cuts; consult sources only for lineage._",
|
|
1309
|
+
""
|
|
1310
|
+
);
|
|
1311
|
+
if (targetIds.length === 0) {
|
|
695
1312
|
sections.push("_No projects selected._", "");
|
|
696
1313
|
return sections.join("\n");
|
|
697
1314
|
}
|
|
698
|
-
sections.push("
|
|
699
|
-
sections.push("
|
|
700
|
-
|
|
701
|
-
|
|
1315
|
+
sections.push("### Selected projects", "");
|
|
1316
|
+
sections.push("| Project | Repository | Recent runs |");
|
|
1317
|
+
sections.push("|---------|------------|-------------|");
|
|
1318
|
+
const runCounts = await Promise.all(
|
|
1319
|
+
targetIds.map((id) => fetchRecentRunCount(rt.params, id))
|
|
1320
|
+
);
|
|
1321
|
+
for (let i = 0; i < targetIds.length; i++) {
|
|
1322
|
+
const id = targetIds[i];
|
|
1323
|
+
const p = projectById.get(id);
|
|
1324
|
+
const runs = runCounts[i] == null ? "-" : String(runCounts[i]);
|
|
702
1325
|
if (!p) {
|
|
703
|
-
sections.push(`| ${id} | - |`);
|
|
1326
|
+
sections.push(`| ${id} | - | ${runs} |`);
|
|
704
1327
|
continue;
|
|
705
1328
|
}
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
1329
|
+
sections.push(
|
|
1330
|
+
`| ${escapePipe(p.name)} | ${escapePipe(repoLabel(p))} | ${runs} |`
|
|
1331
|
+
);
|
|
709
1332
|
}
|
|
710
1333
|
sections.push("");
|
|
1334
|
+
const [{ models, state: modelState }, sources, exposures, metrics] = await Promise.all([
|
|
1335
|
+
fetchModels(rt.params),
|
|
1336
|
+
fetchSources(rt.params),
|
|
1337
|
+
fetchExposures(rt.params),
|
|
1338
|
+
fetchMetrics(rt.params)
|
|
1339
|
+
]);
|
|
1340
|
+
const layered = models.length > 0 ? layerModels(models, exposures) : null;
|
|
1341
|
+
renderInventoryOverview(
|
|
1342
|
+
sections,
|
|
1343
|
+
models,
|
|
1344
|
+
modelState,
|
|
1345
|
+
sources,
|
|
1346
|
+
exposures,
|
|
1347
|
+
metrics,
|
|
1348
|
+
layered
|
|
1349
|
+
);
|
|
1350
|
+
if (layered) {
|
|
1351
|
+
const byMartLikelihood = (a, b) => martLikelihoodScore(b) - martLikelihoodScore(a) || (a.name ?? "").localeCompare(b.name ?? "");
|
|
1352
|
+
const byName = (a, b) => (a.name ?? "").localeCompare(b.name ?? "");
|
|
1353
|
+
layered.mart.sort(byMartLikelihood);
|
|
1354
|
+
layered.other.sort(byMartLikelihood);
|
|
1355
|
+
layered.intermediate.sort(byName);
|
|
1356
|
+
layered.staging.sort(byName);
|
|
1357
|
+
renderExposures(sections, exposures);
|
|
1358
|
+
renderMetrics(sections, metrics);
|
|
1359
|
+
const martLabel = layered.mart.length === 0 && layered.other.length > 0 ? "Models (no layer convention detected \u2014 treat as candidate dashboard inputs)" : "Mart layer \u2014 start here for dashboards";
|
|
1360
|
+
const martBlurb = layered.mart.length === 0 && layered.other.length > 0 ? "_This project doesn't follow the staging/intermediate/marts (or bronze/silver/gold) convention, so layer inference fell back to surfacing every non-staging model here. Use descriptions, columns, and materialization (`table`/`incremental` are stronger dashboard candidates than `view`) to pick the right inputs._" : "_Curated, business-facing tables. Prefer them as the primary input for new dashboards and apps. Reach for the intermediate layer only when a mart model is missing the cut you need._";
|
|
1361
|
+
const martModels = layered.mart.length === 0 && layered.other.length > 0 ? layered.other : layered.mart;
|
|
1362
|
+
renderModelLayerTiered(
|
|
1363
|
+
sections,
|
|
1364
|
+
martLabel,
|
|
1365
|
+
martBlurb,
|
|
1366
|
+
martModels,
|
|
1367
|
+
MART_TIER1_DETAIL,
|
|
1368
|
+
MART_TIER2_NAMES,
|
|
1369
|
+
{ showColumns: modelState === "applied", group: true }
|
|
1370
|
+
);
|
|
1371
|
+
renderModelLayerTiered(
|
|
1372
|
+
sections,
|
|
1373
|
+
"Intermediate layer",
|
|
1374
|
+
"_Joined / derived models used to build the mart layer. Useful when you need a calculation the mart layer doesn't expose; avoid wiring them directly into a dashboard if a mart model already covers the use case._",
|
|
1375
|
+
layered.intermediate,
|
|
1376
|
+
INTERMEDIATE_TIER1_DETAIL,
|
|
1377
|
+
INTERMEDIATE_TIER2_NAMES,
|
|
1378
|
+
{ showColumns: false, group: false }
|
|
1379
|
+
);
|
|
1380
|
+
const otherForOwnSection = layered.mart.length > 0 ? layered.other : [];
|
|
1381
|
+
renderModelLayerTiered(
|
|
1382
|
+
sections,
|
|
1383
|
+
"Other models",
|
|
1384
|
+
"_Models that don't match the staging / intermediate / mart convention. Inspect case-by-case before reusing._",
|
|
1385
|
+
otherForOwnSection,
|
|
1386
|
+
OTHER_TIER1_DETAIL,
|
|
1387
|
+
OTHER_TIER2_NAMES,
|
|
1388
|
+
{ showColumns: false, group: false }
|
|
1389
|
+
);
|
|
1390
|
+
renderStagingLayer(sections, layered.staging);
|
|
1391
|
+
} else {
|
|
1392
|
+
renderExposures(sections, exposures);
|
|
1393
|
+
renderMetrics(sections, metrics);
|
|
1394
|
+
}
|
|
1395
|
+
renderSources(sections, sources);
|
|
711
1396
|
return sections.join("\n");
|
|
712
1397
|
}
|
|
713
1398
|
};
|
|
@@ -812,7 +1497,7 @@ var dbtConnector = new ConnectorPlugin({
|
|
|
812
1497
|
systemPrompt: {
|
|
813
1498
|
en: `### Tools
|
|
814
1499
|
|
|
815
|
-
- \`
|
|
1500
|
+
- \`connector_dbt_request\`: The only way to call the dbt Cloud Discovery API (GraphQL). Use it to explore models, sources, tests, metrics, columns, and lineage. The \`{environmentId}\` placeholder in GraphQL variables is automatically replaced with the prod-env-id. See the dbt Cloud Discovery API Reference below for available GraphQL queries.
|
|
816
1501
|
|
|
817
1502
|
### Business Logic
|
|
818
1503
|
|
|
@@ -921,7 +1606,7 @@ query($environmentId: BigInt!, $uniqueIds: [String!]!) {
|
|
|
921
1606
|
- Get \`ancestors { uniqueId name }\` or \`children { uniqueId name }\` within node`,
|
|
922
1607
|
ja: `### \u30C4\u30FC\u30EB
|
|
923
1608
|
|
|
924
|
-
- \`
|
|
1609
|
+
- \`connector_dbt_request\`: dbt Cloud Discovery API (GraphQL) \u3092\u547C\u3073\u51FA\u3059\u552F\u4E00\u306E\u624B\u6BB5\u3067\u3059\u3002\u30E2\u30C7\u30EB\u3001\u30BD\u30FC\u30B9\u3001\u30C6\u30B9\u30C8\u3001\u30E1\u30C8\u30EA\u30AF\u30B9\u3001\u30AB\u30E9\u30E0\u3001\u30EA\u30CD\u30FC\u30B8\u306E\u63A2\u7D22\u306B\u4F7F\u7528\u3057\u307E\u3059\u3002GraphQL\u5909\u6570\u5185\u306E \`{environmentId}\` \u30D7\u30EC\u30FC\u30B9\u30DB\u30EB\u30C0\u306Fprod-env-id\u3067\u81EA\u52D5\u7684\u306B\u7F6E\u63DB\u3055\u308C\u307E\u3059\u3002\u5229\u7528\u53EF\u80FD\u306AGraphQL\u30AF\u30A8\u30EA\u306F\u4E0B\u90E8\u306E\u300Cdbt Cloud Discovery API \u30EA\u30D5\u30A1\u30EC\u30F3\u30B9\u300D\u3092\u53C2\u7167\u3057\u3066\u304F\u3060\u3055\u3044\u3002
|
|
925
1610
|
|
|
926
1611
|
### Business Logic
|
|
927
1612
|
|