@microsoft/rayfin-semantic-model-pack 0.0.0-stage → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG +14 -0
- package/LICENSE +21 -0
- package/README.md +78 -2
- package/kit/hooks/use-semantic-model-query.spec.ts +361 -0
- package/kit/hooks/use-semantic-model-query.ts +224 -0
- package/kit/scripts/rayfin-connector.mjs +265 -0
- package/kit/scripts/rayfin-connector.test.mjs +243 -0
- package/kit/scripts/schema-sql-batch.test.mjs +2097 -0
- package/kit/scripts/schema-sql-cli.test.mjs +488 -0
- package/kit/scripts/schema-sql.mjs +2650 -0
- package/kit/scripts/schema-sql.test.mjs +1387 -0
- package/kit/scripts/schema.mjs +166 -0
- package/kit/scripts/schema.test.mjs +271 -0
- package/pack.json +44 -0
- package/package.json +12 -3
- package/skills/analytics/SKILL.md +461 -0
- package/skills/dax-authoring/SKILL.md +170 -0
- package/skills/dax-authoring/references/dax-core-reference.md +862 -0
- package/skills/dax-authoring/references/dax-query-patterns.md +396 -0
- package/skills/dax-authoring/references/dax-time-intelligence.md +347 -0
- package/skills/query-design/SKILL.md +114 -0
- package/skills/query-design/references/anti-patterns.md +285 -0
- package/skills/query-design/references/filter-strategy.md +68 -0
- package/skills/query-design/references/format-strings.md +62 -0
- package/skills/query-design/references/highlight-queries.md +122 -0
- package/skills/query-design/references/multi-grain-patterns.md +159 -0
- package/skills/schema-discovery/SKILL.md +382 -0
- package/skills/schema-discovery/references/discovery-queries.md +153 -0
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
# Multi-Grain Patterns
|
|
2
|
+
|
|
3
|
+
When a component needs data at multiple grains (e.g., region detail + grand total, monthly trend + YTD), use separate `.dax` files and separate hook calls.
|
|
4
|
+
|
|
5
|
+
## Contents
|
|
6
|
+
|
|
7
|
+
- [File Organization](#file-organization) — `.dax` + `.ts` layout per visualization
|
|
8
|
+
- [DAX Query Shapes](#dax-query-shapes) — separate detail and one-row summary queries
|
|
9
|
+
- [Rendering in VegaVisual (multi-DataTable)](#rendering-in-vegavisual-multi-datatable) — pass named datasets, layer in spec
|
|
10
|
+
- [Rendering in DataGrid (provided grand total)](#rendering-in-datagrid-provided-grand-total) — pass the summary `DataTable` through `grandTotals`
|
|
11
|
+
- [Consistency Rule](#consistency-rule) — shared filters, measures, scope across split-grain queries
|
|
12
|
+
|
|
13
|
+
## File Organization
|
|
14
|
+
|
|
15
|
+
```text
|
|
16
|
+
packages/frontend/src/queries/sales/
|
|
17
|
+
├── revenue-by-region.dax # Detail grain
|
|
18
|
+
├── revenue-by-region.json # Vega-Lite spec (layers both datasets)
|
|
19
|
+
├── revenue-by-region.ts # Factory: returns detailQuery + vegaLiteSpec
|
|
20
|
+
├── revenue-total.dax # Summary grain (single-row total)
|
|
21
|
+
├── revenue-total.ts # Factory: returns totalQuery (no spec needed)
|
|
22
|
+
└── index.ts
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
## DAX Query Shapes
|
|
26
|
+
|
|
27
|
+
Keep the detail and summary grains in separate query files.
|
|
28
|
+
|
|
29
|
+
**Detail query:**
|
|
30
|
+
|
|
31
|
+
```dax
|
|
32
|
+
EVALUATE
|
|
33
|
+
SUMMARIZECOLUMNS(
|
|
34
|
+
'Region'[Name],
|
|
35
|
+
"Revenue", [Total Revenue],
|
|
36
|
+
"Customer Count", DISTINCTCOUNT('Sales'[Customer Key])
|
|
37
|
+
)
|
|
38
|
+
ORDER BY 'Region'[Name]
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
**Summary query:**
|
|
42
|
+
|
|
43
|
+
```dax
|
|
44
|
+
EVALUATE
|
|
45
|
+
ROW(
|
|
46
|
+
"Revenue", [Total Revenue],
|
|
47
|
+
"Customer Count", DISTINCTCOUNT('Sales'[Customer Key])
|
|
48
|
+
)
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
The `Customer Count` grand total must come from the summary query. Summing the per-region distinct counts would double-count customers that appear in multiple regions.
|
|
52
|
+
|
|
53
|
+
As with detail data, copy the semantic model's format strings into the summary factory's `columnMetadata`; `toDataTable()` places them on the `DataTable`:
|
|
54
|
+
|
|
55
|
+
```typescript
|
|
56
|
+
const columnMetadata: ColumnMetadataMap = {
|
|
57
|
+
"[Revenue]": { name: "Revenue", displayName: "Revenue", format: "$#,##0.00" },
|
|
58
|
+
"[Customer Count]": { name: "Customer Count", displayName: "Customer Count", format: "#,##0" },
|
|
59
|
+
};
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
## Rendering in VegaVisual (multi-DataTable)
|
|
63
|
+
|
|
64
|
+
Use two hook calls and pass both tables as named datasets. The spec references each by name, so no TypeScript stitching is needed:
|
|
65
|
+
|
|
66
|
+
```typescript
|
|
67
|
+
// Two hook calls, two DataTables. Both factories target the same model,
|
|
68
|
+
// so the two connection objects are equivalent — we destructure both
|
|
69
|
+
// (factory signature requires it) but use one for both hooks.
|
|
70
|
+
const { connection, query: detailQuery, columnMetadata: detailMeta, vegaLiteSpec } = revenueByRegion();
|
|
71
|
+
const { connection: _summaryConn, query: totalQuery, columnMetadata: totalMeta } = revenueTotal();
|
|
72
|
+
|
|
73
|
+
const detail = useSemanticModelQuery({ connection, query: detailQuery });
|
|
74
|
+
const summary = useSemanticModelQuery({ connection, query: totalQuery });
|
|
75
|
+
|
|
76
|
+
// After handling both hooks' loading and error states:
|
|
77
|
+
const detailTable = toDataTable(detail.data.table, detailMeta);
|
|
78
|
+
const summaryTable = toDataTable(summary.data.table, totalMeta);
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
```tsx
|
|
82
|
+
<VegaVisual
|
|
83
|
+
spec={vegaLiteSpec}
|
|
84
|
+
data={{ detail: detailTable, summary: summaryTable }}
|
|
85
|
+
theme={theme}
|
|
86
|
+
/>
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
The Vega-Lite spec (in the `.json` file) layers the two datasets:
|
|
90
|
+
|
|
91
|
+
```json
|
|
92
|
+
{
|
|
93
|
+
"layer": [
|
|
94
|
+
{
|
|
95
|
+
"data": { "name": "detail" },
|
|
96
|
+
"mark": "bar",
|
|
97
|
+
"encoding": {
|
|
98
|
+
"x": { "field": "RegionName", "type": "nominal" },
|
|
99
|
+
"y": { "field": "Revenue", "type": "quantitative" }
|
|
100
|
+
}
|
|
101
|
+
},
|
|
102
|
+
{
|
|
103
|
+
"data": { "name": "summary" },
|
|
104
|
+
"mark": { "type": "rule", "color": "firebrick", "strokeDash": [4, 4] },
|
|
105
|
+
"encoding": {
|
|
106
|
+
"y": { "field": "Revenue", "type": "quantitative" }
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
]
|
|
110
|
+
}
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
This pattern works for: reference lines (average, target), cross-highlighting between datasets, annotations on top of detail data.
|
|
114
|
+
|
|
115
|
+
## Rendering in DataGrid (provided grand total)
|
|
116
|
+
|
|
117
|
+
Pass the detail and one-row summary tables separately. DataGrid owns grand-total rendering; do not append the summary to the body rows or use `cellRenderer` for the totals row.
|
|
118
|
+
|
|
119
|
+
DataGrid filtering is enabled by default. Derive the summary query from grid filter state so the provided total matches the visible rows:
|
|
120
|
+
|
|
121
|
+
```tsx
|
|
122
|
+
import type { CellValue } from "@microsoft/fabric-datagrid";
|
|
123
|
+
|
|
124
|
+
const [gridFilters, setGridFilters] = useState<Record<string, CellValue[]>>({});
|
|
125
|
+
const { query: totalQuery, columnMetadata: totalMeta } = revenueTotal({ filters: gridFilters });
|
|
126
|
+
const summary = useSemanticModelQuery({ connection, query: totalQuery });
|
|
127
|
+
|
|
128
|
+
if (summary.error) return <ErrorMessage message={summary.error.message} />;
|
|
129
|
+
|
|
130
|
+
const filteredSummaryTable =
|
|
131
|
+
!summary.isLoading &&
|
|
132
|
+
summary.data?.status === "success"
|
|
133
|
+
? toDataTable(summary.data.table, totalMeta)
|
|
134
|
+
: undefined;
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
```tsx
|
|
138
|
+
<DataGrid
|
|
139
|
+
data={detailTable}
|
|
140
|
+
grandTotals={{ position: "bottom", data: filteredSummaryTable }}
|
|
141
|
+
onFilterChange={(columnId, selectedValues) =>
|
|
142
|
+
setGridFilters((current) => ({ ...current, [columnId]: selectedValues }))
|
|
143
|
+
}
|
|
144
|
+
theme={theme}
|
|
145
|
+
/>
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
Changing `gridFilters` re-queries the summary while DataGrid filters detail rows locally. The summary factory must translate each filterable column ID to its DAX predicate rather than interpolate column IDs directly. See the visuals skill's [DataGrid reference](../../visuals/references/data-grid-visual.md#grand-totals).
|
|
149
|
+
|
|
150
|
+
## Consistency Rule
|
|
151
|
+
|
|
152
|
+
When splitting a visualization across multiple queries, all queries must share the same semantic contract:
|
|
153
|
+
|
|
154
|
+
- Same measures and application-level DAX filters; derive both queries from the same inputs
|
|
155
|
+
- Local DataGrid filters remain enabled; pass `onFilterChange` selections to the summary factory so the provided total matches the visible rows
|
|
156
|
+
- Summary metadata names match the detail table and include formats
|
|
157
|
+
- Use `ROW(...)` for the summary so it returns exactly one row and preserves BLANK values; `SUMMARIZECOLUMNS` can return zero rows when all measures are BLANK
|
|
158
|
+
|
|
159
|
+
Each `useSemanticModelQuery` call is a separate SDK round trip, and every local filter change re-executes the summary query.
|
|
@@ -0,0 +1,382 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: schema-discovery
|
|
3
|
+
description: >
|
|
4
|
+
Discover Power BI semantic-model tables, columns, measures, relationships,
|
|
5
|
+
and column statistics through the analytics pack's local SQLite catalog.
|
|
6
|
+
Use advanced INFO queries only when the fixed catalog does not contain the
|
|
7
|
+
required metadata.
|
|
8
|
+
---
|
|
9
|
+
|
|
10
|
+
# Schema discovery
|
|
11
|
+
|
|
12
|
+
## Use the analytics catalog
|
|
13
|
+
|
|
14
|
+
This skill is the schema-discovery route for the
|
|
15
|
+
[`analytics`](../analytics/SKILL.md) pack.
|
|
16
|
+
|
|
17
|
+
It is a preview capability and ships off. `schema:sql` exists only in an app
|
|
18
|
+
that opted in before applying the pack, so check `package.json` for the script
|
|
19
|
+
before planning around it. To turn it on, add the flag and re-apply:
|
|
20
|
+
|
|
21
|
+
```sh
|
|
22
|
+
# rayfin/.env
|
|
23
|
+
RAYFIN_FEATURE_FLAGS=schema-sqlite
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
```sh
|
|
27
|
+
npm run pack:add -- analytics
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
Without it, discover the model through the read-only queries in
|
|
31
|
+
[`discovery-queries.md`](./references/discovery-queries.md) instead, and say
|
|
32
|
+
that the catalog is unavailable rather than reporting the model as empty.
|
|
33
|
+
|
|
34
|
+
Apply the connectors and analytics packs, connect the intended semantic model,
|
|
35
|
+
then query the local catalog:
|
|
36
|
+
|
|
37
|
+
```sh
|
|
38
|
+
npm run --silent schema:sql -- "SELECT status, incomplete, notes FROM catalog_status"
|
|
39
|
+
npm run --silent schema:sql -- --search "requested business concept"
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
The first command builds the catalog automatically when it is missing or older
|
|
43
|
+
than 15 minutes.
|
|
44
|
+
An explicit rebuild is available when freshness matters:
|
|
45
|
+
|
|
46
|
+
```sh
|
|
47
|
+
npm run schema:refresh
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
One refresh runs five schema-only source queries from a hardcoded list, plus
|
|
51
|
+
retries if the model throttles:
|
|
52
|
+
|
|
53
|
+
- `INFO.VIEW.TABLES()`
|
|
54
|
+
- `INFO.VIEW.COLUMNS()`
|
|
55
|
+
- `INFO.VIEW.MEASURES()`
|
|
56
|
+
- `INFO.VIEW.RELATIONSHIPS()`
|
|
57
|
+
- `COLUMNSTATISTICS()`
|
|
58
|
+
|
|
59
|
+
The builder does not accept agent-supplied DAX.
|
|
60
|
+
Business-data queries cannot enter the catalog-build path.
|
|
61
|
+
After those five requests, discovery is read-only SQL against local SQLite.
|
|
62
|
+
|
|
63
|
+
This resolves the apparent conflict between full and progressive discovery.
|
|
64
|
+
The fixed sources are materialized once for local reuse, but each agent result
|
|
65
|
+
must still be a narrow SQL projection containing only evidence needed for the
|
|
66
|
+
current task.
|
|
67
|
+
Do not dump whole catalog tables into the conversation.
|
|
68
|
+
|
|
69
|
+
## Catalog contents
|
|
70
|
+
|
|
71
|
+
The database is `.schema/models/<itemId>/model.sqlite`.
|
|
72
|
+
Its tables preserve the model's native DAX column names:
|
|
73
|
+
|
|
74
|
+
| SQLite table | Contents |
|
|
75
|
+
| --- | --- |
|
|
76
|
+
| `catalog_status` | Persistent `complete` or `incomplete` build status and notes |
|
|
77
|
+
| `info_view_tables` | One row per semantic-model table |
|
|
78
|
+
| `info_view_columns` | One row per model column |
|
|
79
|
+
| `info_view_measures` | One row per shipped measure |
|
|
80
|
+
| `info_view_relationships` | One row per relationship |
|
|
81
|
+
| `column_statistics` | Actual Min/Max values, cardinality, and maximum text length |
|
|
82
|
+
| `semantic_search` | Searchable projection of tables, columns, measures, and relationships |
|
|
83
|
+
|
|
84
|
+
`column_statistics` is derived from real model rows.
|
|
85
|
+
The catalog is customer data, not metadata-only content.
|
|
86
|
+
The `.schema/` directory is gitignored and must never be copied, committed, or
|
|
87
|
+
exposed.
|
|
88
|
+
|
|
89
|
+
Read `.schema/models/<itemId>/catalog.md` once when orientation is needed.
|
|
90
|
+
It records every table and column and prominently marks an incomplete catalog.
|
|
91
|
+
|
|
92
|
+
## Completeness and truncation
|
|
93
|
+
|
|
94
|
+
Catalog completeness and SQL-result completeness are separate:
|
|
95
|
+
|
|
96
|
+
- `catalog_status.status = 'incomplete'` means at least one source view failed.
|
|
97
|
+
- `truncated: true` means the local SQL output was bounded by row count or byte size.
|
|
98
|
+
- `truncated: false` only means that one SQL result is complete.
|
|
99
|
+
It does not prove the catalog itself is complete.
|
|
100
|
+
|
|
101
|
+
Always check `catalog_status` before using absence as evidence.
|
|
102
|
+
If the catalog is incomplete, read its notes and run `npm run schema:refresh`
|
|
103
|
+
once to retry.
|
|
104
|
+
Concurrent refreshes serialize per model and reuse a complete catalog built by
|
|
105
|
+
the process that won the lock.
|
|
106
|
+
An incomplete refresh never replaces an existing complete catalog.
|
|
107
|
+
If it remains incomplete, continue only with the views that are present and
|
|
108
|
+
state which evidence is missing.
|
|
109
|
+
Do not bypass an incomplete catalog with exploratory `INFO.VIEW.*` calls.
|
|
110
|
+
|
|
111
|
+
`complete` describes which source views answered, not how much of each one you
|
|
112
|
+
were allowed to see. `INFO.VIEW.MEASURES()` returns `Expression` only to a
|
|
113
|
+
caller with write permission, so on a model you have Build access to, measure
|
|
114
|
+
expressions can come back blank while the catalog still reads `complete`. Treat
|
|
115
|
+
a blank `[Expression]` as unknown rather than as proof the measure has no
|
|
116
|
+
definition, and say so instead of rebuilding its logic from physical columns.
|
|
117
|
+
|
|
118
|
+
A SQL result with `truncated: true` cannot be used to choose model objects or
|
|
119
|
+
author DAX.
|
|
120
|
+
Narrow the `WHERE` clause and, for ordinary catalog tables, the projection.
|
|
121
|
+
Queries selecting model tables automatically append compact owned-measure names
|
|
122
|
+
and display folders, so a narrower projection does not reduce that enrichment;
|
|
123
|
+
narrow the selected table rows instead.
|
|
124
|
+
Rerun until the needed evidence is in a result with `truncated: false`.
|
|
125
|
+
Do not use `--allow-truncated` during discovery or DAX authoring.
|
|
126
|
+
|
|
127
|
+
### Reading a single expression that is larger than the budget
|
|
128
|
+
|
|
129
|
+
One cell can exceed the byte budget on its own — localization measures built
|
|
130
|
+
from `SWITCH(USERCULTURE(), ...)` routinely run to tens of thousands of
|
|
131
|
+
characters.
|
|
132
|
+
Narrowing cannot help there: a single cell has nothing left to narrow, so the
|
|
133
|
+
same `SELECT` returns `totalRows: 1` with `returnedRows: 0` however often it is
|
|
134
|
+
repeated.
|
|
135
|
+
|
|
136
|
+
Read it in slices instead, and fetch the length first so the last offset is
|
|
137
|
+
known.
|
|
138
|
+
Use `length([Expression])` as the bound rather than the character count in the
|
|
139
|
+
search summary: `substr` offsets and `length` both count code points, while the
|
|
140
|
+
summary counts UTF-16 units, so the two disagree for text containing emoji.
|
|
141
|
+
|
|
142
|
+
```sh
|
|
143
|
+
npm run --silent schema:sql -- "SELECT length([Expression]) AS chars FROM info_view_measures WHERE [Table] = 'Sales' AND [Name] = 'Localized Label'"
|
|
144
|
+
npm run --silent schema:sql -- "SELECT substr([Expression], 1, 1000) AS part FROM info_view_measures WHERE [Table] = 'Sales' AND [Name] = 'Localized Label'"
|
|
145
|
+
npm run --silent schema:sql -- "SELECT substr([Expression], 1001, 1000) AS part FROM info_view_measures WHERE [Table] = 'Sales' AND [Name] = 'Localized Label'"
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
Use a stride of 1000 characters.
|
|
149
|
+
The budget is counted in bytes, not characters, so a stride that fits ASCII can
|
|
150
|
+
still be three or four times too large for CJK or emoji text.
|
|
151
|
+
|
|
152
|
+
Two conditions both have to hold, and the second is the one that is easy to get
|
|
153
|
+
wrong:
|
|
154
|
+
|
|
155
|
+
- Every chunk returns `returnedRows: 1` and `truncated: false`.
|
|
156
|
+
`returnedRows: 0` means the slice was still too big — halve the stride and
|
|
157
|
+
retry that offset.
|
|
158
|
+
- The offsets cover `1` through the full length with no gap.
|
|
159
|
+
Stopping after one clean chunk looks like success, because the chunks that
|
|
160
|
+
were fetched are all complete; only the coverage is missing.
|
|
161
|
+
|
|
162
|
+
Concatenate the parts in offset order to reconstruct the definition exactly.
|
|
163
|
+
|
|
164
|
+
## Discovery workflow
|
|
165
|
+
|
|
166
|
+
1. Query `catalog_status`.
|
|
167
|
+
2. Run a catalog-wide semantic search for the user's concepts.
|
|
168
|
+
3. Inspect candidate tables and their owned measures.
|
|
169
|
+
4. Query focused columns and measures for every selected table.
|
|
170
|
+
5. Query relationships connecting the selected tables.
|
|
171
|
+
6. Use column statistics only after the exact column is known.
|
|
172
|
+
7. Reuse prior catalog results instead of issuing equivalent SQL again.
|
|
173
|
+
8. Stop only when every requested output has an exact source mapping, and the
|
|
174
|
+
shipped measures, grouping columns, relationship path, **filter grain** and
|
|
175
|
+
**explicit filter values** are all grounded in untruncated results.
|
|
176
|
+
|
|
177
|
+
The catalog holds metadata and per-column extrema, not the distinct values in a
|
|
178
|
+
column. Once the filter column is known and a literal value still has to be
|
|
179
|
+
confirmed, ground it with one bounded query against the model rather than
|
|
180
|
+
looping on catalog SQL that cannot answer it:
|
|
181
|
+
|
|
182
|
+
```sh
|
|
183
|
+
npx rayfin connector invoke <alias> executeQuery --file <probe>.query.json
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
Test the literal you were given, so the answer is decisive:
|
|
187
|
+
|
|
188
|
+
```dax
|
|
189
|
+
EVALUATE
|
|
190
|
+
FILTER(VALUES('Geography'[Country]), 'Geography'[Country] = "France")
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
Zero rows means there is no exact match for that value; report that instead of
|
|
194
|
+
widening the search. If the probe errors, or the mapping is still unresolved
|
|
195
|
+
after it, report the blocker rather than broadening discovery. Do not sample a
|
|
196
|
+
column and read absence from the sample — `TOPN` over a column with more
|
|
197
|
+
distinct values than the bound returns a complete-looking partial answer. This
|
|
198
|
+
is a value check, not more discovery: ask it once, for a column you have
|
|
199
|
+
already selected.
|
|
200
|
+
|
|
201
|
+
Before writing any DAX, write down the output plan: every output field with its
|
|
202
|
+
exact source (a fully qualified column, or an exact shipped measure for every
|
|
203
|
+
business value), the filters and their values, the relationship path, and the
|
|
204
|
+
expected row shape. Then compare what the query returns against that plan.
|
|
205
|
+
A query that runs is not the same as a query at the right grain.
|
|
206
|
+
|
|
207
|
+
One grain trap is worth checking by name. When a measure counts a column —
|
|
208
|
+
`COUNT`, `DISTINCTCOUNT` or `DISTINCTCOUNTNOBLANK` over `'Table'[Column]` —
|
|
209
|
+
filtering or grouping by that same column, or by one that reaches it through an
|
|
210
|
+
active relationship, collapses the result to a trivial one per group. It
|
|
211
|
+
returns rows and looks correct. The catalog already holds both inputs: read the
|
|
212
|
+
measure's `[Expression]` from `info_view_measures` and the path from
|
|
213
|
+
`info_view_relationships` before committing to that filter. If two filter
|
|
214
|
+
columns remain plausible, compare them once with the shipped measures
|
|
215
|
+
unchanged.
|
|
216
|
+
|
|
217
|
+
Start with shipped measures.
|
|
218
|
+
Never reconstruct business logic from physical columns when the model already
|
|
219
|
+
ships a measure.
|
|
220
|
+
Column statistics are not a shortcut around that: `Min`, `Max` and
|
|
221
|
+
`Cardinality` are the raw column extent, ignoring filter context,
|
|
222
|
+
relationships, and any business rule the measure encodes. A cardinality that
|
|
223
|
+
happens to match an expected row count is not evidence that reading the column
|
|
224
|
+
is correct.
|
|
225
|
+
Helper, selector, calculation, and parameter tables may own the measure that
|
|
226
|
+
supplies a displayed value, so do not restrict the first search to a fact-table
|
|
227
|
+
prefix.
|
|
228
|
+
|
|
229
|
+
A selected table is not fully inspected until its owned measures have been
|
|
230
|
+
read. When such a table owns a measure that dispatches on its column — a
|
|
231
|
+
`SELECTEDVALUE` / `SWITCH` over the selector, for example — group by that
|
|
232
|
+
column and evaluate the shipped measure once instead of building one expression
|
|
233
|
+
per row out of similarly named physical columns. That recipe does not apply to
|
|
234
|
+
every helper: a native field-parameter table carries field references that the
|
|
235
|
+
client resolves, so grouping by its label does not swap the underlying field.
|
|
236
|
+
If the owned measures do not explain what the table is for, search measure
|
|
237
|
+
expressions for references to it before falling back to raw aggregation:
|
|
238
|
+
|
|
239
|
+
```sh
|
|
240
|
+
npm run --silent schema:sql -- "SELECT [Table], [Name] FROM info_view_measures WHERE [Expression] LIKE '%ParameterTable%'"
|
|
241
|
+
```
|
|
242
|
+
|
|
243
|
+
Selecting from `info_view_tables` automatically includes a bounded
|
|
244
|
+
`ownedMeasures` preview when `info_view_measures` is available.
|
|
245
|
+
The preview carries names and display folders only so it cannot consume the
|
|
246
|
+
result byte budget with repeated expressions.
|
|
247
|
+
Query `info_view_measures` for each selected table to read expressions,
|
|
248
|
+
descriptions, and format strings.
|
|
249
|
+
Review it before choosing raw columns.
|
|
250
|
+
|
|
251
|
+
## SQL examples
|
|
252
|
+
|
|
253
|
+
Search globally before narrowing:
|
|
254
|
+
|
|
255
|
+
```sh
|
|
256
|
+
npm run --silent schema:sql -- --search "revenue region comparison"
|
|
257
|
+
```
|
|
258
|
+
|
|
259
|
+
Search is an AND across names, descriptions, expressions, and display folders,
|
|
260
|
+
so prefer a few specific terms over a whole sentence.
|
|
261
|
+
When no object matches every term, terms that appear nowhere in the model are
|
|
262
|
+
dropped and the search is answered without them, and the result says which ones
|
|
263
|
+
were dropped.
|
|
264
|
+
A statement sourced with `--file` must live inside the app, and is labelled by
|
|
265
|
+
its path rather than by its contents.
|
|
266
|
+
|
|
267
|
+
Inspect candidate tables:
|
|
268
|
+
|
|
269
|
+
```sh
|
|
270
|
+
npm run --silent schema:sql -- "SELECT [Name], [Description], [IsHidden] FROM info_view_tables WHERE lower([Name]) LIKE '%sales%'"
|
|
271
|
+
```
|
|
272
|
+
|
|
273
|
+
Inspect shipped measures across selected tables in one statement:
|
|
274
|
+
|
|
275
|
+
```sh
|
|
276
|
+
npm run --silent schema:sql -- "SELECT [Table], [Name], [Expression], [Description], [FormatString], [DisplayFolder] FROM info_view_measures WHERE [Table] IN ('Sales', 'Date') ORDER BY [Table], [Name]"
|
|
277
|
+
```
|
|
278
|
+
|
|
279
|
+
Inspect columns only after selecting tables:
|
|
280
|
+
|
|
281
|
+
```sh
|
|
282
|
+
npm run --silent schema:sql -- "SELECT [Table], [Name], [DataType], [Description], [FormatString], [IsHidden] FROM info_view_columns WHERE [Table] IN ('Sales', 'Date') ORDER BY [Table], [Name]"
|
|
283
|
+
```
|
|
284
|
+
|
|
285
|
+
Inspect the relationship path:
|
|
286
|
+
|
|
287
|
+
```sh
|
|
288
|
+
npm run --silent schema:sql -- "SELECT [FromTable], [FromColumn], [ToTable], [ToColumn], [IsActive], [CrossFilteringBehavior] FROM info_view_relationships WHERE [FromTable] IN ('Sales', 'Date') OR [ToTable] IN ('Sales', 'Date')"
|
|
289
|
+
```
|
|
290
|
+
|
|
291
|
+
Inspect statistics for exact columns:
|
|
292
|
+
|
|
293
|
+
```sh
|
|
294
|
+
npm run --silent schema:sql -- "SELECT [Table Name], [Column Name], [Min], [Max], [Cardinality], [Max Length] FROM column_statistics WHERE [Table Name] = 'Date' AND [Column Name] IN ('Date', 'Year')"
|
|
295
|
+
```
|
|
296
|
+
|
|
297
|
+
Column names retain spaces and reserved words.
|
|
298
|
+
Quote them with SQL double quotes or square brackets.
|
|
299
|
+
|
|
300
|
+
## Batch several statements in one invocation
|
|
301
|
+
|
|
302
|
+
Each extra invocation costs a whole model round trip, and a round trip re-sends
|
|
303
|
+
the entire conversation.
|
|
304
|
+
The five statements above are normally run back to back, so run them together
|
|
305
|
+
instead.
|
|
306
|
+
`--search`, `--sql`, and `--file` are repeatable, and searches run before SQL:
|
|
307
|
+
|
|
308
|
+
```sh
|
|
309
|
+
npm run --silent schema:sql -- --search "revenue region comparison" --sql "SELECT [Name], [Description], [IsHidden] FROM info_view_tables WHERE lower([Name]) LIKE '%sales%'" --sql "SELECT [Table], [Name], [Expression], [Description], [FormatString], [DisplayFolder] FROM info_view_measures WHERE [Table] IN ('Sales', 'Date') ORDER BY [Table], [Name]"
|
|
310
|
+
```
|
|
311
|
+
|
|
312
|
+
The reply is one payload: a `#` header naming the batch, then a `##` label line
|
|
313
|
+
per statement followed by that statement's ordinary result block.
|
|
314
|
+
A failed statement stays in the payload beside the ones that worked, so a single
|
|
315
|
+
mistake does not cost the whole batch.
|
|
316
|
+
|
|
317
|
+
At most eight statements per invocation.
|
|
318
|
+
The whole batch shares one byte budget, so batching never costs more context
|
|
319
|
+
than a single statement would.
|
|
320
|
+
Ask everything you already know you need, but keep each statement narrow: the
|
|
321
|
+
shared budget is divided across them, and unused room rolls forward.
|
|
322
|
+
|
|
323
|
+
## Advanced metadata outside the catalog
|
|
324
|
+
|
|
325
|
+
The fixed catalog intentionally covers the common DAX-authoring path.
|
|
326
|
+
It does not include every elevated `INFO.*` function.
|
|
327
|
+
|
|
328
|
+
Only when the current task specifically requires calculation groups, calendars,
|
|
329
|
+
user-defined functions, variations, or model configuration should you use the
|
|
330
|
+
advanced read-only queries in
|
|
331
|
+
[`discovery-queries.md`](./references/discovery-queries.md).
|
|
332
|
+
Run the minimum advanced query needed through:
|
|
333
|
+
|
|
334
|
+
```sh
|
|
335
|
+
npx rayfin connector invoke <alias> executeQuery --file <query>.query.json
|
|
336
|
+
```
|
|
337
|
+
|
|
338
|
+
The file is a JSON operation input such as:
|
|
339
|
+
|
|
340
|
+
```json
|
|
341
|
+
{
|
|
342
|
+
"query": "EVALUATE INFO.CALCULATIONGROUPS()"
|
|
343
|
+
}
|
|
344
|
+
```
|
|
345
|
+
|
|
346
|
+
Do not use advanced queries to re-fetch tables, columns, measures,
|
|
347
|
+
relationships, or column statistics already owned by the catalog.
|
|
348
|
+
If an elevated query fails with a permission error after catalog discovery
|
|
349
|
+
succeeded, mark elevated metadata unavailable and do not probe the remaining
|
|
350
|
+
elevated functions.
|
|
351
|
+
|
|
352
|
+
## Must, prefer, avoid
|
|
353
|
+
|
|
354
|
+
### Must
|
|
355
|
+
|
|
356
|
+
- Check `catalog_status` before treating absence as evidence.
|
|
357
|
+
- Bind every business value to an exact shipped measure, and every label or
|
|
358
|
+
grouping to an exact column, before writing DAX. If exhaustive untruncated
|
|
359
|
+
discovery finds no suitable measure, record which one is missing and the
|
|
360
|
+
catalog evidence for that before using another source.
|
|
361
|
+
- Use fully qualified `'Table'[Column]` references in authored DAX.
|
|
362
|
+
- Use simple `[Measure]` references for measures.
|
|
363
|
+
- Keep every SQL result narrow and untruncated.
|
|
364
|
+
- Reuse discovered metadata.
|
|
365
|
+
|
|
366
|
+
### Prefer
|
|
367
|
+
|
|
368
|
+
- `--search` for the first semantic pass.
|
|
369
|
+
- Batching known statements into one invocation over separate calls.
|
|
370
|
+
- One SQL statement for several known metadata questions.
|
|
371
|
+
- Exact relationship paths and column statistics after candidate selection.
|
|
372
|
+
|
|
373
|
+
### Avoid
|
|
374
|
+
|
|
375
|
+
- Hand-running the catalog's five source queries.
|
|
376
|
+
- Progressive `INFO.VIEW.*` connector round trips.
|
|
377
|
+
- Whole-table SQL dumps.
|
|
378
|
+
- Re-fetching metadata already present in the catalog or conversation.
|
|
379
|
+
- Treating `truncated: false` as proof that an incomplete catalog is complete.
|
|
380
|
+
- Rebuilding a helper table's logic one row at a time from physical columns.
|
|
381
|
+
- Using unavailable `GetSemanticModelSchema`, `GenerateQuery`, or
|
|
382
|
+
`DiscoverArtifacts` tools.
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
# Power BI Semantic Model Discovery Queries
|
|
2
|
+
|
|
3
|
+
Read-only DAX queries for metadata exploration using `INFO.VIEW.*` and `INFO.*` rowsets.
|
|
4
|
+
|
|
5
|
+
The analytics pack materializes tables, columns, measures, relationships, and
|
|
6
|
+
column statistics into its local SQLite catalog.
|
|
7
|
+
Do not run the common `INFO.VIEW.*` queries below when that catalog is
|
|
8
|
+
available.
|
|
9
|
+
They remain here as output-shape reference material and for environments that
|
|
10
|
+
do not have the analytics pack.
|
|
11
|
+
Use the advanced `INFO.*` sections only when the task requires metadata outside
|
|
12
|
+
the fixed catalog.
|
|
13
|
+
|
|
14
|
+
## Scope Estimation Queries
|
|
15
|
+
|
|
16
|
+
```dax
|
|
17
|
+
// Probe object counts to estimate metadata scope before deep discovery
|
|
18
|
+
EVALUATE
|
|
19
|
+
ROW(
|
|
20
|
+
"TableCount", COUNTROWS(INFO.VIEW.TABLES()),
|
|
21
|
+
"ColumnCount", COUNTROWS(INFO.VIEW.COLUMNS()),
|
|
22
|
+
"MeasureCount", COUNTROWS(INFO.VIEW.MEASURES()),
|
|
23
|
+
"RelationshipCount", COUNTROWS(INFO.VIEW.RELATIONSHIPS())
|
|
24
|
+
)
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
## INFO Output Columns
|
|
28
|
+
|
|
29
|
+
### INFO.VIEW.* (preferred first-pass metadata)
|
|
30
|
+
|
|
31
|
+
| Function | High-value columns | What to use them for |
|
|
32
|
+
|---|---|---|
|
|
33
|
+
| `INFO.VIEW.TABLES()` | `Name`, `DataCategory`, `StorageMode`, `IsHidden`, `Expression`, `CalculationGroupPrecedence`, `LineageTag` | Table inventory, calculated-table detection, storage-mode audits, lineage tracking. |
|
|
34
|
+
| `INFO.VIEW.COLUMNS()` | `Table`, `Name`, `DataType`, `DataCategory`, `IsHidden`, `SummarizeBy`, `Expression`, `SortByColumn`, `FormatString`, `LineageTag` | Column dictionary, semantic typing, sort/summarization checks, calculated column review. |
|
|
35
|
+
| `INFO.VIEW.MEASURES()` | `Table`, `Name`, `Expression`, `FormatString`, `State`, `DisplayFolder`, `KPIID`, `LineageTag` | Measure inventory, formula review, formatting/state validation, KPI linkage. |
|
|
36
|
+
| `INFO.VIEW.RELATIONSHIPS()` | `Relationship`, `IsActive`, `FromTable`, `FromColumn`, `ToTable`, `ToColumn`, `FromCardinality`, `ToCardinality`, `CrossFilteringBehavior`, `SecurityFilteringBehavior` | Join topology, cardinality validation, filter-direction and RLS behavior checks. |
|
|
37
|
+
|
|
38
|
+
### Critical INFO.* (deep metadata / diagnostics)
|
|
39
|
+
|
|
40
|
+
| Function | High-value columns | What to use them for |
|
|
41
|
+
|---|---|---|
|
|
42
|
+
| `INFO.MODEL()` | `Name`, `DefaultMode`, `Culture`, `Collation`, `ModifiedTime`, `Version`, `DirectLakeBehavior`, `ValueFilterBehavior`, `SelectionExpressionBehavior` | Model policy/config audits and environment baseline. |
|
|
43
|
+
|
|
44
|
+
```dax
|
|
45
|
+
// Probe output schema of an INFO function (returns column names and types with zero data rows)
|
|
46
|
+
EVALUATE
|
|
47
|
+
TOPN(0, INFO.VIEW.COLUMNS())
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
## Narrowing Results (Projection + Filtering)
|
|
51
|
+
|
|
52
|
+
```dax
|
|
53
|
+
// Pull only needed columns for a single table to reduce output volume
|
|
54
|
+
EVALUATE
|
|
55
|
+
SELECTCOLUMNS(
|
|
56
|
+
FILTER(INFO.VIEW.COLUMNS(), [Table] = "YourTableName"),
|
|
57
|
+
"Column", [Name],
|
|
58
|
+
"DataType", [DataType],
|
|
59
|
+
"Format", [FormatString]
|
|
60
|
+
)
|
|
61
|
+
ORDER BY [Column] ASC
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
```dax
|
|
65
|
+
// Get measures for a specific table
|
|
66
|
+
EVALUATE
|
|
67
|
+
SELECTCOLUMNS(
|
|
68
|
+
FILTER(INFO.VIEW.MEASURES(), [Table] = "YourTableName"),
|
|
69
|
+
"Measure", [Name],
|
|
70
|
+
"Expression", [Expression],
|
|
71
|
+
"Format", [FormatString]
|
|
72
|
+
)
|
|
73
|
+
ORDER BY [Measure] ASC
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
```dax
|
|
77
|
+
// Get relationships involving a specific table
|
|
78
|
+
EVALUATE
|
|
79
|
+
FILTER(
|
|
80
|
+
INFO.VIEW.RELATIONSHIPS(),
|
|
81
|
+
[FromTable] = "YourTableName" || [ToTable] = "YourTableName"
|
|
82
|
+
)
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
## Advanced Metadata Queries
|
|
86
|
+
|
|
87
|
+
These INFO functions may require elevated permissions. If any fail with a permission error after INFO.VIEW.* functions succeeded, skip all remaining elevated queries for this session.
|
|
88
|
+
|
|
89
|
+
### Calculation Groups
|
|
90
|
+
|
|
91
|
+
```dax
|
|
92
|
+
// List all calculation groups in the model
|
|
93
|
+
EVALUATE
|
|
94
|
+
INFO.CALCULATIONGROUPS()
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
```dax
|
|
98
|
+
// List all calculation items (the individual items within calculation groups)
|
|
99
|
+
EVALUATE
|
|
100
|
+
INFO.CALCULATIONITEMS()
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
### Calendars
|
|
104
|
+
|
|
105
|
+
```dax
|
|
106
|
+
// List calendar objects defined in the model
|
|
107
|
+
EVALUATE
|
|
108
|
+
INFO.CALENDARS()
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
```dax
|
|
112
|
+
// List calendar column groups
|
|
113
|
+
EVALUATE
|
|
114
|
+
INFO.CALENDARCOLUMNGROUPS()
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
```dax
|
|
118
|
+
// List calendar column references
|
|
119
|
+
EVALUATE
|
|
120
|
+
INFO.CALENDARCOLUMNREFERENCES()
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
### User-Defined Functions
|
|
124
|
+
|
|
125
|
+
```dax
|
|
126
|
+
// List user-defined functions in the model
|
|
127
|
+
EVALUATE
|
|
128
|
+
INFO.USERDEFINEDFUNCTIONS()
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
### Variations
|
|
132
|
+
|
|
133
|
+
```dax
|
|
134
|
+
// List variations (alternate groupings/display hierarchies)
|
|
135
|
+
EVALUATE
|
|
136
|
+
INFO.VARIATIONS()
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
## Complete INFO Function Catalog (Dynamic)
|
|
140
|
+
|
|
141
|
+
Use this query to enumerate all INFO functions available in the current engine at runtime:
|
|
142
|
+
|
|
143
|
+
```dax
|
|
144
|
+
EVALUATE
|
|
145
|
+
SELECTCOLUMNS(
|
|
146
|
+
INFO.STORAGEFUNCTIONS(),
|
|
147
|
+
"FunctionName", [Name],
|
|
148
|
+
"Description", [Description]
|
|
149
|
+
)
|
|
150
|
+
ORDER BY [FunctionName] ASC
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
If `INFO.STORAGEFUNCTIONS()` is not available, probe individual functions with `TOPN(0, <function>())` to check availability without returning data.
|