@microsoft/rayfin-semantic-model-pack 1.0.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. package/CHANGELOG +8 -0
  2. package/README.md +34 -10
  3. package/kit/hooks/use-query-factory-target.spec.ts +191 -0
  4. package/kit/hooks/use-query-factory-target.ts +123 -0
  5. package/{skills/analytics/kit → kit}/hooks/use-semantic-model-query.spec.ts +59 -23
  6. package/{skills/analytics/kit → kit}/hooks/use-semantic-model-query.ts +30 -54
  7. package/kit/lib/query-factory.spec.ts +97 -0
  8. package/kit/lib/query-factory.ts +68 -0
  9. package/kit/lib/semantic-model-errors.spec.ts +131 -0
  10. package/kit/lib/semantic-model-errors.ts +110 -0
  11. package/kit/lib/to-data-table.spec.ts +444 -0
  12. package/kit/lib/to-data-table.ts +221 -0
  13. package/pack.json +141 -0
  14. package/package.json +3 -1
  15. package/skills/analytics/SKILL.md +126 -421
  16. package/skills/dax-authoring/SKILL.md +99 -34
  17. package/skills/query-design/SKILL.md +22 -29
  18. package/skills/query-design/references/anti-patterns.md +13 -11
  19. package/skills/query-design/references/format-strings.md +6 -12
  20. package/skills/query-design/references/multi-grain-patterns.md +121 -108
  21. package/skills/schema-discovery/SKILL.md +197 -368
  22. package/skills/schema-discovery/references/discovery-queries.md +6 -8
  23. package/skills/visual-interactivity/SKILL.md +68 -0
  24. package/skills/visual-interactivity/references/advanced-query-shapes.md +155 -0
  25. package/skills/visual-interactivity/references/filter-aware-grid-totals.md +101 -0
  26. package/skills/visual-interactivity/references/highlight-queries.md +88 -0
  27. package/skills/visual-interactivity/references/interaction-manager.md +127 -0
  28. package/skills/visual-interactivity/references/native-selections.md +20 -0
  29. package/skills/visual-interactivity/references/query-factories.md +169 -0
  30. package/skills/analytics/kit/lib/connectors.ts +0 -94
  31. package/skills/analytics/kit/scripts/rayfin-connector.mjs +0 -265
  32. package/skills/analytics/kit/scripts/rayfin-connector.test.mjs +0 -243
  33. package/skills/analytics/kit/scripts/schema-sql-batch.test.mjs +0 -2097
  34. package/skills/analytics/kit/scripts/schema-sql-cli.test.mjs +0 -488
  35. package/skills/analytics/kit/scripts/schema-sql.mjs +0 -2650
  36. package/skills/analytics/kit/scripts/schema-sql.test.mjs +0 -1387
  37. package/skills/analytics/kit/scripts/schema.mjs +0 -166
  38. package/skills/analytics/kit/scripts/schema.test.mjs +0 -271
  39. package/skills/analytics/pack.json +0 -49
  40. package/skills/query-design/references/filter-strategy.md +0 -68
  41. package/skills/query-design/references/highlight-queries.md +0 -122
@@ -0,0 +1,169 @@
1
+ # Runtime Query Factories
2
+
3
+ Parameterize factories only when runtime state changes DAX or a Vega-Lite spec; otherwise keep them parameterless. Follow the host template's query-factory conventions for file names, metadata, barrel exports, and validation. Configure sources and targets using [Interaction Manager](interaction-manager.md).
4
+
5
+ ## Factory contract
6
+
7
+ - Accept typed, use-case-specific state, and `ManagedQueryFactoryParams` for Interaction Manager filters; never build filter DAX from selections.
8
+ - Return `connection`, `query`, `columnMetadata`, and `vegaLiteSpec`.
9
+ - Import `.dax` queries with `?raw` and specs from `.json`; never inline a full base query.
10
+ - Map identifiers through closed maps of known model columns; reject unknown IDs.
11
+ - Escape values for their DAX literal type; in strings, replace `"` with `""`.
12
+ - Assemble small value substitutions or `CALCULATETABLE` wrappers in TypeScript, not arbitrary model identifiers.
13
+
14
+ ## Structural variants
15
+
16
+ When grouping, measures, or result shape changes, use separate complete `.dax` files named `{visualization-name}-{variant}.dax`. With shared `connection`, `columnMetadata`, and `vegaLiteSpec` defined:
17
+
18
+ ```ts
19
+ import monthlyQuery from "./revenue-trend-monthly.dax?raw";
20
+ import quarterlyQuery from "./revenue-trend-quarterly.dax?raw";
21
+ import yearlyQuery from "./revenue-trend-yearly.dax?raw";
22
+
23
+ type Granularity = "monthly" | "quarterly" | "yearly";
24
+
25
+ const queryByGranularity: Record<Granularity, string> = {
26
+ monthly: monthlyQuery,
27
+ quarterly: quarterlyQuery,
28
+ yearly: yearlyQuery,
29
+ };
30
+
31
+ export function revenueTrend(params: { granularity: Granularity }) {
32
+ return {
33
+ connection,
34
+ query: queryByGranularity[params.granularity],
35
+ columnMetadata,
36
+ vegaLiteSpec,
37
+ };
38
+ }
39
+ ```
40
+
41
+ Select matching metadata and specs if the result shape differs. Do not add query variants for presentation-only changes.
42
+
43
+ ## Interaction Manager filters
44
+
45
+ Interaction filters must run inside the query, before measures are calculated.
46
+ For example, a baseline row might show Bikes sales of 1,000 across all
47
+ countries, and selecting Canada might reduce that to 200. Filtering the loaded
48
+ row cannot produce 200; the query must run again with the Canada filter.
49
+
50
+ Accept the package's `ManagedQueryFactoryParams`. It declares one property,
51
+ `readonly interactionFilters?: readonly string[]`. Default it to `[]` in your
52
+ factory; an empty list adds no filtering.
53
+
54
+ Each string is one source's complete selection as a DAX table-filter argument.
55
+ Separate arguments in the same call intersect, so each source narrows the
56
+ result. Insert every string unchanged: do not split strings at commas, extract
57
+ predicates, drop entries, or pick filters by field. Those operations change
58
+ which combinations of values are selected.
59
+
60
+ ## Insert filters into a grouped query
61
+
62
+ In `SUMMARIZECOLUMNS`, insert the filters after the grouping columns and before
63
+ the named expressions. Reserve a marker in your authored query:
64
+
65
+ ```dax
66
+ EVALUATE
67
+ SUMMARIZECOLUMNS(
68
+ 'Product'[Category],
69
+ /*__FILTER_ARGUMENTS__*/"Sales", [Total Sales]
70
+ )
71
+ ORDER BY 'Product'[Category]
72
+ ```
73
+
74
+ Keep the DAX in the factory's `.dax` file and insert the filters with the
75
+ shipped, tested marker helper:
76
+
77
+ ```ts
78
+ import type { ManagedQueryFactoryParams } from "@microsoft/fabric-interaction-manager";
79
+ import { insertInteractionFilters } from "@/lib/query-factory";
80
+ import baseQuery from "./category-chart.dax?raw";
81
+ import vegaLiteSpec from "./category-chart.json";
82
+
83
+ export function categoryChart(
84
+ { interactionFilters = [] }: ManagedQueryFactoryParams = {},
85
+ ) {
86
+ return {
87
+ connection: "sales",
88
+ query: insertInteractionFilters(baseQuery, interactionFilters, "/*__FILTER_ARGUMENTS__*/", "beforeMeasures"),
89
+ columnMetadata: {
90
+ "Product[Category]": { name: "Category", displayName: "Category" },
91
+ "[Sales]": { name: "Value", displayName: "Sales", format: "$0" },
92
+ },
93
+ vegaLiteSpec,
94
+ };
95
+ }
96
+ ```
97
+
98
+ The helper adds a comma after each filter in this position, removes the marker
99
+ for an empty list, and throws when the marker is missing or repeated.
100
+
101
+ For measures-only or columns-only queries, multiple bases, Top N, and
102
+ queries with an outer filter context, see [advanced query shapes](advanced-query-shapes.md).
103
+
104
+ ## Load the target
105
+
106
+ `useQueryFactoryTarget` loads a registered target through its query factory.
107
+ It passes the manager's baseline filters as `interactionFilters` and returns
108
+ a load state containing one `DataTable` when data is available:
109
+
110
+ ```tsx
111
+ import { useQueryFactoryTarget } from "@/hooks/use-query-factory-target";
112
+ import { categoryChart } from "@/queries/sales/category-chart";
113
+
114
+ const result = useQueryFactoryTarget("details", categoryChart, {});
115
+ ```
116
+
117
+ Pass the factory's other inputs as the third argument, such as `{ granularity }`
118
+ for a `revenueTrend` factory. The hook compares these inputs by their JSON
119
+ serialization; recreating an inline object with the same serialized contents
120
+ does not itself trigger a reload.
121
+
122
+ The target loads on mount and reloads when its factory inputs or interactions
123
+ affecting it change. Keep the factory function stable unless changing it is
124
+ intended to reload the target.
125
+
126
+ ### Query execution and load states
127
+
128
+ The hook uses `useManagedInteractionTarget` to manage loading and retain
129
+ successful results. Its loader executes factories through `executeQueryFactory`,
130
+ which shares the Rayfin execution path and error normalization with
131
+ `useSemanticModelQuery`, then converts successful query tables using the
132
+ factory's column metadata. Execution caching is template-specific:
133
+ the data-app shares its existing session cache, while Universal App is uncached.
134
+
135
+ `result.status` is `"loading"` while a load is pending with no previous data;
136
+ `"refreshing"` while a replacement load is pending, keeping the previous `data`
137
+ and `isCrossHighlightActive` together; `"ready"` with the latest successful
138
+ load's data; `"refresh-error"` when a load failed, retaining the previous
139
+ successful data and highlight flag and exposing the failure as `error`; or
140
+ `"error"` when a load failed with no previous data. A result from a superseded
141
+ load is ignored but not cancelled.
142
+
143
+ Query failures appear as `result.error` in the `"error"` or `"refresh-error"`
144
+ state. These errors preserve the query category, code, available HTTP status
145
+ and workload details, diagnostic text, and original cause. Handle the returned
146
+ state; query failures do not throw from `useQueryFactoryTarget`.
147
+
148
+ Invalid selections throw during compilation, and the load fails before any
149
+ query runs. A failed query also fails the load. Surface `result.error`; do not
150
+ retry without the interaction filters.
151
+
152
+ ### Present loading and failures
153
+
154
+ Handle each result state using the app's loading and error UI. On
155
+ `"refresh-error"`, keep the previous visual visible and show the failure
156
+ alongside it. Explicitly identify retained rows, highlights and totals as stale,
157
+ never silently present them as the latest result. With no previous data, show a
158
+ load error instead. Preserve accepted selections.
159
+
160
+ ### Target-only components
161
+
162
+ A target-only component, such as a measures-only grid, should declare
163
+ `selectableFields: []` and omit source interaction props. Components that also
164
+ publish selections should follow the skill's source-binding guidance.
165
+
166
+ ### Highlighting
167
+
168
+ A highlight-capable target also loads a highlighted subset and returns
169
+ `{ all, highlighted }`; see [highlight queries](highlight-queries.md).
@@ -1,94 +0,0 @@
1
- //-----------------------------------------------------------------------
2
- // <copyright company="Microsoft Corporation">
3
- // Copyright (c) Microsoft Corporation. All rights reserved.
4
- // Licensed under the MIT license. See LICENSE file in the project root for full license information.
5
- // </copyright>
6
- //-----------------------------------------------------------------------
7
-
8
- import type {
9
- ConnectorConfig,
10
- ConnectorsRuntime,
11
- } from '@microsoft/rayfin-connectors';
12
- import type { FabricSemanticModel } from '@microsoft/rayfin-connector-fabric-semanticmodel';
13
-
14
- /**
15
- * Connectors this app can talk to, keyed by the connector name declared in
16
- * `rayfin/rayfin.yml` under `connectors[].name`.
17
- *
18
- * The index signature keeps every name usable without editing this file each
19
- * time you run `rayfin connector add`. Narrow it to the names you actually use
20
- * if you want the compiler to catch a typo in a connection alias:
21
- *
22
- * @example
23
- * export type AppConnectorsSchema = {
24
- * salesModel: FabricSemanticModel<'executeQuery'>;
25
- * };
26
- */
27
- export type AppConnectorsSchema = Record<
28
- string,
29
- FabricSemanticModel<'executeQuery'>
30
- >;
31
-
32
- /**
33
- * Routing config for each connector, keyed by the same name again.
34
- *
35
- * `rayfin connector add` writes a `connectorConfig` constant into
36
- * `rayfin/connectors/<name>/schema.ts`. Re-export that constant here rather
37
- * than hand-writing the object, so this file cannot drift from `rayfin.yml`.
38
- *
39
- * @example
40
- * import { connectorConfig as salesModel } from '../../../../rayfin/connectors/salesModel/schema';
41
- *
42
- * export const connectorConfigs: Record<string, ConnectorConfig> = { salesModel };
43
- */
44
- export const connectorConfigs: Record<string, ConnectorConfig> = {};
45
-
46
- /**
47
- * Per-connector runtime hooks, keyed by the same name again.
48
- *
49
- * `fabric-semanticmodel` returns an Apache Arrow stream and chooses its
50
- * transport from where the app is running — embedded in the Fabric portal, or
51
- * standalone — and both of those live in the runtime. A connector left out of
52
- * this map falls back to the plain JSON pass-through and will not decode.
53
- * Register one runtime per semantic model; the connectors layer already keys
54
- * instances by name.
55
- *
56
- * @example
57
- * import { fabricSemanticModel } from '@microsoft/rayfin-connector-fabric-semanticmodel';
58
- *
59
- * export const connectorRuntimes: ConnectorsRuntime = {
60
- * salesModel: fabricSemanticModel({}),
61
- * };
62
- *
63
- * @remarks
64
- * Do not pass a `target`. The workspace and item ids come from `rayfin.yml` and
65
- * are injected by the host, so they never ship to the browser. Deriving one from
66
- * a `VITE_*` variable throws at module load, because a per-model URL is not
67
- * among the variables `rayfin env` emits.
68
- */
69
- export const connectorRuntimes: ConnectorsRuntime = {};
70
-
71
- /**
72
- * Fails when a connector is configured without a matching runtime.
73
- *
74
- * A config with no runtime still sends the call, falls back to JSON
75
- * pass-through, and returns an Arrow stream nothing decodes — unreadable rows,
76
- * far from the missing line that caused them. Call this before constructing the
77
- * client to turn that into a startup error naming the connector.
78
- *
79
- * A runtime with no config is fine: it can be registered ahead of the
80
- * `connector add` that declares it.
81
- */
82
- export function assertConnectorRuntimesRegistered(): void {
83
- const missing = Object.keys(connectorConfigs).filter(
84
- (name) => !(name in connectorRuntimes)
85
- );
86
-
87
- if (missing.length > 0) {
88
- throw new Error(
89
- `Connector(s) ${missing.map((n) => `"${n}"`).join(', ')} are in connectorConfigs but have no runtime in connectorRuntimes ` +
90
- `(packages/frontend/src/lib/connectors.ts). Without a runtime the response is not decoded and rows arrive unreadable. ` +
91
- `Add: ${missing.map((n) => `${n}: fabricSemanticModel({})`).join(', ')}.`
92
- );
93
- }
94
- }
@@ -1,265 +0,0 @@
1
- //-----------------------------------------------------------------------
2
- // <copyright company="Microsoft Corporation">
3
- // Copyright (c) Microsoft Corporation. All rights reserved.
4
- // Licensed under the MIT license. See LICENSE file in the project root for full license information.
5
- // </copyright>
6
- //-----------------------------------------------------------------------
7
-
8
- import { spawn } from 'node:child_process';
9
- import { createRequire } from 'node:module';
10
- import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
11
- import { dirname, join, resolve } from 'node:path';
12
- import { StringDecoder } from 'node:string_decoder';
13
-
14
- function parseJson(text) {
15
- try {
16
- return JSON.parse(String(text).trim());
17
- } catch {
18
- return null;
19
- }
20
- }
21
-
22
- function samePath(left, right) {
23
- const normalize = (value) => {
24
- const path = resolve(String(value));
25
- return process.platform === 'win32' ? path.toLocaleLowerCase() : path;
26
- };
27
- return normalize(left) === normalize(right);
28
- }
29
-
30
- export function rayfinErrorMessage(value) {
31
- if (value?.status === 'error') {
32
- return String(
33
- value.error?.message ??
34
- value.error ??
35
- 'Rayfin connector invocation failed.'
36
- );
37
- }
38
- return null;
39
- }
40
-
41
- export function resolveRayfinCliEntry({
42
- moduleUrl = import.meta.url,
43
- createRequireImpl = createRequire,
44
- readFileSyncImpl = readFileSync,
45
- } = {}) {
46
- const require = createRequireImpl(moduleUrl);
47
- const packagePath = require.resolve('@microsoft/rayfin-cli/package.json');
48
- const packageJson = JSON.parse(readFileSyncImpl(packagePath, 'utf8'));
49
- const relativeEntry =
50
- typeof packageJson.bin === 'string'
51
- ? packageJson.bin
52
- : packageJson.bin?.rayfin;
53
- if (typeof relativeEntry !== 'string' || relativeEntry.length === 0) {
54
- throw new Error(
55
- '@microsoft/rayfin-cli does not declare a rayfin executable.'
56
- );
57
- }
58
- return join(dirname(packagePath), relativeEntry);
59
- }
60
-
61
- function launchError(error) {
62
- const code = error?.code ? ` (${error.code})` : '';
63
- return (
64
- `Could not run the Rayfin CLI${code}. ` +
65
- 'Install the app dependencies, then retry. ' +
66
- `Details: ${error?.message ?? String(error)}`
67
- );
68
- }
69
-
70
- export function runRayfinCli({
71
- args,
72
- cwd = process.cwd(),
73
- cliEntry,
74
- spawnImpl = spawn,
75
- environment = process.env,
76
- }) {
77
- return new Promise((resolveResult) => {
78
- let commandEntry;
79
- try {
80
- commandEntry = cliEntry ?? resolveRayfinCliEntry();
81
- } catch (error) {
82
- resolveResult({ ok: false, error: launchError(error) });
83
- return;
84
- }
85
-
86
- let child;
87
- try {
88
- child = spawnImpl(process.execPath, [commandEntry, ...args], {
89
- cwd,
90
- env: environment,
91
- windowsHide: true,
92
- shell: false,
93
- });
94
- } catch (error) {
95
- resolveResult({ ok: false, error: launchError(error) });
96
- return;
97
- }
98
-
99
- let stdout = '';
100
- let stderr = '';
101
- let settled = false;
102
- // Pipe chunks do not respect character boundaries, so a multi-byte
103
- // character split across two reads decodes as U+FFFD under a per-chunk
104
- // toString(). StringDecoder holds the partial sequence back until the rest
105
- // arrives.
106
- const outDecoder = new StringDecoder('utf8');
107
- const errDecoder = new StringDecoder('utf8');
108
- const finish = (value) => {
109
- if (settled) return;
110
- settled = true;
111
- resolveResult(value);
112
- };
113
- child.stdout?.on('data', (chunk) => {
114
- stdout += outDecoder.write(chunk);
115
- });
116
- child.stderr?.on('data', (chunk) => {
117
- stderr += errDecoder.write(chunk);
118
- });
119
- child.on('error', (error) => {
120
- finish({
121
- ok: false,
122
- error: launchError(error),
123
- stdout,
124
- stderr,
125
- });
126
- });
127
- child.on('close', (code, signal) => {
128
- stdout += outDecoder.end();
129
- stderr += errDecoder.end();
130
- const parsed = parseJson(stdout);
131
- if (code === 0 && signal == null) {
132
- finish({ ok: true, value: parsed, stdout, stderr });
133
- return;
134
- }
135
- const rayfinError = parsed ? rayfinErrorMessage(parsed) : null;
136
- const message = rayfinError || stderr.trim() || stdout.trim();
137
- const status = signal
138
- ? `Rayfin terminated by signal ${signal}`
139
- : `Rayfin exited with code ${code ?? 'unknown'}`;
140
- finish({
141
- ok: false,
142
- error: message ? `${status}: ${message}` : `${status}.`,
143
- stdout,
144
- stderr,
145
- exitCode: code,
146
- signal,
147
- });
148
- });
149
- });
150
- }
151
-
152
- export async function listConfiguredConnectors(options = {}) {
153
- const result = await runRayfinCli({
154
- ...options,
155
- args: ['connector', 'list', '--json'],
156
- });
157
- if (!result.ok) {
158
- return result;
159
- }
160
- if (!Array.isArray(result.value)) {
161
- return {
162
- ok: false,
163
- error:
164
- 'Could not resolve connectors because `rayfin connector list --json` ' +
165
- 'did not return a JSON array. Run this command from the app root.',
166
- stdout: result.stdout,
167
- stderr: result.stderr,
168
- };
169
- }
170
- return {
171
- ok: true,
172
- connectors: result.value,
173
- stdout: result.stdout,
174
- stderr: result.stderr,
175
- };
176
- }
177
-
178
- export async function invokeConnector({
179
- connector,
180
- query,
181
- limit,
182
- cwd = process.cwd(),
183
- cliEntry,
184
- spawnImpl = spawn,
185
- }) {
186
- // Rayfin rejects --file paths outside the project root. Keep both the
187
- // request and the explicit full-result file inside the app.
188
- const temporaryDirectory = mkdtempSync(join(cwd, '.rayfin-schema-query-'));
189
- const payloadPath = join(temporaryDirectory, 'query.json');
190
- const outputPath = join(temporaryDirectory, 'result.json');
191
- const payload = {
192
- query,
193
- ...(limit ? { resultSetRowCountLimit: Number(limit) } : {}),
194
- };
195
-
196
- try {
197
- writeFileSync(payloadPath, JSON.stringify(payload), 'utf8');
198
- const result = await runRayfinCli({
199
- args: [
200
- 'connector',
201
- 'invoke',
202
- connector,
203
- 'executeQuery',
204
- '--file',
205
- payloadPath,
206
- '--output-file',
207
- outputPath,
208
- '--json',
209
- ],
210
- cwd,
211
- cliEntry,
212
- spawnImpl,
213
- });
214
- if (!result.ok) {
215
- return result;
216
- }
217
-
218
- const envelope = result.value;
219
- if (!envelope || envelope.status !== 'ok') {
220
- return {
221
- ok: false,
222
- error:
223
- rayfinErrorMessage(envelope) ??
224
- 'Rayfin connector invoke returned an invalid JSON envelope.',
225
- stdout: result.stdout,
226
- stderr: result.stderr,
227
- };
228
- }
229
- if (
230
- typeof envelope.outputFile !== 'string' ||
231
- !samePath(envelope.outputFile, outputPath)
232
- ) {
233
- return {
234
- ok: false,
235
- error:
236
- 'Rayfin did not write the connector result to the requested ' +
237
- 'output file. Refusing to trust the inline preview.',
238
- stdout: result.stdout,
239
- stderr: result.stderr,
240
- };
241
- }
242
-
243
- const output = parseJson(readFileSync(outputPath, 'utf8'));
244
- if (output === null) {
245
- return {
246
- ok: false,
247
- error: 'Rayfin wrote a connector result that was not valid JSON.',
248
- stdout: result.stdout,
249
- stderr: result.stderr,
250
- };
251
- }
252
-
253
- return {
254
- ok: true,
255
- output,
256
- outputFile: envelope.outputFile,
257
- outputBytes: envelope.outputBytes,
258
- outputTruncated: envelope.outputTruncated === true,
259
- stdout: result.stdout,
260
- stderr: result.stderr,
261
- };
262
- } finally {
263
- rmSync(temporaryDirectory, { recursive: true, force: true });
264
- }
265
- }