@microsoft/rayfin-semantic-model-pack 0.0.0-stage → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG +14 -0
- package/LICENSE +21 -0
- package/README.md +78 -2
- package/kit/hooks/use-semantic-model-query.spec.ts +361 -0
- package/kit/hooks/use-semantic-model-query.ts +224 -0
- package/kit/scripts/rayfin-connector.mjs +265 -0
- package/kit/scripts/rayfin-connector.test.mjs +243 -0
- package/kit/scripts/schema-sql-batch.test.mjs +2097 -0
- package/kit/scripts/schema-sql-cli.test.mjs +488 -0
- package/kit/scripts/schema-sql.mjs +2650 -0
- package/kit/scripts/schema-sql.test.mjs +1387 -0
- package/kit/scripts/schema.mjs +166 -0
- package/kit/scripts/schema.test.mjs +271 -0
- package/pack.json +44 -0
- package/package.json +12 -3
- package/skills/analytics/SKILL.md +461 -0
- package/skills/dax-authoring/SKILL.md +170 -0
- package/skills/dax-authoring/references/dax-core-reference.md +862 -0
- package/skills/dax-authoring/references/dax-query-patterns.md +396 -0
- package/skills/dax-authoring/references/dax-time-intelligence.md +347 -0
- package/skills/query-design/SKILL.md +114 -0
- package/skills/query-design/references/anti-patterns.md +285 -0
- package/skills/query-design/references/filter-strategy.md +68 -0
- package/skills/query-design/references/format-strings.md +62 -0
- package/skills/query-design/references/highlight-queries.md +122 -0
- package/skills/query-design/references/multi-grain-patterns.md +159 -0
- package/skills/schema-discovery/SKILL.md +382 -0
- package/skills/schema-discovery/references/discovery-queries.md +153 -0
|
@@ -0,0 +1,1387 @@
|
|
|
1
|
+
//-----------------------------------------------------------------------
|
|
2
|
+
// <copyright company="Microsoft Corporation">
|
|
3
|
+
// Copyright (c) Microsoft Corporation. All rights reserved.
|
|
4
|
+
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
|
5
|
+
// </copyright>
|
|
6
|
+
//-----------------------------------------------------------------------
|
|
7
|
+
|
|
8
|
+
import assert from 'node:assert/strict';
|
|
9
|
+
import { spawn } from 'node:child_process';
|
|
10
|
+
import { once } from 'node:events';
|
|
11
|
+
import {
|
|
12
|
+
existsSync,
|
|
13
|
+
mkdirSync,
|
|
14
|
+
mkdtempSync,
|
|
15
|
+
readFileSync,
|
|
16
|
+
renameSync,
|
|
17
|
+
rmSync,
|
|
18
|
+
statSync,
|
|
19
|
+
writeFileSync,
|
|
20
|
+
} from 'node:fs';
|
|
21
|
+
import { tmpdir } from 'node:os';
|
|
22
|
+
import { join } from 'node:path';
|
|
23
|
+
import test from 'node:test';
|
|
24
|
+
import { DatabaseSync } from 'node:sqlite';
|
|
25
|
+
|
|
26
|
+
import {
|
|
27
|
+
assertSupportedNodeVersion,
|
|
28
|
+
boundRows,
|
|
29
|
+
buildDatabase,
|
|
30
|
+
cacheStatus,
|
|
31
|
+
CATALOG_SOURCES,
|
|
32
|
+
columnNamesOf,
|
|
33
|
+
createTable,
|
|
34
|
+
discoverRaw,
|
|
35
|
+
formatResult,
|
|
36
|
+
inferColumnType,
|
|
37
|
+
isRateLimited,
|
|
38
|
+
materializeCatalog,
|
|
39
|
+
modelCacheDir,
|
|
40
|
+
publishFiles,
|
|
41
|
+
quoteIdentifier,
|
|
42
|
+
rankSemanticSearchRows,
|
|
43
|
+
rawQuery,
|
|
44
|
+
renderCatalog,
|
|
45
|
+
resultExitCode,
|
|
46
|
+
retryDelayMs,
|
|
47
|
+
runSql,
|
|
48
|
+
searchCatalog,
|
|
49
|
+
selectsOnlyTableRows,
|
|
50
|
+
semanticSearchRows,
|
|
51
|
+
toSqlValue,
|
|
52
|
+
ttlMinutes,
|
|
53
|
+
withRateLimitRetry,
|
|
54
|
+
withCatalogLock,
|
|
55
|
+
} from './schema-sql.mjs';
|
|
56
|
+
|
|
57
|
+
function tempDir() {
|
|
58
|
+
return mkdtempSync(join(tmpdir(), 'schema-sql-'));
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
test('every read-only SQLite connection goes through the query-only helper', () => {
|
|
62
|
+
const source = readFileSync(
|
|
63
|
+
new URL('./schema-sql.mjs', import.meta.url),
|
|
64
|
+
'utf8'
|
|
65
|
+
);
|
|
66
|
+
const rawReadOnlyOpens =
|
|
67
|
+
source.match(
|
|
68
|
+
/new DatabaseSync\([^)]*,\s*\{\s*readOnly:\s*true\s*\}\s*\)/gu
|
|
69
|
+
) ?? [];
|
|
70
|
+
assert.equal(
|
|
71
|
+
rawReadOnlyOpens.length,
|
|
72
|
+
1,
|
|
73
|
+
'new read paths must use openReadOnlyDatabase so query_only is applied'
|
|
74
|
+
);
|
|
75
|
+
assert.match(
|
|
76
|
+
source,
|
|
77
|
+
/function openReadOnlyDatabase[\s\S]*PRAGMA query_only = ON/u
|
|
78
|
+
);
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
function sampleRowsFor(source) {
|
|
82
|
+
switch (source.table) {
|
|
83
|
+
case 'info_view_tables':
|
|
84
|
+
return [{ Name: 'Sales', Description: 'Sales facts' }];
|
|
85
|
+
case 'info_view_columns':
|
|
86
|
+
return [
|
|
87
|
+
{
|
|
88
|
+
Table: 'Sales',
|
|
89
|
+
Name: 'Amount',
|
|
90
|
+
DataType: 'Decimal',
|
|
91
|
+
IsHidden: false,
|
|
92
|
+
},
|
|
93
|
+
];
|
|
94
|
+
case 'info_view_measures':
|
|
95
|
+
return [
|
|
96
|
+
{
|
|
97
|
+
Table: 'Sales',
|
|
98
|
+
Name: 'Revenue',
|
|
99
|
+
Expression: "SUM('Sales'[Amount])",
|
|
100
|
+
Description: 'Total revenue',
|
|
101
|
+
FormatString: '$#,0',
|
|
102
|
+
DisplayFolder: 'Finance',
|
|
103
|
+
},
|
|
104
|
+
];
|
|
105
|
+
case 'info_view_relationships':
|
|
106
|
+
return [
|
|
107
|
+
{
|
|
108
|
+
FromTable: 'Sales',
|
|
109
|
+
FromColumn: 'DateKey',
|
|
110
|
+
ToTable: 'Date',
|
|
111
|
+
ToColumn: 'DateKey',
|
|
112
|
+
},
|
|
113
|
+
];
|
|
114
|
+
case 'column_statistics':
|
|
115
|
+
return [
|
|
116
|
+
{
|
|
117
|
+
'Table Name': 'Sales',
|
|
118
|
+
'Column Name': 'Amount',
|
|
119
|
+
Min: '1',
|
|
120
|
+
Max: '99',
|
|
121
|
+
Cardinality: 42,
|
|
122
|
+
'Max Length': 0,
|
|
123
|
+
},
|
|
124
|
+
];
|
|
125
|
+
default:
|
|
126
|
+
throw new Error(`Unknown source: ${source.table}`);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
function sampleColumnsFor(source) {
|
|
131
|
+
return Object.keys(sampleRowsFor(source)[0]).map((name) => ({
|
|
132
|
+
name,
|
|
133
|
+
dataType:
|
|
134
|
+
name === 'IsHidden'
|
|
135
|
+
? 'Boolean'
|
|
136
|
+
: name === 'Cardinality' || name === 'Max Length'
|
|
137
|
+
? 'Double'
|
|
138
|
+
: 'String',
|
|
139
|
+
}));
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
test('the catalog build path contains exactly five hardcoded schema sources', () => {
|
|
143
|
+
assert.equal(Object.isFrozen(CATALOG_SOURCES), true);
|
|
144
|
+
assert.equal(CATALOG_SOURCES.every(Object.isFrozen), true);
|
|
145
|
+
assert.deepEqual(
|
|
146
|
+
CATALOG_SOURCES.map((source) => source.table),
|
|
147
|
+
[
|
|
148
|
+
'info_view_tables',
|
|
149
|
+
'info_view_columns',
|
|
150
|
+
'info_view_measures',
|
|
151
|
+
'info_view_relationships',
|
|
152
|
+
'column_statistics',
|
|
153
|
+
]
|
|
154
|
+
);
|
|
155
|
+
for (const source of CATALOG_SOURCES) {
|
|
156
|
+
assert.equal(
|
|
157
|
+
Object.hasOwn(source, 'batchColumn'),
|
|
158
|
+
false,
|
|
159
|
+
'the Rayfin connector has no default row cap, so batching is not allowed'
|
|
160
|
+
);
|
|
161
|
+
}
|
|
162
|
+
});
|
|
163
|
+
|
|
164
|
+
test('INFO.VIEW sources stay unprojected', () => {
|
|
165
|
+
for (const source of CATALOG_SOURCES.filter(
|
|
166
|
+
({ table }) => table !== 'column_statistics'
|
|
167
|
+
)) {
|
|
168
|
+
assert.equal(
|
|
169
|
+
rawQuery(source.dax).includes('SELECTCOLUMNS'),
|
|
170
|
+
false,
|
|
171
|
+
`${source.table} must preserve native column names`
|
|
172
|
+
);
|
|
173
|
+
}
|
|
174
|
+
assert.equal(rawQuery('INFO.VIEW.TABLES()'), 'EVALUATE INFO.VIEW.TABLES()');
|
|
175
|
+
});
|
|
176
|
+
|
|
177
|
+
test('column statistics normalize values before JSON serialization', () => {
|
|
178
|
+
const source = CATALOG_SOURCES.find(
|
|
179
|
+
({ table }) => table === 'column_statistics'
|
|
180
|
+
);
|
|
181
|
+
assert.match(source.dax, /SELECTCOLUMNS\(\s*COLUMNSTATISTICS\(\)/);
|
|
182
|
+
assert.match(source.dax, /"Min", CONVERT\(\[Min\], STRING\)/);
|
|
183
|
+
assert.match(source.dax, /"Max", CONVERT\(\[Max\], STRING\)/);
|
|
184
|
+
assert.match(source.dax, /"Cardinality", CONVERT\(\[Cardinality\], DOUBLE\)/);
|
|
185
|
+
});
|
|
186
|
+
|
|
187
|
+
test('the Node guard fails with a readable minimum-version error', () => {
|
|
188
|
+
assert.throws(
|
|
189
|
+
() => assertSupportedNodeVersion('22.12.0'),
|
|
190
|
+
/requires Node\.js >=22\.13\.0.*22\.12\.0/i
|
|
191
|
+
);
|
|
192
|
+
assert.doesNotThrow(() => assertSupportedNodeVersion('22.13.0'));
|
|
193
|
+
assert.doesNotThrow(() => assertSupportedNodeVersion('24.0.0'));
|
|
194
|
+
});
|
|
195
|
+
|
|
196
|
+
test('values are coercible by node:sqlite', () => {
|
|
197
|
+
assert.equal(toSqlValue(true), 1);
|
|
198
|
+
assert.equal(toSqlValue(false), 0);
|
|
199
|
+
assert.equal(toSqlValue(null), null);
|
|
200
|
+
assert.equal(toSqlValue(Number.NaN), null);
|
|
201
|
+
assert.equal(toSqlValue({ a: 1 }), '{"a":1}');
|
|
202
|
+
assert.equal(
|
|
203
|
+
toSqlValue(new Date('2024-01-01T00:00:00Z')),
|
|
204
|
+
'2024-01-01T00:00:00.000Z'
|
|
205
|
+
);
|
|
206
|
+
});
|
|
207
|
+
|
|
208
|
+
test('column types preserve numeric ordering', () => {
|
|
209
|
+
assert.equal(inferColumnType([1, 2, 3]), 'INTEGER');
|
|
210
|
+
assert.equal(inferColumnType([1, 2.5]), 'REAL');
|
|
211
|
+
assert.equal(inferColumnType(['a', 1]), 'TEXT');
|
|
212
|
+
assert.equal(inferColumnType([true, false]), 'INTEGER');
|
|
213
|
+
assert.equal(inferColumnType([null, '', null]), 'TEXT');
|
|
214
|
+
});
|
|
215
|
+
|
|
216
|
+
test('identifiers and sparse rows preserve native columns', () => {
|
|
217
|
+
assert.equal(quoteIdentifier('Max Length'), '"Max Length"');
|
|
218
|
+
assert.equal(quoteIdentifier('od"d'), '"od""d"');
|
|
219
|
+
assert.deepEqual(columnNamesOf([{ a: 1 }, { a: 2, b: 3 }]), ['a', 'b']);
|
|
220
|
+
});
|
|
221
|
+
|
|
222
|
+
test('native column spellings and booleans survive SQLite', () => {
|
|
223
|
+
const dir = tempDir();
|
|
224
|
+
try {
|
|
225
|
+
const dbPath = join(dir, 'model.sqlite');
|
|
226
|
+
buildDatabase(
|
|
227
|
+
[
|
|
228
|
+
{
|
|
229
|
+
table: 'info_view_columns',
|
|
230
|
+
grain: '',
|
|
231
|
+
dax: 'INFO.VIEW.COLUMNS()',
|
|
232
|
+
rows: [
|
|
233
|
+
{
|
|
234
|
+
'Table Name': 'Sales',
|
|
235
|
+
'Max Length': 12,
|
|
236
|
+
IsHidden: false,
|
|
237
|
+
},
|
|
238
|
+
],
|
|
239
|
+
},
|
|
240
|
+
],
|
|
241
|
+
dbPath
|
|
242
|
+
);
|
|
243
|
+
const result = runSql(
|
|
244
|
+
dbPath,
|
|
245
|
+
'SELECT "Table Name", "Max Length" FROM info_view_columns WHERE IsHidden = 0'
|
|
246
|
+
);
|
|
247
|
+
assert.equal(result.ok, true);
|
|
248
|
+
assert.equal(result.rows.length, 1);
|
|
249
|
+
assert.equal(result.rows[0]['Table Name'], 'Sales');
|
|
250
|
+
assert.equal(result.rows[0]['Max Length'], 12);
|
|
251
|
+
} finally {
|
|
252
|
+
rmSync(dir, { recursive: true, force: true });
|
|
253
|
+
}
|
|
254
|
+
});
|
|
255
|
+
|
|
256
|
+
test('a successful empty view creates a queryable table from returned columns', () => {
|
|
257
|
+
const dir = tempDir();
|
|
258
|
+
try {
|
|
259
|
+
const dbPath = join(dir, 'model.sqlite');
|
|
260
|
+
buildDatabase(
|
|
261
|
+
[
|
|
262
|
+
{
|
|
263
|
+
table: 'info_view_relationships',
|
|
264
|
+
grain: '',
|
|
265
|
+
dax: 'INFO.VIEW.RELATIONSHIPS()',
|
|
266
|
+
columns: [
|
|
267
|
+
{ name: 'FromTable', dataType: 'String' },
|
|
268
|
+
{ name: 'FromColumn', dataType: 'String' },
|
|
269
|
+
{ name: 'IsActive', dataType: 'Boolean' },
|
|
270
|
+
],
|
|
271
|
+
rows: [],
|
|
272
|
+
},
|
|
273
|
+
],
|
|
274
|
+
dbPath
|
|
275
|
+
);
|
|
276
|
+
const result = runSql(dbPath, 'SELECT * FROM info_view_relationships');
|
|
277
|
+
assert.equal(result.ok, true);
|
|
278
|
+
assert.deepEqual(result.rows, []);
|
|
279
|
+
const db = new DatabaseSync(dbPath, { readOnly: true });
|
|
280
|
+
try {
|
|
281
|
+
assert.deepEqual(
|
|
282
|
+
db
|
|
283
|
+
.prepare('PRAGMA table_info("info_view_relationships")')
|
|
284
|
+
.all()
|
|
285
|
+
.map((column) => [column.name, column.type]),
|
|
286
|
+
[
|
|
287
|
+
['FromTable', 'TEXT'],
|
|
288
|
+
['FromColumn', 'TEXT'],
|
|
289
|
+
['IsActive', 'INTEGER'],
|
|
290
|
+
]
|
|
291
|
+
);
|
|
292
|
+
} finally {
|
|
293
|
+
db.close();
|
|
294
|
+
}
|
|
295
|
+
} finally {
|
|
296
|
+
rmSync(dir, { recursive: true, force: true });
|
|
297
|
+
}
|
|
298
|
+
});
|
|
299
|
+
|
|
300
|
+
test('a failed rebuild leaves the previous complete database', () => {
|
|
301
|
+
const dir = tempDir();
|
|
302
|
+
try {
|
|
303
|
+
const dbPath = join(dir, 'model.sqlite');
|
|
304
|
+
buildDatabase(
|
|
305
|
+
[
|
|
306
|
+
{
|
|
307
|
+
table: 't',
|
|
308
|
+
grain: '',
|
|
309
|
+
dax: 'x',
|
|
310
|
+
rows: [{ value: 'old' }],
|
|
311
|
+
},
|
|
312
|
+
],
|
|
313
|
+
dbPath
|
|
314
|
+
);
|
|
315
|
+
const circular = {};
|
|
316
|
+
circular.self = circular;
|
|
317
|
+
assert.throws(() =>
|
|
318
|
+
buildDatabase(
|
|
319
|
+
[
|
|
320
|
+
{
|
|
321
|
+
table: 't',
|
|
322
|
+
grain: '',
|
|
323
|
+
dax: 'x',
|
|
324
|
+
rows: [{ value: circular }],
|
|
325
|
+
},
|
|
326
|
+
],
|
|
327
|
+
dbPath
|
|
328
|
+
)
|
|
329
|
+
);
|
|
330
|
+
assert.equal(runSql(dbPath, 'SELECT * FROM t').rows[0].value, 'old');
|
|
331
|
+
} finally {
|
|
332
|
+
rmSync(dir, { recursive: true, force: true });
|
|
333
|
+
}
|
|
334
|
+
});
|
|
335
|
+
|
|
336
|
+
test('table population is one transaction and rolls back failed replacement', () => {
|
|
337
|
+
const dir = tempDir();
|
|
338
|
+
const dbPath = join(dir, 'model.sqlite');
|
|
339
|
+
const db = new DatabaseSync(dbPath);
|
|
340
|
+
try {
|
|
341
|
+
createTable(db, 't', [{ value: 'old' }]);
|
|
342
|
+
const circular = {};
|
|
343
|
+
circular.self = circular;
|
|
344
|
+
assert.throws(() => createTable(db, 't', [{ value: circular }]));
|
|
345
|
+
assert.equal(db.prepare('SELECT value FROM t').get().value, 'old');
|
|
346
|
+
} finally {
|
|
347
|
+
db.close();
|
|
348
|
+
rmSync(dir, { recursive: true, force: true });
|
|
349
|
+
}
|
|
350
|
+
});
|
|
351
|
+
|
|
352
|
+
test('a failed multi-file publication restores the previous generation', () => {
|
|
353
|
+
const dir = tempDir();
|
|
354
|
+
try {
|
|
355
|
+
const dbPath = join(dir, 'model.sqlite');
|
|
356
|
+
const catalogPath = join(dir, 'catalog.md');
|
|
357
|
+
const temporaryDbPath = join(dir, 'next.sqlite');
|
|
358
|
+
const temporaryCatalogPath = join(dir, 'next.md');
|
|
359
|
+
writeFileSync(dbPath, 'old database');
|
|
360
|
+
writeFileSync(catalogPath, 'old catalog');
|
|
361
|
+
writeFileSync(temporaryDbPath, 'new database');
|
|
362
|
+
writeFileSync(temporaryCatalogPath, 'new catalog');
|
|
363
|
+
|
|
364
|
+
assert.throws(
|
|
365
|
+
() =>
|
|
366
|
+
publishFiles(
|
|
367
|
+
[
|
|
368
|
+
{ temporaryPath: temporaryDbPath, destinationPath: dbPath },
|
|
369
|
+
{
|
|
370
|
+
temporaryPath: temporaryCatalogPath,
|
|
371
|
+
destinationPath: catalogPath,
|
|
372
|
+
},
|
|
373
|
+
],
|
|
374
|
+
{
|
|
375
|
+
rename(from, to) {
|
|
376
|
+
if (from === temporaryCatalogPath && to === catalogPath) {
|
|
377
|
+
const error = new Error('simulated second publish failure');
|
|
378
|
+
error.code = 'EPERM';
|
|
379
|
+
throw error;
|
|
380
|
+
}
|
|
381
|
+
renameSync(from, to);
|
|
382
|
+
},
|
|
383
|
+
}
|
|
384
|
+
),
|
|
385
|
+
/simulated second publish failure/
|
|
386
|
+
);
|
|
387
|
+
assert.equal(readFileSync(dbPath, 'utf8'), 'old database');
|
|
388
|
+
assert.equal(readFileSync(catalogPath, 'utf8'), 'old catalog');
|
|
389
|
+
} finally {
|
|
390
|
+
rmSync(dir, { recursive: true, force: true });
|
|
391
|
+
}
|
|
392
|
+
});
|
|
393
|
+
|
|
394
|
+
test('selected tables expose bounded owned-measure context', () => {
|
|
395
|
+
const dir = tempDir();
|
|
396
|
+
try {
|
|
397
|
+
const dbPath = join(dir, 'model.sqlite');
|
|
398
|
+
buildDatabase(
|
|
399
|
+
[
|
|
400
|
+
{
|
|
401
|
+
table: 'info_view_tables',
|
|
402
|
+
grain: '',
|
|
403
|
+
dax: '',
|
|
404
|
+
rows: [{ Name: 'Sales' }],
|
|
405
|
+
},
|
|
406
|
+
{
|
|
407
|
+
table: 'info_view_measures',
|
|
408
|
+
grain: '',
|
|
409
|
+
dax: '',
|
|
410
|
+
rows: sampleRowsFor({
|
|
411
|
+
table: 'info_view_measures',
|
|
412
|
+
}),
|
|
413
|
+
},
|
|
414
|
+
],
|
|
415
|
+
dbPath
|
|
416
|
+
);
|
|
417
|
+
const result = runSql(dbPath, 'SELECT "Name" FROM info_view_tables');
|
|
418
|
+
assert.equal(result.rows[0].ownedMeasureCount, 1);
|
|
419
|
+
assert.equal(result.rows[0].ownedMeasures[0].name, 'Revenue');
|
|
420
|
+
} finally {
|
|
421
|
+
rmSync(dir, { recursive: true, force: true });
|
|
422
|
+
}
|
|
423
|
+
});
|
|
424
|
+
|
|
425
|
+
test('a second statement is rejected rather than silently ignored', () => {
|
|
426
|
+
const dir = tempDir();
|
|
427
|
+
try {
|
|
428
|
+
const dbPath = join(dir, 'model.sqlite');
|
|
429
|
+
buildDatabase(
|
|
430
|
+
[
|
|
431
|
+
{
|
|
432
|
+
table: 'info_view_tables',
|
|
433
|
+
grain: '',
|
|
434
|
+
dax: '',
|
|
435
|
+
rows: [{ Name: 'Sales' }, { Name: 'Date' }],
|
|
436
|
+
},
|
|
437
|
+
],
|
|
438
|
+
dbPath
|
|
439
|
+
);
|
|
440
|
+
|
|
441
|
+
// prepare() compiles only the first statement, so the agent asked two
|
|
442
|
+
// questions, got one answer, and nothing said the rest never ran.
|
|
443
|
+
const two = runSql(
|
|
444
|
+
dbPath,
|
|
445
|
+
'SELECT "Name" FROM info_view_tables; SELECT 1 AS other'
|
|
446
|
+
);
|
|
447
|
+
assert.equal(two.ok, false);
|
|
448
|
+
assert.match(two.error, /only one statement runs per --sql/iu);
|
|
449
|
+
assert.match(two.error, /SELECT/u, 'the ignored text is named');
|
|
450
|
+
|
|
451
|
+
// A semicolon that is not a separator must still run.
|
|
452
|
+
for (const sql of [
|
|
453
|
+
'SELECT "Name" FROM info_view_tables;',
|
|
454
|
+
'SELECT "Name" FROM info_view_tables WHERE "Name" <> \'a;b\'',
|
|
455
|
+
'SELECT "Name" FROM info_view_tables -- ; SELECT 1',
|
|
456
|
+
]) {
|
|
457
|
+
assert.equal(runSql(dbPath, sql).ok, true, sql);
|
|
458
|
+
}
|
|
459
|
+
} finally {
|
|
460
|
+
rmSync(dir, { recursive: true, force: true });
|
|
461
|
+
}
|
|
462
|
+
});
|
|
463
|
+
|
|
464
|
+
test('enrichment does not decorate rows a join returned from another table', () => {
|
|
465
|
+
const dir = tempDir();
|
|
466
|
+
try {
|
|
467
|
+
const dbPath = join(dir, 'model.sqlite');
|
|
468
|
+
buildDatabase(
|
|
469
|
+
[
|
|
470
|
+
{
|
|
471
|
+
table: 'info_view_tables',
|
|
472
|
+
grain: '',
|
|
473
|
+
dax: '',
|
|
474
|
+
rows: [{ Name: 'Sales', Description: 'facts' }],
|
|
475
|
+
},
|
|
476
|
+
{
|
|
477
|
+
table: 'info_view_measures',
|
|
478
|
+
grain: '',
|
|
479
|
+
dax: '',
|
|
480
|
+
rows: Array.from({ length: 10 }, (_, index) => ({
|
|
481
|
+
Table: 'Sales',
|
|
482
|
+
Name: `Revenue ${index}`,
|
|
483
|
+
Expression: 'SUM(x)',
|
|
484
|
+
})),
|
|
485
|
+
},
|
|
486
|
+
{
|
|
487
|
+
table: 'info_view_columns',
|
|
488
|
+
grain: '',
|
|
489
|
+
dax: '',
|
|
490
|
+
rows: Array.from({ length: 6 }, (_, index) => ({
|
|
491
|
+
Table: 'Sales',
|
|
492
|
+
Name: `Column ${index}`,
|
|
493
|
+
})),
|
|
494
|
+
},
|
|
495
|
+
],
|
|
496
|
+
dbPath
|
|
497
|
+
);
|
|
498
|
+
|
|
499
|
+
// catalog.md tells the agent these tables join, so this is a query it is
|
|
500
|
+
// invited to write. The rows are column rows; treating each one as a
|
|
501
|
+
// selected table attached ten measures apiece and blew the byte budget.
|
|
502
|
+
const joined = runSql(
|
|
503
|
+
dbPath,
|
|
504
|
+
'SELECT c."Table", c."Name" FROM info_view_columns c ' +
|
|
505
|
+
'JOIN info_view_tables t ON c."Table" = t."Name"'
|
|
506
|
+
);
|
|
507
|
+
assert.equal(joined.ok, true);
|
|
508
|
+
assert.equal(joined.rows.length, 6);
|
|
509
|
+
for (const row of joined.rows) {
|
|
510
|
+
assert.deepEqual(
|
|
511
|
+
Object.keys(row),
|
|
512
|
+
['Table', 'Name'],
|
|
513
|
+
'a column row carries no owned-measure fields'
|
|
514
|
+
);
|
|
515
|
+
}
|
|
516
|
+
|
|
517
|
+
// The ordinary table query is unaffected.
|
|
518
|
+
const direct = runSql(dbPath, 'SELECT "Name" FROM info_view_tables');
|
|
519
|
+
assert.equal(direct.rows[0].ownedMeasureCount, 10);
|
|
520
|
+
} finally {
|
|
521
|
+
rmSync(dir, { recursive: true, force: true });
|
|
522
|
+
}
|
|
523
|
+
});
|
|
524
|
+
|
|
525
|
+
test('the enrichment gate reads SQL structure, not the table name', () => {
|
|
526
|
+
// Matching the name anywhere fired on comments, string literals and joins.
|
|
527
|
+
for (const sql of [
|
|
528
|
+
'SELECT [Name] FROM info_view_tables',
|
|
529
|
+
"SELECT t.[Name] FROM info_view_tables t WHERE t.[Name] = 'Sales'",
|
|
530
|
+
'SELECT t.[Name] FROM info_view_tables AS t',
|
|
531
|
+
'SELECT "Name" FROM "info_view_tables"',
|
|
532
|
+
'SELECT [Name] FROM info_view_tables;',
|
|
533
|
+
'SELECT [Name] FROM info_view_tables ORDER BY [Name]',
|
|
534
|
+
]) {
|
|
535
|
+
assert.equal(selectsOnlyTableRows(sql), true, sql);
|
|
536
|
+
}
|
|
537
|
+
for (const sql of [
|
|
538
|
+
'SELECT * FROM info_view_columns c JOIN info_view_tables t ON 1 = 1',
|
|
539
|
+
'SELECT * FROM info_view_tables, info_view_measures',
|
|
540
|
+
'SELECT [Name] FROM info_view_tables UNION SELECT [Name] FROM info_view_measures',
|
|
541
|
+
"SELECT [Name] FROM info_view_measures WHERE [Expression] LIKE '%info_view_tables%'",
|
|
542
|
+
'SELECT [Name] FROM info_view_measures -- info_view_tables',
|
|
543
|
+
'SELECT [Name] FROM info_view_measures /* info_view_tables */',
|
|
544
|
+
'SELECT [Name] FROM info_view_measures WHERE [Table] IN (SELECT [Name] FROM info_view_tables)',
|
|
545
|
+
]) {
|
|
546
|
+
assert.equal(selectsOnlyTableRows(sql), false, sql);
|
|
547
|
+
}
|
|
548
|
+
});
|
|
549
|
+
|
|
550
|
+
test('owned-measure enrichment keeps the documented table query under budget', () => {
|
|
551
|
+
const dir = tempDir();
|
|
552
|
+
try {
|
|
553
|
+
const dbPath = join(dir, 'model.sqlite');
|
|
554
|
+
const tables = ['Sales North', 'Sales South'];
|
|
555
|
+
buildDatabase(
|
|
556
|
+
[
|
|
557
|
+
{
|
|
558
|
+
table: 'info_view_tables',
|
|
559
|
+
grain: '',
|
|
560
|
+
dax: '',
|
|
561
|
+
rows: tables.map((Name) => ({
|
|
562
|
+
Name,
|
|
563
|
+
Description: `${Name} facts`,
|
|
564
|
+
IsHidden: false,
|
|
565
|
+
})),
|
|
566
|
+
},
|
|
567
|
+
{
|
|
568
|
+
table: 'info_view_measures',
|
|
569
|
+
grain: '',
|
|
570
|
+
dax: '',
|
|
571
|
+
rows: tables.flatMap((Table) =>
|
|
572
|
+
Array.from({ length: 10 }, (_, index) => ({
|
|
573
|
+
Table,
|
|
574
|
+
Name: `Revenue ${index}`,
|
|
575
|
+
Expression: `CALCULATE(SUM('Sales'[Amount]), ${'x'.repeat(1200)})`,
|
|
576
|
+
Description: 'y'.repeat(600),
|
|
577
|
+
DisplayFolder: 'Revenue',
|
|
578
|
+
}))
|
|
579
|
+
),
|
|
580
|
+
},
|
|
581
|
+
],
|
|
582
|
+
dbPath
|
|
583
|
+
);
|
|
584
|
+
|
|
585
|
+
const result = runSql(
|
|
586
|
+
dbPath,
|
|
587
|
+
"SELECT [Name], [Description], [IsHidden] FROM info_view_tables WHERE lower([Name]) LIKE '%sales%'"
|
|
588
|
+
);
|
|
589
|
+
assert.equal(result.totalRows, 2);
|
|
590
|
+
assert.equal(result.returnedRows, 2);
|
|
591
|
+
assert.equal(result.truncated, false);
|
|
592
|
+
assert.equal(result.rows[0].ownedMeasureCount, 10);
|
|
593
|
+
assert.deepEqual(Object.keys(result.rows[0].ownedMeasures[0]), [
|
|
594
|
+
'name',
|
|
595
|
+
'displayFolder',
|
|
596
|
+
]);
|
|
597
|
+
assert.equal(result.rows[0].ownedMeasuresNeedDetail, true);
|
|
598
|
+
} finally {
|
|
599
|
+
rmSync(dir, { recursive: true, force: true });
|
|
600
|
+
}
|
|
601
|
+
});
|
|
602
|
+
|
|
603
|
+
test('discoverRaw performs exactly one request per source', async () => {
|
|
604
|
+
const calls = [];
|
|
605
|
+
const result = await discoverRaw({
|
|
606
|
+
run: async (dax) => {
|
|
607
|
+
calls.push(dax);
|
|
608
|
+
const source = CATALOG_SOURCES.find(
|
|
609
|
+
(candidate) => rawQuery(candidate.dax) === dax
|
|
610
|
+
);
|
|
611
|
+
return {
|
|
612
|
+
ok: true,
|
|
613
|
+
columns: sampleColumnsFor(source),
|
|
614
|
+
rows: sampleRowsFor(source),
|
|
615
|
+
};
|
|
616
|
+
},
|
|
617
|
+
});
|
|
618
|
+
assert.equal(result.ok, true);
|
|
619
|
+
assert.equal(calls.length, 5);
|
|
620
|
+
assert.deepEqual(
|
|
621
|
+
calls,
|
|
622
|
+
CATALOG_SOURCES.map((source) => rawQuery(source.dax))
|
|
623
|
+
);
|
|
624
|
+
assert.equal(result.incomplete, false);
|
|
625
|
+
assert.equal(result.resultSets.at(-1).table, 'catalog_status');
|
|
626
|
+
});
|
|
627
|
+
|
|
628
|
+
test('a failed non-table view is cached with a persistent incomplete marker', async () => {
|
|
629
|
+
const result = await discoverRaw({
|
|
630
|
+
run: async (dax) => {
|
|
631
|
+
if (dax.includes('INFO.VIEW.MEASURES')) {
|
|
632
|
+
return { ok: false, error: '429 rate limit' };
|
|
633
|
+
}
|
|
634
|
+
const source = CATALOG_SOURCES.find(
|
|
635
|
+
(candidate) => rawQuery(candidate.dax) === dax
|
|
636
|
+
);
|
|
637
|
+
return {
|
|
638
|
+
ok: true,
|
|
639
|
+
columns: sampleColumnsFor(source),
|
|
640
|
+
rows: sampleRowsFor(source),
|
|
641
|
+
};
|
|
642
|
+
},
|
|
643
|
+
});
|
|
644
|
+
assert.equal(result.ok, true);
|
|
645
|
+
assert.equal(result.incomplete, true);
|
|
646
|
+
assert.equal(
|
|
647
|
+
result.resultSets.some((set) => set.table === 'info_view_measures'),
|
|
648
|
+
false
|
|
649
|
+
);
|
|
650
|
+
assert.deepEqual(result.resultSets.at(-1).rows, [
|
|
651
|
+
{
|
|
652
|
+
status: 'incomplete',
|
|
653
|
+
incomplete: true,
|
|
654
|
+
notes: JSON.stringify(result.notes),
|
|
655
|
+
},
|
|
656
|
+
]);
|
|
657
|
+
});
|
|
658
|
+
|
|
659
|
+
test('successful empty discovery remains distinct from a failed view', async () => {
|
|
660
|
+
const successful = await discoverRaw({
|
|
661
|
+
run: async (dax) => {
|
|
662
|
+
const source = CATALOG_SOURCES.find(
|
|
663
|
+
(candidate) => rawQuery(candidate.dax) === dax
|
|
664
|
+
);
|
|
665
|
+
return {
|
|
666
|
+
ok: true,
|
|
667
|
+
columns: sampleColumnsFor(source),
|
|
668
|
+
rows:
|
|
669
|
+
source.table === 'info_view_relationships'
|
|
670
|
+
? []
|
|
671
|
+
: sampleRowsFor(source),
|
|
672
|
+
};
|
|
673
|
+
},
|
|
674
|
+
});
|
|
675
|
+
assert.equal(successful.incomplete, false);
|
|
676
|
+
const relationships = successful.resultSets.find(
|
|
677
|
+
(set) => set.table === 'info_view_relationships'
|
|
678
|
+
);
|
|
679
|
+
assert.deepEqual(relationships.rows, []);
|
|
680
|
+
assert.ok(relationships.columns.length > 0);
|
|
681
|
+
|
|
682
|
+
const failed = await discoverRaw({
|
|
683
|
+
run: async (dax) => {
|
|
684
|
+
const source = CATALOG_SOURCES.find(
|
|
685
|
+
(candidate) => rawQuery(candidate.dax) === dax
|
|
686
|
+
);
|
|
687
|
+
if (source.table === 'info_view_relationships') {
|
|
688
|
+
return { ok: false, error: 'denied' };
|
|
689
|
+
}
|
|
690
|
+
return {
|
|
691
|
+
ok: true,
|
|
692
|
+
columns: sampleColumnsFor(source),
|
|
693
|
+
rows: sampleRowsFor(source),
|
|
694
|
+
};
|
|
695
|
+
},
|
|
696
|
+
});
|
|
697
|
+
assert.equal(failed.incomplete, true);
|
|
698
|
+
assert.equal(
|
|
699
|
+
failed.resultSets.some((set) => set.table === 'info_view_relationships'),
|
|
700
|
+
false
|
|
701
|
+
);
|
|
702
|
+
});
|
|
703
|
+
|
|
704
|
+
test('table discovery failure remains fatal', async () => {
|
|
705
|
+
const result = await discoverRaw({
|
|
706
|
+
run: async () => ({ ok: false, error: 'denied' }),
|
|
707
|
+
});
|
|
708
|
+
assert.deepEqual(result, {
|
|
709
|
+
ok: false,
|
|
710
|
+
error: 'Table discovery failed: denied',
|
|
711
|
+
});
|
|
712
|
+
});
|
|
713
|
+
|
|
714
|
+
test('materializeCatalog writes the database, catalog and status table', async () => {
|
|
715
|
+
const dir = tempDir();
|
|
716
|
+
const calls = [];
|
|
717
|
+
try {
|
|
718
|
+
const result = await materializeCatalog({
|
|
719
|
+
dir,
|
|
720
|
+
datasetId: 'ds-1',
|
|
721
|
+
alias: 'sales',
|
|
722
|
+
retry: (run) => run,
|
|
723
|
+
run: async (dax) => {
|
|
724
|
+
calls.push(dax);
|
|
725
|
+
const source = CATALOG_SOURCES.find(
|
|
726
|
+
(candidate) => rawQuery(candidate.dax) === dax
|
|
727
|
+
);
|
|
728
|
+
return {
|
|
729
|
+
ok: true,
|
|
730
|
+
columns: sampleColumnsFor(source),
|
|
731
|
+
rows: sampleRowsFor(source),
|
|
732
|
+
};
|
|
733
|
+
},
|
|
734
|
+
});
|
|
735
|
+
assert.equal(calls.length, 5);
|
|
736
|
+
assert.equal(result.ok, true);
|
|
737
|
+
assert.equal(
|
|
738
|
+
runSql(result.dbPath, 'SELECT status, incomplete FROM catalog_status')
|
|
739
|
+
.rows[0].status,
|
|
740
|
+
'complete'
|
|
741
|
+
);
|
|
742
|
+
const catalog = readFileSync(result.catalogPath, 'utf8');
|
|
743
|
+
assert.match(catalog, /Catalog status: \*\*complete\*\*/);
|
|
744
|
+
assert.match(catalog, /Dataset: `ds-1`/);
|
|
745
|
+
assert.match(catalog, /actual Min\/Max/i);
|
|
746
|
+
} finally {
|
|
747
|
+
rmSync(dir, { recursive: true, force: true });
|
|
748
|
+
}
|
|
749
|
+
});
|
|
750
|
+
|
|
751
|
+
test('concurrent refreshes coalesce after the first complete catalog', async () => {
|
|
752
|
+
const dir = tempDir();
|
|
753
|
+
let releaseFirst;
|
|
754
|
+
let markFirstStarted;
|
|
755
|
+
const firstStarted = new Promise((resolve) => {
|
|
756
|
+
markFirstStarted = resolve;
|
|
757
|
+
});
|
|
758
|
+
|
|
759
|
+
test('an abandoned refresh lock is reclaimed when its owner process is gone', async () => {
|
|
760
|
+
const dir = tempDir();
|
|
761
|
+
const child = spawn(process.execPath, [
|
|
762
|
+
'-e',
|
|
763
|
+
'setInterval(() => {}, 1000)',
|
|
764
|
+
]);
|
|
765
|
+
try {
|
|
766
|
+
await once(child, 'spawn');
|
|
767
|
+
const deadPid = child.pid;
|
|
768
|
+
child.kill();
|
|
769
|
+
await once(child, 'exit');
|
|
770
|
+
|
|
771
|
+
const lockPath = join(dir, '.refresh.lock');
|
|
772
|
+
mkdirSync(lockPath);
|
|
773
|
+
writeFileSync(
|
|
774
|
+
join(lockPath, 'owner.json'),
|
|
775
|
+
JSON.stringify({ pid: deadPid }),
|
|
776
|
+
'utf8'
|
|
777
|
+
);
|
|
778
|
+
|
|
779
|
+
const result = await withCatalogLock(
|
|
780
|
+
dir,
|
|
781
|
+
async ({ waited }) => ({ acquired: true, waited }),
|
|
782
|
+
{
|
|
783
|
+
sleep: async () => {
|
|
784
|
+
throw new Error('a dead owner should be reclaimed without polling');
|
|
785
|
+
},
|
|
786
|
+
}
|
|
787
|
+
);
|
|
788
|
+
assert.deepEqual(result, { acquired: true, waited: true });
|
|
789
|
+
assert.equal(existsSync(lockPath), false);
|
|
790
|
+
} finally {
|
|
791
|
+
if (child.exitCode === null) {
|
|
792
|
+
child.kill();
|
|
793
|
+
}
|
|
794
|
+
rmSync(dir, { recursive: true, force: true });
|
|
795
|
+
}
|
|
796
|
+
});
|
|
797
|
+
const firstGate = new Promise((resolve) => {
|
|
798
|
+
releaseFirst = resolve;
|
|
799
|
+
});
|
|
800
|
+
let firstCalls = 0;
|
|
801
|
+
let secondCalls = 0;
|
|
802
|
+
const response = (dax) => {
|
|
803
|
+
const source = CATALOG_SOURCES.find(
|
|
804
|
+
(candidate) => rawQuery(candidate.dax) === dax
|
|
805
|
+
);
|
|
806
|
+
return {
|
|
807
|
+
ok: true,
|
|
808
|
+
columns: sampleColumnsFor(source),
|
|
809
|
+
rows: sampleRowsFor(source),
|
|
810
|
+
};
|
|
811
|
+
};
|
|
812
|
+
|
|
813
|
+
try {
|
|
814
|
+
const first = materializeCatalog({
|
|
815
|
+
dir,
|
|
816
|
+
datasetId: 'ds-1',
|
|
817
|
+
alias: 'sales',
|
|
818
|
+
retry: (run) => run,
|
|
819
|
+
lockOptions: { pollMs: 1, jitter: () => 0 },
|
|
820
|
+
run: async (dax) => {
|
|
821
|
+
firstCalls += 1;
|
|
822
|
+
if (firstCalls === 1) {
|
|
823
|
+
markFirstStarted();
|
|
824
|
+
await firstGate;
|
|
825
|
+
}
|
|
826
|
+
return response(dax);
|
|
827
|
+
},
|
|
828
|
+
});
|
|
829
|
+
await firstStarted;
|
|
830
|
+
const second = materializeCatalog({
|
|
831
|
+
dir,
|
|
832
|
+
datasetId: 'ds-1',
|
|
833
|
+
alias: 'sales',
|
|
834
|
+
retry: (run) => run,
|
|
835
|
+
lockOptions: { pollMs: 1, jitter: () => 0 },
|
|
836
|
+
run: async (dax) => {
|
|
837
|
+
secondCalls += 1;
|
|
838
|
+
return response(dax);
|
|
839
|
+
},
|
|
840
|
+
});
|
|
841
|
+
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
842
|
+
releaseFirst();
|
|
843
|
+
|
|
844
|
+
const [firstResult, secondResult] = await Promise.all([first, second]);
|
|
845
|
+
assert.equal(firstResult.reused, undefined);
|
|
846
|
+
assert.equal(secondResult.reused, true);
|
|
847
|
+
assert.equal(firstCalls, 5);
|
|
848
|
+
assert.equal(secondCalls, 0);
|
|
849
|
+
assert.match(secondResult.notes[0], /another process refreshed/i);
|
|
850
|
+
} finally {
|
|
851
|
+
rmSync(dir, { recursive: true, force: true });
|
|
852
|
+
}
|
|
853
|
+
});
|
|
854
|
+
|
|
855
|
+
test('an incomplete refresh cannot overwrite a complete catalog', async () => {
|
|
856
|
+
const dir = tempDir();
|
|
857
|
+
const response = (dax) => {
|
|
858
|
+
const source = CATALOG_SOURCES.find(
|
|
859
|
+
(candidate) => rawQuery(candidate.dax) === dax
|
|
860
|
+
);
|
|
861
|
+
return {
|
|
862
|
+
ok: true,
|
|
863
|
+
columns: sampleColumnsFor(source),
|
|
864
|
+
rows: sampleRowsFor(source),
|
|
865
|
+
};
|
|
866
|
+
};
|
|
867
|
+
try {
|
|
868
|
+
const complete = await materializeCatalog({
|
|
869
|
+
dir,
|
|
870
|
+
datasetId: 'ds-1',
|
|
871
|
+
alias: 'sales',
|
|
872
|
+
retry: (run) => run,
|
|
873
|
+
run: async (dax) => response(dax),
|
|
874
|
+
});
|
|
875
|
+
assert.equal(complete.incomplete, false);
|
|
876
|
+
|
|
877
|
+
const preserved = await materializeCatalog({
|
|
878
|
+
dir,
|
|
879
|
+
datasetId: 'ds-1',
|
|
880
|
+
alias: 'sales',
|
|
881
|
+
retry: (run) => run,
|
|
882
|
+
run: async (dax) =>
|
|
883
|
+
dax.includes('INFO.VIEW.MEASURES')
|
|
884
|
+
? { ok: false, error: 'rate limited' }
|
|
885
|
+
: response(dax),
|
|
886
|
+
});
|
|
887
|
+
assert.equal(preserved.reused, true);
|
|
888
|
+
assert.equal(preserved.incomplete, false);
|
|
889
|
+
assert.match(preserved.notes[0], /previous catalog was preserved/i);
|
|
890
|
+
assert.equal(
|
|
891
|
+
runSql(complete.dbPath, 'SELECT status FROM catalog_status').rows[0]
|
|
892
|
+
.status,
|
|
893
|
+
'complete'
|
|
894
|
+
);
|
|
895
|
+
assert.equal(
|
|
896
|
+
runSql(
|
|
897
|
+
complete.dbPath,
|
|
898
|
+
'SELECT COUNT(*) AS count FROM info_view_measures'
|
|
899
|
+
).rows[0].count,
|
|
900
|
+
1
|
|
901
|
+
);
|
|
902
|
+
} finally {
|
|
903
|
+
rmSync(dir, { recursive: true, force: true });
|
|
904
|
+
}
|
|
905
|
+
});
|
|
906
|
+
|
|
907
|
+
test('an optional view the previous catalog never had does not freeze it', async () => {
|
|
908
|
+
const dir = tempDir();
|
|
909
|
+
const response = (dax) => {
|
|
910
|
+
const source = CATALOG_SOURCES.find(
|
|
911
|
+
(candidate) => rawQuery(candidate.dax) === dax
|
|
912
|
+
);
|
|
913
|
+
return {
|
|
914
|
+
ok: true,
|
|
915
|
+
columns: sampleColumnsFor(source),
|
|
916
|
+
rows: sampleRowsFor(source),
|
|
917
|
+
};
|
|
918
|
+
};
|
|
919
|
+
// COLUMNSTATISTICS is optional and has never worked on this model.
|
|
920
|
+
const withoutStatistics = async (dax) =>
|
|
921
|
+
dax.includes('COLUMNSTATISTICS')
|
|
922
|
+
? { ok: false, error: 'not supported' }
|
|
923
|
+
: response(dax);
|
|
924
|
+
try {
|
|
925
|
+
const first = await materializeCatalog({
|
|
926
|
+
dir,
|
|
927
|
+
datasetId: 'ds-1',
|
|
928
|
+
alias: 'sales',
|
|
929
|
+
retry: (run) => run,
|
|
930
|
+
run: withoutStatistics,
|
|
931
|
+
});
|
|
932
|
+
assert.equal(first.incomplete, true, 'the optional view is missing');
|
|
933
|
+
|
|
934
|
+
// A later rebuild is still incomplete for the same reason. Preserving the
|
|
935
|
+
// earlier snapshot here would discard every future refresh, so the fresh
|
|
936
|
+
// measures and columns must be published.
|
|
937
|
+
const second = await materializeCatalog({
|
|
938
|
+
dir,
|
|
939
|
+
datasetId: 'ds-1',
|
|
940
|
+
alias: 'sales',
|
|
941
|
+
retry: (run) => run,
|
|
942
|
+
run: withoutStatistics,
|
|
943
|
+
});
|
|
944
|
+
assert.notEqual(
|
|
945
|
+
second.reused,
|
|
946
|
+
true,
|
|
947
|
+
'a rebuild that loses nothing is published'
|
|
948
|
+
);
|
|
949
|
+
|
|
950
|
+
// But losing a view the catalog does have still preserves it.
|
|
951
|
+
const losesMeasures = await materializeCatalog({
|
|
952
|
+
dir,
|
|
953
|
+
datasetId: 'ds-1',
|
|
954
|
+
alias: 'sales',
|
|
955
|
+
retry: (run) => run,
|
|
956
|
+
run: async (dax) =>
|
|
957
|
+
dax.includes('INFO.VIEW.MEASURES')
|
|
958
|
+
? { ok: false, error: 'rate limited' }
|
|
959
|
+
: withoutStatistics(dax),
|
|
960
|
+
});
|
|
961
|
+
assert.equal(losesMeasures.reused, true);
|
|
962
|
+
|
|
963
|
+
// The bug this guards: with an optional view permanently missing the
|
|
964
|
+
// status never reaches "complete", which used to switch the protection
|
|
965
|
+
// off entirely and let this rebuild delete info_view_measures.
|
|
966
|
+
assert.equal(
|
|
967
|
+
runSql(first.dbPath, 'SELECT COUNT(*) AS count FROM info_view_measures')
|
|
968
|
+
.rows[0].count,
|
|
969
|
+
1,
|
|
970
|
+
'the measures the previous catalog had survive'
|
|
971
|
+
);
|
|
972
|
+
} finally {
|
|
973
|
+
rmSync(dir, { recursive: true, force: true });
|
|
974
|
+
}
|
|
975
|
+
});
|
|
976
|
+
|
|
977
|
+
test('a refresh that throws still answers from the previous catalog', async () => {
|
|
978
|
+
const dir = tempDir();
|
|
979
|
+
const response = (dax) => {
|
|
980
|
+
const source = CATALOG_SOURCES.find(
|
|
981
|
+
(candidate) => rawQuery(candidate.dax) === dax
|
|
982
|
+
);
|
|
983
|
+
return {
|
|
984
|
+
ok: true,
|
|
985
|
+
columns: sampleColumnsFor(source),
|
|
986
|
+
rows: sampleRowsFor(source),
|
|
987
|
+
};
|
|
988
|
+
};
|
|
989
|
+
try {
|
|
990
|
+
await materializeCatalog({
|
|
991
|
+
dir,
|
|
992
|
+
datasetId: 'ds-1',
|
|
993
|
+
alias: 'sales',
|
|
994
|
+
retry: (run) => run,
|
|
995
|
+
run: async (dax) => response(dax),
|
|
996
|
+
});
|
|
997
|
+
|
|
998
|
+
// On Windows a process holding model.sqlite open makes the publish rename
|
|
999
|
+
// fail with EPERM after its retries, and the exception escaped the whole
|
|
1000
|
+
// command rather than falling back like a returned error does.
|
|
1001
|
+
const threw = await materializeCatalog({
|
|
1002
|
+
dir,
|
|
1003
|
+
datasetId: 'ds-1',
|
|
1004
|
+
alias: 'sales',
|
|
1005
|
+
retry: (run) => run,
|
|
1006
|
+
run: async () => {
|
|
1007
|
+
throw new Error('EPERM: operation not permitted');
|
|
1008
|
+
},
|
|
1009
|
+
});
|
|
1010
|
+
assert.equal(threw.ok, true, 'the command is not lost');
|
|
1011
|
+
assert.equal(threw.reused, true);
|
|
1012
|
+
assert.equal(threw.refreshIncomplete, true, 'a cooldown is still recorded');
|
|
1013
|
+
assert.match(threw.notes[1], /EPERM/u, 'the reason survives');
|
|
1014
|
+
} finally {
|
|
1015
|
+
rmSync(dir, { recursive: true, force: true });
|
|
1016
|
+
}
|
|
1017
|
+
});
|
|
1018
|
+
|
|
1019
|
+
test('a refresh that throws with no catalog on disk is still fatal', async () => {
|
|
1020
|
+
const dir = tempDir();
|
|
1021
|
+
try {
|
|
1022
|
+
await assert.rejects(
|
|
1023
|
+
materializeCatalog({
|
|
1024
|
+
dir,
|
|
1025
|
+
datasetId: 'ds-1',
|
|
1026
|
+
alias: 'sales',
|
|
1027
|
+
retry: (run) => run,
|
|
1028
|
+
run: async () => {
|
|
1029
|
+
throw new Error('EPERM: operation not permitted');
|
|
1030
|
+
},
|
|
1031
|
+
}),
|
|
1032
|
+
/EPERM/u
|
|
1033
|
+
);
|
|
1034
|
+
} finally {
|
|
1035
|
+
rmSync(dir, { recursive: true, force: true });
|
|
1036
|
+
}
|
|
1037
|
+
});
|
|
1038
|
+
|
|
1039
|
+
test('a failed refresh answers from the previous complete catalog', async () => {
|
|
1040
|
+
const dir = tempDir();
|
|
1041
|
+
const response = (dax) => {
|
|
1042
|
+
const source = CATALOG_SOURCES.find(
|
|
1043
|
+
(candidate) => rawQuery(candidate.dax) === dax
|
|
1044
|
+
);
|
|
1045
|
+
return {
|
|
1046
|
+
ok: true,
|
|
1047
|
+
columns: sampleColumnsFor(source),
|
|
1048
|
+
rows: sampleRowsFor(source),
|
|
1049
|
+
};
|
|
1050
|
+
};
|
|
1051
|
+
try {
|
|
1052
|
+
const complete = await materializeCatalog({
|
|
1053
|
+
dir,
|
|
1054
|
+
datasetId: 'ds-1',
|
|
1055
|
+
alias: 'sales',
|
|
1056
|
+
retry: (run) => run,
|
|
1057
|
+
run: async (dax) => response(dax),
|
|
1058
|
+
});
|
|
1059
|
+
assert.equal(complete.incomplete, false);
|
|
1060
|
+
|
|
1061
|
+
// Table discovery itself fails, which is what throttling looks like on the
|
|
1062
|
+
// first request after the TTL expires. With a good catalog on disk the
|
|
1063
|
+
// command must still answer instead of losing the caller's query.
|
|
1064
|
+
const survived = await materializeCatalog({
|
|
1065
|
+
dir,
|
|
1066
|
+
datasetId: 'ds-1',
|
|
1067
|
+
alias: 'sales',
|
|
1068
|
+
retry: (run) => run,
|
|
1069
|
+
run: async () => ({ ok: false, error: 'rate limited' }),
|
|
1070
|
+
});
|
|
1071
|
+
assert.equal(survived.ok, true, 'the command is not lost');
|
|
1072
|
+
assert.equal(survived.reused, true);
|
|
1073
|
+
assert.equal(survived.incomplete, false);
|
|
1074
|
+
assert.equal(
|
|
1075
|
+
survived.refreshIncomplete,
|
|
1076
|
+
true,
|
|
1077
|
+
'the caller still records a cooldown'
|
|
1078
|
+
);
|
|
1079
|
+
assert.match(survived.notes[0], /refresh failed/i);
|
|
1080
|
+
assert.match(
|
|
1081
|
+
survived.notes[1],
|
|
1082
|
+
/rate limited/i,
|
|
1083
|
+
'the reason survives for the cooldown note'
|
|
1084
|
+
);
|
|
1085
|
+
assert.equal(
|
|
1086
|
+
runSql(
|
|
1087
|
+
complete.dbPath,
|
|
1088
|
+
'SELECT COUNT(*) AS count FROM info_view_measures'
|
|
1089
|
+
).rows[0].count,
|
|
1090
|
+
1,
|
|
1091
|
+
'the earlier snapshot is intact'
|
|
1092
|
+
);
|
|
1093
|
+
} finally {
|
|
1094
|
+
rmSync(dir, { recursive: true, force: true });
|
|
1095
|
+
}
|
|
1096
|
+
});
|
|
1097
|
+
|
|
1098
|
+
test('a failed refresh with no catalog on disk is still fatal', async () => {
|
|
1099
|
+
const dir = tempDir();
|
|
1100
|
+
try {
|
|
1101
|
+
const result = await materializeCatalog({
|
|
1102
|
+
dir,
|
|
1103
|
+
datasetId: 'ds-1',
|
|
1104
|
+
alias: 'sales',
|
|
1105
|
+
retry: (run) => run,
|
|
1106
|
+
run: async () => ({ ok: false, error: 'rate limited' }),
|
|
1107
|
+
});
|
|
1108
|
+
assert.equal(result.ok, false, 'a cold start has nothing to fall back to');
|
|
1109
|
+
assert.match(result.error, /rate limited/i);
|
|
1110
|
+
} finally {
|
|
1111
|
+
rmSync(dir, { recursive: true, force: true });
|
|
1112
|
+
}
|
|
1113
|
+
});
|
|
1114
|
+
|
|
1115
|
+
test('renderCatalog surfaces incomplete status prominently', () => {
|
|
1116
|
+
const catalog = renderCatalog(
|
|
1117
|
+
[
|
|
1118
|
+
{
|
|
1119
|
+
table: 'catalog_status',
|
|
1120
|
+
rowCount: 1,
|
|
1121
|
+
columns: [{ name: 'status', type: 'TEXT' }],
|
|
1122
|
+
grain: 'status',
|
|
1123
|
+
dax: 'derived',
|
|
1124
|
+
},
|
|
1125
|
+
],
|
|
1126
|
+
{
|
|
1127
|
+
incomplete: true,
|
|
1128
|
+
notes: ['measures failed'],
|
|
1129
|
+
}
|
|
1130
|
+
);
|
|
1131
|
+
assert.match(catalog, /Catalog status: \*\*INCOMPLETE\*\*/);
|
|
1132
|
+
assert.match(catalog, /measures failed/);
|
|
1133
|
+
});
|
|
1134
|
+
|
|
1135
|
+
test('rate-limit retry honors hints and then succeeds', async () => {
|
|
1136
|
+
let calls = 0;
|
|
1137
|
+
const waits = [];
|
|
1138
|
+
const run = withRateLimitRetry(
|
|
1139
|
+
async () => {
|
|
1140
|
+
calls += 1;
|
|
1141
|
+
return calls < 3
|
|
1142
|
+
? {
|
|
1143
|
+
ok: false,
|
|
1144
|
+
error: '429 rate limit. Retry in 60 seconds.',
|
|
1145
|
+
}
|
|
1146
|
+
: { ok: true, rows: [] };
|
|
1147
|
+
},
|
|
1148
|
+
{
|
|
1149
|
+
sleep: async (milliseconds) => {
|
|
1150
|
+
waits.push(milliseconds);
|
|
1151
|
+
},
|
|
1152
|
+
jitter: () => 1,
|
|
1153
|
+
}
|
|
1154
|
+
);
|
|
1155
|
+
assert.equal(isRateLimited(await run('x')), false);
|
|
1156
|
+
assert.equal(calls, 3);
|
|
1157
|
+
assert.deepEqual(waits, [62_000, 64_000]);
|
|
1158
|
+
assert.equal(
|
|
1159
|
+
retryDelayMs({ ok: false, error: '429. Retry in 60 seconds.' }, 0, {
|
|
1160
|
+
base: 2_000,
|
|
1161
|
+
jitter: () => 0,
|
|
1162
|
+
}),
|
|
1163
|
+
60_000,
|
|
1164
|
+
'the service hint is a minimum, not a jitter window'
|
|
1165
|
+
);
|
|
1166
|
+
assert.equal(
|
|
1167
|
+
retryDelayMs({ ok: false, error: '429. Retry in 60 seconds.' }, 0, {
|
|
1168
|
+
base: 2_000,
|
|
1169
|
+
jitter: () => 1,
|
|
1170
|
+
}),
|
|
1171
|
+
62_000
|
|
1172
|
+
);
|
|
1173
|
+
assert.equal(
|
|
1174
|
+
retryDelayMs(
|
|
1175
|
+
{
|
|
1176
|
+
ok: false,
|
|
1177
|
+
error:
|
|
1178
|
+
'Fabric API error: 429. Request is blocked by the upstream ' +
|
|
1179
|
+
'service until: 9/22/2026 4:15:38 AM (UTC)',
|
|
1180
|
+
},
|
|
1181
|
+
0,
|
|
1182
|
+
{
|
|
1183
|
+
base: 2_000,
|
|
1184
|
+
jitter: () => 0,
|
|
1185
|
+
now: () => Date.UTC(2026, 8, 22, 4, 14, 38),
|
|
1186
|
+
}
|
|
1187
|
+
),
|
|
1188
|
+
60_000,
|
|
1189
|
+
'an absolute UTC deadline is also a minimum wait'
|
|
1190
|
+
);
|
|
1191
|
+
assert.equal(
|
|
1192
|
+
retryDelayMs(
|
|
1193
|
+
{
|
|
1194
|
+
ok: false,
|
|
1195
|
+
error:
|
|
1196
|
+
'Fabric API error: 429. Request is blocked by the upstream ' +
|
|
1197
|
+
'service until: 9/22/2026 4:15:38 AM (UTC)',
|
|
1198
|
+
},
|
|
1199
|
+
0,
|
|
1200
|
+
{
|
|
1201
|
+
base: 2_000,
|
|
1202
|
+
jitter: () => 1,
|
|
1203
|
+
now: () => Date.UTC(2026, 8, 22, 4, 14, 38),
|
|
1204
|
+
}
|
|
1205
|
+
),
|
|
1206
|
+
62_000
|
|
1207
|
+
);
|
|
1208
|
+
assert.equal(
|
|
1209
|
+
retryDelayMs({ ok: false, error: 'too many requests' }, 1, {
|
|
1210
|
+
base: 100,
|
|
1211
|
+
jitter: () => 0,
|
|
1212
|
+
}),
|
|
1213
|
+
100
|
|
1214
|
+
);
|
|
1215
|
+
assert.equal(
|
|
1216
|
+
retryDelayMs({ ok: false, error: 'too many requests' }, 1, {
|
|
1217
|
+
base: 100,
|
|
1218
|
+
jitter: () => 1,
|
|
1219
|
+
}),
|
|
1220
|
+
200
|
|
1221
|
+
);
|
|
1222
|
+
});
|
|
1223
|
+
|
|
1224
|
+
test('result formatting uses TSV only when lossless', () => {
|
|
1225
|
+
const tsv = formatResult({
|
|
1226
|
+
ok: true,
|
|
1227
|
+
totalRows: 1,
|
|
1228
|
+
returnedRows: 1,
|
|
1229
|
+
truncated: false,
|
|
1230
|
+
rows: [{ Name: 'Revenue', Count: 3 }],
|
|
1231
|
+
});
|
|
1232
|
+
assert.match(tsv, /^# \{"ok":true/u);
|
|
1233
|
+
assert.match(tsv, /Name\tCount\nRevenue\t3\n$/u);
|
|
1234
|
+
|
|
1235
|
+
const json = formatResult({
|
|
1236
|
+
ok: true,
|
|
1237
|
+
totalRows: 1,
|
|
1238
|
+
returnedRows: 1,
|
|
1239
|
+
truncated: false,
|
|
1240
|
+
rows: [{ Name: 'line\nbreak' }],
|
|
1241
|
+
});
|
|
1242
|
+
assert.match(json, /^\{"ok":true/u);
|
|
1243
|
+
});
|
|
1244
|
+
|
|
1245
|
+
test('row and byte bounds cannot silently report a complete preview', () => {
|
|
1246
|
+
const rowBound = boundRows(
|
|
1247
|
+
Array.from({ length: 5 }, (_, index) => ({
|
|
1248
|
+
index,
|
|
1249
|
+
})),
|
|
1250
|
+
{ limit: 2 }
|
|
1251
|
+
);
|
|
1252
|
+
assert.equal(rowBound.truncated, true);
|
|
1253
|
+
assert.equal(rowBound.truncation.reason, 'row_limit');
|
|
1254
|
+
|
|
1255
|
+
const byteBound = boundRows(
|
|
1256
|
+
[{ text: 'x'.repeat(1000) }, { text: 'y'.repeat(1000) }],
|
|
1257
|
+
{ maxBytes: 300 }
|
|
1258
|
+
);
|
|
1259
|
+
assert.equal(byteBound.truncated, true);
|
|
1260
|
+
assert.equal(byteBound.truncation.reason, 'byte_budget');
|
|
1261
|
+
assert.equal(resultExitCode(byteBound), 2);
|
|
1262
|
+
assert.equal(resultExitCode(byteBound, { allowTruncated: true }), 0);
|
|
1263
|
+
});
|
|
1264
|
+
|
|
1265
|
+
test('semantic search ranks exact shipped measures first', () => {
|
|
1266
|
+
const resultSets = [
|
|
1267
|
+
{
|
|
1268
|
+
table: 'info_view_tables',
|
|
1269
|
+
rows: [{ Name: 'Sales' }],
|
|
1270
|
+
},
|
|
1271
|
+
{
|
|
1272
|
+
table: 'info_view_columns',
|
|
1273
|
+
rows: [
|
|
1274
|
+
{
|
|
1275
|
+
Table: 'Sales',
|
|
1276
|
+
Name: 'Revenue Category',
|
|
1277
|
+
},
|
|
1278
|
+
],
|
|
1279
|
+
},
|
|
1280
|
+
{
|
|
1281
|
+
table: 'info_view_measures',
|
|
1282
|
+
rows: [
|
|
1283
|
+
{
|
|
1284
|
+
Table: 'Sales',
|
|
1285
|
+
Name: 'Revenue',
|
|
1286
|
+
Expression: "SUM('Sales'[Amount])",
|
|
1287
|
+
},
|
|
1288
|
+
],
|
|
1289
|
+
},
|
|
1290
|
+
];
|
|
1291
|
+
const rows = semanticSearchRows(resultSets);
|
|
1292
|
+
const ranked = rankSemanticSearchRows(rows, 'revenue');
|
|
1293
|
+
assert.equal(ranked[0].kind, 'measure');
|
|
1294
|
+
assert.equal(ranked[0].object_name, 'Revenue');
|
|
1295
|
+
});
|
|
1296
|
+
|
|
1297
|
+
test('searchCatalog queries the derived search table', () => {
|
|
1298
|
+
const dir = tempDir();
|
|
1299
|
+
try {
|
|
1300
|
+
const dbPath = join(dir, 'model.sqlite');
|
|
1301
|
+
buildDatabase(
|
|
1302
|
+
[
|
|
1303
|
+
{
|
|
1304
|
+
table: 'info_view_measures',
|
|
1305
|
+
grain: '',
|
|
1306
|
+
dax: '',
|
|
1307
|
+
rows: sampleRowsFor({
|
|
1308
|
+
table: 'info_view_measures',
|
|
1309
|
+
}),
|
|
1310
|
+
},
|
|
1311
|
+
],
|
|
1312
|
+
dbPath
|
|
1313
|
+
);
|
|
1314
|
+
const result = searchCatalog(dbPath, 'revenue');
|
|
1315
|
+
assert.equal(result.ok, true);
|
|
1316
|
+
assert.equal(result.rows[0].object_name, 'Revenue');
|
|
1317
|
+
} finally {
|
|
1318
|
+
rmSync(dir, { recursive: true, force: true });
|
|
1319
|
+
}
|
|
1320
|
+
});
|
|
1321
|
+
|
|
1322
|
+
test('runSql is read-only', () => {
|
|
1323
|
+
const dir = tempDir();
|
|
1324
|
+
try {
|
|
1325
|
+
const dbPath = join(dir, 'model.sqlite');
|
|
1326
|
+
const db = new DatabaseSync(dbPath);
|
|
1327
|
+
createTable(db, 't', [{ value: 1 }]);
|
|
1328
|
+
db.close();
|
|
1329
|
+
const result = runSql(dbPath, 'INSERT INTO t VALUES (2)');
|
|
1330
|
+
assert.equal(result.ok, false);
|
|
1331
|
+
assert.match(result.error, /read.?only|readonly/i);
|
|
1332
|
+
|
|
1333
|
+
const copiedPath = join(dir, 'exfiltrated.sqlite');
|
|
1334
|
+
const escapedPath = copiedPath.replace(/'/gu, "''");
|
|
1335
|
+
const vacuum = runSql(dbPath, `VACUUM INTO '${escapedPath}'`);
|
|
1336
|
+
assert.equal(vacuum.ok, false);
|
|
1337
|
+
assert.match(vacuum.error, /read.?only|readonly/i);
|
|
1338
|
+
assert.equal(
|
|
1339
|
+
existsSync(copiedPath) ? statSync(copiedPath).size : 0,
|
|
1340
|
+
0,
|
|
1341
|
+
'query_only may leave an empty destination, but no catalog data may escape'
|
|
1342
|
+
);
|
|
1343
|
+
|
|
1344
|
+
const bypassPath = join(dir, 'multi-statement.sqlite');
|
|
1345
|
+
const escapedBypassPath = bypassPath.replace(/'/gu, "''");
|
|
1346
|
+
runSql(
|
|
1347
|
+
dbPath,
|
|
1348
|
+
`PRAGMA query_only = OFF; VACUUM INTO '${escapedBypassPath}'`
|
|
1349
|
+
);
|
|
1350
|
+
// The first statement succeeds as a no-op PRAGMA. The guarantee is that
|
|
1351
|
+
// prepare().all() never compiles the second statement; exec() would.
|
|
1352
|
+
assert.equal(
|
|
1353
|
+
existsSync(bypassPath) && statSync(bypassPath).size > 0,
|
|
1354
|
+
false,
|
|
1355
|
+
'one prepared statement per connection is load-bearing: exec would run both statements'
|
|
1356
|
+
);
|
|
1357
|
+
} finally {
|
|
1358
|
+
rmSync(dir, { recursive: true, force: true });
|
|
1359
|
+
}
|
|
1360
|
+
});
|
|
1361
|
+
|
|
1362
|
+
test('TTL defaults to 15 minutes and staleness is timestamp-only', () => {
|
|
1363
|
+
assert.equal(ttlMinutes(undefined), 15);
|
|
1364
|
+
assert.equal(ttlMinutes('0'), 0);
|
|
1365
|
+
assert.equal(cacheStatus(null).reason, 'no cached catalog on disk');
|
|
1366
|
+
assert.equal(
|
|
1367
|
+
cacheStatus(Date.now() - 14 * 60_000, {
|
|
1368
|
+
ttl: 15,
|
|
1369
|
+
}).fresh,
|
|
1370
|
+
true
|
|
1371
|
+
);
|
|
1372
|
+
assert.equal(
|
|
1373
|
+
cacheStatus(Date.now() - 16 * 60_000, {
|
|
1374
|
+
ttl: 15,
|
|
1375
|
+
}).fresh,
|
|
1376
|
+
false
|
|
1377
|
+
);
|
|
1378
|
+
});
|
|
1379
|
+
|
|
1380
|
+
test('model cache paths require a GUID and stay inside the cache root', () => {
|
|
1381
|
+
const id = '11111111-2222-3333-4444-555555555555';
|
|
1382
|
+
assert.equal(modelCacheDir('.schema', id), join('.schema', 'models', id));
|
|
1383
|
+
assert.throws(
|
|
1384
|
+
() => modelCacheDir('.schema', '../outside'),
|
|
1385
|
+
/Invalid.*itemId/i
|
|
1386
|
+
);
|
|
1387
|
+
});
|