@microsoft/rayfin-semantic-model-pack 0.0.0-stage → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG +10 -0
- package/LICENSE +21 -0
- package/README.md +65 -2
- package/package.json +10 -3
- package/skills/analytics/SKILL.md +460 -0
- package/skills/analytics/kit/hooks/use-semantic-model-query.spec.ts +342 -0
- package/skills/analytics/kit/hooks/use-semantic-model-query.ts +229 -0
- package/skills/analytics/kit/lib/connectors.ts +94 -0
- package/skills/analytics/kit/scripts/rayfin-connector.mjs +265 -0
- package/skills/analytics/kit/scripts/rayfin-connector.test.mjs +243 -0
- package/skills/analytics/kit/scripts/schema-sql-batch.test.mjs +2097 -0
- package/skills/analytics/kit/scripts/schema-sql-cli.test.mjs +488 -0
- package/skills/analytics/kit/scripts/schema-sql.mjs +2650 -0
- package/skills/analytics/kit/scripts/schema-sql.test.mjs +1387 -0
- package/skills/analytics/kit/scripts/schema.mjs +166 -0
- package/skills/analytics/kit/scripts/schema.test.mjs +271 -0
- package/skills/analytics/pack.json +49 -0
- package/skills/dax-authoring/SKILL.md +172 -0
- package/skills/dax-authoring/references/dax-core-reference.md +862 -0
- package/skills/dax-authoring/references/dax-query-patterns.md +396 -0
- package/skills/dax-authoring/references/dax-time-intelligence.md +347 -0
- package/skills/query-design/SKILL.md +114 -0
- package/skills/query-design/references/anti-patterns.md +285 -0
- package/skills/query-design/references/filter-strategy.md +68 -0
- package/skills/query-design/references/format-strings.md +62 -0
- package/skills/query-design/references/highlight-queries.md +122 -0
- package/skills/query-design/references/multi-grain-patterns.md +159 -0
- package/skills/schema-discovery/SKILL.md +382 -0
- package/skills/schema-discovery/references/discovery-queries.md +153 -0
|
@@ -0,0 +1,2650 @@
|
|
|
1
|
+
//-----------------------------------------------------------------------
|
|
2
|
+
// <copyright company="Microsoft Corporation">
|
|
3
|
+
// Copyright (c) Microsoft Corporation. All rights reserved.
|
|
4
|
+
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
|
5
|
+
// </copyright>
|
|
6
|
+
//-----------------------------------------------------------------------
|
|
7
|
+
|
|
8
|
+
// Materialize the semantic model's schema into SQLite, then answer read-only SQL
|
|
9
|
+
// locally. The catalog builder has no agent-supplied DAX input: every remote
|
|
10
|
+
// query comes from CATALOG_SOURCES below.
|
|
11
|
+
//
|
|
12
|
+
// --search, --sql, and --file are repeatable and answered in one process. One
|
|
13
|
+
// invocation costs one model round trip, and a round trip re-sends the whole
|
|
14
|
+
// conversation, so the batch is the cheap shape. The whole batch shares the
|
|
15
|
+
// single-statement byte budget.
|
|
16
|
+
|
|
17
|
+
import { createHash, randomUUID } from 'node:crypto';
|
|
18
|
+
import {
|
|
19
|
+
existsSync,
|
|
20
|
+
linkSync,
|
|
21
|
+
mkdirSync,
|
|
22
|
+
readFileSync,
|
|
23
|
+
realpathSync,
|
|
24
|
+
renameSync,
|
|
25
|
+
rmSync,
|
|
26
|
+
statSync,
|
|
27
|
+
writeFileSync,
|
|
28
|
+
} from 'node:fs';
|
|
29
|
+
import { join, relative, resolve, sep } from 'node:path';
|
|
30
|
+
import { parseArgs } from 'node:util';
|
|
31
|
+
import { pathToFileURL } from 'node:url';
|
|
32
|
+
|
|
33
|
+
import {
|
|
34
|
+
isFabricGuid,
|
|
35
|
+
resolveAlias,
|
|
36
|
+
resolveIdentity,
|
|
37
|
+
runFabricQuery,
|
|
38
|
+
} from './schema.mjs';
|
|
39
|
+
|
|
40
|
+
const MINIMUM_NODE = Object.freeze({ major: 22, minor: 13, patch: 0 });
|
|
41
|
+
|
|
42
|
+
export function assertSupportedNodeVersion(version = process.versions.node) {
|
|
43
|
+
const [major = 0, minor = 0, patch = 0] = String(version)
|
|
44
|
+
.split('.')
|
|
45
|
+
.map(Number);
|
|
46
|
+
const supported =
|
|
47
|
+
major > MINIMUM_NODE.major ||
|
|
48
|
+
(major === MINIMUM_NODE.major &&
|
|
49
|
+
(minor > MINIMUM_NODE.minor ||
|
|
50
|
+
(minor === MINIMUM_NODE.minor && patch >= MINIMUM_NODE.patch)));
|
|
51
|
+
if (!supported) {
|
|
52
|
+
throw new Error(
|
|
53
|
+
`The semantic-model catalog requires Node.js >=22.13.0; ` +
|
|
54
|
+
`this process is using ${version}. Upgrade Node.js, reinstall ` +
|
|
55
|
+
'the app dependencies, and retry.'
|
|
56
|
+
);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
assertSupportedNodeVersion();
|
|
61
|
+
const { DatabaseSync } = await import('node:sqlite');
|
|
62
|
+
|
|
63
|
+
function openReadOnlyDatabase(path) {
|
|
64
|
+
// DatabaseSync readOnly still permits VACUUM INTO; query_only prevents a
|
|
65
|
+
// catalog copy from being written to an arbitrary path.
|
|
66
|
+
const db = new DatabaseSync(path, { readOnly: true });
|
|
67
|
+
db.exec('PRAGMA query_only = ON');
|
|
68
|
+
return db;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export const CATALOG_FILE = 'catalog.md';
|
|
72
|
+
export const DATABASE_FILE = 'model.sqlite';
|
|
73
|
+
export const CACHE_ROOT = '.schema';
|
|
74
|
+
export const DEFAULT_TTL_MINUTES = 15;
|
|
75
|
+
|
|
76
|
+
const DEFAULT_PRINT_LIMIT = 200;
|
|
77
|
+
export const MAX_RESULT_BYTES = 20_000;
|
|
78
|
+
const MAX_OWNED_MEASURE_PREVIEW = 10;
|
|
79
|
+
const RATE_LIMIT_RETRIES = 3;
|
|
80
|
+
const RATE_LIMIT_BASE_DELAY_MS = 2_000;
|
|
81
|
+
const RATE_LIMIT_MAX_DELAY_MS = 75_000;
|
|
82
|
+
const CATALOG_LOCK_DIRECTORY = '.refresh.lock';
|
|
83
|
+
const CATALOG_LOCK_POLL_MS = 250;
|
|
84
|
+
const CATALOG_LOCK_TIMEOUT_MS = 30 * 60_000;
|
|
85
|
+
const CATALOG_LOCK_STALE_MS = 60 * 60_000;
|
|
86
|
+
// A rename over a live destination fails with a sharing violation on Windows
|
|
87
|
+
// while a concurrent reader still holds the file open. The A4 hard-link backup
|
|
88
|
+
// keeps the path visible but does not make the replacement non-exclusive, so a
|
|
89
|
+
// completed refresh could still be thrown away because someone was reading.
|
|
90
|
+
const PUBLISH_RETRY_ATTEMPTS = 20;
|
|
91
|
+
const PUBLISH_RETRY_DELAY_MS = 50;
|
|
92
|
+
// A failed automatic refresh must not re-fetch every source on the next
|
|
93
|
+
// command: the usual cause is throttling, and retrying immediately extends the
|
|
94
|
+
// block it is reacting to. The cooldown is a separate marker, never the
|
|
95
|
+
// catalog's mtime - faking freshness would make the catalog lie about its age.
|
|
96
|
+
export const REFRESH_COOLDOWN_MINUTES = 15;
|
|
97
|
+
const REFRESH_COOLDOWN_FILE = '.refresh-cooldown.json';
|
|
98
|
+
const TRUNCATION_REQUIRED_ACTION =
|
|
99
|
+
'Required: rerun with a narrower WHERE clause before using this result to plan DAX; a smaller projection also helps except for automatic table-measure enrichment.';
|
|
100
|
+
|
|
101
|
+
// This hardcoded list is the schema-only boundary. Do not accept DAX from a
|
|
102
|
+
// caller on the catalog-build path.
|
|
103
|
+
export const CATALOG_SOURCES = Object.freeze([
|
|
104
|
+
Object.freeze({
|
|
105
|
+
table: 'info_view_tables',
|
|
106
|
+
dax: 'INFO.VIEW.TABLES()',
|
|
107
|
+
grain: 'One row per semantic-model table',
|
|
108
|
+
}),
|
|
109
|
+
Object.freeze({
|
|
110
|
+
table: 'info_view_columns',
|
|
111
|
+
dax: 'INFO.VIEW.COLUMNS()',
|
|
112
|
+
grain: 'One row per model column',
|
|
113
|
+
}),
|
|
114
|
+
Object.freeze({
|
|
115
|
+
table: 'info_view_measures',
|
|
116
|
+
dax: 'INFO.VIEW.MEASURES()',
|
|
117
|
+
grain: 'One row per measure',
|
|
118
|
+
}),
|
|
119
|
+
Object.freeze({
|
|
120
|
+
table: 'info_view_relationships',
|
|
121
|
+
dax: 'INFO.VIEW.RELATIONSHIPS()',
|
|
122
|
+
grain: 'One row per model relationship',
|
|
123
|
+
}),
|
|
124
|
+
Object.freeze({
|
|
125
|
+
// COLUMNSTATISTICS returns real Min/Max values derived from model rows,
|
|
126
|
+
// plus cardinality and maximum text length. BigInt-bearing values are
|
|
127
|
+
// normalized before they cross Rayfin's JSON boundary.
|
|
128
|
+
table: 'column_statistics',
|
|
129
|
+
dax: `SELECTCOLUMNS(
|
|
130
|
+
COLUMNSTATISTICS(),
|
|
131
|
+
"Table Name", [Table Name],
|
|
132
|
+
"Column Name", [Column Name],
|
|
133
|
+
"Min", CONVERT([Min], STRING),
|
|
134
|
+
"Max", CONVERT([Max], STRING),
|
|
135
|
+
"Cardinality", CONVERT([Cardinality], DOUBLE),
|
|
136
|
+
"Max Length", CONVERT([Max Length], DOUBLE)
|
|
137
|
+
)`,
|
|
138
|
+
optional: true,
|
|
139
|
+
grain: 'One row per statistically described column',
|
|
140
|
+
}),
|
|
141
|
+
]);
|
|
142
|
+
|
|
143
|
+
export function rawQuery(dax) {
|
|
144
|
+
return `EVALUATE ${dax}`;
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
export function isRateLimited(result) {
|
|
148
|
+
return (
|
|
149
|
+
!result?.ok &&
|
|
150
|
+
/\b429\b|rate limit|too many requests/i.test(String(result?.error || ''))
|
|
151
|
+
);
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
function absoluteRetryDelayMs(error, now) {
|
|
155
|
+
const match =
|
|
156
|
+
/blocked by the upstream service until:\s*(\d{1,2})\/(\d{1,2})\/(\d{4})\s+(\d{1,2}):(\d{2}):(\d{2})\s+(AM|PM)\s+\(UTC\)/i.exec(
|
|
157
|
+
String(error || '')
|
|
158
|
+
);
|
|
159
|
+
if (!match) {
|
|
160
|
+
return 0;
|
|
161
|
+
}
|
|
162
|
+
let hour = Number(match[4]) % 12;
|
|
163
|
+
if (match[7].toUpperCase() === 'PM') {
|
|
164
|
+
hour += 12;
|
|
165
|
+
}
|
|
166
|
+
const deadline = Date.UTC(
|
|
167
|
+
Number(match[3]),
|
|
168
|
+
Number(match[1]) - 1,
|
|
169
|
+
Number(match[2]),
|
|
170
|
+
hour,
|
|
171
|
+
Number(match[5]),
|
|
172
|
+
Number(match[6])
|
|
173
|
+
);
|
|
174
|
+
return Math.max(0, deadline - now);
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
export function retryDelayMs(
|
|
178
|
+
result,
|
|
179
|
+
attempt,
|
|
180
|
+
{
|
|
181
|
+
base = RATE_LIMIT_BASE_DELAY_MS,
|
|
182
|
+
max = RATE_LIMIT_MAX_DELAY_MS,
|
|
183
|
+
jitter = Math.random,
|
|
184
|
+
now = Date.now,
|
|
185
|
+
} = {}
|
|
186
|
+
) {
|
|
187
|
+
const hint = /retry in (\d+)\s*second/i.exec(String(result?.error || ''));
|
|
188
|
+
const backoff = Math.min(base * 2 ** attempt, max);
|
|
189
|
+
const relativeMinimum = hint ? Number(hint[1]) * 1000 : 0;
|
|
190
|
+
const absoluteMinimum = absoluteRetryDelayMs(result?.error, now());
|
|
191
|
+
const minimum = Math.max(relativeMinimum, absoluteMinimum);
|
|
192
|
+
if (minimum > 0) {
|
|
193
|
+
return Math.round(minimum + backoff * jitter());
|
|
194
|
+
}
|
|
195
|
+
return Math.round(backoff * (0.5 + 0.5 * jitter()));
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
export function withRateLimitRetry(
|
|
199
|
+
run,
|
|
200
|
+
{
|
|
201
|
+
retries = RATE_LIMIT_RETRIES,
|
|
202
|
+
sleep = (milliseconds) =>
|
|
203
|
+
new Promise((resolve) => setTimeout(resolve, milliseconds)),
|
|
204
|
+
onRetry = null,
|
|
205
|
+
...delayOptions
|
|
206
|
+
} = {}
|
|
207
|
+
) {
|
|
208
|
+
return async (dax) => {
|
|
209
|
+
let result = await run(dax);
|
|
210
|
+
for (
|
|
211
|
+
let attempt = 0;
|
|
212
|
+
attempt < retries && isRateLimited(result);
|
|
213
|
+
attempt += 1
|
|
214
|
+
) {
|
|
215
|
+
const wait = retryDelayMs(result, attempt, delayOptions);
|
|
216
|
+
onRetry?.({ attempt, wait, error: result.error });
|
|
217
|
+
await sleep(wait);
|
|
218
|
+
result = await run(dax);
|
|
219
|
+
}
|
|
220
|
+
return result;
|
|
221
|
+
};
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
export function toSqlValue(value) {
|
|
225
|
+
if (value === undefined || value === null) {
|
|
226
|
+
return null;
|
|
227
|
+
}
|
|
228
|
+
if (typeof value === 'boolean') {
|
|
229
|
+
return value ? 1 : 0;
|
|
230
|
+
}
|
|
231
|
+
if (typeof value === 'number') {
|
|
232
|
+
return Number.isFinite(value) ? value : null;
|
|
233
|
+
}
|
|
234
|
+
if (typeof value === 'bigint' || typeof value === 'string') {
|
|
235
|
+
return value;
|
|
236
|
+
}
|
|
237
|
+
if (value instanceof Date) {
|
|
238
|
+
return value.toISOString();
|
|
239
|
+
}
|
|
240
|
+
return JSON.stringify(value);
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
export function inferColumnType(values) {
|
|
244
|
+
let seen = false;
|
|
245
|
+
let integral = true;
|
|
246
|
+
for (const value of values) {
|
|
247
|
+
if (value === null || value === undefined || value === '') {
|
|
248
|
+
continue;
|
|
249
|
+
}
|
|
250
|
+
if (typeof value === 'boolean') {
|
|
251
|
+
seen = true;
|
|
252
|
+
continue;
|
|
253
|
+
}
|
|
254
|
+
if (typeof value === 'number' || typeof value === 'bigint') {
|
|
255
|
+
seen = true;
|
|
256
|
+
if (typeof value === 'number' && !Number.isInteger(value)) {
|
|
257
|
+
integral = false;
|
|
258
|
+
}
|
|
259
|
+
continue;
|
|
260
|
+
}
|
|
261
|
+
return 'TEXT';
|
|
262
|
+
}
|
|
263
|
+
if (!seen) {
|
|
264
|
+
return 'TEXT';
|
|
265
|
+
}
|
|
266
|
+
return integral ? 'INTEGER' : 'REAL';
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
export function quoteIdentifier(name) {
|
|
270
|
+
return `"${String(name).replace(/"/g, '""')}"`;
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
function jsonWithoutNullFields(value) {
|
|
274
|
+
return JSON.stringify(value, (_key, item) =>
|
|
275
|
+
item === null ? undefined : item
|
|
276
|
+
);
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
function isTsvScalar(value) {
|
|
280
|
+
return (
|
|
281
|
+
value === null ||
|
|
282
|
+
value === undefined ||
|
|
283
|
+
typeof value === 'string' ||
|
|
284
|
+
typeof value === 'number' ||
|
|
285
|
+
typeof value === 'boolean' ||
|
|
286
|
+
typeof value === 'bigint'
|
|
287
|
+
);
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
function canFormatRowsAsTsv(rows) {
|
|
291
|
+
if (!Array.isArray(rows) || rows.length === 0) {
|
|
292
|
+
return false;
|
|
293
|
+
}
|
|
294
|
+
const columns = Object.keys(rows[0]);
|
|
295
|
+
return rows.every((row) => {
|
|
296
|
+
if (row === null || typeof row !== 'object' || Array.isArray(row)) {
|
|
297
|
+
return false;
|
|
298
|
+
}
|
|
299
|
+
const rowColumns = Object.keys(row);
|
|
300
|
+
if (
|
|
301
|
+
rowColumns.length !== columns.length ||
|
|
302
|
+
rowColumns.some((column, index) => column !== columns[index])
|
|
303
|
+
) {
|
|
304
|
+
return false;
|
|
305
|
+
}
|
|
306
|
+
return columns.every((column) => {
|
|
307
|
+
const value = row[column];
|
|
308
|
+
return (
|
|
309
|
+
isTsvScalar(value) &&
|
|
310
|
+
!/[\t\r\n]/.test(value == null ? '' : String(value))
|
|
311
|
+
);
|
|
312
|
+
});
|
|
313
|
+
});
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
export function formatResult(result) {
|
|
317
|
+
if (!result?.ok || !canFormatRowsAsTsv(result.rows)) {
|
|
318
|
+
return `${jsonWithoutNullFields(result)}\n`;
|
|
319
|
+
}
|
|
320
|
+
const columns = Object.keys(result.rows[0]);
|
|
321
|
+
const metadata = {
|
|
322
|
+
ok: true,
|
|
323
|
+
totalRows: result.totalRows,
|
|
324
|
+
returnedRows: result.returnedRows,
|
|
325
|
+
truncated: result.truncated,
|
|
326
|
+
...(result.truncation ? { truncation: result.truncation } : {}),
|
|
327
|
+
...(result.notes?.length ? { notes: result.notes } : {}),
|
|
328
|
+
encoding: 'tsv',
|
|
329
|
+
};
|
|
330
|
+
const lines = [
|
|
331
|
+
`# ${JSON.stringify(metadata)}`,
|
|
332
|
+
columns.join('\t'),
|
|
333
|
+
...result.rows.map((row) =>
|
|
334
|
+
columns
|
|
335
|
+
.map((column) => (row[column] == null ? '' : String(row[column])))
|
|
336
|
+
.join('\t')
|
|
337
|
+
),
|
|
338
|
+
];
|
|
339
|
+
return `${lines.join('\n')}\n`;
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
export function columnNamesOf(rows) {
|
|
343
|
+
const names = [];
|
|
344
|
+
const seen = new Set();
|
|
345
|
+
for (const row of rows || []) {
|
|
346
|
+
for (const key of Object.keys(row)) {
|
|
347
|
+
if (!seen.has(key)) {
|
|
348
|
+
seen.add(key);
|
|
349
|
+
names.push(key);
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
return names;
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
function sourceColumnType(dataType) {
|
|
357
|
+
const normalized = String(dataType ?? '').toLocaleLowerCase();
|
|
358
|
+
if (
|
|
359
|
+
normalized.includes('int') ||
|
|
360
|
+
normalized.includes('boolean') ||
|
|
361
|
+
normalized.includes('whole')
|
|
362
|
+
) {
|
|
363
|
+
return 'INTEGER';
|
|
364
|
+
}
|
|
365
|
+
if (
|
|
366
|
+
normalized.includes('double') ||
|
|
367
|
+
normalized.includes('decimal') ||
|
|
368
|
+
normalized.includes('single') ||
|
|
369
|
+
normalized.includes('currency') ||
|
|
370
|
+
normalized.includes('number')
|
|
371
|
+
) {
|
|
372
|
+
return 'REAL';
|
|
373
|
+
}
|
|
374
|
+
if (normalized.includes('binary')) {
|
|
375
|
+
return 'BLOB';
|
|
376
|
+
}
|
|
377
|
+
return 'TEXT';
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
export function createTable(db, table, rows, sourceColumns = []) {
|
|
381
|
+
const names = [];
|
|
382
|
+
const seen = new Set();
|
|
383
|
+
for (const column of sourceColumns) {
|
|
384
|
+
if (
|
|
385
|
+
typeof column?.name === 'string' &&
|
|
386
|
+
column.name.length > 0 &&
|
|
387
|
+
!seen.has(column.name)
|
|
388
|
+
) {
|
|
389
|
+
seen.add(column.name);
|
|
390
|
+
names.push(column.name);
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
for (const name of columnNamesOf(rows)) {
|
|
394
|
+
if (!seen.has(name)) {
|
|
395
|
+
seen.add(name);
|
|
396
|
+
names.push(name);
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
if (names.length === 0) {
|
|
400
|
+
return { table, columns: [], rowCount: 0 };
|
|
401
|
+
}
|
|
402
|
+
const sourceTypes = new Map(
|
|
403
|
+
sourceColumns.map((column) => [column.name, column.dataType])
|
|
404
|
+
);
|
|
405
|
+
const columns = names.map((name) => ({
|
|
406
|
+
name,
|
|
407
|
+
type: rows.some(
|
|
408
|
+
(row) => row[name] !== null && row[name] !== undefined && row[name] !== ''
|
|
409
|
+
)
|
|
410
|
+
? inferColumnType(rows.map((row) => row[name]))
|
|
411
|
+
: sourceColumnType(sourceTypes.get(name)),
|
|
412
|
+
}));
|
|
413
|
+
const ddl = columns
|
|
414
|
+
.map((column) => `${quoteIdentifier(column.name)} ${column.type}`)
|
|
415
|
+
.join(', ');
|
|
416
|
+
db.exec('BEGIN');
|
|
417
|
+
try {
|
|
418
|
+
db.exec(`DROP TABLE IF EXISTS ${quoteIdentifier(table)}`);
|
|
419
|
+
db.exec(`CREATE TABLE ${quoteIdentifier(table)} (${ddl})`);
|
|
420
|
+
|
|
421
|
+
const placeholders = names.map(() => '?').join(', ');
|
|
422
|
+
const insert = db.prepare(
|
|
423
|
+
`INSERT INTO ${quoteIdentifier(table)} VALUES (${placeholders})`
|
|
424
|
+
);
|
|
425
|
+
for (const row of rows) {
|
|
426
|
+
insert.run(...names.map((name) => toSqlValue(row[name])));
|
|
427
|
+
}
|
|
428
|
+
db.exec('COMMIT');
|
|
429
|
+
} catch (error) {
|
|
430
|
+
db.exec('ROLLBACK');
|
|
431
|
+
throw error;
|
|
432
|
+
}
|
|
433
|
+
return { table, columns, rowCount: rows.length };
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
function textField(row, names) {
|
|
437
|
+
for (const name of names) {
|
|
438
|
+
if (typeof row?.[name] === 'string' && row[name].length > 0) {
|
|
439
|
+
return row[name];
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
return '';
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
export function semanticSearchRows(resultSets) {
|
|
446
|
+
const rows = [];
|
|
447
|
+
for (const resultSet of resultSets ?? []) {
|
|
448
|
+
for (const row of resultSet.rows ?? []) {
|
|
449
|
+
let entity;
|
|
450
|
+
if (resultSet.table === 'info_view_tables') {
|
|
451
|
+
const name = textField(row, ['Name']);
|
|
452
|
+
entity = {
|
|
453
|
+
kind: 'table',
|
|
454
|
+
parent_name: name,
|
|
455
|
+
object_name: name,
|
|
456
|
+
description: textField(row, ['Description']),
|
|
457
|
+
expression: textField(row, ['Expression']),
|
|
458
|
+
display_folder: '',
|
|
459
|
+
data_type: '',
|
|
460
|
+
synonyms: textField(row, ['Synonyms']),
|
|
461
|
+
};
|
|
462
|
+
} else if (resultSet.table === 'info_view_measures') {
|
|
463
|
+
entity = {
|
|
464
|
+
kind: 'measure',
|
|
465
|
+
parent_name: textField(row, ['Table']),
|
|
466
|
+
object_name: textField(row, ['Name']),
|
|
467
|
+
description: textField(row, ['Description']),
|
|
468
|
+
expression: textField(row, ['Expression']),
|
|
469
|
+
display_folder: textField(row, ['DisplayFolder']),
|
|
470
|
+
data_type: '',
|
|
471
|
+
synonyms: textField(row, ['Synonyms']),
|
|
472
|
+
};
|
|
473
|
+
} else if (resultSet.table === 'info_view_columns') {
|
|
474
|
+
entity = {
|
|
475
|
+
kind: 'column',
|
|
476
|
+
parent_name: textField(row, ['Table']),
|
|
477
|
+
object_name: textField(row, ['Name']),
|
|
478
|
+
description: textField(row, ['Description']),
|
|
479
|
+
expression: textField(row, ['Expression']),
|
|
480
|
+
display_folder: textField(row, ['DisplayFolder']),
|
|
481
|
+
data_type: textField(row, ['DataType']),
|
|
482
|
+
synonyms: textField(row, ['Synonyms']),
|
|
483
|
+
};
|
|
484
|
+
} else if (resultSet.table === 'info_view_relationships') {
|
|
485
|
+
const fromTable = textField(row, ['FromTable']);
|
|
486
|
+
const fromColumn = textField(row, ['FromColumn']);
|
|
487
|
+
const toTable = textField(row, ['ToTable']);
|
|
488
|
+
const toColumn = textField(row, ['ToColumn']);
|
|
489
|
+
entity = {
|
|
490
|
+
kind: 'relationship',
|
|
491
|
+
parent_name: fromTable,
|
|
492
|
+
object_name:
|
|
493
|
+
`${fromTable}.${fromColumn} -> ` + `${toTable}.${toColumn}`,
|
|
494
|
+
description: '',
|
|
495
|
+
expression: '',
|
|
496
|
+
display_folder: '',
|
|
497
|
+
data_type: '',
|
|
498
|
+
synonyms: '',
|
|
499
|
+
};
|
|
500
|
+
} else {
|
|
501
|
+
continue;
|
|
502
|
+
}
|
|
503
|
+
if (!entity.object_name) {
|
|
504
|
+
continue;
|
|
505
|
+
}
|
|
506
|
+
rows.push({
|
|
507
|
+
...entity,
|
|
508
|
+
search_text: Object.values(entity).join(' ').toLocaleLowerCase(),
|
|
509
|
+
});
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
return rows;
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
function createSemanticSearchTable(db, rows) {
|
|
516
|
+
db.exec('BEGIN');
|
|
517
|
+
try {
|
|
518
|
+
db.exec('DROP TABLE IF EXISTS "semantic_search"');
|
|
519
|
+
db.exec(
|
|
520
|
+
'CREATE TABLE "semantic_search" (' +
|
|
521
|
+
'"kind" TEXT, "parent_name" TEXT, "object_name" TEXT, ' +
|
|
522
|
+
'"description" TEXT, "expression" TEXT, "display_folder" TEXT, ' +
|
|
523
|
+
'"data_type" TEXT, "synonyms" TEXT, "search_text" TEXT)'
|
|
524
|
+
);
|
|
525
|
+
const insert = db.prepare(
|
|
526
|
+
'INSERT INTO "semantic_search" VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)'
|
|
527
|
+
);
|
|
528
|
+
for (const row of rows) {
|
|
529
|
+
insert.run(
|
|
530
|
+
row.kind,
|
|
531
|
+
row.parent_name,
|
|
532
|
+
row.object_name,
|
|
533
|
+
row.description,
|
|
534
|
+
row.expression,
|
|
535
|
+
row.display_folder,
|
|
536
|
+
row.data_type,
|
|
537
|
+
row.synonyms,
|
|
538
|
+
row.search_text
|
|
539
|
+
);
|
|
540
|
+
}
|
|
541
|
+
db.exec('COMMIT');
|
|
542
|
+
} catch (error) {
|
|
543
|
+
db.exec('ROLLBACK');
|
|
544
|
+
throw error;
|
|
545
|
+
}
|
|
546
|
+
return {
|
|
547
|
+
table: 'semantic_search',
|
|
548
|
+
columns: [
|
|
549
|
+
'kind',
|
|
550
|
+
'parent_name',
|
|
551
|
+
'object_name',
|
|
552
|
+
'description',
|
|
553
|
+
'expression',
|
|
554
|
+
'display_folder',
|
|
555
|
+
'data_type',
|
|
556
|
+
'synonyms',
|
|
557
|
+
'search_text',
|
|
558
|
+
].map((name) => ({ name, type: 'TEXT' })),
|
|
559
|
+
rowCount: rows.length,
|
|
560
|
+
grain: 'One row per searchable model entity',
|
|
561
|
+
dax: 'Derived from the native metadata tables',
|
|
562
|
+
note: 'Use --search for ranked retrieval across this table.',
|
|
563
|
+
};
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
function populateDatabase(resultSets, path) {
|
|
567
|
+
const db = new DatabaseSync(path);
|
|
568
|
+
const built = [];
|
|
569
|
+
try {
|
|
570
|
+
const searchable = semanticSearchRows(resultSets);
|
|
571
|
+
for (const { table, rows, columns, grain, dax, note } of resultSets) {
|
|
572
|
+
const info = createTable(db, table, rows || [], columns || []);
|
|
573
|
+
built.push({
|
|
574
|
+
...info,
|
|
575
|
+
grain,
|
|
576
|
+
dax,
|
|
577
|
+
note: note ?? null,
|
|
578
|
+
});
|
|
579
|
+
}
|
|
580
|
+
built.push(createSemanticSearchTable(db, searchable));
|
|
581
|
+
return built;
|
|
582
|
+
} finally {
|
|
583
|
+
db.close();
|
|
584
|
+
}
|
|
585
|
+
}
|
|
586
|
+
|
|
587
|
+
function sleepSync(milliseconds) {
|
|
588
|
+
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, milliseconds);
|
|
589
|
+
}
|
|
590
|
+
|
|
591
|
+
/**
|
|
592
|
+
* Replace `to` with `from`, retrying a sharing violation.
|
|
593
|
+
*
|
|
594
|
+
* Only a concurrent reader produces these codes here; the destination is ours
|
|
595
|
+
* and the source is a temporary file we just wrote. Retrying briefly is enough
|
|
596
|
+
* because a reader holds the catalog open for the length of one query.
|
|
597
|
+
*/
|
|
598
|
+
function renameWithRetry(
|
|
599
|
+
rename,
|
|
600
|
+
from,
|
|
601
|
+
to,
|
|
602
|
+
{
|
|
603
|
+
attempts = PUBLISH_RETRY_ATTEMPTS,
|
|
604
|
+
delayMs = PUBLISH_RETRY_DELAY_MS,
|
|
605
|
+
sleep = sleepSync,
|
|
606
|
+
} = {}
|
|
607
|
+
) {
|
|
608
|
+
for (let attempt = 1; ; attempt += 1) {
|
|
609
|
+
try {
|
|
610
|
+
rename(from, to);
|
|
611
|
+
return;
|
|
612
|
+
} catch (error) {
|
|
613
|
+
const sharingViolation =
|
|
614
|
+
error?.code === 'EPERM' ||
|
|
615
|
+
error?.code === 'EBUSY' ||
|
|
616
|
+
error?.code === 'EACCES';
|
|
617
|
+
if (!sharingViolation || attempt >= attempts) {
|
|
618
|
+
throw error;
|
|
619
|
+
}
|
|
620
|
+
sleep(delayMs);
|
|
621
|
+
}
|
|
622
|
+
}
|
|
623
|
+
}
|
|
624
|
+
|
|
625
|
+
/**
|
|
626
|
+
* Publish one or more completed files without deleting the previous generation.
|
|
627
|
+
* If any rename fails, every prior destination is restored from its backup.
|
|
628
|
+
*/
|
|
629
|
+
export function publishFiles(
|
|
630
|
+
files,
|
|
631
|
+
{
|
|
632
|
+
exists = existsSync,
|
|
633
|
+
rename = renameSync,
|
|
634
|
+
remove = rmSync,
|
|
635
|
+
link = linkSync,
|
|
636
|
+
uuid = randomUUID,
|
|
637
|
+
retry = {},
|
|
638
|
+
} = {}
|
|
639
|
+
) {
|
|
640
|
+
const entries = files.map(({ temporaryPath, destinationPath }) => ({
|
|
641
|
+
temporaryPath,
|
|
642
|
+
destinationPath,
|
|
643
|
+
backupPath: `${destinationPath}.${process.pid}.${uuid()}.previous`,
|
|
644
|
+
hadPrevious: false,
|
|
645
|
+
linked: false,
|
|
646
|
+
published: false,
|
|
647
|
+
}));
|
|
648
|
+
|
|
649
|
+
try {
|
|
650
|
+
for (const entry of entries) {
|
|
651
|
+
if (!exists(entry.destinationPath)) {
|
|
652
|
+
continue;
|
|
653
|
+
}
|
|
654
|
+
// Back the previous generation up by hard-linking rather than renaming,
|
|
655
|
+
// so the destination never stops existing. Renaming it away first left a
|
|
656
|
+
// window in which a concurrent reader saw no database at all, and with
|
|
657
|
+
// batching one such reader discarded up to eight statements of work.
|
|
658
|
+
try {
|
|
659
|
+
link(entry.destinationPath, entry.backupPath);
|
|
660
|
+
entry.linked = true;
|
|
661
|
+
} catch {
|
|
662
|
+
// A filesystem without hard links still gets the old guarantee: the
|
|
663
|
+
// previous generation is preserved and rollback works. What it does
|
|
664
|
+
// NOT get is the gap-free publication above - the destination briefly
|
|
665
|
+
// stops existing again, and this fallback is silent about it. Verified
|
|
666
|
+
// supported on NTFS, which is where scaffolded apps live; an exotic
|
|
667
|
+
// filesystem would degrade here without any signal.
|
|
668
|
+
rename(entry.destinationPath, entry.backupPath);
|
|
669
|
+
}
|
|
670
|
+
entry.hadPrevious = true;
|
|
671
|
+
}
|
|
672
|
+
|
|
673
|
+
for (const entry of entries) {
|
|
674
|
+
// rename() replaces an existing destination atomically, so a reader sees
|
|
675
|
+
// either the whole previous generation or the whole new one. On Windows
|
|
676
|
+
// it still needs exclusive access, so a reader mid-query is waited out
|
|
677
|
+
// rather than being allowed to discard a completed refresh.
|
|
678
|
+
renameWithRetry(
|
|
679
|
+
rename,
|
|
680
|
+
entry.temporaryPath,
|
|
681
|
+
entry.destinationPath,
|
|
682
|
+
retry
|
|
683
|
+
);
|
|
684
|
+
entry.published = true;
|
|
685
|
+
}
|
|
686
|
+
} catch (error) {
|
|
687
|
+
const rollbackErrors = [];
|
|
688
|
+
for (const entry of [...entries].reverse()) {
|
|
689
|
+
try {
|
|
690
|
+
if (entry.published) {
|
|
691
|
+
remove(entry.destinationPath, { force: true });
|
|
692
|
+
}
|
|
693
|
+
if (entry.hadPrevious) {
|
|
694
|
+
if (entry.linked && !entry.published) {
|
|
695
|
+
// The destination was never replaced; drop the extra link only.
|
|
696
|
+
remove(entry.backupPath, { force: true });
|
|
697
|
+
} else {
|
|
698
|
+
rename(entry.backupPath, entry.destinationPath);
|
|
699
|
+
}
|
|
700
|
+
}
|
|
701
|
+
} catch (rollbackError) {
|
|
702
|
+
rollbackErrors.push(rollbackError);
|
|
703
|
+
}
|
|
704
|
+
}
|
|
705
|
+
if (rollbackErrors.length > 0) {
|
|
706
|
+
throw new AggregateError(
|
|
707
|
+
[error, ...rollbackErrors],
|
|
708
|
+
'Catalog publication failed and the previous generation could not be fully restored.'
|
|
709
|
+
);
|
|
710
|
+
}
|
|
711
|
+
throw error;
|
|
712
|
+
}
|
|
713
|
+
|
|
714
|
+
for (const entry of entries) {
|
|
715
|
+
if (entry.hadPrevious) {
|
|
716
|
+
remove(entry.backupPath, { force: true });
|
|
717
|
+
}
|
|
718
|
+
}
|
|
719
|
+
}
|
|
720
|
+
|
|
721
|
+
export function buildDatabase(resultSets, dbPath) {
|
|
722
|
+
mkdirSync(join(dbPath, '..'), { recursive: true });
|
|
723
|
+
const temporaryPath = `${dbPath}.${process.pid}.${randomUUID()}.tmp`;
|
|
724
|
+
rmSync(temporaryPath, { force: true });
|
|
725
|
+
try {
|
|
726
|
+
const built = populateDatabase(resultSets, temporaryPath);
|
|
727
|
+
publishFiles([
|
|
728
|
+
{
|
|
729
|
+
temporaryPath,
|
|
730
|
+
destinationPath: dbPath,
|
|
731
|
+
},
|
|
732
|
+
]);
|
|
733
|
+
return built;
|
|
734
|
+
} finally {
|
|
735
|
+
rmSync(temporaryPath, { force: true });
|
|
736
|
+
}
|
|
737
|
+
}
|
|
738
|
+
|
|
739
|
+
export async function withCatalogLock(
|
|
740
|
+
dir,
|
|
741
|
+
action,
|
|
742
|
+
{
|
|
743
|
+
now = Date.now,
|
|
744
|
+
sleep = (milliseconds) =>
|
|
745
|
+
new Promise((resolve) => setTimeout(resolve, milliseconds)),
|
|
746
|
+
pollMs = CATALOG_LOCK_POLL_MS,
|
|
747
|
+
timeoutMs = CATALOG_LOCK_TIMEOUT_MS,
|
|
748
|
+
staleMs = CATALOG_LOCK_STALE_MS,
|
|
749
|
+
jitter = Math.random,
|
|
750
|
+
processAlive = (pid) => {
|
|
751
|
+
try {
|
|
752
|
+
process.kill(pid, 0);
|
|
753
|
+
return true;
|
|
754
|
+
} catch (error) {
|
|
755
|
+
if (error?.code === 'ESRCH') {
|
|
756
|
+
return false;
|
|
757
|
+
}
|
|
758
|
+
if (error?.code === 'EPERM') {
|
|
759
|
+
return true;
|
|
760
|
+
}
|
|
761
|
+
throw error;
|
|
762
|
+
}
|
|
763
|
+
},
|
|
764
|
+
} = {}
|
|
765
|
+
) {
|
|
766
|
+
mkdirSync(dir, { recursive: true });
|
|
767
|
+
const lockPath = join(dir, CATALOG_LOCK_DIRECTORY);
|
|
768
|
+
// A per-acquisition token, not just a pid: pids are reused, and two reclaimers
|
|
769
|
+
// that both decided a lock was abandoned would otherwise each remove whatever
|
|
770
|
+
// lock happens to be there when they get round to it - including a live one
|
|
771
|
+
// created in between. Every removal below is conditional on the token that
|
|
772
|
+
// was observed when the decision was made.
|
|
773
|
+
const ownerToken = randomUUID();
|
|
774
|
+
const waitStartedAt = now();
|
|
775
|
+
let acquired = false;
|
|
776
|
+
let waited = false;
|
|
777
|
+
|
|
778
|
+
const readOwner = () => {
|
|
779
|
+
try {
|
|
780
|
+
return JSON.parse(readFileSync(join(lockPath, 'owner.json'), 'utf8'));
|
|
781
|
+
} catch (ownerError) {
|
|
782
|
+
if (ownerError?.code === 'ENOENT' || ownerError instanceof SyntaxError) {
|
|
783
|
+
return null;
|
|
784
|
+
}
|
|
785
|
+
throw ownerError;
|
|
786
|
+
}
|
|
787
|
+
};
|
|
788
|
+
|
|
789
|
+
// Remove the lock only if it is still the one the caller decided about.
|
|
790
|
+
const removeIfUnchanged = (expected) => {
|
|
791
|
+
const current = readOwner();
|
|
792
|
+
if (current?.token !== expected?.token) {
|
|
793
|
+
return false;
|
|
794
|
+
}
|
|
795
|
+
rmSync(lockPath, { recursive: true, force: true });
|
|
796
|
+
return true;
|
|
797
|
+
};
|
|
798
|
+
|
|
799
|
+
while (!acquired) {
|
|
800
|
+
try {
|
|
801
|
+
mkdirSync(lockPath);
|
|
802
|
+
try {
|
|
803
|
+
writeFileSync(
|
|
804
|
+
join(lockPath, 'owner.json'),
|
|
805
|
+
JSON.stringify({
|
|
806
|
+
pid: process.pid,
|
|
807
|
+
token: ownerToken,
|
|
808
|
+
startedAt: new Date(now()).toISOString(),
|
|
809
|
+
}),
|
|
810
|
+
'utf8'
|
|
811
|
+
);
|
|
812
|
+
} catch (error) {
|
|
813
|
+
rmSync(lockPath, { recursive: true, force: true });
|
|
814
|
+
throw error;
|
|
815
|
+
}
|
|
816
|
+
acquired = true;
|
|
817
|
+
} catch (error) {
|
|
818
|
+
if (error?.code !== 'EEXIST') {
|
|
819
|
+
throw error;
|
|
820
|
+
}
|
|
821
|
+
waited = true;
|
|
822
|
+
let age;
|
|
823
|
+
try {
|
|
824
|
+
age = now() - statSync(lockPath).mtimeMs;
|
|
825
|
+
} catch (statError) {
|
|
826
|
+
if (statError?.code === 'ENOENT') {
|
|
827
|
+
continue;
|
|
828
|
+
}
|
|
829
|
+
throw statError;
|
|
830
|
+
}
|
|
831
|
+
const owner = readOwner();
|
|
832
|
+
if (
|
|
833
|
+
Number.isInteger(owner?.pid) &&
|
|
834
|
+
owner.pid > 0 &&
|
|
835
|
+
owner.pid !== process.pid &&
|
|
836
|
+
!processAlive(owner.pid)
|
|
837
|
+
) {
|
|
838
|
+
// Re-read before removing: the owner we judged dead may already have
|
|
839
|
+
// been cleared and replaced by a live holder.
|
|
840
|
+
removeIfUnchanged(owner);
|
|
841
|
+
continue;
|
|
842
|
+
}
|
|
843
|
+
if (age > staleMs) {
|
|
844
|
+
removeIfUnchanged(owner);
|
|
845
|
+
continue;
|
|
846
|
+
}
|
|
847
|
+
if (now() - waitStartedAt >= timeoutMs) {
|
|
848
|
+
throw new Error(
|
|
849
|
+
`Timed out waiting for another catalog refresh to finish after ` +
|
|
850
|
+
`${Math.round(timeoutMs / 1000)} seconds.`
|
|
851
|
+
);
|
|
852
|
+
}
|
|
853
|
+
await sleep(Math.round(pollMs * (0.75 + 0.5 * jitter())));
|
|
854
|
+
}
|
|
855
|
+
}
|
|
856
|
+
|
|
857
|
+
try {
|
|
858
|
+
return await action({ waited });
|
|
859
|
+
} finally {
|
|
860
|
+
// Release only our own lock. If another process reclaimed it while we were
|
|
861
|
+
// working, its lock is live and removing it would let a third process run
|
|
862
|
+
// concurrently with it; the staleness path reclaims a genuinely orphaned
|
|
863
|
+
// one instead.
|
|
864
|
+
removeIfUnchanged({ token: ownerToken });
|
|
865
|
+
}
|
|
866
|
+
}
|
|
867
|
+
|
|
868
|
+
export function renderCatalog(
|
|
869
|
+
built,
|
|
870
|
+
{ datasetId, notes = [], incomplete = false } = {}
|
|
871
|
+
) {
|
|
872
|
+
const lines = [];
|
|
873
|
+
lines.push('# Semantic model catalog');
|
|
874
|
+
lines.push('');
|
|
875
|
+
lines.push(`Catalog status: **${incomplete ? 'INCOMPLETE' : 'complete'}**`);
|
|
876
|
+
lines.push('');
|
|
877
|
+
lines.push(`SQLite database: \`${DATABASE_FILE}\``);
|
|
878
|
+
if (datasetId) {
|
|
879
|
+
lines.push(`Dataset: \`${datasetId}\``);
|
|
880
|
+
}
|
|
881
|
+
lines.push('');
|
|
882
|
+
lines.push('| Table | Shape | Grain | Source |');
|
|
883
|
+
lines.push('| --- | ---: | --- | --- |');
|
|
884
|
+
for (const table of built) {
|
|
885
|
+
const shape =
|
|
886
|
+
`${table.rowCount.toLocaleString('en-US')} rows x ` +
|
|
887
|
+
`${table.columns.length} columns`;
|
|
888
|
+
const source = String(table.dax).replace(/\s+/g, ' ').replace(/\|/g, '\\|');
|
|
889
|
+
lines.push(
|
|
890
|
+
`| \`${table.table}\` | ${shape} | ${table.grain} | ` + `\`${source}\` |`
|
|
891
|
+
);
|
|
892
|
+
}
|
|
893
|
+
lines.push('');
|
|
894
|
+
lines.push('## Columns');
|
|
895
|
+
lines.push('');
|
|
896
|
+
for (const table of built) {
|
|
897
|
+
if (table.columns.length === 0) {
|
|
898
|
+
lines.push(`- \`${table.table}\` - empty.`);
|
|
899
|
+
continue;
|
|
900
|
+
}
|
|
901
|
+
const columns = table.columns
|
|
902
|
+
.map((column) => `\`${column.name}\``)
|
|
903
|
+
.join(', ');
|
|
904
|
+
lines.push(`- \`${table.table}\`: ${columns}`);
|
|
905
|
+
}
|
|
906
|
+
lines.push('');
|
|
907
|
+
lines.push('## How the tables relate');
|
|
908
|
+
lines.push('');
|
|
909
|
+
lines.push(
|
|
910
|
+
'- `info_view_columns."Table"` and `info_view_measures."Table"` ' +
|
|
911
|
+
'join to `info_view_tables."Name"`.'
|
|
912
|
+
);
|
|
913
|
+
lines.push(
|
|
914
|
+
'- `column_statistics.("Table Name", "Column Name")` joins to ' +
|
|
915
|
+
'`info_view_columns.("Table", "Name")`.'
|
|
916
|
+
);
|
|
917
|
+
lines.push(
|
|
918
|
+
'- Relationship endpoints join to columns by (table, column): ' +
|
|
919
|
+
'`("FromTable", "FromColumn")` and ' +
|
|
920
|
+
'`("ToTable", "ToColumn")`.'
|
|
921
|
+
);
|
|
922
|
+
lines.push(
|
|
923
|
+
'- `catalog_status` records whether every source view was captured. ' +
|
|
924
|
+
'Query it before treating an untruncated SQL result as proof that ' +
|
|
925
|
+
'an object is absent.'
|
|
926
|
+
);
|
|
927
|
+
lines.push('');
|
|
928
|
+
lines.push(
|
|
929
|
+
"Column names are the model's own spellings, exactly as DAX returned " +
|
|
930
|
+
'them, so names containing spaces need quoting.'
|
|
931
|
+
);
|
|
932
|
+
lines.push(
|
|
933
|
+
'`column_statistics` contains actual Min/Max values derived from model ' +
|
|
934
|
+
'rows, plus cardinality and maximum text length. Treat this cache as ' +
|
|
935
|
+
'customer data.'
|
|
936
|
+
);
|
|
937
|
+
if (notes.length > 0) {
|
|
938
|
+
lines.push('');
|
|
939
|
+
lines.push('## Notes');
|
|
940
|
+
lines.push('');
|
|
941
|
+
for (const note of notes) {
|
|
942
|
+
lines.push(`- ${note}`);
|
|
943
|
+
}
|
|
944
|
+
}
|
|
945
|
+
lines.push('');
|
|
946
|
+
return lines.join('\n');
|
|
947
|
+
}
|
|
948
|
+
|
|
949
|
+
export async function discoverRaw({ run } = {}) {
|
|
950
|
+
const notes = [];
|
|
951
|
+
const resultSets = [];
|
|
952
|
+
let incomplete = false;
|
|
953
|
+
const missing = [];
|
|
954
|
+
|
|
955
|
+
for (const source of CATALOG_SOURCES) {
|
|
956
|
+
const result = await run(rawQuery(source.dax));
|
|
957
|
+
if (!result.ok) {
|
|
958
|
+
if (source.table === 'info_view_tables') {
|
|
959
|
+
return {
|
|
960
|
+
ok: false,
|
|
961
|
+
error: `Table discovery failed: ${result.error}`,
|
|
962
|
+
};
|
|
963
|
+
}
|
|
964
|
+
incomplete = true;
|
|
965
|
+
missing.push(source.table);
|
|
966
|
+
const scope = source.optional ? '' : ' The catalog is incomplete.';
|
|
967
|
+
notes.push(
|
|
968
|
+
`${source.dax} failed, so \`${source.table}\` is not in the catalog.` +
|
|
969
|
+
`${scope} ${result.error}`
|
|
970
|
+
);
|
|
971
|
+
continue;
|
|
972
|
+
}
|
|
973
|
+
if (!Array.isArray(result.columns) || result.columns.length === 0) {
|
|
974
|
+
if (source.table === 'info_view_tables') {
|
|
975
|
+
return {
|
|
976
|
+
ok: false,
|
|
977
|
+
error: 'Table discovery failed: the query returned no column schema.',
|
|
978
|
+
};
|
|
979
|
+
}
|
|
980
|
+
incomplete = true;
|
|
981
|
+
missing.push(source.table);
|
|
982
|
+
notes.push(
|
|
983
|
+
`${source.dax} returned no column schema, so ` +
|
|
984
|
+
`\`${source.table}\` is not in the catalog.`
|
|
985
|
+
);
|
|
986
|
+
continue;
|
|
987
|
+
}
|
|
988
|
+
resultSets.push({
|
|
989
|
+
...source,
|
|
990
|
+
columns: result.columns,
|
|
991
|
+
rows: result.rows,
|
|
992
|
+
});
|
|
993
|
+
}
|
|
994
|
+
|
|
995
|
+
resultSets.push({
|
|
996
|
+
table: 'catalog_status',
|
|
997
|
+
rows: [
|
|
998
|
+
{
|
|
999
|
+
status: incomplete ? 'incomplete' : 'complete',
|
|
1000
|
+
incomplete,
|
|
1001
|
+
notes: JSON.stringify(notes),
|
|
1002
|
+
},
|
|
1003
|
+
],
|
|
1004
|
+
grain: 'One row describing catalog completeness',
|
|
1005
|
+
dax: 'Derived from catalog build outcomes',
|
|
1006
|
+
note: 'Check this row before treating absence from another table as proof.',
|
|
1007
|
+
});
|
|
1008
|
+
|
|
1009
|
+
return { ok: true, resultSets, notes, incomplete, missing };
|
|
1010
|
+
}
|
|
1011
|
+
|
|
1012
|
+
function catalogFileVersion(path) {
|
|
1013
|
+
try {
|
|
1014
|
+
const stat = statSync(path, { bigint: true });
|
|
1015
|
+
return `${stat.ino}:${stat.size}:${stat.mtimeNs}`;
|
|
1016
|
+
} catch {
|
|
1017
|
+
return null;
|
|
1018
|
+
}
|
|
1019
|
+
}
|
|
1020
|
+
|
|
1021
|
+
function readCatalogStatus(dbPath) {
|
|
1022
|
+
if (!existsSync(dbPath)) {
|
|
1023
|
+
return null;
|
|
1024
|
+
}
|
|
1025
|
+
let db;
|
|
1026
|
+
try {
|
|
1027
|
+
db = openReadOnlyDatabase(dbPath);
|
|
1028
|
+
const row = db
|
|
1029
|
+
.prepare('SELECT status, incomplete, notes FROM "catalog_status" LIMIT 1')
|
|
1030
|
+
.get();
|
|
1031
|
+
if (!row) {
|
|
1032
|
+
return null;
|
|
1033
|
+
}
|
|
1034
|
+
return {
|
|
1035
|
+
status: row.status,
|
|
1036
|
+
incomplete: Boolean(row.incomplete),
|
|
1037
|
+
notes: row.notes,
|
|
1038
|
+
};
|
|
1039
|
+
} catch {
|
|
1040
|
+
return null;
|
|
1041
|
+
} finally {
|
|
1042
|
+
db?.close();
|
|
1043
|
+
}
|
|
1044
|
+
}
|
|
1045
|
+
|
|
1046
|
+
function catalogHasTable(dbPath, table) {
|
|
1047
|
+
if (!existsSync(dbPath)) {
|
|
1048
|
+
return false;
|
|
1049
|
+
}
|
|
1050
|
+
let db;
|
|
1051
|
+
try {
|
|
1052
|
+
db = openReadOnlyDatabase(dbPath);
|
|
1053
|
+
return (
|
|
1054
|
+
db
|
|
1055
|
+
.prepare(
|
|
1056
|
+
"SELECT 1 FROM sqlite_schema WHERE type = 'table' AND name = ? LIMIT 1"
|
|
1057
|
+
)
|
|
1058
|
+
.get(table) !== undefined
|
|
1059
|
+
);
|
|
1060
|
+
} catch {
|
|
1061
|
+
// An unreadable database cannot be shown to hold the view, and the caller
|
|
1062
|
+
// only uses this to decide whether publishing would lose one.
|
|
1063
|
+
return false;
|
|
1064
|
+
} finally {
|
|
1065
|
+
db?.close();
|
|
1066
|
+
}
|
|
1067
|
+
}
|
|
1068
|
+
|
|
1069
|
+
function reusedCatalogResult(dbPath, catalogPath, notes, extra = {}) {
|
|
1070
|
+
const status = readCatalogStatus(dbPath);
|
|
1071
|
+
return {
|
|
1072
|
+
ok: true,
|
|
1073
|
+
reused: true,
|
|
1074
|
+
dbPath,
|
|
1075
|
+
catalogPath,
|
|
1076
|
+
built: [],
|
|
1077
|
+
notes: Array.isArray(notes) ? notes : [notes],
|
|
1078
|
+
incomplete: status?.status !== 'complete',
|
|
1079
|
+
...extra,
|
|
1080
|
+
};
|
|
1081
|
+
}
|
|
1082
|
+
|
|
1083
|
+
export async function materializeCatalog({
|
|
1084
|
+
dir,
|
|
1085
|
+
datasetId,
|
|
1086
|
+
alias,
|
|
1087
|
+
run = (dax) => runFabricQuery(dax, { alias }),
|
|
1088
|
+
retry = withRateLimitRetry,
|
|
1089
|
+
lockOptions,
|
|
1090
|
+
} = {}) {
|
|
1091
|
+
mkdirSync(dir, { recursive: true });
|
|
1092
|
+
const dbPath = join(dir, DATABASE_FILE);
|
|
1093
|
+
const catalogPath = join(dir, CATALOG_FILE);
|
|
1094
|
+
const initialVersion = catalogFileVersion(dbPath);
|
|
1095
|
+
|
|
1096
|
+
const refreshCatalog = async ({ waited }) => {
|
|
1097
|
+
const currentVersion = catalogFileVersion(dbPath);
|
|
1098
|
+
const currentStatus = readCatalogStatus(dbPath);
|
|
1099
|
+
if (
|
|
1100
|
+
waited &&
|
|
1101
|
+
currentVersion !== null &&
|
|
1102
|
+
currentVersion !== initialVersion &&
|
|
1103
|
+
currentStatus?.status === 'complete'
|
|
1104
|
+
) {
|
|
1105
|
+
return reusedCatalogResult(
|
|
1106
|
+
dbPath,
|
|
1107
|
+
catalogPath,
|
|
1108
|
+
'Another process refreshed this catalog while this process was waiting.'
|
|
1109
|
+
);
|
|
1110
|
+
}
|
|
1111
|
+
|
|
1112
|
+
const discovery = await discoverRaw({
|
|
1113
|
+
run: retry(run),
|
|
1114
|
+
});
|
|
1115
|
+
if (!discovery.ok) {
|
|
1116
|
+
// A cold start has nothing to answer from, so a failed discovery stays
|
|
1117
|
+
// fatal. With a complete catalog already on disk it must not be:
|
|
1118
|
+
// throttling and transient outages hit the first request after the TTL
|
|
1119
|
+
// expires, and failing there costs the whole command even though the
|
|
1120
|
+
// previous snapshot would have answered it. The cooldown is still
|
|
1121
|
+
// recorded from these notes, so the next call does not re-hammer a
|
|
1122
|
+
// model that is already refusing.
|
|
1123
|
+
if (currentStatus?.status !== 'complete') {
|
|
1124
|
+
return discovery;
|
|
1125
|
+
}
|
|
1126
|
+
return reusedCatalogResult(
|
|
1127
|
+
dbPath,
|
|
1128
|
+
catalogPath,
|
|
1129
|
+
[
|
|
1130
|
+
'The refresh failed, so the previous complete catalog was ' +
|
|
1131
|
+
'preserved. No new data was fetched; the catalog below is the ' +
|
|
1132
|
+
'earlier snapshot.',
|
|
1133
|
+
discovery.error,
|
|
1134
|
+
],
|
|
1135
|
+
{ refreshIncomplete: true }
|
|
1136
|
+
);
|
|
1137
|
+
}
|
|
1138
|
+
// Preserve the previous catalog when republishing would drop a view it
|
|
1139
|
+
// actually has. Gating on "the previous catalog was complete" instead
|
|
1140
|
+
// failed both ways: an optional view that never works (COLUMNSTATISTICS
|
|
1141
|
+
// on a model that does not support it) keeps the status at incomplete
|
|
1142
|
+
// forever, which disabled this protection entirely and let a throttled
|
|
1143
|
+
// rebuild delete info_view_measures; and once the status was complete, a
|
|
1144
|
+
// rebuild missing only that same optional view was discarded for good,
|
|
1145
|
+
// freezing the catalog on the last snapshot that included it.
|
|
1146
|
+
const lost = (discovery.missing ?? []).filter((table) =>
|
|
1147
|
+
catalogHasTable(dbPath, table)
|
|
1148
|
+
);
|
|
1149
|
+
if (discovery.incomplete && lost.length > 0) {
|
|
1150
|
+
// Keep the previous catalog - a partial rebuild must never replace good
|
|
1151
|
+
// data. Keep the reason too: `catalog_status` describes the retained
|
|
1152
|
+
// snapshot, not this refresh, so these notes are the ONLY evidence that
|
|
1153
|
+
// the fetch failed, and they are what tells throttling (retry later)
|
|
1154
|
+
// apart from a permission or configuration problem (stop and fix
|
|
1155
|
+
// access).
|
|
1156
|
+
return reusedCatalogResult(
|
|
1157
|
+
dbPath,
|
|
1158
|
+
catalogPath,
|
|
1159
|
+
[
|
|
1160
|
+
`The refresh was incomplete and would have dropped ` +
|
|
1161
|
+
`${lost.map((table) => `\`${table}\``).join(', ')}, so the ` +
|
|
1162
|
+
'previous catalog was preserved. No new data was fetched; the ' +
|
|
1163
|
+
'catalog below is the earlier snapshot.',
|
|
1164
|
+
...discovery.notes,
|
|
1165
|
+
],
|
|
1166
|
+
{ refreshIncomplete: true }
|
|
1167
|
+
);
|
|
1168
|
+
}
|
|
1169
|
+
|
|
1170
|
+
const temporaryDbPath = `${dbPath}.${process.pid}.${randomUUID()}.tmp`;
|
|
1171
|
+
const temporaryCatalogPath = `${catalogPath}.${process.pid}.${randomUUID()}.tmp`;
|
|
1172
|
+
rmSync(temporaryDbPath, { force: true });
|
|
1173
|
+
rmSync(temporaryCatalogPath, { force: true });
|
|
1174
|
+
try {
|
|
1175
|
+
const built = populateDatabase(discovery.resultSets, temporaryDbPath);
|
|
1176
|
+
writeFileSync(
|
|
1177
|
+
temporaryCatalogPath,
|
|
1178
|
+
renderCatalog(built, {
|
|
1179
|
+
datasetId,
|
|
1180
|
+
notes: discovery.notes,
|
|
1181
|
+
incomplete: discovery.incomplete,
|
|
1182
|
+
}),
|
|
1183
|
+
'utf8'
|
|
1184
|
+
);
|
|
1185
|
+
publishFiles([
|
|
1186
|
+
{
|
|
1187
|
+
temporaryPath: temporaryDbPath,
|
|
1188
|
+
destinationPath: dbPath,
|
|
1189
|
+
},
|
|
1190
|
+
{
|
|
1191
|
+
temporaryPath: temporaryCatalogPath,
|
|
1192
|
+
destinationPath: catalogPath,
|
|
1193
|
+
},
|
|
1194
|
+
]);
|
|
1195
|
+
return {
|
|
1196
|
+
ok: true,
|
|
1197
|
+
dbPath,
|
|
1198
|
+
catalogPath,
|
|
1199
|
+
built,
|
|
1200
|
+
notes: discovery.notes,
|
|
1201
|
+
incomplete: discovery.incomplete,
|
|
1202
|
+
};
|
|
1203
|
+
} finally {
|
|
1204
|
+
rmSync(temporaryDbPath, { force: true });
|
|
1205
|
+
rmSync(temporaryCatalogPath, { force: true });
|
|
1206
|
+
}
|
|
1207
|
+
};
|
|
1208
|
+
|
|
1209
|
+
return withCatalogLock(
|
|
1210
|
+
dir,
|
|
1211
|
+
async (context) => {
|
|
1212
|
+
// A refresh can fail by throwing rather than by returning an error. On
|
|
1213
|
+
// Windows another process holding model.sqlite open - a second
|
|
1214
|
+
// schema:sql, a database browser - makes the publish rename fail with
|
|
1215
|
+
// EPERM after its retries. With a catalog already on disk that must not
|
|
1216
|
+
// cost the caller their query either, so it is handled exactly like the
|
|
1217
|
+
// failed refresh above: answer from the previous snapshot and let the
|
|
1218
|
+
// cooldown stop the next call from trying again immediately.
|
|
1219
|
+
const previousStatus = readCatalogStatus(dbPath);
|
|
1220
|
+
try {
|
|
1221
|
+
return await refreshCatalog(context);
|
|
1222
|
+
} catch (error) {
|
|
1223
|
+
if (previousStatus === null) {
|
|
1224
|
+
throw error;
|
|
1225
|
+
}
|
|
1226
|
+
return reusedCatalogResult(
|
|
1227
|
+
dbPath,
|
|
1228
|
+
catalogPath,
|
|
1229
|
+
[
|
|
1230
|
+
'The refresh could not be completed, so the previous catalog was ' +
|
|
1231
|
+
'preserved. No new data was fetched; the catalog below is the ' +
|
|
1232
|
+
'earlier snapshot.',
|
|
1233
|
+
error instanceof Error ? error.message : String(error),
|
|
1234
|
+
],
|
|
1235
|
+
{ refreshIncomplete: true }
|
|
1236
|
+
);
|
|
1237
|
+
}
|
|
1238
|
+
},
|
|
1239
|
+
lockOptions
|
|
1240
|
+
);
|
|
1241
|
+
}
|
|
1242
|
+
|
|
1243
|
+
function tableNameFromResultRow(row) {
|
|
1244
|
+
if (row.kind != null && String(row.kind).toLocaleLowerCase() !== 'table') {
|
|
1245
|
+
return null;
|
|
1246
|
+
}
|
|
1247
|
+
for (const key of [
|
|
1248
|
+
'table_name',
|
|
1249
|
+
'tableName',
|
|
1250
|
+
'object_name',
|
|
1251
|
+
'Table',
|
|
1252
|
+
'Name',
|
|
1253
|
+
]) {
|
|
1254
|
+
if (typeof row[key] === 'string' && row[key].length > 0) {
|
|
1255
|
+
return row[key];
|
|
1256
|
+
}
|
|
1257
|
+
}
|
|
1258
|
+
return null;
|
|
1259
|
+
}
|
|
1260
|
+
|
|
1261
|
+
/**
|
|
1262
|
+
* Split SQL into coarse tokens: words, quoted identifiers, string literals and
|
|
1263
|
+
* single punctuation characters. Comments and whitespace are dropped.
|
|
1264
|
+
*
|
|
1265
|
+
* This is not a parser. It only has to be exact about where literals,
|
|
1266
|
+
* identifiers and comments end, so text inside them can never be mistaken for a
|
|
1267
|
+
* keyword, a table name, or a statement separator.
|
|
1268
|
+
*/
|
|
1269
|
+
export function sqlTokens(sql) {
|
|
1270
|
+
const text = String(sql);
|
|
1271
|
+
const tokens = [];
|
|
1272
|
+
let i = 0;
|
|
1273
|
+
const quoted = (open, close, type) => {
|
|
1274
|
+
let value = '';
|
|
1275
|
+
let j = i + 1;
|
|
1276
|
+
while (j < text.length) {
|
|
1277
|
+
if (text[j] === close) {
|
|
1278
|
+
// A doubled delimiter is an escaped one, not the end.
|
|
1279
|
+
if (close === open && text[j + 1] === close) {
|
|
1280
|
+
value += close;
|
|
1281
|
+
j += 2;
|
|
1282
|
+
continue;
|
|
1283
|
+
}
|
|
1284
|
+
break;
|
|
1285
|
+
}
|
|
1286
|
+
value += text[j];
|
|
1287
|
+
j += 1;
|
|
1288
|
+
}
|
|
1289
|
+
tokens.push({ type, value });
|
|
1290
|
+
i = j + 1;
|
|
1291
|
+
};
|
|
1292
|
+
while (i < text.length) {
|
|
1293
|
+
const ch = text[i];
|
|
1294
|
+
if (/\s/u.test(ch)) {
|
|
1295
|
+
i += 1;
|
|
1296
|
+
} else if (ch === '-' && text[i + 1] === '-') {
|
|
1297
|
+
const end = text.indexOf('\n', i);
|
|
1298
|
+
i = end < 0 ? text.length : end + 1;
|
|
1299
|
+
} else if (ch === '/' && text[i + 1] === '*') {
|
|
1300
|
+
const end = text.indexOf('*/', i + 2);
|
|
1301
|
+
i = end < 0 ? text.length : end + 2;
|
|
1302
|
+
} else if (ch === "'") {
|
|
1303
|
+
quoted("'", "'", 'string');
|
|
1304
|
+
} else if (ch === '"') {
|
|
1305
|
+
quoted('"', '"', 'identifier');
|
|
1306
|
+
} else if (ch === '`') {
|
|
1307
|
+
quoted('`', '`', 'identifier');
|
|
1308
|
+
} else if (ch === '[') {
|
|
1309
|
+
quoted('[', ']', 'identifier');
|
|
1310
|
+
} else if (/[\p{L}_]/u.test(ch)) {
|
|
1311
|
+
const match = /^[\p{L}\p{N}_$]+/u.exec(text.slice(i));
|
|
1312
|
+
tokens.push({ type: 'word', value: match[0] });
|
|
1313
|
+
i += match[0].length;
|
|
1314
|
+
} else if (/[0-9]/u.test(ch)) {
|
|
1315
|
+
const match = /^[0-9][0-9A-Za-z_.]*/u.exec(text.slice(i));
|
|
1316
|
+
tokens.push({ type: 'number', value: match[0] });
|
|
1317
|
+
i += match[0].length;
|
|
1318
|
+
} else {
|
|
1319
|
+
tokens.push({ type: 'punct', value: ch });
|
|
1320
|
+
i += 1;
|
|
1321
|
+
}
|
|
1322
|
+
}
|
|
1323
|
+
return tokens;
|
|
1324
|
+
}
|
|
1325
|
+
|
|
1326
|
+
const FROM_CLAUSE_TERMINATORS = new Set([
|
|
1327
|
+
'where',
|
|
1328
|
+
'group',
|
|
1329
|
+
'order',
|
|
1330
|
+
'limit',
|
|
1331
|
+
'having',
|
|
1332
|
+
'window',
|
|
1333
|
+
]);
|
|
1334
|
+
|
|
1335
|
+
function isKeyword(token, ...words) {
|
|
1336
|
+
return (
|
|
1337
|
+
token?.type === 'word' && words.includes(token.value.toLocaleLowerCase())
|
|
1338
|
+
);
|
|
1339
|
+
}
|
|
1340
|
+
|
|
1341
|
+
function isNamed(token, name) {
|
|
1342
|
+
return (
|
|
1343
|
+
(token?.type === 'word' || token?.type === 'identifier') &&
|
|
1344
|
+
token.value.toLocaleLowerCase() === name
|
|
1345
|
+
);
|
|
1346
|
+
}
|
|
1347
|
+
|
|
1348
|
+
/**
|
|
1349
|
+
* True when every result row is an `info_view_tables` row: one SELECT whose
|
|
1350
|
+
* only top-level FROM source is `info_view_tables`, optionally aliased, with no
|
|
1351
|
+
* JOIN, comma join or compound operator.
|
|
1352
|
+
*
|
|
1353
|
+
* Subqueries in parentheses cannot change which table the rows come from, so
|
|
1354
|
+
* only top-level tokens are inspected.
|
|
1355
|
+
*/
|
|
1356
|
+
export function selectsOnlyTableRows(sql) {
|
|
1357
|
+
const topLevel = [];
|
|
1358
|
+
let depth = 0;
|
|
1359
|
+
for (const token of sqlTokens(sql)) {
|
|
1360
|
+
if (token.type === 'punct' && token.value === '(') {
|
|
1361
|
+
depth += 1;
|
|
1362
|
+
} else if (token.type === 'punct' && token.value === ')') {
|
|
1363
|
+
depth = Math.max(0, depth - 1);
|
|
1364
|
+
} else if (depth === 0) {
|
|
1365
|
+
topLevel.push(token);
|
|
1366
|
+
}
|
|
1367
|
+
}
|
|
1368
|
+
while (topLevel.at(-1)?.type === 'punct' && topLevel.at(-1).value === ';') {
|
|
1369
|
+
topLevel.pop();
|
|
1370
|
+
}
|
|
1371
|
+
if (
|
|
1372
|
+
topLevel.some((token) =>
|
|
1373
|
+
isKeyword(token, 'join', 'union', 'intersect', 'except')
|
|
1374
|
+
)
|
|
1375
|
+
) {
|
|
1376
|
+
return false;
|
|
1377
|
+
}
|
|
1378
|
+
const fromIndexes = topLevel
|
|
1379
|
+
.map((token, index) => (isKeyword(token, 'from') ? index : -1))
|
|
1380
|
+
.filter((index) => index >= 0);
|
|
1381
|
+
if (fromIndexes.length !== 1) {
|
|
1382
|
+
return false;
|
|
1383
|
+
}
|
|
1384
|
+
let next = fromIndexes[0] + 1;
|
|
1385
|
+
if (!isNamed(topLevel[next], 'info_view_tables')) {
|
|
1386
|
+
return false;
|
|
1387
|
+
}
|
|
1388
|
+
next += 1;
|
|
1389
|
+
if (isKeyword(topLevel[next], 'as')) {
|
|
1390
|
+
next += 1;
|
|
1391
|
+
}
|
|
1392
|
+
// An alias, if present. A bare word that begins the next clause is not one.
|
|
1393
|
+
const alias = topLevel[next];
|
|
1394
|
+
if (
|
|
1395
|
+
alias?.type === 'identifier' ||
|
|
1396
|
+
(alias?.type === 'word' &&
|
|
1397
|
+
!FROM_CLAUSE_TERMINATORS.has(alias.value.toLocaleLowerCase()))
|
|
1398
|
+
) {
|
|
1399
|
+
next += 1;
|
|
1400
|
+
}
|
|
1401
|
+
// A comma here is a comma join, so the rows are no longer only table rows.
|
|
1402
|
+
return (
|
|
1403
|
+
next >= topLevel.length ||
|
|
1404
|
+
(topLevel[next].type === 'word' &&
|
|
1405
|
+
FROM_CLAUSE_TERMINATORS.has(topLevel[next].value.toLocaleLowerCase()))
|
|
1406
|
+
);
|
|
1407
|
+
}
|
|
1408
|
+
|
|
1409
|
+
function enrichSelectedTablesWithOwnedMeasures(db, sql, rows) {
|
|
1410
|
+
// Only rows that ARE table rows may be enriched. Matching the table name
|
|
1411
|
+
// anywhere in the statement also fired on
|
|
1412
|
+
// `info_view_columns c JOIN info_view_tables t`, which returns column rows:
|
|
1413
|
+
// every one was read as a selected table, decorated with up to ten measures,
|
|
1414
|
+
// and the result blew the byte budget. The catalog itself documents that
|
|
1415
|
+
// join, so this is a query the agent is told to write.
|
|
1416
|
+
if (!selectsOnlyTableRows(sql)) {
|
|
1417
|
+
return rows;
|
|
1418
|
+
}
|
|
1419
|
+
const measureColumns = new Set(
|
|
1420
|
+
db
|
|
1421
|
+
.prepare('PRAGMA table_info("info_view_measures")')
|
|
1422
|
+
.all()
|
|
1423
|
+
.map((column) => column.name)
|
|
1424
|
+
);
|
|
1425
|
+
if (!measureColumns.has('Table') || !measureColumns.has('Name')) {
|
|
1426
|
+
return rows.map((row) =>
|
|
1427
|
+
tableNameFromResultRow(row)
|
|
1428
|
+
? {
|
|
1429
|
+
...row,
|
|
1430
|
+
ownedMeasuresUnavailable: true,
|
|
1431
|
+
ownedMeasureCount: null,
|
|
1432
|
+
ownedMeasures: null,
|
|
1433
|
+
}
|
|
1434
|
+
: row
|
|
1435
|
+
);
|
|
1436
|
+
}
|
|
1437
|
+
const fields = ['Name', 'DisplayFolder'].filter((field) =>
|
|
1438
|
+
measureColumns.has(field)
|
|
1439
|
+
);
|
|
1440
|
+
const lookup = db.prepare(
|
|
1441
|
+
`SELECT ${fields.map(quoteIdentifier).join(', ')} ` +
|
|
1442
|
+
`FROM "info_view_measures" ` +
|
|
1443
|
+
`WHERE lower("Table") = lower(?) ORDER BY "Name"`
|
|
1444
|
+
);
|
|
1445
|
+
|
|
1446
|
+
return rows.map((row) => {
|
|
1447
|
+
const tableName = tableNameFromResultRow(row);
|
|
1448
|
+
if (!tableName) {
|
|
1449
|
+
return row;
|
|
1450
|
+
}
|
|
1451
|
+
const measures = lookup.all(tableName);
|
|
1452
|
+
return {
|
|
1453
|
+
...row,
|
|
1454
|
+
ownedMeasureCount: measures.length,
|
|
1455
|
+
ownedMeasuresTruncated: measures.length > MAX_OWNED_MEASURE_PREVIEW,
|
|
1456
|
+
ownedMeasures: measures
|
|
1457
|
+
.slice(0, MAX_OWNED_MEASURE_PREVIEW)
|
|
1458
|
+
.map((measure) => ({
|
|
1459
|
+
name: measure.Name,
|
|
1460
|
+
displayFolder: measure.DisplayFolder ?? null,
|
|
1461
|
+
})),
|
|
1462
|
+
ownedMeasuresNeedDetail: measures.length > 0,
|
|
1463
|
+
};
|
|
1464
|
+
});
|
|
1465
|
+
}
|
|
1466
|
+
|
|
1467
|
+
export function boundRows(
|
|
1468
|
+
rows,
|
|
1469
|
+
{ limit = DEFAULT_PRINT_LIMIT, maxBytes = MAX_RESULT_BYTES, notes = [] } = {}
|
|
1470
|
+
) {
|
|
1471
|
+
const total = rows.length;
|
|
1472
|
+
const kept = rows.slice(0, Math.max(0, limit));
|
|
1473
|
+
const rowLimitBound = kept.length < total;
|
|
1474
|
+
return boundResultRows(
|
|
1475
|
+
{
|
|
1476
|
+
ok: true,
|
|
1477
|
+
totalRows: total,
|
|
1478
|
+
returnedRows: kept.length,
|
|
1479
|
+
truncated: rowLimitBound,
|
|
1480
|
+
...(rowLimitBound
|
|
1481
|
+
? {
|
|
1482
|
+
truncation: {
|
|
1483
|
+
reason: 'row_limit',
|
|
1484
|
+
requiredAction: TRUNCATION_REQUIRED_ACTION,
|
|
1485
|
+
},
|
|
1486
|
+
}
|
|
1487
|
+
: {}),
|
|
1488
|
+
...(notes.length ? { notes } : {}),
|
|
1489
|
+
rows: kept,
|
|
1490
|
+
},
|
|
1491
|
+
maxBytes
|
|
1492
|
+
);
|
|
1493
|
+
}
|
|
1494
|
+
|
|
1495
|
+
// Drop whole rows until the serialized payload fits. Splitting the work out of
|
|
1496
|
+
// boundRows lets a batch re-bound an already-built result against a smaller
|
|
1497
|
+
// share without rebuilding it from the source rows.
|
|
1498
|
+
function boundResultRows(result, maxBytes) {
|
|
1499
|
+
let candidate = result;
|
|
1500
|
+
let kept = result.rows;
|
|
1501
|
+
let bytes = Buffer.byteLength(formatResult(candidate), 'utf8');
|
|
1502
|
+
while (kept.length > 0 && bytes > maxBytes) {
|
|
1503
|
+
kept = kept.slice(0, Math.ceil(kept.length / 2));
|
|
1504
|
+
if (kept.length === candidate.returnedRows) {
|
|
1505
|
+
kept = kept.slice(0, -1);
|
|
1506
|
+
}
|
|
1507
|
+
candidate = {
|
|
1508
|
+
...result,
|
|
1509
|
+
returnedRows: kept.length,
|
|
1510
|
+
truncated: true,
|
|
1511
|
+
truncation: {
|
|
1512
|
+
reason: result.truncation?.reason?.includes('row_limit')
|
|
1513
|
+
? 'row_limit_and_byte_budget'
|
|
1514
|
+
: 'byte_budget',
|
|
1515
|
+
requiredAction: TRUNCATION_REQUIRED_ACTION,
|
|
1516
|
+
},
|
|
1517
|
+
rows: kept,
|
|
1518
|
+
};
|
|
1519
|
+
bytes = Buffer.byteLength(formatResult(candidate), 'utf8');
|
|
1520
|
+
}
|
|
1521
|
+
return candidate;
|
|
1522
|
+
}
|
|
1523
|
+
|
|
1524
|
+
/**
|
|
1525
|
+
* Statement text after the first `;`, or null when there is none.
|
|
1526
|
+
*
|
|
1527
|
+
* `db.prepare()` compiles only the first statement, so anything after it never
|
|
1528
|
+
* runs while the answer still looks complete. Literals, quoted and bracketed
|
|
1529
|
+
* identifiers and comments are tokenized away first, so a `;` inside them is
|
|
1530
|
+
* not mistaken for a separator. A trailing `;` is fine.
|
|
1531
|
+
*/
|
|
1532
|
+
export function trailingStatement(sql) {
|
|
1533
|
+
const tokens = sqlTokens(sql);
|
|
1534
|
+
const end = tokens.findIndex(
|
|
1535
|
+
(token) => token.type === 'punct' && token.value === ';'
|
|
1536
|
+
);
|
|
1537
|
+
if (end < 0) {
|
|
1538
|
+
return null;
|
|
1539
|
+
}
|
|
1540
|
+
const rest = tokens
|
|
1541
|
+
.slice(end + 1)
|
|
1542
|
+
.filter((token) => !(token.type === 'punct' && token.value === ';'));
|
|
1543
|
+
return rest.length > 0 ? rest[0].value : null;
|
|
1544
|
+
}
|
|
1545
|
+
|
|
1546
|
+
export function runSql(
|
|
1547
|
+
dbPath,
|
|
1548
|
+
sql,
|
|
1549
|
+
{ limit = DEFAULT_PRINT_LIMIT, maxBytes = MAX_RESULT_BYTES } = {}
|
|
1550
|
+
) {
|
|
1551
|
+
// Reject a second statement rather than answering the first and discarding
|
|
1552
|
+
// it silently: the agent asked two questions and would otherwise be told the
|
|
1553
|
+
// single answer is complete.
|
|
1554
|
+
const extra = trailingStatement(sql);
|
|
1555
|
+
if (extra !== null) {
|
|
1556
|
+
return {
|
|
1557
|
+
ok: false,
|
|
1558
|
+
error:
|
|
1559
|
+
`Only one statement runs per --sql, and text after the first ';' ` +
|
|
1560
|
+
`was ignored (starting at \`${extra}\`). Pass each statement as its ` +
|
|
1561
|
+
'own --sql; they still answer in one invocation.',
|
|
1562
|
+
};
|
|
1563
|
+
}
|
|
1564
|
+
// The open is inside the try: publication, a missing catalog, or a permission
|
|
1565
|
+
// problem must become this statement's error, not an exception that escapes
|
|
1566
|
+
// runBatch and discards every other statement's completed work.
|
|
1567
|
+
let db;
|
|
1568
|
+
try {
|
|
1569
|
+
db = openReadOnlyDatabase(dbPath);
|
|
1570
|
+
const rows = enrichSelectedTablesWithOwnedMeasures(
|
|
1571
|
+
db,
|
|
1572
|
+
sql,
|
|
1573
|
+
db.prepare(sql).all()
|
|
1574
|
+
);
|
|
1575
|
+
return boundRows(rows, { limit, maxBytes });
|
|
1576
|
+
} catch (error) {
|
|
1577
|
+
return {
|
|
1578
|
+
ok: false,
|
|
1579
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1580
|
+
};
|
|
1581
|
+
} finally {
|
|
1582
|
+
db?.close();
|
|
1583
|
+
}
|
|
1584
|
+
}
|
|
1585
|
+
|
|
1586
|
+
function searchTokens(query) {
|
|
1587
|
+
if (Array.isArray(query)) {
|
|
1588
|
+
return [...new Set(query.filter(Boolean))];
|
|
1589
|
+
}
|
|
1590
|
+
return [
|
|
1591
|
+
...new Set(
|
|
1592
|
+
String(query)
|
|
1593
|
+
.toLocaleLowerCase()
|
|
1594
|
+
.split(/[^\p{L}\p{N}_]+/u)
|
|
1595
|
+
.map((token) => token.trim())
|
|
1596
|
+
.filter(Boolean)
|
|
1597
|
+
),
|
|
1598
|
+
];
|
|
1599
|
+
}
|
|
1600
|
+
|
|
1601
|
+
// One definition of "what search looks at", shared by ranking and by the
|
|
1602
|
+
// relaxation below, so the two can never disagree about whether a term matches.
|
|
1603
|
+
function searchableFields(row) {
|
|
1604
|
+
return {
|
|
1605
|
+
object_name: String(row.object_name ?? '').toLocaleLowerCase(),
|
|
1606
|
+
parent_name: String(row.parent_name ?? '').toLocaleLowerCase(),
|
|
1607
|
+
display_folder: String(row.display_folder ?? '').toLocaleLowerCase(),
|
|
1608
|
+
description: String(row.description ?? '').toLocaleLowerCase(),
|
|
1609
|
+
expression: String(row.expression ?? '').toLocaleLowerCase(),
|
|
1610
|
+
data_type: String(row.data_type ?? '').toLocaleLowerCase(),
|
|
1611
|
+
synonyms: String(row.synonyms ?? '').toLocaleLowerCase(),
|
|
1612
|
+
};
|
|
1613
|
+
}
|
|
1614
|
+
|
|
1615
|
+
/** Terms that appear somewhere in this catalog, in the order supplied. */
|
|
1616
|
+
export function matchedSearchTokens(rows, tokens) {
|
|
1617
|
+
const fields = (rows || []).map(searchableFields);
|
|
1618
|
+
return tokens.filter((token) =>
|
|
1619
|
+
fields.some((row) =>
|
|
1620
|
+
Object.values(row).some((value) => value.includes(token))
|
|
1621
|
+
)
|
|
1622
|
+
);
|
|
1623
|
+
}
|
|
1624
|
+
|
|
1625
|
+
export function rankSemanticSearchRows(rows, query) {
|
|
1626
|
+
const tokens = searchTokens(query);
|
|
1627
|
+
if (tokens.length === 0) {
|
|
1628
|
+
throw new Error(
|
|
1629
|
+
'search requires at least one letter, number, or underscore.'
|
|
1630
|
+
);
|
|
1631
|
+
}
|
|
1632
|
+
const kindWeight = {
|
|
1633
|
+
measure: 12,
|
|
1634
|
+
table: 8,
|
|
1635
|
+
column: 4,
|
|
1636
|
+
relationship: 0,
|
|
1637
|
+
};
|
|
1638
|
+
return rows
|
|
1639
|
+
.map((row) => {
|
|
1640
|
+
const fields = searchableFields(row);
|
|
1641
|
+
const matchedFields = new Set();
|
|
1642
|
+
let score = kindWeight[row.kind] ?? 0;
|
|
1643
|
+
for (const token of tokens) {
|
|
1644
|
+
let tokenMatched = false;
|
|
1645
|
+
for (const [field, value] of Object.entries(fields)) {
|
|
1646
|
+
if (!value.includes(token)) {
|
|
1647
|
+
continue;
|
|
1648
|
+
}
|
|
1649
|
+
tokenMatched = true;
|
|
1650
|
+
matchedFields.add(field);
|
|
1651
|
+
const weight =
|
|
1652
|
+
field === 'object_name'
|
|
1653
|
+
? value === token
|
|
1654
|
+
? 100
|
|
1655
|
+
: value.startsWith(token)
|
|
1656
|
+
? 70
|
|
1657
|
+
: 50
|
|
1658
|
+
: field === 'parent_name'
|
|
1659
|
+
? value === token
|
|
1660
|
+
? 80
|
|
1661
|
+
: value.startsWith(token)
|
|
1662
|
+
? 55
|
|
1663
|
+
: 35
|
|
1664
|
+
: field === 'display_folder'
|
|
1665
|
+
? 20
|
|
1666
|
+
: field === 'synonyms'
|
|
1667
|
+
? 18
|
|
1668
|
+
: field === 'description'
|
|
1669
|
+
? 12
|
|
1670
|
+
: field === 'expression'
|
|
1671
|
+
? 8
|
|
1672
|
+
: 3;
|
|
1673
|
+
score += weight;
|
|
1674
|
+
}
|
|
1675
|
+
if (!tokenMatched) {
|
|
1676
|
+
return null;
|
|
1677
|
+
}
|
|
1678
|
+
}
|
|
1679
|
+
return {
|
|
1680
|
+
score,
|
|
1681
|
+
reason: [...matchedFields].join(', '),
|
|
1682
|
+
kind: row.kind,
|
|
1683
|
+
parent_name: row.parent_name,
|
|
1684
|
+
object_name: row.object_name,
|
|
1685
|
+
description: row.description,
|
|
1686
|
+
expression: row.expression,
|
|
1687
|
+
display_folder: row.display_folder,
|
|
1688
|
+
data_type: row.data_type,
|
|
1689
|
+
synonyms: row.synonyms,
|
|
1690
|
+
};
|
|
1691
|
+
})
|
|
1692
|
+
.filter(Boolean)
|
|
1693
|
+
.sort(
|
|
1694
|
+
(left, right) =>
|
|
1695
|
+
right.score - left.score ||
|
|
1696
|
+
left.kind.localeCompare(right.kind) ||
|
|
1697
|
+
left.parent_name.localeCompare(right.parent_name) ||
|
|
1698
|
+
left.object_name.localeCompare(right.object_name)
|
|
1699
|
+
);
|
|
1700
|
+
}
|
|
1701
|
+
|
|
1702
|
+
// --- search payload trimming ------------------------------------------------
|
|
1703
|
+
//
|
|
1704
|
+
// Search is a findability surface, not a reading surface: the agent uses it to
|
|
1705
|
+
// learn which objects exist, then reads detail with SQL. Two fields break that
|
|
1706
|
+
// contract by carrying whole documents into every result page.
|
|
1707
|
+
//
|
|
1708
|
+
// Both are trimmed in the OUTPUT ONLY. Ranking above still sees the full text,
|
|
1709
|
+
// so nothing becomes unfindable - the failure mode of simply dropping the rows.
|
|
1710
|
+
//
|
|
1711
|
+
// This is separate from, and complementary to, the owned-measure preview trim in
|
|
1712
|
+
// enrichSelectedTablesWithOwnedMeasures: that bounds info_view_tables rows on the
|
|
1713
|
+
// SQL path, this bounds semantic_search rows on the search path.
|
|
1714
|
+
|
|
1715
|
+
/** Expressions at or under this stay verbatim; a real measure definition is short. */
|
|
1716
|
+
export const SEARCH_EXPRESSION_LIMIT = 120;
|
|
1717
|
+
|
|
1718
|
+
/**
|
|
1719
|
+
* A description shared by this many rows, and by this share of the described
|
|
1720
|
+
* rows, discriminates nothing between them.
|
|
1721
|
+
*/
|
|
1722
|
+
export const BOILERPLATE_MIN_ROWS = 8;
|
|
1723
|
+
export const BOILERPLATE_MIN_SHARE = 0.2;
|
|
1724
|
+
|
|
1725
|
+
/**
|
|
1726
|
+
* Keep the head of a long expression and say how to get the rest.
|
|
1727
|
+
*
|
|
1728
|
+
* Localization measures are the pathological case - a `SWITCH(USERCULTURE(), ...)`
|
|
1729
|
+
* holding one sentence per locale runs to thousands of characters and computes
|
|
1730
|
+
* nothing - but the rule is general: search never needs a full definition.
|
|
1731
|
+
*/
|
|
1732
|
+
export function summarizeSearchExpression(expression) {
|
|
1733
|
+
const text = String(expression ?? '')
|
|
1734
|
+
.replace(/\s+/gu, ' ')
|
|
1735
|
+
.trim();
|
|
1736
|
+
if (text.length <= SEARCH_EXPRESSION_LIMIT) {
|
|
1737
|
+
return text;
|
|
1738
|
+
}
|
|
1739
|
+
let head = text.slice(0, SEARCH_EXPRESSION_LIMIT);
|
|
1740
|
+
// Cut on a token boundary so the head stays readable DAX rather than half an
|
|
1741
|
+
// identifier the agent might copy.
|
|
1742
|
+
const boundary = Math.max(
|
|
1743
|
+
head.lastIndexOf(' '),
|
|
1744
|
+
head.lastIndexOf(','),
|
|
1745
|
+
head.lastIndexOf('(')
|
|
1746
|
+
);
|
|
1747
|
+
if (boundary > SEARCH_EXPRESSION_LIMIT / 2) {
|
|
1748
|
+
head = head.slice(0, boundary);
|
|
1749
|
+
}
|
|
1750
|
+
// A head with no separator in it at all gets no boundary rescue, so the raw
|
|
1751
|
+
// slice can land between a surrogate pair. Same rule as the batch labels.
|
|
1752
|
+
head = dropTrailingHighSurrogate(head);
|
|
1753
|
+
return (
|
|
1754
|
+
`${head.trimEnd()} \u2026 ` +
|
|
1755
|
+
`(${text.length} chars - read via substr([Expression], <offset>, 1000))`
|
|
1756
|
+
);
|
|
1757
|
+
}
|
|
1758
|
+
|
|
1759
|
+
/** Description values repeated widely enough to carry no information. */
|
|
1760
|
+
export function boilerplateDescriptions(rows) {
|
|
1761
|
+
const counts = new Map();
|
|
1762
|
+
let described = 0;
|
|
1763
|
+
for (const row of rows || []) {
|
|
1764
|
+
const text = String(row?.description ?? '').trim();
|
|
1765
|
+
if (!text) {
|
|
1766
|
+
continue;
|
|
1767
|
+
}
|
|
1768
|
+
described += 1;
|
|
1769
|
+
counts.set(text, (counts.get(text) ?? 0) + 1);
|
|
1770
|
+
}
|
|
1771
|
+
const boilerplate = new Set();
|
|
1772
|
+
for (const [text, count] of counts) {
|
|
1773
|
+
if (
|
|
1774
|
+
count >= BOILERPLATE_MIN_ROWS &&
|
|
1775
|
+
count >= described * BOILERPLATE_MIN_SHARE
|
|
1776
|
+
) {
|
|
1777
|
+
boilerplate.add(text);
|
|
1778
|
+
}
|
|
1779
|
+
}
|
|
1780
|
+
return boilerplate;
|
|
1781
|
+
}
|
|
1782
|
+
|
|
1783
|
+
/**
|
|
1784
|
+
* Apply both trims and report what was done once, rather than per row.
|
|
1785
|
+
*
|
|
1786
|
+
* `catalogRows` is the whole table so "boilerplate" means the same thing no
|
|
1787
|
+
* matter which subset a query happened to match.
|
|
1788
|
+
*/
|
|
1789
|
+
export function trimSearchRows(rankedRows, catalogRows) {
|
|
1790
|
+
const boilerplate = boilerplateDescriptions(catalogRows);
|
|
1791
|
+
let droppedDescriptions = 0;
|
|
1792
|
+
let summarizedExpressions = 0;
|
|
1793
|
+
|
|
1794
|
+
const rows = rankedRows.map((row) => {
|
|
1795
|
+
const next = { ...row };
|
|
1796
|
+
const description = String(next.description ?? '').trim();
|
|
1797
|
+
if (description && boilerplate.has(description)) {
|
|
1798
|
+
next.description = null;
|
|
1799
|
+
droppedDescriptions += 1;
|
|
1800
|
+
}
|
|
1801
|
+
const expression = String(next.expression ?? '');
|
|
1802
|
+
if (expression) {
|
|
1803
|
+
const summary = summarizeSearchExpression(expression);
|
|
1804
|
+
if (summary !== expression) {
|
|
1805
|
+
summarizedExpressions += 1;
|
|
1806
|
+
}
|
|
1807
|
+
next.expression = summary;
|
|
1808
|
+
}
|
|
1809
|
+
return next;
|
|
1810
|
+
});
|
|
1811
|
+
|
|
1812
|
+
return {
|
|
1813
|
+
rows,
|
|
1814
|
+
notes: searchTrimNotes(droppedDescriptions, summarizedExpressions),
|
|
1815
|
+
};
|
|
1816
|
+
}
|
|
1817
|
+
|
|
1818
|
+
function searchTrimNotes(droppedDescriptions, summarizedExpressions) {
|
|
1819
|
+
const notes = [];
|
|
1820
|
+
if (droppedDescriptions > 0) {
|
|
1821
|
+
notes.push(
|
|
1822
|
+
`description omitted on ${droppedDescriptions} row(s): identical ` +
|
|
1823
|
+
'boilerplate shared across the model. It is still searched, just not ' +
|
|
1824
|
+
'reprinted.'
|
|
1825
|
+
);
|
|
1826
|
+
}
|
|
1827
|
+
if (summarizedExpressions > 0) {
|
|
1828
|
+
notes.push(
|
|
1829
|
+
`expression shortened on ${summarizedExpressions} row(s). Search ` +
|
|
1830
|
+
'matches the full text; SELECT the row for it.'
|
|
1831
|
+
);
|
|
1832
|
+
}
|
|
1833
|
+
return notes;
|
|
1834
|
+
}
|
|
1835
|
+
|
|
1836
|
+
// Notes are reserved out of the batch byte budget before any statement runs, so
|
|
1837
|
+
// a note built from agent-supplied query text has to have a computable maximum.
|
|
1838
|
+
// The cap is in serialized UTF-8 BYTES, not UTF-16 length: forty CJK terms cost
|
|
1839
|
+
// three times forty ASCII ones, and a length-based cap measured against a
|
|
1840
|
+
// byte-based reserve overruns the share and aborts the whole batch.
|
|
1841
|
+
const MAX_NOTE_TOKENS = 8;
|
|
1842
|
+
const MAX_NOTE_TOKEN_CHARS = 40;
|
|
1843
|
+
const MAX_NOTE_BYTES = 400;
|
|
1844
|
+
|
|
1845
|
+
/** A lone high surrogate left at a cut is not a character; drop it. */
|
|
1846
|
+
function dropTrailingHighSurrogate(text) {
|
|
1847
|
+
return text.replace(/[\uD800-\uDBFF]$/u, '');
|
|
1848
|
+
}
|
|
1849
|
+
|
|
1850
|
+
function clampNoteBytes(text) {
|
|
1851
|
+
if (Buffer.byteLength(text, 'utf8') <= MAX_NOTE_BYTES) {
|
|
1852
|
+
return text;
|
|
1853
|
+
}
|
|
1854
|
+
return `${fittingPrefix(text, (value) => `${value}\u2026`, MAX_NOTE_BYTES)}\u2026`;
|
|
1855
|
+
}
|
|
1856
|
+
|
|
1857
|
+
function renderSearchTokens(tokens) {
|
|
1858
|
+
const shown = tokens
|
|
1859
|
+
.slice(0, MAX_NOTE_TOKENS)
|
|
1860
|
+
.map(
|
|
1861
|
+
(token) =>
|
|
1862
|
+
`"${dropTrailingHighSurrogate(String(token).slice(0, MAX_NOTE_TOKEN_CHARS))}"`
|
|
1863
|
+
);
|
|
1864
|
+
const extra = tokens.length - shown.length;
|
|
1865
|
+
return extra > 0 ? `${shown.join(', ')} and ${extra} more` : shown.join(', ');
|
|
1866
|
+
}
|
|
1867
|
+
|
|
1868
|
+
function droppedTermsNote(termCount, dropped, useful) {
|
|
1869
|
+
return clampNoteBytes(
|
|
1870
|
+
`no object matched all ${termCount} terms, so ` +
|
|
1871
|
+
`${renderSearchTokens(dropped)} ` +
|
|
1872
|
+
`${dropped.length === 1 ? 'was' : 'were'} dropped: nothing in this ` +
|
|
1873
|
+
`model contains ${dropped.length === 1 ? 'it' : 'them'}. These rows ` +
|
|
1874
|
+
`matched ${renderSearchTokens(useful)}.`
|
|
1875
|
+
);
|
|
1876
|
+
}
|
|
1877
|
+
|
|
1878
|
+
const NO_MATCH_NOTE =
|
|
1879
|
+
'no object matched every term. Search is an AND across names, ' +
|
|
1880
|
+
'descriptions, expressions, folders, and synonyms, so retry with fewer and ' +
|
|
1881
|
+
'more specific terms rather than a whole phrase.';
|
|
1882
|
+
|
|
1883
|
+
/**
|
|
1884
|
+
* The longest note set a search runner can produce, for budget reservation.
|
|
1885
|
+
*
|
|
1886
|
+
* The relaxation note is clamped to MAX_NOTE_BYTES whatever script it quotes,
|
|
1887
|
+
* so one ASCII byte per character is an exact upper bound rather than an
|
|
1888
|
+
* estimate that a non-Latin query can exceed.
|
|
1889
|
+
*/
|
|
1890
|
+
function worstCaseSearchNotes() {
|
|
1891
|
+
return [
|
|
1892
|
+
'x'.repeat(MAX_NOTE_BYTES),
|
|
1893
|
+
...searchTrimNotes(Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER),
|
|
1894
|
+
];
|
|
1895
|
+
}
|
|
1896
|
+
|
|
1897
|
+
/**
|
|
1898
|
+
* Rank, and if nothing matched, say so usefully.
|
|
1899
|
+
*
|
|
1900
|
+
* Ranking is a strict AND: every term must appear somewhere in a row. That is
|
|
1901
|
+
* the right default - it is what makes a two-word query precise - but it turns
|
|
1902
|
+
* natural phrasing into a silent dead end, because one term the model has never
|
|
1903
|
+
* heard of empties the whole result. "sales amount by product" returns nothing
|
|
1904
|
+
* on a model that answers "sales amount" with two rows.
|
|
1905
|
+
*
|
|
1906
|
+
* So when, and only when, the strict pass found nothing, drop the terms that
|
|
1907
|
+
* match no row anywhere in this catalog and rank once more. Dropping a term
|
|
1908
|
+
* because THIS model does not contain it keeps the rule independent of language
|
|
1909
|
+
* and of any stop-word list, and the retry happens inside the same invocation,
|
|
1910
|
+
* which is the whole point of this file. A search that already found something
|
|
1911
|
+
* is never touched.
|
|
1912
|
+
*/
|
|
1913
|
+
export function rankSearchWithFallback(rows, query) {
|
|
1914
|
+
const ranked = rankSemanticSearchRows(rows, query);
|
|
1915
|
+
if (ranked.length > 0) {
|
|
1916
|
+
return { rows: ranked, notes: [] };
|
|
1917
|
+
}
|
|
1918
|
+
|
|
1919
|
+
const tokens = searchTokens(query);
|
|
1920
|
+
const useful = matchedSearchTokens(rows, tokens);
|
|
1921
|
+
const dropped = tokens.filter((token) => !useful.includes(token));
|
|
1922
|
+
|
|
1923
|
+
if (useful.length > 0 && dropped.length > 0) {
|
|
1924
|
+
const relaxed = rankSemanticSearchRows(rows, useful);
|
|
1925
|
+
if (relaxed.length > 0) {
|
|
1926
|
+
return {
|
|
1927
|
+
rows: relaxed,
|
|
1928
|
+
notes: [droppedTermsNote(tokens.length, dropped, useful)],
|
|
1929
|
+
};
|
|
1930
|
+
}
|
|
1931
|
+
}
|
|
1932
|
+
|
|
1933
|
+
return { rows: [], notes: [NO_MATCH_NOTE] };
|
|
1934
|
+
}
|
|
1935
|
+
|
|
1936
|
+
export function searchCatalog(dbPath, query, options = {}) {
|
|
1937
|
+
// Opened inside the try for the same reason as runSql.
|
|
1938
|
+
let db;
|
|
1939
|
+
try {
|
|
1940
|
+
db = openReadOnlyDatabase(dbPath);
|
|
1941
|
+
const rows = db
|
|
1942
|
+
.prepare(
|
|
1943
|
+
'SELECT kind, parent_name, object_name, description, ' +
|
|
1944
|
+
'expression, display_folder, data_type, synonyms ' +
|
|
1945
|
+
'FROM "semantic_search"'
|
|
1946
|
+
)
|
|
1947
|
+
.all();
|
|
1948
|
+
const ranked = rankSearchWithFallback(rows, query);
|
|
1949
|
+
const trimmed = trimSearchRows(ranked.rows, rows);
|
|
1950
|
+
return boundRows(trimmed.rows, {
|
|
1951
|
+
...options,
|
|
1952
|
+
notes: [...ranked.notes, ...trimmed.notes],
|
|
1953
|
+
});
|
|
1954
|
+
} catch (error) {
|
|
1955
|
+
return {
|
|
1956
|
+
ok: false,
|
|
1957
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1958
|
+
};
|
|
1959
|
+
} finally {
|
|
1960
|
+
db?.close();
|
|
1961
|
+
}
|
|
1962
|
+
}
|
|
1963
|
+
|
|
1964
|
+
export function resultExitCode(result, { allowTruncated = false } = {}) {
|
|
1965
|
+
if (!result?.ok) {
|
|
1966
|
+
return 1;
|
|
1967
|
+
}
|
|
1968
|
+
return result.truncated && !allowTruncated ? 2 : 0;
|
|
1969
|
+
}
|
|
1970
|
+
|
|
1971
|
+
// --- batching ---------------------------------------------------------------
|
|
1972
|
+
//
|
|
1973
|
+
// One invocation costs one model round trip, and a round trip re-sends the
|
|
1974
|
+
// entire conversation. Discovery is rarely a single question, so the batch is
|
|
1975
|
+
// the cheap shape and the single statement is just a batch of one.
|
|
1976
|
+
|
|
1977
|
+
export const MAX_BATCH_STATEMENTS = 8;
|
|
1978
|
+
const MAX_BATCH_LABEL_BYTES = 512;
|
|
1979
|
+
|
|
1980
|
+
// --file is an input channel, not a new capability: its contents run through the
|
|
1981
|
+
// same runSql -> openReadOnlyDatabase path as positional SQL, so query_only
|
|
1982
|
+
// still applies. What it does add is an unbounded read of an arbitrary path, so
|
|
1983
|
+
// the size is capped before the file is loaded. A statement larger than this is
|
|
1984
|
+
// a mistake, not a query.
|
|
1985
|
+
const MAX_SQL_FILE_BYTES = 1_000_000;
|
|
1986
|
+
|
|
1987
|
+
function batchHeader(count, ok) {
|
|
1988
|
+
return `# ${JSON.stringify({ ok, batch: true, statements: count })}\n`;
|
|
1989
|
+
}
|
|
1990
|
+
|
|
1991
|
+
// Longest prefix of `text` whose serialized form still fits `maxBytes`. The
|
|
1992
|
+
// search is over characters but the budget is over UTF-8 bytes, so the two
|
|
1993
|
+
// cannot be assumed proportional.
|
|
1994
|
+
function fittingPrefix(text, serialize, maxBytes) {
|
|
1995
|
+
if (Buffer.byteLength(serialize(''), 'utf8') > maxBytes) {
|
|
1996
|
+
throw new Error('The byte budget cannot fit the required metadata.');
|
|
1997
|
+
}
|
|
1998
|
+
let low = 0;
|
|
1999
|
+
let high = Math.min(text.length, maxBytes);
|
|
2000
|
+
const prefix = (length) => dropTrailingHighSurrogate(text.slice(0, length));
|
|
2001
|
+
while (low < high) {
|
|
2002
|
+
const middle = Math.ceil((low + high) / 2);
|
|
2003
|
+
if (Buffer.byteLength(serialize(prefix(middle)), 'utf8') <= maxBytes) {
|
|
2004
|
+
low = middle;
|
|
2005
|
+
} else {
|
|
2006
|
+
high = middle - 1;
|
|
2007
|
+
}
|
|
2008
|
+
}
|
|
2009
|
+
return prefix(low);
|
|
2010
|
+
}
|
|
2011
|
+
|
|
2012
|
+
function batchLabel(entry, index) {
|
|
2013
|
+
// A --file statement is named by its path, not by its contents: the path is
|
|
2014
|
+
// what the caller actually typed, it is shorter, and it keeps file bytes out
|
|
2015
|
+
// of a payload that the agent did not ask to read.
|
|
2016
|
+
const field = entry.source ? 'source' : 'query';
|
|
2017
|
+
const text = entry.source ?? entry.query;
|
|
2018
|
+
const label = { index, kind: entry.kind, [field]: text };
|
|
2019
|
+
const serialize = (value) => `## ${JSON.stringify(value)}\n`;
|
|
2020
|
+
const full = serialize(label);
|
|
2021
|
+
if (Buffer.byteLength(full, 'utf8') <= MAX_BATCH_LABEL_BYTES) {
|
|
2022
|
+
return full;
|
|
2023
|
+
}
|
|
2024
|
+
const textBytes = Buffer.byteLength(text, 'utf8');
|
|
2025
|
+
const textHash = createHash('sha256').update(text).digest('hex').slice(0, 16);
|
|
2026
|
+
const summarized = (value) =>
|
|
2027
|
+
serialize({
|
|
2028
|
+
...label,
|
|
2029
|
+
[field]: value,
|
|
2030
|
+
[`${field}Truncated`]: true,
|
|
2031
|
+
[`${field}Bytes`]: textBytes,
|
|
2032
|
+
[`${field}Hash`]: textHash,
|
|
2033
|
+
});
|
|
2034
|
+
return summarized(fittingPrefix(text, summarized, MAX_BATCH_LABEL_BYTES));
|
|
2035
|
+
}
|
|
2036
|
+
|
|
2037
|
+
/**
|
|
2038
|
+
* Format several results as one payload.
|
|
2039
|
+
*
|
|
2040
|
+
* Each block is exactly what {@link formatResult} would have printed on its
|
|
2041
|
+
* own, introduced by a `##` line naming the statement. Keeping the
|
|
2042
|
+
* per-statement body byte-identical to the single-statement form means nothing
|
|
2043
|
+
* has to learn a second encoding, and TSV rows stay TSV rather than being
|
|
2044
|
+
* re-wrapped as JSON.
|
|
2045
|
+
*/
|
|
2046
|
+
export function formatBatch(entries) {
|
|
2047
|
+
const header = batchHeader(
|
|
2048
|
+
entries.length,
|
|
2049
|
+
entries.every((entry) => entry.result?.ok)
|
|
2050
|
+
);
|
|
2051
|
+
const blocks = entries.map(
|
|
2052
|
+
(entry, index) => `${batchLabel(entry, index)}${formatResult(entry.result)}`
|
|
2053
|
+
);
|
|
2054
|
+
return `${header}${blocks.join('')}`;
|
|
2055
|
+
}
|
|
2056
|
+
|
|
2057
|
+
/**
|
|
2058
|
+
* Worst outcome across the batch, using the same codes as a single statement:
|
|
2059
|
+
* 1 for a failure, 2 for an untolerated truncation, 0 otherwise.
|
|
2060
|
+
*
|
|
2061
|
+
* A failure outranks a truncation because a truncated result is still evidence,
|
|
2062
|
+
* whereas a failed one is not.
|
|
2063
|
+
*/
|
|
2064
|
+
export function batchExitCode(entries, { allowTruncated = false } = {}) {
|
|
2065
|
+
let code = 0;
|
|
2066
|
+
for (const entry of entries) {
|
|
2067
|
+
const entryCode = resultExitCode(entry.result, { allowTruncated });
|
|
2068
|
+
if (entryCode === 1) {
|
|
2069
|
+
return 1;
|
|
2070
|
+
}
|
|
2071
|
+
if (entryCode === 2) {
|
|
2072
|
+
code = 2;
|
|
2073
|
+
}
|
|
2074
|
+
}
|
|
2075
|
+
return code;
|
|
2076
|
+
}
|
|
2077
|
+
|
|
2078
|
+
/** The `## {...}` label line each batch block carries, in bytes. */
|
|
2079
|
+
function batchLabelBytes(batch) {
|
|
2080
|
+
return batch.reduce(
|
|
2081
|
+
(total, entry, index) =>
|
|
2082
|
+
total + Buffer.byteLength(batchLabel(entry, index), 'utf8'),
|
|
2083
|
+
// A failing header is one byte longer than a successful one.
|
|
2084
|
+
Buffer.byteLength(batchHeader(batch.length, false), 'utf8')
|
|
2085
|
+
);
|
|
2086
|
+
}
|
|
2087
|
+
|
|
2088
|
+
function batchResultReserve(kind) {
|
|
2089
|
+
// Reserve the largest empty-result metadata our runners can produce,
|
|
2090
|
+
// including search notes and row counts, before giving any statement room for
|
|
2091
|
+
// row data. An empty row set formats through the JSON branch, which is the
|
|
2092
|
+
// floor boundResultRows can always reach.
|
|
2093
|
+
return Buffer.byteLength(
|
|
2094
|
+
formatResult({
|
|
2095
|
+
ok: true,
|
|
2096
|
+
totalRows: Number.MAX_SAFE_INTEGER,
|
|
2097
|
+
returnedRows: 0,
|
|
2098
|
+
truncated: true,
|
|
2099
|
+
truncation: {
|
|
2100
|
+
reason: 'row_limit_and_byte_budget',
|
|
2101
|
+
requiredAction: TRUNCATION_REQUIRED_ACTION,
|
|
2102
|
+
},
|
|
2103
|
+
...(kind === 'search' ? { notes: worstCaseSearchNotes() } : {}),
|
|
2104
|
+
rows: [],
|
|
2105
|
+
}),
|
|
2106
|
+
'utf8'
|
|
2107
|
+
);
|
|
2108
|
+
}
|
|
2109
|
+
|
|
2110
|
+
function boundBatchResult(result, maxBytes) {
|
|
2111
|
+
if (Buffer.byteLength(formatResult(result), 'utf8') <= maxBytes) {
|
|
2112
|
+
return result;
|
|
2113
|
+
}
|
|
2114
|
+
if (result.ok) {
|
|
2115
|
+
return boundResultRows(result, maxBytes);
|
|
2116
|
+
}
|
|
2117
|
+
// A SQLite diagnostic quotes the offending SQL back, so a large statement
|
|
2118
|
+
// produces a large error. Keep it indexed and inside the budget.
|
|
2119
|
+
const text = String(result.error);
|
|
2120
|
+
const errorBytes = Buffer.byteLength(text, 'utf8');
|
|
2121
|
+
const summarized = (error) => ({
|
|
2122
|
+
...result,
|
|
2123
|
+
error,
|
|
2124
|
+
errorTruncated: true,
|
|
2125
|
+
errorBytes,
|
|
2126
|
+
});
|
|
2127
|
+
return summarized(
|
|
2128
|
+
fittingPrefix(text, (error) => formatResult(summarized(error)), maxBytes)
|
|
2129
|
+
);
|
|
2130
|
+
}
|
|
2131
|
+
|
|
2132
|
+
/**
|
|
2133
|
+
* Run a planned batch inside ONE byte budget.
|
|
2134
|
+
*
|
|
2135
|
+
* The per-statement cap exists because an answer that buries the session's
|
|
2136
|
+
* context is worse than no answer. Batching must not multiply that cap by the
|
|
2137
|
+
* number of statements, so the whole payload shares the single-statement
|
|
2138
|
+
* budget: each statement gets an equal share of what is left, and whatever it
|
|
2139
|
+
* does not use rolls forward to the next. A statement can therefore never
|
|
2140
|
+
* starve a later one, and a cheap statement makes an expensive one roomier.
|
|
2141
|
+
*/
|
|
2142
|
+
export function runBatch(
|
|
2143
|
+
batch,
|
|
2144
|
+
{ runners, limit = DEFAULT_PRINT_LIMIT, maxBytes = MAX_RESULT_BYTES } = {}
|
|
2145
|
+
) {
|
|
2146
|
+
const labels = batchLabelBytes(batch);
|
|
2147
|
+
const reserves = batch.map((entry) => batchResultReserve(entry.kind));
|
|
2148
|
+
let reserved = reserves.reduce((total, bytes) => total + bytes, 0);
|
|
2149
|
+
const minimumBytes = labels + reserved;
|
|
2150
|
+
if (!Number.isSafeInteger(maxBytes) || maxBytes < minimumBytes) {
|
|
2151
|
+
throw new Error(
|
|
2152
|
+
`Batch byte budget must be an integer of at least ${minimumBytes} bytes.`
|
|
2153
|
+
);
|
|
2154
|
+
}
|
|
2155
|
+
let remaining = maxBytes - labels;
|
|
2156
|
+
const entries = [];
|
|
2157
|
+
batch.forEach((entry, index) => {
|
|
2158
|
+
const share =
|
|
2159
|
+
reserves[index] +
|
|
2160
|
+
Math.floor((remaining - reserved) / (batch.length - index));
|
|
2161
|
+
const result = boundBatchResult(
|
|
2162
|
+
runners[entry.kind](entry.query, { limit, maxBytes: share }),
|
|
2163
|
+
share
|
|
2164
|
+
);
|
|
2165
|
+
const bytes = Buffer.byteLength(formatResult(result), 'utf8');
|
|
2166
|
+
if (bytes > share) {
|
|
2167
|
+
throw new Error(
|
|
2168
|
+
`Statement ${index} metadata exceeds its batch byte budget. ` +
|
|
2169
|
+
TRUNCATION_REQUIRED_ACTION
|
|
2170
|
+
);
|
|
2171
|
+
}
|
|
2172
|
+
remaining -= bytes;
|
|
2173
|
+
reserved -= reserves[index];
|
|
2174
|
+
entries.push({ ...entry, result });
|
|
2175
|
+
});
|
|
2176
|
+
return entries;
|
|
2177
|
+
}
|
|
2178
|
+
|
|
2179
|
+
function isInsideRoot(base, candidate) {
|
|
2180
|
+
return candidate === base || candidate.startsWith(base + sep);
|
|
2181
|
+
}
|
|
2182
|
+
|
|
2183
|
+
function readSqlFile(path, root) {
|
|
2184
|
+
// Match the Rayfin CLI's own --file convention (connector-invoke.ts
|
|
2185
|
+
// readJsonInputFile): a statement file must live inside the app. Without
|
|
2186
|
+
// this, a mistyped path pulls unrelated file bytes into the agent's context
|
|
2187
|
+
// and SQLite quotes a fragment of them back in its syntax error.
|
|
2188
|
+
const base = resolve(root);
|
|
2189
|
+
const absolute = resolve(base, path);
|
|
2190
|
+
if (!isInsideRoot(base, absolute)) {
|
|
2191
|
+
return { error: `--file must be inside the app: ${path}` };
|
|
2192
|
+
}
|
|
2193
|
+
try {
|
|
2194
|
+
// Then check again after resolving links. A junction or symlink inside the
|
|
2195
|
+
// app that points outside it satisfies the purely lexical test above, so a
|
|
2196
|
+
// lexical check alone is containment that can be walked around. The CLI's
|
|
2197
|
+
// own check has this gap; we do not inherit it, because here the thing
|
|
2198
|
+
// being guarded is a catalog holding real column values.
|
|
2199
|
+
const realBase = realpathSync(base);
|
|
2200
|
+
const realPath = realpathSync(absolute);
|
|
2201
|
+
if (!isInsideRoot(realBase, realPath)) {
|
|
2202
|
+
return { error: `--file must be inside the app: ${path}` };
|
|
2203
|
+
}
|
|
2204
|
+
const stats = statSync(realPath);
|
|
2205
|
+
if (!stats.isFile()) {
|
|
2206
|
+
throw new Error('not a regular file');
|
|
2207
|
+
}
|
|
2208
|
+
if (stats.size > MAX_SQL_FILE_BYTES) {
|
|
2209
|
+
throw new Error(
|
|
2210
|
+
`${stats.size} bytes exceeds the ${MAX_SQL_FILE_BYTES}-byte limit ` +
|
|
2211
|
+
'for a single statement'
|
|
2212
|
+
);
|
|
2213
|
+
}
|
|
2214
|
+
return {
|
|
2215
|
+
query: readFileSync(realPath, 'utf8'),
|
|
2216
|
+
source: relative(realBase, realPath) || path,
|
|
2217
|
+
};
|
|
2218
|
+
} catch (error) {
|
|
2219
|
+
return {
|
|
2220
|
+
error:
|
|
2221
|
+
`Could not read --file ${path}: ` +
|
|
2222
|
+
`${error instanceof Error ? error.message : String(error)}`,
|
|
2223
|
+
};
|
|
2224
|
+
}
|
|
2225
|
+
}
|
|
2226
|
+
|
|
2227
|
+
/**
|
|
2228
|
+
* Turn parsed CLI values into the ordered list of statements to run.
|
|
2229
|
+
*
|
|
2230
|
+
* Searches come before SQL because that is the order discovery actually happens
|
|
2231
|
+
* in: find the objects, then read their detail. `parseArgs` does not preserve
|
|
2232
|
+
* interleaved order across different option names, so each entry carries its
|
|
2233
|
+
* own query text and the caller never has to infer which result is which.
|
|
2234
|
+
*
|
|
2235
|
+
* Returns `{ error }`, or `{ batch, mode }` where mode is:
|
|
2236
|
+
* "single" - one positional statement; print the original single-result payload
|
|
2237
|
+
* "batch" - one or more --search/--sql/--file statements; print the envelope
|
|
2238
|
+
* "none" - nothing to run
|
|
2239
|
+
*/
|
|
2240
|
+
export function planStatements({
|
|
2241
|
+
positionalSql = '',
|
|
2242
|
+
search = [],
|
|
2243
|
+
sql = [],
|
|
2244
|
+
file = [],
|
|
2245
|
+
root = process.cwd(),
|
|
2246
|
+
} = {}) {
|
|
2247
|
+
const fromFiles = [];
|
|
2248
|
+
for (const path of file) {
|
|
2249
|
+
const read = readSqlFile(path, root);
|
|
2250
|
+
if (read.error) {
|
|
2251
|
+
return { error: read.error };
|
|
2252
|
+
}
|
|
2253
|
+
fromFiles.push({ kind: 'sql', query: read.query, source: read.source });
|
|
2254
|
+
}
|
|
2255
|
+
const batch = [
|
|
2256
|
+
...search.map((query) => ({ kind: 'search', query })),
|
|
2257
|
+
...sql.map((query) => ({ kind: 'sql', query })),
|
|
2258
|
+
...fromFiles,
|
|
2259
|
+
];
|
|
2260
|
+
|
|
2261
|
+
if (positionalSql && batch.length > 0) {
|
|
2262
|
+
return {
|
|
2263
|
+
error: 'Use either positional SQL or --search/--sql/--file, not both.',
|
|
2264
|
+
};
|
|
2265
|
+
}
|
|
2266
|
+
if (batch.length > MAX_BATCH_STATEMENTS) {
|
|
2267
|
+
return {
|
|
2268
|
+
error:
|
|
2269
|
+
`Too many statements (${batch.length}); the limit is ` +
|
|
2270
|
+
`${MAX_BATCH_STATEMENTS}. Split them across two invocations.`,
|
|
2271
|
+
};
|
|
2272
|
+
}
|
|
2273
|
+
const empty = batch.find((entry) => !String(entry.query).trim());
|
|
2274
|
+
if (empty) {
|
|
2275
|
+
return {
|
|
2276
|
+
error:
|
|
2277
|
+
`An empty ${empty.kind} statement was supplied` +
|
|
2278
|
+
`${empty.source ? ` from ${empty.source}` : ''}.`,
|
|
2279
|
+
};
|
|
2280
|
+
}
|
|
2281
|
+
if (positionalSql) {
|
|
2282
|
+
return {
|
|
2283
|
+
batch: [{ kind: 'sql', query: positionalSql }],
|
|
2284
|
+
mode: 'single',
|
|
2285
|
+
};
|
|
2286
|
+
}
|
|
2287
|
+
return { batch, mode: batch.length > 0 ? 'batch' : 'none' };
|
|
2288
|
+
}
|
|
2289
|
+
|
|
2290
|
+
export function modelCacheDir(root, datasetId) {
|
|
2291
|
+
if (!isFabricGuid(datasetId)) {
|
|
2292
|
+
throw new Error(
|
|
2293
|
+
`Invalid semantic-model itemId "${datasetId}". ` +
|
|
2294
|
+
'Expected a Fabric GUID from `rayfin connector list --json`.'
|
|
2295
|
+
);
|
|
2296
|
+
}
|
|
2297
|
+
const modelsRoot = resolve(root, 'models');
|
|
2298
|
+
const absolute = resolve(modelsRoot, datasetId);
|
|
2299
|
+
const rel = relative(modelsRoot, absolute);
|
|
2300
|
+
if (rel.startsWith('..') || resolve(modelsRoot, rel) !== absolute) {
|
|
2301
|
+
throw new Error('Resolved catalog path escaped the schema cache root.');
|
|
2302
|
+
}
|
|
2303
|
+
return join(root, 'models', datasetId);
|
|
2304
|
+
}
|
|
2305
|
+
|
|
2306
|
+
export function ttlMinutes(explicit) {
|
|
2307
|
+
const raw = explicit ?? process.env.RAYFIN_SCHEMA_TTL_MINUTES;
|
|
2308
|
+
const value = Number(raw);
|
|
2309
|
+
return Number.isFinite(value) && value >= 0 ? value : DEFAULT_TTL_MINUTES;
|
|
2310
|
+
}
|
|
2311
|
+
|
|
2312
|
+
export function modelMtime(path) {
|
|
2313
|
+
try {
|
|
2314
|
+
return statSync(path).mtimeMs;
|
|
2315
|
+
} catch {
|
|
2316
|
+
return null;
|
|
2317
|
+
}
|
|
2318
|
+
}
|
|
2319
|
+
|
|
2320
|
+
/**
|
|
2321
|
+
* A cooldown after a failed refresh, kept separate from the catalog's mtime.
|
|
2322
|
+
*
|
|
2323
|
+
* The catalog keeps its real age - faking freshness would make it lie about
|
|
2324
|
+
* itself - so without this marker every subsequent command sees "stale" and
|
|
2325
|
+
* re-fetches all five sources. The usual cause of a failed fetch is throttling,
|
|
2326
|
+
* and retrying immediately extends the block it is reacting to.
|
|
2327
|
+
*/
|
|
2328
|
+
export function refreshCooldown(dir, { now = Date.now } = {}) {
|
|
2329
|
+
try {
|
|
2330
|
+
const raw = JSON.parse(
|
|
2331
|
+
readFileSync(join(dir, REFRESH_COOLDOWN_FILE), 'utf8')
|
|
2332
|
+
);
|
|
2333
|
+
const until = Date.parse(raw?.retryAfter);
|
|
2334
|
+
if (!Number.isFinite(until) || until <= now()) {
|
|
2335
|
+
return null;
|
|
2336
|
+
}
|
|
2337
|
+
return { retryAfter: raw.retryAfter, reason: raw.reason ?? null };
|
|
2338
|
+
} catch {
|
|
2339
|
+
return null;
|
|
2340
|
+
}
|
|
2341
|
+
}
|
|
2342
|
+
|
|
2343
|
+
export function recordRefreshCooldown(
|
|
2344
|
+
dir,
|
|
2345
|
+
reason,
|
|
2346
|
+
{ now = Date.now, minutes = REFRESH_COOLDOWN_MINUTES } = {}
|
|
2347
|
+
) {
|
|
2348
|
+
try {
|
|
2349
|
+
writeFileSync(
|
|
2350
|
+
join(dir, REFRESH_COOLDOWN_FILE),
|
|
2351
|
+
JSON.stringify({
|
|
2352
|
+
retryAfter: new Date(now() + minutes * 60_000).toISOString(),
|
|
2353
|
+
reason: reason ?? null,
|
|
2354
|
+
}),
|
|
2355
|
+
'utf8'
|
|
2356
|
+
);
|
|
2357
|
+
return { ok: true };
|
|
2358
|
+
} catch (error) {
|
|
2359
|
+
// Not worth failing a usable query over, but not worth hiding either: the
|
|
2360
|
+
// visible symptom would be the repeated-refresh behaviour this prevents,
|
|
2361
|
+
// with nothing to explain it.
|
|
2362
|
+
return {
|
|
2363
|
+
ok: false,
|
|
2364
|
+
error: error instanceof Error ? error.message : String(error),
|
|
2365
|
+
};
|
|
2366
|
+
}
|
|
2367
|
+
}
|
|
2368
|
+
|
|
2369
|
+
export function clearRefreshCooldown(dir) {
|
|
2370
|
+
rmSync(join(dir, REFRESH_COOLDOWN_FILE), { force: true });
|
|
2371
|
+
}
|
|
2372
|
+
|
|
2373
|
+
export function cacheStatus(
|
|
2374
|
+
mtimeMs,
|
|
2375
|
+
{ ttl = DEFAULT_TTL_MINUTES, now = Date.now() } = {}
|
|
2376
|
+
) {
|
|
2377
|
+
if (mtimeMs == null) {
|
|
2378
|
+
return {
|
|
2379
|
+
fresh: false,
|
|
2380
|
+
reason: 'no cached catalog on disk',
|
|
2381
|
+
};
|
|
2382
|
+
}
|
|
2383
|
+
const ageMinutes = (now - mtimeMs) / 60_000;
|
|
2384
|
+
if (!Number.isFinite(ageMinutes)) {
|
|
2385
|
+
return {
|
|
2386
|
+
fresh: false,
|
|
2387
|
+
reason: 'cached catalog has an unreadable timestamp',
|
|
2388
|
+
};
|
|
2389
|
+
}
|
|
2390
|
+
return {
|
|
2391
|
+
fresh: ageMinutes <= ttl,
|
|
2392
|
+
reason:
|
|
2393
|
+
`cached catalog is ${ageMinutes.toFixed(1)} min old ` +
|
|
2394
|
+
`(TTL ${ttl} min)`,
|
|
2395
|
+
ageMinutes,
|
|
2396
|
+
};
|
|
2397
|
+
}
|
|
2398
|
+
|
|
2399
|
+
export function parseCliArgs(argv) {
|
|
2400
|
+
return parseArgs({
|
|
2401
|
+
args: argv,
|
|
2402
|
+
allowPositionals: true,
|
|
2403
|
+
options: {
|
|
2404
|
+
alias: { type: 'string', short: 'a' },
|
|
2405
|
+
limit: { type: 'string', short: 'l' },
|
|
2406
|
+
search: { type: 'string', multiple: true },
|
|
2407
|
+
sql: { type: 'string', multiple: true },
|
|
2408
|
+
file: { type: 'string', multiple: true },
|
|
2409
|
+
'allow-truncated': {
|
|
2410
|
+
type: 'boolean',
|
|
2411
|
+
default: false,
|
|
2412
|
+
},
|
|
2413
|
+
refresh: { type: 'boolean', default: false },
|
|
2414
|
+
help: {
|
|
2415
|
+
type: 'boolean',
|
|
2416
|
+
short: 'h',
|
|
2417
|
+
default: false,
|
|
2418
|
+
},
|
|
2419
|
+
},
|
|
2420
|
+
});
|
|
2421
|
+
}
|
|
2422
|
+
|
|
2423
|
+
const USAGE = `Usage:
|
|
2424
|
+
npm run --silent schema:sql -- "SELECT ..." Run SQL against the model catalog
|
|
2425
|
+
npm run --silent schema:sql -- --search "terms" Search all model metadata by relevance
|
|
2426
|
+
npm run --silent schema:refresh Rebuild the catalog from the model
|
|
2427
|
+
|
|
2428
|
+
Batch - one invocation, several answers. Prefer this: each extra invocation costs a
|
|
2429
|
+
whole model round trip, so ask everything you already know you need at once.
|
|
2430
|
+
npm run --silent schema:sql -- --search "revenue" --sql "SELECT [Name] FROM info_view_tables" --sql "SELECT [Table], [Name] FROM info_view_measures"
|
|
2431
|
+
|
|
2432
|
+
Options:
|
|
2433
|
+
-a, --alias <name> Rayfin semantic-model connector name
|
|
2434
|
+
-l, --limit <n> Max rows to print per statement (default ${DEFAULT_PRINT_LIMIT})
|
|
2435
|
+
--search <terms> Ranked catalog-wide semantic search (repeatable)
|
|
2436
|
+
--sql <text> A SQL statement (repeatable)
|
|
2437
|
+
--file <path> Read one SQL statement from a file (repeatable)
|
|
2438
|
+
--allow-truncated Exit successfully with an incomplete preview
|
|
2439
|
+
--refresh Rebuild even when the cache is fresh
|
|
2440
|
+
-h, --help Show this help
|
|
2441
|
+
|
|
2442
|
+
At most ${MAX_BATCH_STATEMENTS} statements per invocation. The whole batch shares one byte budget,
|
|
2443
|
+
so batching never costs more context than a single statement. Exit code is the worst
|
|
2444
|
+
outcome: 1 if any statement failed, 2 if any was truncated OR an explicitly requested
|
|
2445
|
+
--refresh could not fetch fresh data, 0 otherwise.
|
|
2446
|
+
|
|
2447
|
+
Environment:
|
|
2448
|
+
RAYFIN_SCHEMA_TTL_MINUTES Catalog freshness window (default ${DEFAULT_TTL_MINUTES})
|
|
2449
|
+
`;
|
|
2450
|
+
|
|
2451
|
+
async function main(argv) {
|
|
2452
|
+
const { values, positionals } = parseCliArgs(argv);
|
|
2453
|
+
if (values.help) {
|
|
2454
|
+
process.stdout.write(USAGE);
|
|
2455
|
+
return 0;
|
|
2456
|
+
}
|
|
2457
|
+
|
|
2458
|
+
// Plan before touching the model: an argument mistake should not cost a
|
|
2459
|
+
// catalog build first.
|
|
2460
|
+
const plan = planStatements({
|
|
2461
|
+
positionalSql: positionals.join(' ').trim(),
|
|
2462
|
+
search: values.search ?? [],
|
|
2463
|
+
sql: values.sql ?? [],
|
|
2464
|
+
file: values.file ?? [],
|
|
2465
|
+
});
|
|
2466
|
+
if (plan.error) {
|
|
2467
|
+
process.stderr.write(`${plan.error}\n`);
|
|
2468
|
+
return 1;
|
|
2469
|
+
}
|
|
2470
|
+
if (plan.mode === 'none' && !values.refresh) {
|
|
2471
|
+
process.stdout.write(USAGE);
|
|
2472
|
+
return 0;
|
|
2473
|
+
}
|
|
2474
|
+
|
|
2475
|
+
const resolved = await resolveAlias(process.cwd(), values.alias);
|
|
2476
|
+
if (resolved.error) {
|
|
2477
|
+
process.stderr.write(`${resolved.error}\n`);
|
|
2478
|
+
return 1;
|
|
2479
|
+
}
|
|
2480
|
+
const alias = resolved.alias;
|
|
2481
|
+
const identity = await resolveIdentity({
|
|
2482
|
+
alias,
|
|
2483
|
+
connector: resolved.connector,
|
|
2484
|
+
cwd: process.cwd(),
|
|
2485
|
+
});
|
|
2486
|
+
if (identity?.error) {
|
|
2487
|
+
process.stderr.write(`${identity.error}\n`);
|
|
2488
|
+
return 1;
|
|
2489
|
+
}
|
|
2490
|
+
if (!identity?.datasetId) {
|
|
2491
|
+
process.stderr.write(
|
|
2492
|
+
`Could not resolve which dataset "${alias}" points at. ` +
|
|
2493
|
+
'Run `npx rayfin connector list --json` from the app root.\n'
|
|
2494
|
+
);
|
|
2495
|
+
return 1;
|
|
2496
|
+
}
|
|
2497
|
+
|
|
2498
|
+
const dir = modelCacheDir(CACHE_ROOT, identity.datasetId);
|
|
2499
|
+
const dbPath = join(dir, DATABASE_FILE);
|
|
2500
|
+
const status = cacheStatus(modelMtime(dbPath), {
|
|
2501
|
+
ttl: ttlMinutes(),
|
|
2502
|
+
});
|
|
2503
|
+
|
|
2504
|
+
// Reusing a catalog is not by itself a failure. Three outcomes look alike
|
|
2505
|
+
// from the outside and must not be collapsed:
|
|
2506
|
+
// - an explicit --refresh whose fetch failed: the requested action did not
|
|
2507
|
+
// happen, so the status has to say so;
|
|
2508
|
+
// - an automatic TTL rebuild that fell back: nobody asked for freshness,
|
|
2509
|
+
// and a usable catalog is the right outcome;
|
|
2510
|
+
// - a waiter reusing another process's newly completed refresh: the
|
|
2511
|
+
// refresh did happen, just not here, so this is success.
|
|
2512
|
+
// Only the first sets refreshIncomplete, so that is what we branch on.
|
|
2513
|
+
let refreshUnsatisfied = false;
|
|
2514
|
+
// The cooldown exists to stop re-fetching when there is already something
|
|
2515
|
+
// good to serve. With no usable catalog there is nothing to serve, so
|
|
2516
|
+
// skipping the rebuild would lock the user out of a feature that has never
|
|
2517
|
+
// worked yet - strictly worse than the repeated fetching it prevents. An
|
|
2518
|
+
// explicit --refresh always retries regardless.
|
|
2519
|
+
const retained = readCatalogStatus(dbPath);
|
|
2520
|
+
const cooldown = values.refresh || !retained ? null : refreshCooldown(dir);
|
|
2521
|
+
const noteCooldown = (reason) => {
|
|
2522
|
+
if (!retained) {
|
|
2523
|
+
return;
|
|
2524
|
+
}
|
|
2525
|
+
const recorded = recordRefreshCooldown(dir, reason);
|
|
2526
|
+
if (!recorded.ok) {
|
|
2527
|
+
process.stderr.write(
|
|
2528
|
+
`Warning: could not record the refresh cooldown, so the next command ` +
|
|
2529
|
+
`will try to fetch again. ${recorded.error}\n`
|
|
2530
|
+
);
|
|
2531
|
+
}
|
|
2532
|
+
};
|
|
2533
|
+
if (values.refresh || (!status.fresh && !cooldown)) {
|
|
2534
|
+
process.stderr.write(`Building the model catalog - ${status.reason}.\n`);
|
|
2535
|
+
const result = await materializeCatalog({
|
|
2536
|
+
dir,
|
|
2537
|
+
datasetId: identity.datasetId,
|
|
2538
|
+
alias,
|
|
2539
|
+
});
|
|
2540
|
+
if (!result.ok) {
|
|
2541
|
+
noteCooldown(result.error);
|
|
2542
|
+
process.stderr.write(`${result.error}\n`);
|
|
2543
|
+
return 1;
|
|
2544
|
+
}
|
|
2545
|
+
if (result.refreshIncomplete) {
|
|
2546
|
+
noteCooldown(result.notes[1] ?? result.notes[0] ?? null);
|
|
2547
|
+
} else if (!result.reused) {
|
|
2548
|
+
clearRefreshCooldown(dir);
|
|
2549
|
+
}
|
|
2550
|
+
if (result.reused) {
|
|
2551
|
+
process.stderr.write(`Using ${result.dbPath}.\n`);
|
|
2552
|
+
} else {
|
|
2553
|
+
const shape = result.built
|
|
2554
|
+
.map((table) => `${table.table} (${table.rowCount})`)
|
|
2555
|
+
.join(', ');
|
|
2556
|
+
process.stderr.write(`Wrote ${result.dbPath}: ${shape}\n`);
|
|
2557
|
+
}
|
|
2558
|
+
for (const note of result.notes) {
|
|
2559
|
+
process.stderr.write(`Note: ${note}\n`);
|
|
2560
|
+
}
|
|
2561
|
+
if (result.refreshIncomplete && values.refresh) {
|
|
2562
|
+
refreshUnsatisfied = true;
|
|
2563
|
+
process.stderr.write(
|
|
2564
|
+
'Refresh not completed: you asked for fresh data and it could not be ' +
|
|
2565
|
+
'fetched. The notes above name the source that failed and why. The ' +
|
|
2566
|
+
'catalog below is the previous snapshot, so `catalog_status` still ' +
|
|
2567
|
+
'reads complete because it describes that earlier generation.\n'
|
|
2568
|
+
);
|
|
2569
|
+
}
|
|
2570
|
+
if (result.incomplete) {
|
|
2571
|
+
process.stderr.write(
|
|
2572
|
+
'Warning: this catalog is incomplete. Query `catalog_status` ' +
|
|
2573
|
+
'and read catalog.md before using absence as evidence. ' +
|
|
2574
|
+
'Run `npm run schema:refresh` to retry the missing views.\n'
|
|
2575
|
+
);
|
|
2576
|
+
}
|
|
2577
|
+
process.stderr.write(`Catalog: ${result.catalogPath}\n`);
|
|
2578
|
+
} else if (cooldown && !status.fresh) {
|
|
2579
|
+
process.stderr.write(
|
|
2580
|
+
`Note: the last refresh did not complete, so it is not retrying until ` +
|
|
2581
|
+
`${cooldown.retryAfter}. Answering from the catalog on disk. Run ` +
|
|
2582
|
+
'with --refresh to retry now.\n'
|
|
2583
|
+
);
|
|
2584
|
+
if (cooldown.reason) {
|
|
2585
|
+
process.stderr.write(`Note: ${cooldown.reason}\n`);
|
|
2586
|
+
}
|
|
2587
|
+
}
|
|
2588
|
+
|
|
2589
|
+
if (plan.mode === 'none') {
|
|
2590
|
+
return refreshUnsatisfied ? 2 : 0;
|
|
2591
|
+
}
|
|
2592
|
+
|
|
2593
|
+
const limit = Number(values.limit);
|
|
2594
|
+
const printLimit =
|
|
2595
|
+
Number.isFinite(limit) && limit > 0 ? limit : DEFAULT_PRINT_LIMIT;
|
|
2596
|
+
const runners = {
|
|
2597
|
+
search: (query, options) => searchCatalog(dbPath, query, options),
|
|
2598
|
+
sql: (query, options) => runSql(dbPath, query, options),
|
|
2599
|
+
};
|
|
2600
|
+
// A failed freshness request is not swallowed just because the SQL half
|
|
2601
|
+
// succeeded. Same precedence as a batch: failure outranks truncation.
|
|
2602
|
+
const withRefreshStatus = (code) => {
|
|
2603
|
+
if (code === 1) {
|
|
2604
|
+
return 1;
|
|
2605
|
+
}
|
|
2606
|
+
return refreshUnsatisfied || code === 2 ? 2 : 0;
|
|
2607
|
+
};
|
|
2608
|
+
|
|
2609
|
+
// A lone positional statement keeps the original payload byte for byte, so
|
|
2610
|
+
// every existing instruction, parser, and truncation rule still holds.
|
|
2611
|
+
if (plan.mode === 'single') {
|
|
2612
|
+
const result = runners.sql(plan.batch[0].query, { limit: printLimit });
|
|
2613
|
+
if (!result.ok) {
|
|
2614
|
+
process.stderr.write(`${result.error}\n`);
|
|
2615
|
+
return 1;
|
|
2616
|
+
}
|
|
2617
|
+
process.stdout.write(formatResult(result));
|
|
2618
|
+
return withRefreshStatus(
|
|
2619
|
+
resultExitCode(result, {
|
|
2620
|
+
allowTruncated: values['allow-truncated'],
|
|
2621
|
+
})
|
|
2622
|
+
);
|
|
2623
|
+
}
|
|
2624
|
+
|
|
2625
|
+
const entries = runBatch(plan.batch, { runners, limit: printLimit });
|
|
2626
|
+
|
|
2627
|
+
// A failed statement inside a batch stays in the payload rather than going to
|
|
2628
|
+
// stderr: the other statements succeeded, and the agent needs to see which
|
|
2629
|
+
// one did not next to the ones that did.
|
|
2630
|
+
process.stdout.write(formatBatch(entries));
|
|
2631
|
+
return withRefreshStatus(
|
|
2632
|
+
batchExitCode(entries, {
|
|
2633
|
+
allowTruncated: values['allow-truncated'],
|
|
2634
|
+
})
|
|
2635
|
+
);
|
|
2636
|
+
}
|
|
2637
|
+
|
|
2638
|
+
if (
|
|
2639
|
+
process.argv[1] &&
|
|
2640
|
+
import.meta.url === pathToFileURL(process.argv[1]).href
|
|
2641
|
+
) {
|
|
2642
|
+
main(process.argv.slice(2))
|
|
2643
|
+
.then((code) => {
|
|
2644
|
+
process.exitCode = code;
|
|
2645
|
+
})
|
|
2646
|
+
.catch((error) => {
|
|
2647
|
+
process.stderr.write(`${error?.stack || error}\n`);
|
|
2648
|
+
process.exitCode = 1;
|
|
2649
|
+
});
|
|
2650
|
+
}
|