@microsoft/rayfin-semantic-model-pack 0.0.0-stage → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (29) hide show
  1. package/CHANGELOG +10 -0
  2. package/LICENSE +21 -0
  3. package/README.md +65 -2
  4. package/package.json +10 -3
  5. package/skills/analytics/SKILL.md +460 -0
  6. package/skills/analytics/kit/hooks/use-semantic-model-query.spec.ts +342 -0
  7. package/skills/analytics/kit/hooks/use-semantic-model-query.ts +229 -0
  8. package/skills/analytics/kit/lib/connectors.ts +94 -0
  9. package/skills/analytics/kit/scripts/rayfin-connector.mjs +265 -0
  10. package/skills/analytics/kit/scripts/rayfin-connector.test.mjs +243 -0
  11. package/skills/analytics/kit/scripts/schema-sql-batch.test.mjs +2097 -0
  12. package/skills/analytics/kit/scripts/schema-sql-cli.test.mjs +488 -0
  13. package/skills/analytics/kit/scripts/schema-sql.mjs +2650 -0
  14. package/skills/analytics/kit/scripts/schema-sql.test.mjs +1387 -0
  15. package/skills/analytics/kit/scripts/schema.mjs +166 -0
  16. package/skills/analytics/kit/scripts/schema.test.mjs +271 -0
  17. package/skills/analytics/pack.json +49 -0
  18. package/skills/dax-authoring/SKILL.md +172 -0
  19. package/skills/dax-authoring/references/dax-core-reference.md +862 -0
  20. package/skills/dax-authoring/references/dax-query-patterns.md +396 -0
  21. package/skills/dax-authoring/references/dax-time-intelligence.md +347 -0
  22. package/skills/query-design/SKILL.md +114 -0
  23. package/skills/query-design/references/anti-patterns.md +285 -0
  24. package/skills/query-design/references/filter-strategy.md +68 -0
  25. package/skills/query-design/references/format-strings.md +62 -0
  26. package/skills/query-design/references/highlight-queries.md +122 -0
  27. package/skills/query-design/references/multi-grain-patterns.md +159 -0
  28. package/skills/schema-discovery/SKILL.md +382 -0
  29. package/skills/schema-discovery/references/discovery-queries.md +153 -0
@@ -0,0 +1,2097 @@
1
+ //-----------------------------------------------------------------------
2
+ // <copyright company="Microsoft Corporation">
3
+ // Copyright (c) Microsoft Corporation. All rights reserved.
4
+ // Licensed under the MIT license. See LICENSE file in the project root for full license information.
5
+ // </copyright>
6
+ //-----------------------------------------------------------------------
7
+
8
+ // Batching and search-payload trimming.
9
+ //
10
+ // Both exist for the same reason: one invocation costs one model round trip, and
11
+ // a round trip re-sends the whole conversation. Batching removes round trips;
12
+ // trimming shrinks what each one carries forward. The trimming tests care most
13
+ // about what must NOT change - an object that was findable before has to stay
14
+ // findable.
15
+
16
+ import assert from 'node:assert/strict';
17
+ import { createHash } from 'node:crypto';
18
+ import {
19
+ existsSync,
20
+ linkSync,
21
+ mkdirSync,
22
+ mkdtempSync,
23
+ readFileSync,
24
+ renameSync,
25
+ rmSync,
26
+ statSync,
27
+ symlinkSync,
28
+ writeFileSync,
29
+ } from 'node:fs';
30
+ import { tmpdir } from 'node:os';
31
+ import { join, sep } from 'node:path';
32
+ import test from 'node:test';
33
+
34
+ import {
35
+ batchExitCode,
36
+ BOILERPLATE_MIN_ROWS,
37
+ BOILERPLATE_MIN_SHARE,
38
+ boilerplateDescriptions,
39
+ buildDatabase,
40
+ formatBatch,
41
+ formatResult,
42
+ MAX_BATCH_STATEMENTS,
43
+ MAX_RESULT_BYTES,
44
+ matchedSearchTokens,
45
+ parseCliArgs,
46
+ planStatements,
47
+ publishFiles,
48
+ rankSearchWithFallback,
49
+ rankSemanticSearchRows,
50
+ runBatch,
51
+ runSql,
52
+ searchCatalog,
53
+ SEARCH_EXPRESSION_LIMIT,
54
+ summarizeSearchExpression,
55
+ trimSearchRows,
56
+ withCatalogLock,
57
+ } from './schema-sql.mjs';
58
+
59
+ function tempDir() {
60
+ return mkdtempSync(join(tmpdir(), 'schema-sql-batch-'));
61
+ }
62
+
63
+ function runnersFor(dbPath) {
64
+ return {
65
+ sql: (query, options) => runSql(dbPath, query, options),
66
+ search: (query, options) => searchCatalog(dbPath, query, options),
67
+ };
68
+ }
69
+
70
+ // -- Statement planning --
71
+
72
+ test('repeated --search/--sql/--file parse into arrays', () => {
73
+ const { values, positionals } = parseCliArgs([
74
+ '--search',
75
+ 'revenue',
76
+ '--search',
77
+ 'margin',
78
+ '--sql',
79
+ 'SELECT 1',
80
+ '--file',
81
+ 'a.sql',
82
+ ]);
83
+ assert.deepEqual(values.search, ['revenue', 'margin']);
84
+ assert.deepEqual(values.sql, ['SELECT 1']);
85
+ assert.deepEqual(values.file, ['a.sql']);
86
+ assert.deepEqual(positionals, []);
87
+ });
88
+
89
+ test('--profile stays removed rather than returning with the batch flags', () => {
90
+ // It was deprecated on arrival and ignored by both resolvers. Upstream still
91
+ // parses it; reintroducing it would give the agent an option that does nothing.
92
+ assert.throws(
93
+ () => parseCliArgs(['--profile', 'anything']),
94
+ /unknown option|not (a )?known/iu
95
+ );
96
+ });
97
+
98
+ test('a lone positional statement still plans as a single statement', () => {
99
+ const plan = planStatements({ positionalSql: 'SELECT 1' });
100
+ assert.equal(plan.mode, 'single');
101
+ assert.equal(plan.batch.length, 1);
102
+ assert.equal(plan.batch[0].query, 'SELECT 1');
103
+ });
104
+
105
+ test('searches are planned before SQL so results read in discovery order', () => {
106
+ const plan = planStatements({
107
+ search: ['revenue'],
108
+ sql: ['SELECT 1', 'SELECT 2'],
109
+ });
110
+ assert.equal(plan.mode, 'batch');
111
+ assert.deepEqual(
112
+ plan.batch.map((entry) => entry.kind),
113
+ ['search', 'sql', 'sql']
114
+ );
115
+ assert.deepEqual(
116
+ plan.batch.map((entry) => entry.query),
117
+ ['revenue', 'SELECT 1', 'SELECT 2']
118
+ );
119
+ });
120
+
121
+ test('--file contributes its contents as a statement and remembers its source', () => {
122
+ const dir = tempDir();
123
+ try {
124
+ const path = join(dir, 'q.sql');
125
+ writeFileSync(path, 'SELECT [Name] FROM info_view_measures', 'utf8');
126
+ const plan = planStatements({ file: [path], root: dir });
127
+ assert.equal(plan.mode, 'batch');
128
+ assert.equal(plan.batch[0].query, 'SELECT [Name] FROM info_view_measures');
129
+ assert.equal(plan.batch[0].source, 'q.sql');
130
+ } finally {
131
+ rmSync(dir, { recursive: true, force: true });
132
+ }
133
+ });
134
+
135
+ test('an unreadable --file is reported rather than thrown', () => {
136
+ const dir = tempDir();
137
+ try {
138
+ const plan = planStatements({
139
+ file: [join(dir, 'missing.sql')],
140
+ root: dir,
141
+ });
142
+ assert.match(plan.error, /Could not read --file/u);
143
+ } finally {
144
+ rmSync(dir, { recursive: true, force: true });
145
+ }
146
+ });
147
+
148
+ test('positional SQL and batch flags cannot be mixed', () => {
149
+ const plan = planStatements({
150
+ positionalSql: 'SELECT 1',
151
+ sql: ['SELECT 2'],
152
+ });
153
+ assert.match(plan.error, /not both/u);
154
+ });
155
+
156
+ test('the batch size is capped so one call cannot become a whole workload', () => {
157
+ const sql = Array.from(
158
+ { length: MAX_BATCH_STATEMENTS + 1 },
159
+ (_, index) => `SELECT ${index}`
160
+ );
161
+ const plan = planStatements({ sql });
162
+ assert.match(plan.error, new RegExp(`limit is ${MAX_BATCH_STATEMENTS}`, 'u'));
163
+ });
164
+
165
+ test('an empty statement is rejected instead of running as blank SQL', () => {
166
+ assert.match(planStatements({ sql: [' '] }).error, /empty sql statement/u);
167
+ assert.match(
168
+ planStatements({ search: [''] }).error,
169
+ /empty search statement/u
170
+ );
171
+ });
172
+
173
+ test('no statements and no refresh plans nothing', () => {
174
+ assert.equal(planStatements({}).mode, 'none');
175
+ });
176
+
177
+ // -- --file is an input channel, not a new capability --
178
+ //
179
+ // Everything reachable through --file was already reachable through positional
180
+ // SQL. Two things still needed closing, because --file is the only option that
181
+ // reads bytes the caller did not type:
182
+ //
183
+ // 1. the path is constrained to the app, matching the Rayfin CLI's own --file
184
+ // convention (connector-invoke.ts readJsonInputFile), and
185
+ // 2. the printed label names the file rather than quoting its contents.
186
+ //
187
+ // Together those mean a mistyped path cannot pull unrelated file bytes into the
188
+ // agent's context, whether through the label or through a SQLite syntax error
189
+ // that quotes the offending token back.
190
+
191
+ test('a --file outside the app is refused before it is opened', () => {
192
+ const dir = tempDir();
193
+ const outside = tempDir();
194
+ try {
195
+ const path = join(outside, 'secret.txt');
196
+ writeFileSync(path, 'NOT-A-REAL-CREDENTIAL-abcdef1234567890', 'utf8');
197
+
198
+ const plan = planStatements({ file: [path], root: dir });
199
+ assert.match(plan.error, /--file must be inside the app/u);
200
+ assert.ok(
201
+ !plan.error.includes('NOT-A-REAL-CREDENTIAL'),
202
+ 'the refusal must not quote what it refused to read'
203
+ );
204
+
205
+ // Traversal out of the app is the same refusal, not a different one.
206
+ assert.match(
207
+ planStatements({ file: ['../secret.txt'], root: dir }).error,
208
+ /--file must be inside the app/u
209
+ );
210
+ } finally {
211
+ rmSync(dir, { recursive: true, force: true });
212
+ rmSync(outside, { recursive: true, force: true });
213
+ }
214
+ });
215
+
216
+ test('a --file statement is labelled by its path, not by the bytes it holds', (t) => {
217
+ // The label is bounded at 512 bytes, so quoting file contents would spill up
218
+ // to 512 bytes of an arbitrary in-app file into every batch payload. Naming
219
+ // the file is both safer and a better label: it is what the caller typed.
220
+ const marker = 'DISTINCTIVE-FILE-CONTENT-MARKER';
221
+ const queries = [
222
+ `SELECT marker FROM t WHERE id = 1 /* ${marker} ${'x'.repeat(4_000)} */`,
223
+ ];
224
+ const { labels, payload, executed } = batchFixture(t, queries, {
225
+ via: 'file',
226
+ });
227
+
228
+ assert.equal(labels[0].source, 'statement 0.sql');
229
+ assert.equal(labels[0].query, undefined);
230
+ assert.ok(
231
+ !payload.includes(marker),
232
+ 'no part of the file body may appear in the printed label'
233
+ );
234
+ assert.deepEqual(
235
+ executed,
236
+ queries,
237
+ 'the full file text still runs - only the label is shortened'
238
+ );
239
+ });
240
+
241
+ test('file-sourced statements still share the one batch byte budget', (t) => {
242
+ const rows = Array.from({ length: 200 }, (_, id) => ({
243
+ id,
244
+ marker: `${id} ${'value '.repeat(50)}`,
245
+ }));
246
+ const queries = Array.from({ length: 8 }, () => 'SELECT * FROM t');
247
+ const { entries, bytes } = batchFixture(t, queries, { via: 'file', rows });
248
+
249
+ assert.ok(bytes <= MAX_RESULT_BYTES, `batch emitted ${bytes} bytes`);
250
+ for (const entry of entries) {
251
+ assert.equal(entry.result.totalRows, rows.length);
252
+ assert.ok(entry.result.returnedRows > 0, 'no statement was starved');
253
+ }
254
+ t.diagnostic(`eight file statements: ${bytes}/${MAX_RESULT_BYTES} bytes`);
255
+ });
256
+
257
+ test('SQL supplied through --file cannot escape the read-only connection', () => {
258
+ const dir = tempDir();
259
+ try {
260
+ const dbPath = join(dir, 'model.sqlite');
261
+ buildDatabase(
262
+ [{ table: 't', grain: '', dax: '', rows: [{ secret: 'customer row' }] }],
263
+ dbPath
264
+ );
265
+
266
+ const copiedPath = join(dir, 'exfiltrated.sqlite');
267
+ const statementPath = join(dir, 'exfiltrate.sql');
268
+ writeFileSync(
269
+ statementPath,
270
+ `VACUUM INTO '${copiedPath.replace(/'/gu, "''")}'`,
271
+ 'utf8'
272
+ );
273
+
274
+ const plan = planStatements({ file: [statementPath], root: dir });
275
+ assert.equal(plan.mode, 'batch');
276
+ const [entry] = runBatch(plan.batch, { runners: runnersFor(dbPath) });
277
+
278
+ assert.equal(entry.result.ok, false);
279
+ assert.match(entry.result.error, /read.?only|readonly/iu);
280
+ assert.equal(
281
+ existsSync(copiedPath) ? statSync(copiedPath).size : 0,
282
+ 0,
283
+ 'no catalog data may escape through a --file statement'
284
+ );
285
+ } finally {
286
+ rmSync(dir, { recursive: true, force: true });
287
+ }
288
+ });
289
+
290
+ test('a query_only bypass cannot carry across two batch statements', () => {
291
+ const dir = tempDir();
292
+ try {
293
+ const dbPath = join(dir, 'model.sqlite');
294
+ buildDatabase(
295
+ [{ table: 't', grain: '', dax: '', rows: [{ secret: 'customer row' }] }],
296
+ dbPath
297
+ );
298
+
299
+ // On one reused connection PRAGMA query_only=OFF re-enables writes and the
300
+ // VACUUM INTO that follows succeeds. Each batch entry opens its own
301
+ // read-only connection, so the PRAGMA cannot carry into the next statement.
302
+ const copiedPath = join(dir, 'exfiltrated.sqlite');
303
+ const plan = planStatements({
304
+ sql: [
305
+ 'PRAGMA query_only=OFF',
306
+ `VACUUM INTO '${copiedPath.replace(/'/gu, "''")}'`,
307
+ ],
308
+ root: dir,
309
+ });
310
+ assert.equal(plan.mode, 'batch');
311
+ const entries = runBatch(plan.batch, { runners: runnersFor(dbPath) });
312
+
313
+ assert.equal(entries.length, 2);
314
+ assert.equal(entries[1].result.ok, false);
315
+ assert.match(entries[1].result.error, /read.?only|readonly/iu);
316
+ assert.equal(
317
+ existsSync(copiedPath) ? statSync(copiedPath).size : 0,
318
+ 0,
319
+ 'a PRAGMA in an earlier statement may not unlock a later write'
320
+ );
321
+ } finally {
322
+ rmSync(dir, { recursive: true, force: true });
323
+ }
324
+ });
325
+
326
+ test('an oversized --file is rejected before it is read into memory', () => {
327
+ const dir = tempDir();
328
+ try {
329
+ const path = join(dir, 'huge.sql');
330
+ // Upstream reads --file unboundedly, which turns a wrong path into a memory
331
+ // blowup. The cap is stated in the error so the limit is discoverable.
332
+ writeFileSync(path, `-- ${'x'.repeat(1_000_001)}`, 'utf8');
333
+ const plan = planStatements({ file: [path], root: dir });
334
+ assert.match(plan.error, /Could not read --file/u);
335
+ assert.match(plan.error, /exceeds the 1000000-byte limit/u);
336
+ } finally {
337
+ rmSync(dir, { recursive: true, force: true });
338
+ }
339
+ });
340
+
341
+ test('a --file that is not a regular file is reported, not read', () => {
342
+ const dir = tempDir();
343
+ try {
344
+ const plan = planStatements({ file: [dir], root: dir });
345
+ // Assert the guard's own message rather than just "could not read": a
346
+ // directory would fail inside readFileSync anyway, so a loose match here
347
+ // would be a test that cannot fail.
348
+ assert.match(plan.error, /Could not read --file/u);
349
+ assert.match(plan.error, /not a regular file/u);
350
+ } finally {
351
+ rmSync(dir, { recursive: true, force: true });
352
+ }
353
+ });
354
+
355
+ // -- Batch payload --
356
+
357
+ test('each batch block is byte-identical to the single-statement payload', () => {
358
+ const result = {
359
+ ok: true,
360
+ totalRows: 1,
361
+ returnedRows: 1,
362
+ truncated: false,
363
+ rows: [{ name: 'Revenue' }],
364
+ };
365
+ const out = formatBatch([{ kind: 'sql', query: 'SELECT 1', result }]);
366
+ const lines = out.split('\n');
367
+ assert.deepEqual(JSON.parse(lines[0].slice(2)), {
368
+ ok: true,
369
+ batch: true,
370
+ statements: 1,
371
+ });
372
+ assert.deepEqual(JSON.parse(lines[1].slice(3)), {
373
+ index: 0,
374
+ kind: 'sql',
375
+ query: 'SELECT 1',
376
+ });
377
+ // The body is still TSV - batching must not force results into JSON.
378
+ assert.match(lines[2], /^# \{/u);
379
+ assert.equal(lines[3], 'name');
380
+ assert.equal(lines[4], 'Revenue');
381
+ });
382
+
383
+ test('a failed statement stays in the payload beside the ones that worked', () => {
384
+ const out = formatBatch([
385
+ {
386
+ kind: 'sql',
387
+ query: 'SELECT 1',
388
+ result: {
389
+ ok: true,
390
+ totalRows: 1,
391
+ returnedRows: 1,
392
+ truncated: false,
393
+ rows: [{ n: 1 }],
394
+ },
395
+ },
396
+ {
397
+ kind: 'sql',
398
+ query: 'SELEKT',
399
+ result: { ok: false, error: 'no such function: SELEKT' },
400
+ },
401
+ ]);
402
+ assert.match(out, /"statements":2/u);
403
+ assert.match(out, /"ok":false/u);
404
+ assert.match(out, /no such function/u);
405
+ assert.match(out, /"index":1/u);
406
+ });
407
+
408
+ test('the batch exit code is the worst outcome, and failure outranks truncation', () => {
409
+ const ok = { ok: true, truncated: false };
410
+ const cut = { ok: true, truncated: true };
411
+ const bad = { ok: false, error: 'x' };
412
+ const wrap = (results) =>
413
+ results.map((result) => ({ kind: 'sql', query: 'q', result }));
414
+
415
+ assert.equal(batchExitCode(wrap([ok, ok])), 0);
416
+ assert.equal(batchExitCode(wrap([ok, cut])), 2);
417
+ assert.equal(batchExitCode(wrap([cut, bad])), 1);
418
+ assert.equal(batchExitCode(wrap([ok, cut]), { allowTruncated: true }), 0);
419
+ assert.equal(batchExitCode(wrap([])), 0);
420
+ });
421
+
422
+ // -- The whole batch shares one byte budget --
423
+ //
424
+ // The per-statement cap exists so a result cannot bury the session's context.
425
+ // Batching must not multiply it by the number of statements, or it would make
426
+ // the very problem it is meant to reduce strictly worse.
427
+
428
+ test('a full batch of large results still fits the single-statement byte budget', () => {
429
+ const dir = tempDir();
430
+ try {
431
+ const dbPath = join(dir, 'm.sqlite');
432
+ buildDatabase(
433
+ [
434
+ {
435
+ table: 'info_view_measures',
436
+ grain: '',
437
+ dax: 'INFO.VIEW.MEASURES()',
438
+ // Each row alone would blow a meaningful part of the budget.
439
+ rows: Array.from({ length: 200 }, (_, index) => ({
440
+ Table: 'Facts',
441
+ Name: `Measure ${index}`,
442
+ Description: `unique description ${index} ${'padding '.repeat(40)}`,
443
+ })),
444
+ },
445
+ ],
446
+ dbPath
447
+ );
448
+
449
+ const batch = Array.from({ length: MAX_BATCH_STATEMENTS }, () => ({
450
+ kind: 'sql',
451
+ query: 'SELECT * FROM info_view_measures',
452
+ }));
453
+ const entries = runBatch(batch, { runners: runnersFor(dbPath) });
454
+
455
+ assert.equal(entries.length, MAX_BATCH_STATEMENTS);
456
+ const payload = Buffer.byteLength(formatBatch(entries), 'utf8');
457
+ assert.ok(
458
+ payload <= MAX_RESULT_BYTES,
459
+ `the whole batch must fit ${MAX_RESULT_BYTES} bytes, got ${payload}`
460
+ );
461
+
462
+ // The property that makes this worth asserting: eight statements did not
463
+ // cost eight budgets. A per-statement cap would have allowed 8x this.
464
+ const single = Buffer.byteLength(
465
+ formatResult(runSql(dbPath, 'SELECT * FROM info_view_measures')),
466
+ 'utf8'
467
+ );
468
+ assert.ok(
469
+ payload < single * 2,
470
+ `a batch of ${MAX_BATCH_STATEMENTS} must not approach ${MAX_BATCH_STATEMENTS}x ` +
471
+ `one statement: ${payload} vs ${single} each`
472
+ );
473
+
474
+ // Every statement still answered - bounded, not dropped.
475
+ for (const entry of entries) {
476
+ assert.equal(entry.result.ok, true);
477
+ assert.ok(
478
+ entry.result.returnedRows > 0,
479
+ 'no statement was starved to zero rows'
480
+ );
481
+ }
482
+ } finally {
483
+ rmSync(dir, { recursive: true, force: true });
484
+ }
485
+ });
486
+
487
+ test('an unused share rolls forward, so a cheap statement makes a later one roomier', () => {
488
+ const sizes = [];
489
+ const runners = {
490
+ sql: (query, options) => {
491
+ sizes.push(options.maxBytes);
492
+ // The first statement answers in almost nothing.
493
+ return query === 'tiny'
494
+ ? {
495
+ ok: true,
496
+ totalRows: 1,
497
+ returnedRows: 1,
498
+ truncated: false,
499
+ rows: [{ a: 1 }],
500
+ }
501
+ : {
502
+ ok: true,
503
+ totalRows: 1,
504
+ returnedRows: 1,
505
+ truncated: false,
506
+ rows: [{ a: 'x'.repeat(200) }],
507
+ };
508
+ },
509
+ search: () => ({ ok: false, error: 'unused' }),
510
+ };
511
+ runBatch(
512
+ [
513
+ { kind: 'sql', query: 'tiny' },
514
+ { kind: 'sql', query: 'big' },
515
+ ],
516
+ { runners, maxBytes: 10_000 }
517
+ );
518
+
519
+ assert.ok(
520
+ sizes[1] > sizes[0],
521
+ `expected the second share to grow: ${sizes[0]} -> ${sizes[1]}`
522
+ );
523
+ });
524
+
525
+ function batchFixture(
526
+ t,
527
+ queries,
528
+ { via = 'sql', rows = [{ id: 1, marker: 'unchanged SQL' }], ...options } = {}
529
+ ) {
530
+ const dir = tempDir();
531
+ t.after(() => rmSync(dir, { recursive: true, force: true }));
532
+ const dbPath = join(dir, 'budget.sqlite');
533
+ buildDatabase([{ table: 't', grain: '', dax: '', rows }], dbPath);
534
+ const args = queries.flatMap((query, index) => {
535
+ if (via === 'sql') {
536
+ return ['--sql', query];
537
+ }
538
+ const path = join(dir, `statement ${index}.sql`);
539
+ writeFileSync(path, query, 'utf8');
540
+ return ['--file', path];
541
+ });
542
+ const { values } = parseCliArgs(args);
543
+ const plan = planStatements({ ...values, root: dir });
544
+ assert.equal(plan.mode, 'batch');
545
+ const executed = [];
546
+ const runners = {
547
+ sql: (query, limits) => {
548
+ executed.push(query);
549
+ return runSql(dbPath, query, limits);
550
+ },
551
+ search: (query, limits) => searchCatalog(dbPath, query, limits),
552
+ };
553
+ const entries = runBatch(plan.batch, { runners, ...options });
554
+ assert.deepEqual(
555
+ executed,
556
+ queries,
557
+ 'execute the full original statement text, not its printed label'
558
+ );
559
+ const payload = formatBatch(entries);
560
+ const bytes = Buffer.byteLength(payload, 'utf8');
561
+ assert.ok(
562
+ bytes <= (options.maxBytes ?? MAX_RESULT_BYTES),
563
+ `batch emitted ${bytes} bytes`
564
+ );
565
+ const labels = payload
566
+ .split('\n')
567
+ .filter((line) => line.startsWith('## '))
568
+ .map((line) => JSON.parse(line.slice(3)));
569
+ return { entries, payload, bytes, labels, plan, runners, executed, dir };
570
+ }
571
+
572
+ for (const [name, count, padding] of [
573
+ ['one oversized statement', 1, 'x'.repeat(30_000)],
574
+ ['eight large statements', 8, 'x'.repeat(4_100)],
575
+ ['Unicode and JSON escapes', 1, '\u03b1\u{1f642}"\\\t\n'.repeat(4_000)],
576
+ ]) {
577
+ test(`batch bounds printed labels from ${name} without altering SQL`, (t) => {
578
+ const queries = Array.from(
579
+ { length: count },
580
+ (_, index) =>
581
+ `SELECT marker FROM t /* ${index} ${padding} */ WHERE id = 1`
582
+ );
583
+ const { entries, labels, bytes } = batchFixture(t, queries);
584
+ assert.equal(batchExitCode(entries), 0);
585
+ for (const [index, entry] of entries.entries()) {
586
+ assert.deepEqual(
587
+ entry.result.rows.map((row) => ({ ...row })),
588
+ [{ marker: 'unchanged SQL' }]
589
+ );
590
+ assert.equal(labels[index].index, index);
591
+ assert.equal(labels[index].kind, 'sql');
592
+ assert.ok(queries[index].startsWith(labels[index].query));
593
+ assert.equal(labels[index].queryTruncated, true);
594
+ assert.equal(
595
+ labels[index].queryBytes,
596
+ Buffer.byteLength(queries[index], 'utf8')
597
+ );
598
+ assert.equal(
599
+ labels[index].queryHash,
600
+ createHash('sha256').update(queries[index]).digest('hex').slice(0, 16)
601
+ );
602
+ assert.ok(
603
+ !/[\uD800-\uDBFF]$/u.test(labels[index].query),
604
+ 'do not split a surrogate pair'
605
+ );
606
+ }
607
+ t.diagnostic(`${name}: ${bytes}/${MAX_RESULT_BYTES} UTF-8 bytes`);
608
+ });
609
+ }
610
+
611
+ test('eight long labels leave fair shares for rows and preserve recovery metadata', (t) => {
612
+ const rows = Array.from({ length: 200 }, (_, id) => ({
613
+ id,
614
+ marker: `${id} ${'value '.repeat(50)}`,
615
+ }));
616
+ const queries = Array.from(
617
+ { length: 8 },
618
+ () => `SELECT * FROM t /* ${'x'.repeat(4_100)} */`
619
+ );
620
+ const { entries, bytes } = batchFixture(t, queries, { rows });
621
+ assert.equal(batchExitCode(entries), 2);
622
+ for (const entry of entries) {
623
+ assert.equal(entry.result.totalRows, rows.length);
624
+ assert.ok(
625
+ entry.result.returnedRows > 0,
626
+ 'labels cannot starve later statements'
627
+ );
628
+ assert.equal(entry.result.truncated, true);
629
+ assert.match(
630
+ entry.result.truncation.requiredAction,
631
+ /narrower WHERE|smaller projection/u
632
+ );
633
+ assert.deepEqual(Object.keys(entry.result.rows[0]), ['id', 'marker']);
634
+ }
635
+ t.diagnostic(`labels and row data: ${bytes}/${MAX_RESULT_BYTES} UTF-8 bytes`);
636
+ });
637
+
638
+ test('large SQLite diagnostics stay indexed errors inside the final budget', (t) => {
639
+ const missing = 'missing_\u03b1\u{1f642}\\"'
640
+ .repeat(4_000)
641
+ .replace(/"/gu, '""');
642
+ const { entries, payload, bytes } = batchFixture(t, [
643
+ 'SELECT marker FROM t',
644
+ `SELECT * FROM "${missing}"`,
645
+ 'SELEKT broken',
646
+ 'SELECT marker FROM t WHERE id = 1',
647
+ ]);
648
+ assert.equal(batchExitCode(entries, { allowTruncated: true }), 1);
649
+ assert.equal(entries[0].result.ok, true);
650
+ assert.equal(entries[3].result.ok, true);
651
+ assert.equal(entries[1].result.ok, false);
652
+ assert.match(entries[1].result.error, /no such table/u);
653
+ assert.equal(entries[1].result.errorTruncated, true);
654
+ assert.ok(entries[1].result.errorBytes > MAX_RESULT_BYTES);
655
+ assert.match(entries[2].result.error, /syntax error/u);
656
+ assert.equal(JSON.parse(payload.split('\n')[0].slice(2)).ok, false);
657
+ t.diagnostic(`long error: ${bytes}/${MAX_RESULT_BYTES} UTF-8 bytes`);
658
+ });
659
+
660
+ test('an all-error batch can fill the exact cap without a one-byte header overrun', (t) => {
661
+ const queries = Array.from(
662
+ { length: 2 },
663
+ (_, index) => `SELECT * FROM "missing_${index}_${'x'.repeat(30_000)}"`
664
+ );
665
+ const { entries, bytes } = batchFixture(t, queries);
666
+ assert.equal(bytes, MAX_RESULT_BYTES);
667
+ assert.equal(batchExitCode(entries), 1);
668
+ assert.ok(
669
+ entries.every(
670
+ (entry) => entry.result.ok === false && entry.result.errorTruncated
671
+ )
672
+ );
673
+ assert.ok(
674
+ entries.every((entry) =>
675
+ entry.result.error.startsWith('no such table: missing_')
676
+ )
677
+ );
678
+ t.diagnostic(`all-error batch: ${bytes}/${MAX_RESULT_BYTES} UTF-8 bytes`);
679
+ });
680
+
681
+ test('short query labels and result blocks retain their original serialized form', (t) => {
682
+ const queries = [
683
+ 'SELECT marker FROM t',
684
+ 'SELECT marker FROM t WHERE id = 0',
685
+ 'SELEKT broken',
686
+ ];
687
+ const { entries, payload, labels } = batchFixture(t, queries);
688
+ const expected =
689
+ '# {"ok":false,"batch":true,"statements":3}\n' +
690
+ entries
691
+ .map(
692
+ (entry, index) =>
693
+ `## ${JSON.stringify({ index, kind: 'sql', query: queries[index] })}\n` +
694
+ formatResult(entry.result)
695
+ )
696
+ .join('');
697
+ assert.equal(payload, expected);
698
+ assert.deepEqual(
699
+ labels,
700
+ queries.map((query, index) => ({ index, kind: 'sql', query }))
701
+ );
702
+ assert.equal(entries[1].result.totalRows, 0);
703
+ assert.equal(entries[1].result.truncated, false);
704
+ });
705
+
706
+ test('batch byte boundaries include escaped labels, errors and result metadata', (t) => {
707
+ const queries = [
708
+ 'SELECT marker FROM t /* "\u{1f642}\\ */',
709
+ 'SELECT * FROM t WHERE id = 0',
710
+ 'SELEKT broken',
711
+ ];
712
+ for (const maxBytes of [
713
+ 2_000,
714
+ 2_001,
715
+ 4_096,
716
+ MAX_RESULT_BYTES - 1,
717
+ MAX_RESULT_BYTES,
718
+ ]) {
719
+ const { entries } = batchFixture(t, queries, { maxBytes });
720
+ assert.equal(batchExitCode(entries), 1);
721
+ assert.equal(entries[1].result.truncated, false);
722
+ }
723
+ });
724
+
725
+ test('budgets too small for batch metadata fail explicitly before execution', () => {
726
+ let executions = 0;
727
+ const runners = {
728
+ sql: () => {
729
+ executions += 1;
730
+ return {
731
+ ok: true,
732
+ totalRows: 0,
733
+ returnedRows: 0,
734
+ truncated: false,
735
+ rows: [],
736
+ };
737
+ },
738
+ };
739
+ for (const maxBytes of [0, 1, 100, -1, NaN, Infinity]) {
740
+ assert.throws(
741
+ () =>
742
+ runBatch([{ kind: 'sql', query: 'SELECT 1' }], { runners, maxBytes }),
743
+ /budget|bytes/iu
744
+ );
745
+ }
746
+ assert.equal(executions, 0);
747
+ });
748
+
749
+ test('label limits use serialized UTF-8 bytes and only summarize above the boundary', (t) => {
750
+ const base = 'SELECT marker FROM t /* "\\\u{1f642} */';
751
+ const labelBytes = Buffer.byteLength(
752
+ `## ${JSON.stringify({ index: 0, kind: 'sql', query: base })}\n`
753
+ );
754
+ for (const offset of [-1, 0, 1]) {
755
+ const query = base + ' '.repeat(512 - labelBytes + offset);
756
+ const { payload, labels } = batchFixture(t, [query]);
757
+ const label = `${payload.split('\n')[1]}\n`;
758
+ assert.ok(Buffer.byteLength(label) <= 512);
759
+ assert.equal(labels[0].queryTruncated, offset > 0 ? true : undefined);
760
+ if (offset <= 0) {
761
+ assert.equal(labels[0].query, query);
762
+ assert.equal(Buffer.byteLength(label), 512 + offset);
763
+ }
764
+ }
765
+ });
766
+
767
+ test('exact final budget includes the failing header and trims whole rows one byte below it', (t) => {
768
+ const fixture = batchFixture(t, ['SELEKT', 'SELECT marker FROM t'], {
769
+ rows: [{ marker: 'x'.repeat(2_000) }],
770
+ });
771
+ const exact = runBatch(fixture.plan.batch, {
772
+ runners: fixture.runners,
773
+ maxBytes: fixture.bytes,
774
+ });
775
+ assert.equal(formatBatch(exact), fixture.payload);
776
+ assert.equal(Buffer.byteLength(formatBatch(exact)), fixture.bytes);
777
+ const below = runBatch(fixture.plan.batch, {
778
+ runners: fixture.runners,
779
+ maxBytes: fixture.bytes - 1,
780
+ });
781
+ assert.ok(Buffer.byteLength(formatBatch(below)) <= fixture.bytes - 1);
782
+ assert.equal(below[0].result.error, exact[0].result.error);
783
+ assert.equal(below[1].result.totalRows, 1);
784
+ assert.equal(below[1].result.returnedRows, 0);
785
+ assert.equal(below[1].result.truncated, true);
786
+ assert.equal(below[1].result.truncation.reason, 'byte_budget');
787
+ assert.equal(batchExitCode(below), 1);
788
+ });
789
+
790
+ test('minimum supported budget reserves all metadata before running any statement', (t) => {
791
+ const fixture = batchFixture(
792
+ t,
793
+ [
794
+ 'SELECT marker FROM t WHERE id = 0',
795
+ 'SELECT marker FROM t',
796
+ 'SELECT marker FROM t',
797
+ ],
798
+ { rows: [{ id: 1, marker: 'x'.repeat(30_000) }] }
799
+ );
800
+ fixture.executed.length = 0;
801
+ let minimum;
802
+ assert.throws(
803
+ () =>
804
+ runBatch(fixture.plan.batch, { runners: fixture.runners, maxBytes: 0 }),
805
+ (error) => {
806
+ minimum = Number(/at least (\d+) bytes/u.exec(error.message)?.[1]);
807
+ return Number.isSafeInteger(minimum) && minimum > 0;
808
+ }
809
+ );
810
+ assert.throws(
811
+ () =>
812
+ runBatch(fixture.plan.batch, {
813
+ runners: fixture.runners,
814
+ maxBytes: minimum - 1,
815
+ }),
816
+ /byte budget/u
817
+ );
818
+ assert.deepEqual(fixture.executed, []);
819
+ for (const maxBytes of [minimum, minimum + 1]) {
820
+ const entries = runBatch(fixture.plan.batch, {
821
+ runners: fixture.runners,
822
+ maxBytes,
823
+ });
824
+ assert.ok(Buffer.byteLength(formatBatch(entries)) <= maxBytes);
825
+ assert.equal(
826
+ entries[0].result.truncated,
827
+ false,
828
+ 'empty is complete, not truncated'
829
+ );
830
+ for (const entry of entries.slice(1)) {
831
+ assert.equal(entry.result.totalRows, 1);
832
+ assert.equal(entry.result.returnedRows, 0);
833
+ assert.equal(entry.result.truncated, true);
834
+ assert.match(
835
+ entry.result.truncation.requiredAction,
836
+ /before using this result/u
837
+ );
838
+ }
839
+ }
840
+ t.diagnostic(`minimum supported three-statement budget: ${minimum} bytes`);
841
+ });
842
+
843
+ test('bounded UTF-8 batches retain complete row values or explicit truncation across encodings', (t) => {
844
+ const cells = ['plain', '"\\\u03b1\u{1f642}', 'line1\nline2\t"\\\u{1f642}'];
845
+ for (const [index, cell] of cells.entries()) {
846
+ for (const count of [1, 2, 8]) {
847
+ const rows = Array.from({ length: 30 }, (_, id) => ({
848
+ id,
849
+ marker: cell.repeat(20 + id),
850
+ }));
851
+ const queries = Array.from(
852
+ { length: count },
853
+ (_, position) =>
854
+ `SELECT * FROM t /* ${position} ${cell.repeat(300 + index)} */`
855
+ );
856
+ const fixture = batchFixture(t, queries, {
857
+ rows,
858
+ maxBytes: 8_000 + index,
859
+ limit: 20,
860
+ });
861
+ for (const entry of fixture.entries) {
862
+ assert.equal(entry.result.totalRows, rows.length);
863
+ assert.equal(entry.result.returnedRows, entry.result.rows.length);
864
+ assert.equal(entry.result.truncated, true);
865
+ assert.deepEqual(
866
+ entry.result.rows.map((row) => ({ ...row })),
867
+ rows.slice(0, entry.result.returnedRows)
868
+ );
869
+ assert.match(entry.result.truncation.reason, /row_limit/u);
870
+ assert.match(entry.result.truncation.requiredAction, /narrower WHERE/u);
871
+ }
872
+ }
873
+ }
874
+ });
875
+
876
+ test('minimum search shares preserve notes and indexed errors beside other statements', (t) => {
877
+ const dir = tempDir();
878
+ t.after(() => rmSync(dir, { recursive: true, force: true }));
879
+ const dbPath = join(dir, 'search.sqlite');
880
+ buildDatabase(
881
+ [
882
+ {
883
+ table: 'info_view_measures',
884
+ grain: '',
885
+ dax: '',
886
+ rows: Array.from({ length: 30 }, (_, index) => ({
887
+ Table: 'Facts',
888
+ Name: `Metric ${index}`,
889
+ Description: 'Shared description',
890
+ Expression: 'measure '.repeat(100),
891
+ })),
892
+ },
893
+ ],
894
+ dbPath
895
+ );
896
+ const { values } = parseCliArgs([
897
+ '--search',
898
+ 'metric',
899
+ '--search',
900
+ '!!!',
901
+ '--sql',
902
+ 'SELECT 1 AS n',
903
+ ]);
904
+ const plan = planStatements(values);
905
+ const runners = runnersFor(dbPath);
906
+ let minimum;
907
+ assert.throws(
908
+ () => runBatch(plan.batch, { runners, maxBytes: 0 }),
909
+ (error) => {
910
+ minimum = Number(/at least (\d+) bytes/u.exec(error.message)?.[1]);
911
+ return Number.isSafeInteger(minimum);
912
+ }
913
+ );
914
+ const entries = runBatch(plan.batch, { runners, maxBytes: minimum });
915
+ assert.ok(Buffer.byteLength(formatBatch(entries)) <= minimum);
916
+ assert.equal(entries[0].result.totalRows, 30);
917
+ assert.equal(entries[0].result.truncated, true);
918
+ assert.equal(entries[0].result.notes.length, 2);
919
+ assert.match(entries[0].result.notes[0], /description omitted/u);
920
+ assert.match(entries[0].result.notes[1], /expression shortened/u);
921
+ assert.match(entries[1].result.error, /search requires/u);
922
+ assert.equal(entries[2].result.rows[0].n, 1);
923
+ assert.equal(batchExitCode(entries), 1);
924
+ });
925
+
926
+ // -- The workload batching exists for --
927
+ //
928
+ // schema-discovery/SKILL.md documents a five-query sequence that an agent runs
929
+ // back to back after one --search. That is the shape this must hold at, not a
930
+ // pair of toy SELECTs, and it is the shape where the owned-measure enrichment
931
+ // on info_view_tables competes for the same budget.
932
+
933
+ function realisticModel() {
934
+ const tables = ['Sales', 'Date', 'Product', 'Customer', 'Store', 'Geography'];
935
+ return [
936
+ {
937
+ table: 'info_view_tables',
938
+ grain: '',
939
+ dax: 'INFO.VIEW.TABLES()',
940
+ rows: tables.map((name) => ({
941
+ Name: name,
942
+ Description: `${name} dimension or fact table for the retail model`,
943
+ IsHidden: false,
944
+ })),
945
+ },
946
+ {
947
+ table: 'info_view_measures',
948
+ grain: '',
949
+ dax: 'INFO.VIEW.MEASURES()',
950
+ rows: [
951
+ // Headline business measures, so the --search pass matches the way it
952
+ // would on a real model rather than returning an empty set.
953
+ ...Array.from({ length: 4 }, (_, index) => ({
954
+ Table: 'Sales',
955
+ Name: `Revenue by Region ${index}`,
956
+ Expression: `CALCULATE(SUM('Sales'[Amount]), ALLEXCEPT('Geography', 'Geography'[Region]))`,
957
+ Description: 'Revenue comparison across region for the period.',
958
+ FormatString: '$#,0',
959
+ DisplayFolder: 'Sales\\Revenue',
960
+ })),
961
+ ...tables.flatMap((table) =>
962
+ Array.from({ length: 12 }, (_, index) => ({
963
+ Table: table,
964
+ Name: `${table} Measure ${index}`,
965
+ Expression: `CALCULATE(SUM('${table}'[Amount]), ALLEXCEPT('${table}', '${table}'[Key]))`,
966
+ Description: `Aggregates ${table} amount for reporting purposes.`,
967
+ FormatString: '$#,0',
968
+ DisplayFolder: `${table}\\Core`,
969
+ }))
970
+ ),
971
+ ],
972
+ },
973
+ {
974
+ table: 'info_view_columns',
975
+ grain: '',
976
+ dax: 'INFO.VIEW.COLUMNS()',
977
+ rows: tables.flatMap((table) =>
978
+ Array.from({ length: 14 }, (_, index) => ({
979
+ Table: table,
980
+ Name: `${table} Column ${index}`,
981
+ DataType: index % 2 === 0 ? 'String' : 'Decimal',
982
+ Description: `Column ${index} of ${table}.`,
983
+ FormatString: '',
984
+ IsHidden: false,
985
+ }))
986
+ ),
987
+ },
988
+ {
989
+ table: 'info_view_relationships',
990
+ grain: '',
991
+ dax: 'INFO.VIEW.RELATIONSHIPS()',
992
+ rows: tables.slice(1).map((table) => ({
993
+ FromTable: 'Sales',
994
+ FromColumn: `${table}Key`,
995
+ ToTable: table,
996
+ ToColumn: `${table}Key`,
997
+ IsActive: true,
998
+ CrossFilteringBehavior: 'OneDirection',
999
+ })),
1000
+ },
1001
+ {
1002
+ table: 'column_statistics',
1003
+ grain: '',
1004
+ dax: 'COLUMNSTATISTICS()',
1005
+ rows: tables.flatMap((table) =>
1006
+ Array.from({ length: 6 }, (_, index) => ({
1007
+ 'Table Name': table,
1008
+ 'Column Name': `${table} Column ${index}`,
1009
+ Min: '1',
1010
+ Max: '99999',
1011
+ Cardinality: 1000 + index,
1012
+ 'Max Length': 64,
1013
+ }))
1014
+ ),
1015
+ },
1016
+ ];
1017
+ }
1018
+
1019
+ test('the documented six-step discovery sequence answers in one shared budget', (t) => {
1020
+ const dir = tempDir();
1021
+ t.after(() => rmSync(dir, { recursive: true, force: true }));
1022
+ const dbPath = join(dir, 'model.sqlite');
1023
+ buildDatabase(realisticModel(), dbPath);
1024
+
1025
+ // Verbatim shapes from schema-discovery/SKILL.md, narrowed the way that file
1026
+ // instructs, plus the --search pass that precedes them.
1027
+ const { values } = parseCliArgs([
1028
+ '--search',
1029
+ 'revenue region comparison',
1030
+ '--sql',
1031
+ "SELECT [Name], [Description], [IsHidden] FROM info_view_tables WHERE lower([Name]) LIKE '%sales%'",
1032
+ '--sql',
1033
+ "SELECT [Table], [Name], [Expression], [Description], [FormatString], [DisplayFolder] FROM info_view_measures WHERE [Table] IN ('Sales', 'Date') ORDER BY [Table], [Name]",
1034
+ '--sql',
1035
+ "SELECT [Table], [Name], [DataType], [Description], [FormatString], [IsHidden] FROM info_view_columns WHERE [Table] IN ('Sales', 'Date') ORDER BY [Table], [Name]",
1036
+ '--sql',
1037
+ "SELECT [FromTable], [FromColumn], [ToTable], [ToColumn], [IsActive], [CrossFilteringBehavior] FROM info_view_relationships WHERE [FromTable] IN ('Sales', 'Date') OR [ToTable] IN ('Sales', 'Date')",
1038
+ '--sql',
1039
+ "SELECT [Table Name], [Column Name], [Min], [Max], [Cardinality], [Max Length] FROM column_statistics WHERE [Table Name] = 'Date'",
1040
+ ]);
1041
+ const plan = planStatements(values);
1042
+ assert.equal(plan.batch.length, 6);
1043
+
1044
+ const entries = runBatch(plan.batch, { runners: runnersFor(dbPath) });
1045
+ const bytes = Buffer.byteLength(formatBatch(entries), 'utf8');
1046
+
1047
+ assert.ok(
1048
+ bytes <= MAX_RESULT_BYTES,
1049
+ `six real discovery statements must fit ${MAX_RESULT_BYTES} bytes, got ${bytes}`
1050
+ );
1051
+ for (const [index, entry] of entries.entries()) {
1052
+ assert.equal(entry.result.ok, true, `statement ${index} failed`);
1053
+ assert.ok(
1054
+ entry.result.returnedRows > 0,
1055
+ `statement ${index} was starved to zero rows`
1056
+ );
1057
+ }
1058
+
1059
+ // The info_view_tables statement still carries the owned-measure enrichment,
1060
+ // so P5 and the H2 preview trim coexist inside the same shared budget.
1061
+ const tablesRow = entries[1].result.rows[0];
1062
+ assert.equal(tablesRow.ownedMeasureCount, 16);
1063
+ assert.equal(tablesRow.ownedMeasuresTruncated, true);
1064
+ assert.equal(tablesRow.ownedMeasuresNeedDetail, true);
1065
+ assert.equal(tablesRow.ownedMeasures.length, 10);
1066
+ assert.deepEqual(Object.keys(tablesRow.ownedMeasures[0]), [
1067
+ 'name',
1068
+ 'displayFolder',
1069
+ ]);
1070
+
1071
+ // And the search block reports its own trimming rather than silently
1072
+ // shortening what it printed.
1073
+ assert.ok(entries[0].result.totalRows > 0);
1074
+
1075
+ t.diagnostic(
1076
+ `six documented discovery statements: ${bytes}/${MAX_RESULT_BYTES} UTF-8 bytes`
1077
+ );
1078
+ });
1079
+
1080
+ // -- A zero-result search must not be a silent dead end (F1) --
1081
+
1082
+ test('a term this model has never heard of does not empty the whole result', () => {
1083
+ const dir = tempDir();
1084
+ try {
1085
+ const dbPath = join(dir, 'm.sqlite');
1086
+ buildDatabase(
1087
+ [
1088
+ {
1089
+ table: 'info_view_measures',
1090
+ grain: '',
1091
+ dax: '',
1092
+ rows: [
1093
+ {
1094
+ Table: 'Sales',
1095
+ Name: 'Total Sales Amount',
1096
+ Expression: 'SUM(1)',
1097
+ },
1098
+ { Table: 'Sales', Name: 'Sales Amount LY', Expression: 'SUM(2)' },
1099
+ ],
1100
+ },
1101
+ ],
1102
+ dbPath
1103
+ );
1104
+
1105
+ // The strict AND is unchanged and still precise.
1106
+ assert.equal(searchCatalog(dbPath, 'sales amount').totalRows, 2);
1107
+
1108
+ // "by" appears nowhere in this catalog, so upstream returns zero rows for
1109
+ // the user's natural phrasing. The relaxed pass answers instead.
1110
+ const natural = searchCatalog(dbPath, 'sales amount by');
1111
+ assert.equal(natural.ok, true);
1112
+ assert.equal(natural.totalRows, 2);
1113
+ assert.match(natural.notes[0], /no object matched all 3 terms/u);
1114
+ assert.match(natural.notes[0], /"by"/u);
1115
+ assert.match(natural.notes[0], /was dropped/u);
1116
+ } finally {
1117
+ rmSync(dir, { recursive: true, force: true });
1118
+ }
1119
+ });
1120
+
1121
+ test('a search that already found rows is never relaxed', () => {
1122
+ const rows = [
1123
+ { kind: 'measure', object_name: 'Revenue', parent_name: 'Sales' },
1124
+ { kind: 'measure', object_name: 'Cost', parent_name: 'Sales' },
1125
+ ];
1126
+ const strict = rankSearchWithFallback(rows, 'sales revenue');
1127
+ assert.deepEqual(
1128
+ strict.rows,
1129
+ rankSemanticSearchRows(rows, 'sales revenue'),
1130
+ 'the successful path must be byte-identical to plain ranking'
1131
+ );
1132
+ assert.deepEqual(strict.notes, [], 'and must say nothing extra');
1133
+ });
1134
+
1135
+ test('a search that matches nothing at all says how to retry', () => {
1136
+ const rows = [{ kind: 'measure', object_name: 'Revenue', parent_name: 'S' }];
1137
+ const missed = rankSearchWithFallback(rows, 'zzz qqq');
1138
+ assert.deepEqual(missed.rows, []);
1139
+ assert.equal(missed.notes.length, 1);
1140
+ assert.match(missed.notes[0], /retry with fewer and more specific terms/u);
1141
+ });
1142
+
1143
+ test('relaxation only drops terms absent from this model, not a word list', () => {
1144
+ // "by" is a stop word in English and nothing in the other direction: the rule
1145
+ // has to be about the catalog, or it stops working in another locale.
1146
+ const rows = [
1147
+ { kind: 'column', object_name: 'Byte Size', parent_name: 'Files' },
1148
+ ];
1149
+ assert.deepEqual(matchedSearchTokens(rows, ['by']), ['by']);
1150
+ assert.deepEqual(matchedSearchTokens(rows, ['by', 'zzz']), ['by']);
1151
+ // A term the model does contain is kept even when it reads like noise, so a
1152
+ // query of only such terms still answers strictly.
1153
+ assert.equal(rankSearchWithFallback(rows, 'by').rows.length, 1);
1154
+ });
1155
+
1156
+ test('the relaxation note is bounded, so the batch reserve can cover it', (t) => {
1157
+ // The note quotes query terms, and the query is agent-supplied. An unbounded
1158
+ // note would overrun the share reserved for it before any row is printed.
1159
+ const dir = tempDir();
1160
+ t.after(() => rmSync(dir, { recursive: true, force: true }));
1161
+ const dbPath = join(dir, 'm.sqlite');
1162
+ buildDatabase(
1163
+ [
1164
+ {
1165
+ table: 'info_view_measures',
1166
+ grain: '',
1167
+ dax: '',
1168
+ rows: [{ Table: 'Sales', Name: 'Revenue', Expression: 'SUM(1)' }],
1169
+ },
1170
+ ],
1171
+ dbPath
1172
+ );
1173
+
1174
+ const absurd = `revenue ${Array.from({ length: 50 }, (_, index) => `${'z'.repeat(200)}${index}`).join(' ')}`;
1175
+ const out = searchCatalog(dbPath, absurd);
1176
+ assert.equal(out.totalRows, 1, 'the one real term still answers');
1177
+ assert.ok(
1178
+ Buffer.byteLength(out.notes.join(''), 'utf8') < 1_000,
1179
+ `the note must stay bounded, got ${out.notes.join('').length} chars`
1180
+ );
1181
+
1182
+ // And the whole thing still fits a batch beside another statement.
1183
+ const entries = runBatch(
1184
+ [
1185
+ { kind: 'search', query: absurd },
1186
+ { kind: 'sql', query: 'SELECT 1 AS n' },
1187
+ ],
1188
+ { runners: runnersFor(dbPath) }
1189
+ );
1190
+ assert.equal(entries[0].result.totalRows, 1);
1191
+ assert.equal(entries[1].result.rows[0].n, 1);
1192
+ assert.ok(
1193
+ Buffer.byteLength(formatBatch(entries), 'utf8') <= MAX_RESULT_BYTES
1194
+ );
1195
+ });
1196
+
1197
+ // -- Owned-measure detail is a per-row signal, not a banner (F2) --
1198
+
1199
+ test('a table with no owned measures is not told to go fetch them', () => {
1200
+ const dir = tempDir();
1201
+ try {
1202
+ const dbPath = join(dir, 'm.sqlite');
1203
+ buildDatabase(
1204
+ [
1205
+ {
1206
+ table: 'info_view_tables',
1207
+ grain: '',
1208
+ dax: '',
1209
+ rows: [{ Name: 'Sales' }, { Name: 'Sales Territory' }],
1210
+ },
1211
+ {
1212
+ table: 'info_view_measures',
1213
+ grain: '',
1214
+ dax: '',
1215
+ rows: [{ Table: 'Sales', Name: 'Revenue', DisplayFolder: 'Finance' }],
1216
+ },
1217
+ ],
1218
+ dbPath
1219
+ );
1220
+
1221
+ const out = runSql(dbPath, 'SELECT [Name] FROM info_view_tables');
1222
+ const byName = Object.fromEntries(out.rows.map((row) => [row.Name, row]));
1223
+
1224
+ assert.equal(byName.Sales.ownedMeasureCount, 1);
1225
+ assert.equal(byName.Sales.ownedMeasuresNeedDetail, true);
1226
+
1227
+ assert.equal(byName['Sales Territory'].ownedMeasureCount, 0);
1228
+ assert.equal(
1229
+ byName['Sales Territory'].ownedMeasuresNeedDetail,
1230
+ false,
1231
+ 'there is nothing to fetch, so do not send the agent to fetch it'
1232
+ );
1233
+ // The key set stays identical across rows either way.
1234
+ assert.deepEqual(
1235
+ Object.keys(byName.Sales),
1236
+ Object.keys(byName['Sales Territory'])
1237
+ );
1238
+ } finally {
1239
+ rmSync(dir, { recursive: true, force: true });
1240
+ }
1241
+ });
1242
+
1243
+ // -- Publication must stay visible, and a failed open is one statement's
1244
+ // problem, not the batch's (A4) --
1245
+
1246
+ test('publication never leaves the destination missing', () => {
1247
+ const dir = tempDir();
1248
+ try {
1249
+ const dbPath = join(dir, 'model.sqlite');
1250
+ const catalogPath = join(dir, 'catalog.md');
1251
+ writeFileSync(dbPath, 'previous database', 'utf8');
1252
+ writeFileSync(catalogPath, 'previous catalog', 'utf8');
1253
+ const temporaryDb = join(dir, 'next.sqlite.tmp');
1254
+ const temporaryCatalog = join(dir, 'next.md.tmp');
1255
+ writeFileSync(temporaryDb, 'next database', 'utf8');
1256
+ writeFileSync(temporaryCatalog, 'next catalog', 'utf8');
1257
+
1258
+ // Observe the live path after every filesystem step. Backing the previous
1259
+ // generation up by renaming it away made this false in the middle, and a
1260
+ // reader landing there got "unable to open database file".
1261
+ const seen = [];
1262
+ const observe =
1263
+ (operation) =>
1264
+ (...args) => {
1265
+ const result = operation(...args);
1266
+ seen.push(existsSync(dbPath));
1267
+ return result;
1268
+ };
1269
+ publishFiles(
1270
+ [
1271
+ { temporaryPath: temporaryDb, destinationPath: dbPath },
1272
+ { temporaryPath: temporaryCatalog, destinationPath: catalogPath },
1273
+ ],
1274
+ { rename: observe(renameSync), link: observe(linkSync) }
1275
+ );
1276
+
1277
+ assert.ok(seen.length >= 4, 'the publication really ran');
1278
+ assert.ok(
1279
+ seen.every(Boolean),
1280
+ `the database path must exist at every step, got ${JSON.stringify(seen)}`
1281
+ );
1282
+ assert.equal(readFileSync(dbPath, 'utf8'), 'next database');
1283
+ assert.equal(readFileSync(catalogPath, 'utf8'), 'next catalog');
1284
+ } finally {
1285
+ rmSync(dir, { recursive: true, force: true });
1286
+ }
1287
+ });
1288
+
1289
+ test('a statement that cannot open the catalog does not discard the batch', () => {
1290
+ const dir = tempDir();
1291
+ try {
1292
+ const dbPath = join(dir, 'model.sqlite');
1293
+ buildDatabase(
1294
+ [{ table: 't', grain: '', dax: '', rows: [{ n: 1 }] }],
1295
+ dbPath
1296
+ );
1297
+
1298
+ // Model a reader that lands on the catalog exactly while it is being
1299
+ // replaced: the first statement succeeds, then the file goes away.
1300
+ const runners = {
1301
+ search: (query, options) => searchCatalog(dbPath, query, options),
1302
+ sql: (query, options) => {
1303
+ if (query.includes('second')) {
1304
+ rmSync(dbPath, { force: true });
1305
+ }
1306
+ return runSql(dbPath, query, options);
1307
+ },
1308
+ };
1309
+ const entries = runBatch(
1310
+ [
1311
+ { kind: 'sql', query: 'SELECT n FROM t' },
1312
+ { kind: 'sql', query: 'SELECT n FROM t -- second' },
1313
+ { kind: 'sql', query: 'SELECT n FROM t' },
1314
+ ],
1315
+ { runners }
1316
+ );
1317
+
1318
+ assert.equal(entries.length, 3, 'every statement is still accounted for');
1319
+ assert.equal(entries[0].result.ok, true);
1320
+ assert.equal(
1321
+ entries[0].result.rows[0].n,
1322
+ 1,
1323
+ 'its work was not thrown away'
1324
+ );
1325
+ assert.equal(entries[1].result.ok, false);
1326
+ assert.match(entries[1].result.error, /unable to open database file/u);
1327
+ assert.equal(batchExitCode(entries), 1);
1328
+ } finally {
1329
+ rmSync(dir, { recursive: true, force: true });
1330
+ }
1331
+ });
1332
+
1333
+ // -- Cuts must not split a surrogate pair (A5) --
1334
+
1335
+ test('an expression cut mid-emoji still emits well-formed text', () => {
1336
+ // 119 ASCII characters then an emoji: the slice lands between the pair, and
1337
+ // there is no space, comma or paren in the head for the boundary rule to
1338
+ // fall back on.
1339
+ const expression = `${'a'.repeat(119)}\u{1F600} trailing text`;
1340
+ const summary = summarizeSearchExpression(expression);
1341
+
1342
+ assert.equal(summary.isWellFormed(), true, 'no lone surrogate may survive');
1343
+ assert.equal(
1344
+ Buffer.from(summary, 'utf8').toString('utf8'),
1345
+ summary,
1346
+ 'the summary must round-trip through UTF-8'
1347
+ );
1348
+ assert.ok(!summary.includes('\uFFFD'), 'and must not contain U+FFFD');
1349
+ });
1350
+
1351
+ test('a BMP-only expression is unaffected by the surrogate rule', () => {
1352
+ // Every character here is one UTF-16 unit, so the cut was already safe and
1353
+ // must stay exactly where it was.
1354
+ const expression = '\u6f22\u5b57'.repeat(200);
1355
+ const summary = summarizeSearchExpression(expression);
1356
+
1357
+ assert.equal(summary.isWellFormed(), true);
1358
+ assert.equal(
1359
+ summary.split(' \u2026 ')[0],
1360
+ expression.slice(0, SEARCH_EXPRESSION_LIMIT),
1361
+ 'the BMP cut is untouched'
1362
+ );
1363
+ });
1364
+
1365
+ // -- Note budgets are counted in UTF-8 bytes, not UTF-16 length (A1) --
1366
+
1367
+ test('eight non-Latin searches stay inside the reserved note budget', (t) => {
1368
+ const dir = tempDir();
1369
+ t.after(() => rmSync(dir, { recursive: true, force: true }));
1370
+ const dbPath = join(dir, 'm.sqlite');
1371
+ buildDatabase(
1372
+ [
1373
+ {
1374
+ table: 'info_view_measures',
1375
+ grain: '',
1376
+ dax: '',
1377
+ rows: Array.from({ length: 20 }, (_, index) => ({
1378
+ Table: '\u58f2\u4e0a',
1379
+ Name: `\u6307\u6a19 ${index}`,
1380
+ Expression: 'SUM(1)',
1381
+ })),
1382
+ },
1383
+ ],
1384
+ dbPath
1385
+ );
1386
+
1387
+ // One term the model has (so the relaxed pass answers and emits the note),
1388
+ // plus terms it does not. Every term is 3 UTF-8 bytes per character, which
1389
+ // is what a UTF-16 length cap under a byte reserve gets wrong.
1390
+ const absent = Array.from(
1391
+ { length: 6 },
1392
+ (_, index) => `${'\u3042'.repeat(40)}${index}`
1393
+ ).join(' ');
1394
+ const batch = Array.from({ length: MAX_BATCH_STATEMENTS }, () => ({
1395
+ kind: 'search',
1396
+ query: `\u6307\u6a19 ${absent}`,
1397
+ }));
1398
+
1399
+ const entries = runBatch(batch, { runners: runnersFor(dbPath) });
1400
+ const bytes = Buffer.byteLength(formatBatch(entries), 'utf8');
1401
+
1402
+ assert.equal(entries.length, MAX_BATCH_STATEMENTS);
1403
+ assert.ok(bytes <= MAX_RESULT_BYTES, `batch emitted ${bytes} bytes`);
1404
+ for (const entry of entries) {
1405
+ assert.equal(entry.result.ok, true);
1406
+ assert.equal(entry.result.totalRows, 20, 'the relaxed pass still answered');
1407
+ assert.match(entry.result.notes[0], /no object matched all/u);
1408
+ assert.ok(
1409
+ Buffer.byteLength(entry.result.notes[0], 'utf8') <= 400,
1410
+ 'each note is bounded in bytes, whatever script it quotes'
1411
+ );
1412
+ }
1413
+ t.diagnostic(`eight CJK searches: ${bytes}/${MAX_RESULT_BYTES} bytes`);
1414
+ });
1415
+
1416
+ // -- Containment has to survive a link (A2) --
1417
+
1418
+ test('a --file reached through an in-app link to outside is refused', (t) => {
1419
+ const app = tempDir();
1420
+ const outside = tempDir();
1421
+ t.after(() => {
1422
+ rmSync(app, { recursive: true, force: true });
1423
+ rmSync(outside, { recursive: true, force: true });
1424
+ });
1425
+ writeFileSync(join(outside, 'secret.txt'), 'NOT-A-REAL-CREDENTIAL', 'utf8');
1426
+
1427
+ const linkPath = join(app, 'link');
1428
+ try {
1429
+ symlinkSync(outside, linkPath, 'junction');
1430
+ } catch {
1431
+ t.skip('this filesystem or account cannot create a directory link');
1432
+ return;
1433
+ }
1434
+
1435
+ // Lexically the path is under the app root, so a resolve+startsWith check
1436
+ // passes it. Containment that a link walks around is not containment.
1437
+ const escaped = join(linkPath, 'secret.txt');
1438
+ assert.ok(escaped.startsWith(app + sep), 'the test targets the lexical gap');
1439
+
1440
+ const plan = planStatements({ file: [escaped], root: app });
1441
+ assert.match(plan.error, /--file must be inside the app/u);
1442
+ assert.ok(!plan.error.includes('NOT-A-REAL-CREDENTIAL'));
1443
+ });
1444
+
1445
+ test('an ordinary in-app --file still resolves through a linked app root', (t) => {
1446
+ // tmpdir() is itself a link on some platforms, so the root must be resolved
1447
+ // too or every legitimate read would be refused.
1448
+ const app = tempDir();
1449
+ t.after(() => rmSync(app, { recursive: true, force: true }));
1450
+ writeFileSync(join(app, 'q.sql'), 'SELECT 1', 'utf8');
1451
+
1452
+ const plan = planStatements({ file: ['q.sql'], root: app });
1453
+ assert.equal(plan.mode, 'batch');
1454
+ assert.equal(plan.batch[0].query, 'SELECT 1');
1455
+ assert.equal(plan.batch[0].source, 'q.sql');
1456
+ });
1457
+
1458
+ // -- A reclaimer must not remove a lock it did not judge (A3) --
1459
+
1460
+ test('releasing a lock does not remove one another process now holds', async () => {
1461
+ const dir = tempDir();
1462
+ try {
1463
+ const lockPath = join(dir, '.refresh.lock');
1464
+ await withCatalogLock(dir, async () => {
1465
+ // Another process reclaimed and took the lock while we were working.
1466
+ writeFileSync(
1467
+ join(lockPath, 'owner.json'),
1468
+ JSON.stringify({ pid: process.pid, token: 'someone-else' }),
1469
+ 'utf8'
1470
+ );
1471
+ });
1472
+
1473
+ assert.equal(
1474
+ existsSync(lockPath),
1475
+ true,
1476
+ 'releasing must not delete a live lock that replaced ours'
1477
+ );
1478
+ assert.equal(
1479
+ JSON.parse(readFileSync(join(lockPath, 'owner.json'), 'utf8')).token,
1480
+ 'someone-else'
1481
+ );
1482
+ } finally {
1483
+ rmSync(dir, { recursive: true, force: true });
1484
+ }
1485
+ });
1486
+
1487
+ test('a stale reclaim re-reads before removing, so it cannot steal a new lock', async () => {
1488
+ const dir = tempDir();
1489
+ try {
1490
+ mkdirSync(dir, { recursive: true });
1491
+ const lockPath = join(dir, '.refresh.lock');
1492
+ mkdirSync(lockPath);
1493
+ writeFileSync(
1494
+ join(lockPath, 'owner.json'),
1495
+ JSON.stringify({ pid: 999_999, token: 'abandoned' }),
1496
+ 'utf8'
1497
+ );
1498
+
1499
+ let checks = 0;
1500
+ await assert.rejects(
1501
+ withCatalogLock(dir, async () => 'must not run', {
1502
+ timeoutMs: 50,
1503
+ pollMs: 1,
1504
+ processAlive: () => {
1505
+ checks += 1;
1506
+ if (checks === 1) {
1507
+ // Another reclaimer wins the race and installs its own live lock
1508
+ // between our decision and our removal.
1509
+ writeFileSync(
1510
+ join(lockPath, 'owner.json'),
1511
+ JSON.stringify({ pid: process.pid, token: 'winner' }),
1512
+ 'utf8'
1513
+ );
1514
+ }
1515
+ return false;
1516
+ },
1517
+ }),
1518
+ /Timed out waiting/u
1519
+ );
1520
+
1521
+ assert.equal(existsSync(lockPath), true, 'the winner keeps its lock');
1522
+ assert.equal(
1523
+ JSON.parse(readFileSync(join(lockPath, 'owner.json'), 'utf8')).token,
1524
+ 'winner'
1525
+ );
1526
+ } finally {
1527
+ rmSync(dir, { recursive: true, force: true });
1528
+ }
1529
+ });
1530
+
1531
+ test('publication waits out a reader holding the destination open', () => {
1532
+ // Windows reports a sharing violation under more than one code, so all three
1533
+ // have to be waited out rather than whichever one a local run happened to
1534
+ // produce.
1535
+ for (const code of ['EPERM', 'EBUSY', 'EACCES']) {
1536
+ const dir = tempDir();
1537
+ try {
1538
+ const dbPath = join(dir, 'model.sqlite');
1539
+ const temporaryPath = join(dir, 'next.tmp');
1540
+ writeFileSync(dbPath, 'previous', 'utf8');
1541
+ writeFileSync(temporaryPath, 'next', 'utf8');
1542
+
1543
+ let attempts = 0;
1544
+ const rename = (from, to) => {
1545
+ if (to === dbPath && attempts++ < 3) {
1546
+ const error = new Error(`${code}: rename blocked`);
1547
+ error.code = code;
1548
+ throw error;
1549
+ }
1550
+ renameSync(from, to);
1551
+ };
1552
+
1553
+ publishFiles([{ temporaryPath, destinationPath: dbPath }], {
1554
+ rename,
1555
+ retry: { attempts: 10, delayMs: 0, sleep: () => {} },
1556
+ });
1557
+
1558
+ assert.equal(attempts, 4, `${code} retried rather than giving up`);
1559
+ assert.equal(readFileSync(dbPath, 'utf8'), 'next');
1560
+ } finally {
1561
+ rmSync(dir, { recursive: true, force: true });
1562
+ }
1563
+ }
1564
+ });
1565
+
1566
+ test('publication still gives up once the retry bound is reached', () => {
1567
+ const dir = tempDir();
1568
+ try {
1569
+ const dbPath = join(dir, 'model.sqlite');
1570
+ const temporaryPath = join(dir, 'next.tmp');
1571
+ writeFileSync(dbPath, 'previous', 'utf8');
1572
+ writeFileSync(temporaryPath, 'next', 'utf8');
1573
+
1574
+ const rename = (from, to) => {
1575
+ if (to === dbPath) {
1576
+ const error = new Error('EPERM: operation not permitted, rename');
1577
+ error.code = 'EPERM';
1578
+ throw error;
1579
+ }
1580
+ renameSync(from, to);
1581
+ };
1582
+
1583
+ assert.throws(
1584
+ () =>
1585
+ publishFiles([{ temporaryPath, destinationPath: dbPath }], {
1586
+ rename,
1587
+ retry: { attempts: 3, delayMs: 0, sleep: () => {} },
1588
+ }),
1589
+ /EPERM/u
1590
+ );
1591
+ // The previous generation is restored, not left behind a backup link.
1592
+ assert.equal(readFileSync(dbPath, 'utf8'), 'previous');
1593
+ } finally {
1594
+ rmSync(dir, { recursive: true, force: true });
1595
+ }
1596
+ });
1597
+
1598
+ test('a rename failure that is not a sharing violation is not retried', () => {
1599
+ let attempts = 0;
1600
+ const rename = () => {
1601
+ attempts += 1;
1602
+ const error = new Error('ENOSPC: no space left on device');
1603
+ error.code = 'ENOSPC';
1604
+ throw error;
1605
+ };
1606
+ assert.throws(
1607
+ () =>
1608
+ publishFiles([{ temporaryPath: 'a', destinationPath: 'b' }], {
1609
+ exists: () => false,
1610
+ rename,
1611
+ retry: { attempts: 10, delayMs: 0, sleep: () => {} },
1612
+ }),
1613
+ /ENOSPC/u
1614
+ );
1615
+ assert.equal(attempts, 1, 'only a sharing violation is worth waiting out');
1616
+ });
1617
+
1618
+ // -- An over-budget expression must still be recoverable --
1619
+
1620
+ test('an expression larger than the budget is reconstructible in chunks', () => {
1621
+ const dir = tempDir();
1622
+ try {
1623
+ const dbPath = join(dir, 'm.sqlite');
1624
+ // Non-ASCII, quotes and newlines, so a reconstruction that mangles
1625
+ // encoding or drops the tail cannot pass. Well past MAX_RESULT_BYTES.
1626
+ const unit = `"\u6f22\u5b57" & \u{1F600} & '\u00e9\u00e0'\nRETURN `;
1627
+ const expression = `SWITCH(USERCULTURE(), ${unit.repeat(2_000)})`;
1628
+ assert.ok(
1629
+ Buffer.byteLength(expression, 'utf8') > MAX_RESULT_BYTES * 2,
1630
+ 'the fixture has to actually exceed the budget'
1631
+ );
1632
+ buildDatabase(
1633
+ [
1634
+ {
1635
+ table: 'info_view_measures',
1636
+ grain: '',
1637
+ dax: '',
1638
+ rows: [
1639
+ { Table: 'Sales', Name: 'Localized Label', Expression: expression },
1640
+ ],
1641
+ },
1642
+ ],
1643
+ dbPath
1644
+ );
1645
+
1646
+ const where =
1647
+ "FROM info_view_measures WHERE [Table] = 'Sales' " +
1648
+ "AND [Name] = 'Localized Label'";
1649
+
1650
+ // The dead end the documented recovery used to walk into: a single cell
1651
+ // has nothing left to narrow, so the row matches but cannot be returned.
1652
+ const whole = runSql(dbPath, `SELECT [Expression] ${where}`);
1653
+ assert.equal(whole.ok, true);
1654
+ assert.equal(whole.totalRows, 1);
1655
+ assert.equal(
1656
+ whole.returnedRows,
1657
+ 0,
1658
+ 'whole rows are dropped, not shortened'
1659
+ );
1660
+ assert.equal(whole.truncated, true);
1661
+
1662
+ const size = runSql(
1663
+ dbPath,
1664
+ `SELECT length([Expression]) AS chars ${where}`
1665
+ );
1666
+ const chars = size.rows[0].chars;
1667
+ // SQLite counts code points; the size in the search summary comes from
1668
+ // JavaScript and counts UTF-16 units, so an expression with astral
1669
+ // characters reports fewer here. length() is the authoritative bound for
1670
+ // substr() offsets, which are themselves code-point based.
1671
+ assert.ok(chars > 0);
1672
+ assert.equal(
1673
+ chars,
1674
+ [...expression].length,
1675
+ 'length() counts code points, not UTF-16 units'
1676
+ );
1677
+ assert.ok(chars < expression.length, 'the fixture exercises that gap');
1678
+
1679
+ const stride = 1000;
1680
+ const parts = [];
1681
+ for (let offset = 1; offset <= chars; offset += stride) {
1682
+ const chunk = runSql(
1683
+ dbPath,
1684
+ `SELECT substr([Expression], ${offset}, ${stride}) AS part ${where}`
1685
+ );
1686
+ assert.equal(chunk.ok, true, `chunk at ${offset} failed`);
1687
+ assert.equal(chunk.truncated, false, `chunk at ${offset} was truncated`);
1688
+ assert.equal(chunk.returnedRows, 1, `chunk at ${offset} was dropped`);
1689
+ parts.push(chunk.rows[0].part);
1690
+ }
1691
+
1692
+ assert.equal(
1693
+ parts.join(''),
1694
+ expression,
1695
+ 'the reconstruction must equal the original exactly, tail included'
1696
+ );
1697
+ } finally {
1698
+ rmSync(dir, { recursive: true, force: true });
1699
+ }
1700
+ });
1701
+
1702
+ // -- Search payload trimming --
1703
+
1704
+ test('a short expression survives verbatim apart from whitespace collapsing', () => {
1705
+ assert.equal(
1706
+ summarizeSearchExpression('SUM(Sales[Amount])'),
1707
+ 'SUM(Sales[Amount])'
1708
+ );
1709
+ assert.equal(
1710
+ summarizeSearchExpression('SUM(\n Sales[Amount]\n)'),
1711
+ 'SUM( Sales[Amount] )'
1712
+ );
1713
+ assert.equal(summarizeSearchExpression(null), '');
1714
+ });
1715
+
1716
+ test('a long expression keeps a readable head and says where the rest is', () => {
1717
+ const long = `SWITCH(USERCULTURE(), ${Array.from(
1718
+ { length: 40 },
1719
+ (_, index) => `"loc-${index}", "a translated sentence"`
1720
+ ).join(', ')})`;
1721
+ const summary = summarizeSearchExpression(long);
1722
+
1723
+ assert.ok(
1724
+ summary.length < long.length / 4,
1725
+ 'the summary must actually be smaller'
1726
+ );
1727
+ assert.match(
1728
+ summary,
1729
+ /^SWITCH\(USERCULTURE\(\)/u,
1730
+ 'the head identifies what the measure does'
1731
+ );
1732
+ assert.match(
1733
+ summary,
1734
+ new RegExp(`${long.length} chars`, 'u'),
1735
+ 'the original size is stated'
1736
+ );
1737
+ assert.match(
1738
+ summary,
1739
+ /substr\(\[Expression\], <offset>, 1000\)/u,
1740
+ 'the named next step must be one that can actually succeed'
1741
+ );
1742
+ });
1743
+
1744
+ test('the head is cut at a token boundary when the text offers one', () => {
1745
+ // Only a guarantee when a separator falls in the second half of the head; a
1746
+ // single unbroken token has nowhere to cut and is truncated as-is.
1747
+ const separated = `CALCULATE(${'SUM(Sales[Amount]), '.repeat(30)})`;
1748
+ const head = summarizeSearchExpression(separated).split(' \u2026 ')[0];
1749
+ // The cut lands ON the separator, so the head ends with a whole token and the
1750
+ // next source character is the separator itself - `SUM` rather than `SU`.
1751
+ // Asserting the head's own last character instead would be wrong: a cut at
1752
+ // `(` leaves `SUM`, which ends in no separator at all.
1753
+ assert.ok(separated.startsWith(head), 'the head is a prefix of the source');
1754
+ assert.match(
1755
+ separated[head.length],
1756
+ /[\s,(]/u,
1757
+ 'the character the cut landed on is a separator'
1758
+ );
1759
+ assert.ok(head.length <= SEARCH_EXPRESSION_LIMIT);
1760
+
1761
+ const unbroken = 'A'.repeat(400);
1762
+ const unbrokenHead = summarizeSearchExpression(unbroken).split(' \u2026 ')[0];
1763
+ assert.equal(unbrokenHead, 'A'.repeat(SEARCH_EXPRESSION_LIMIT));
1764
+ });
1765
+
1766
+ test('expressions at the limit are untouched, one character over are not', () => {
1767
+ const atLimit = 'A'.repeat(SEARCH_EXPRESSION_LIMIT);
1768
+ assert.equal(summarizeSearchExpression(atLimit), atLimit);
1769
+ assert.notEqual(summarizeSearchExpression(`${atLimit}B`), `${atLimit}B`);
1770
+ });
1771
+
1772
+ test('boilerplate is a description repeated across the model, not merely a common word', () => {
1773
+ const shared = 'Localized label. Do not use for calculations.';
1774
+ const rows = [
1775
+ ...Array.from({ length: 20 }, () => ({ description: shared })),
1776
+ { description: 'Net revenue after returns' },
1777
+ { description: 'Gross margin percentage' },
1778
+ { description: '' },
1779
+ ];
1780
+ const boilerplate = boilerplateDescriptions(rows);
1781
+ assert.equal(boilerplate.has(shared), true);
1782
+ assert.equal(boilerplate.has('Net revenue after returns'), false);
1783
+ assert.equal(boilerplate.size, 1);
1784
+ });
1785
+
1786
+ test('a description repeated only a handful of times is left alone', () => {
1787
+ const rows = Array.from({ length: 4 }, () => ({
1788
+ description: 'Shared by a few',
1789
+ }));
1790
+ assert.equal(boilerplateDescriptions(rows).size, 0);
1791
+ });
1792
+
1793
+ test('the row floor is exact: one below keeps the description, one at it drops', () => {
1794
+ // Both thresholds are shipped constants, so the boundary is part of the
1795
+ // contract rather than an implementation detail. Asserting it against the
1796
+ // exported value means retuning the constant cannot silently change behaviour
1797
+ // without this test being read.
1798
+ const shared = 'Repeated';
1799
+ const below = Array.from({ length: BOILERPLATE_MIN_ROWS - 1 }, () => ({
1800
+ description: shared,
1801
+ }));
1802
+ const at = Array.from({ length: BOILERPLATE_MIN_ROWS }, () => ({
1803
+ description: shared,
1804
+ }));
1805
+
1806
+ assert.equal(boilerplateDescriptions(below).has(shared), false);
1807
+ assert.equal(boilerplateDescriptions(at).has(shared), true);
1808
+ });
1809
+
1810
+ test('a description on many rows is still kept when it describes a small share of them', () => {
1811
+ // The count alone is not enough. On a large model a genuine description can
1812
+ // easily appear on eight measures while still telling them apart from the
1813
+ // other hundreds, and suppressing it there would remove information rather
1814
+ // than noise.
1815
+ const shared = 'Appears often but not everywhere';
1816
+ const repeats = BOILERPLATE_MIN_ROWS;
1817
+ // Sized so the share lands clearly under the floor rather than adjacent to it.
1818
+ const others = Math.ceil(repeats / BOILERPLATE_MIN_SHARE);
1819
+ const rows = [
1820
+ ...Array.from({ length: repeats }, () => ({ description: shared })),
1821
+ ...Array.from({ length: others }, (_, index) => ({
1822
+ description: `Distinct ${index}`,
1823
+ })),
1824
+ ];
1825
+
1826
+ const share = repeats / (repeats + others);
1827
+ assert.ok(share < BOILERPLATE_MIN_SHARE, `share ${share} is below the floor`);
1828
+ assert.equal(boilerplateDescriptions(rows).has(shared), false);
1829
+ });
1830
+
1831
+ test('the share floor is exact: at it the description drops, just under it survives', () => {
1832
+ // The row floor alone is asserted above. This pins the other half of the
1833
+ // conjunction, so neither threshold can be retuned without a test being read.
1834
+ const shared = 'Repeated';
1835
+ const repeats = BOILERPLATE_MIN_ROWS;
1836
+ const describedAtFloor = Math.floor(repeats / BOILERPLATE_MIN_SHARE);
1837
+ const atFloor = [
1838
+ ...Array.from({ length: repeats }, () => ({ description: shared })),
1839
+ ...Array.from({ length: describedAtFloor - repeats }, (_, index) => ({
1840
+ description: `Distinct ${index}`,
1841
+ })),
1842
+ ];
1843
+ const justUnder = [
1844
+ ...atFloor,
1845
+ { description: `Distinct ${describedAtFloor}` },
1846
+ ];
1847
+
1848
+ assert.equal(repeats / describedAtFloor >= BOILERPLATE_MIN_SHARE, true);
1849
+ assert.equal(boilerplateDescriptions(atFloor).has(shared), true);
1850
+
1851
+ assert.equal(repeats / (describedAtFloor + 1) < BOILERPLATE_MIN_SHARE, true);
1852
+ assert.equal(boilerplateDescriptions(justUnder).has(shared), false);
1853
+ });
1854
+
1855
+ // -- Trimming has to be proportional to the model, not tuned to one of them --
1856
+
1857
+ test('trimming is inert on a model whose expressions are already short', () => {
1858
+ // Measured upstream against three bound models by reading INFO.VIEW.MEASURES()
1859
+ // directly:
1860
+ //
1861
+ // model measures median max over the cap chars removed
1862
+ // EYReport_PBIR 2560 400 104916 2366 (92.4%) 81.3%
1863
+ // RetailBenchmark 56 38.5 205 1 (1.8%) 3.5%
1864
+ // HRBenchmark 31 34 111 0 (0.0%) 0.0%
1865
+ //
1866
+ // The cap earns its keep on the model that has bytes to give up and does
1867
+ // nothing on the ones that do not. This encodes the HR shape so a future
1868
+ // retune cannot start charging small models for a problem they do not have.
1869
+ const catalog = Array.from({ length: 31 }, (_, index) => ({
1870
+ object_name: `Measure ${index}`,
1871
+ description: `Distinct description ${index}`,
1872
+ expression: 'A'.repeat(34 + (index % 3)),
1873
+ }));
1874
+
1875
+ const { rows, notes } = trimSearchRows(catalog, catalog);
1876
+
1877
+ assert.deepEqual(rows, catalog, 'nothing should have been rewritten');
1878
+ assert.deepEqual(notes, [], 'and nothing should be reported');
1879
+ });
1880
+
1881
+ test('trimming engages on a model whose expressions are long', () => {
1882
+ // The EY shape: expressions well past the cap, and one description repeated
1883
+ // across enough of the model to discriminate nothing.
1884
+ const shared = 'Localized label. Do not use for calculations.';
1885
+ const catalog = Array.from({ length: 40 }, (_, index) => ({
1886
+ object_name: `Measure ${index}`,
1887
+ description: shared,
1888
+ expression: 'A'.repeat(400),
1889
+ }));
1890
+
1891
+ const { rows, notes } = trimSearchRows(catalog, catalog);
1892
+
1893
+ assert.equal(
1894
+ rows.every((row) => row.description === null),
1895
+ true
1896
+ );
1897
+ assert.equal(
1898
+ rows.every((row) => row.expression.length < 400),
1899
+ true
1900
+ );
1901
+ assert.equal(
1902
+ notes.length,
1903
+ 2,
1904
+ 'one note for descriptions, one for expressions'
1905
+ );
1906
+ });
1907
+
1908
+ test('trimming nulls boilerplate, keeps real descriptions, and reports itself once', () => {
1909
+ const shared = 'Localized label. Do not use for calculations.';
1910
+ const catalog = [
1911
+ ...Array.from({ length: 20 }, (_, index) => ({
1912
+ object_name: `loc-${index}`,
1913
+ description: shared,
1914
+ expression: 'x',
1915
+ })),
1916
+ {
1917
+ object_name: 'Revenue',
1918
+ description: 'Net revenue after returns',
1919
+ expression: 'SUM(Sales[Amount])',
1920
+ },
1921
+ ];
1922
+ const { rows, notes } = trimSearchRows(catalog, catalog);
1923
+
1924
+ assert.equal(rows[0].description, null);
1925
+ assert.equal(rows.at(-1).description, 'Net revenue after returns');
1926
+ assert.equal(notes.length, 1, 'no expression was long enough to summarize');
1927
+ assert.match(notes[0], /description omitted on 20 row\(s\)/u);
1928
+ // Every row keeps every key, or the TSV encoding collapses into JSON.
1929
+ assert.deepEqual(Object.keys(rows[0]), Object.keys(rows.at(-1)));
1930
+ });
1931
+
1932
+ test('trimming reports a summarized expression separately from a dropped description', () => {
1933
+ const catalog = [
1934
+ { object_name: 'm', description: 'unique', expression: 'A'.repeat(500) },
1935
+ ];
1936
+ const { rows, notes } = trimSearchRows(catalog, catalog);
1937
+ assert.equal(rows[0].description, 'unique');
1938
+ assert.equal(notes.length, 1);
1939
+ assert.match(notes[0], /expression shortened on 1 row\(s\)/u);
1940
+ });
1941
+
1942
+ // -- The regression this whole design exists to avoid --
1943
+
1944
+ test('ranking sees the full expression, not the summarized one', () => {
1945
+ // The sharp form of the property. The discriminating term appears ONLY past
1946
+ // the cut, so a ranker fed trimmed text could not possibly match it. Placing
1947
+ // the term near the front - as the upstream fixture does - would let this
1948
+ // test pass even if trimming happened before ranking.
1949
+ const term = 'reconciliation';
1950
+ const prefix = `SWITCH(USERCULTURE(), ${'"padding-locale", "unrelated text", '.repeat(8)}`;
1951
+ assert.ok(
1952
+ prefix.length > SEARCH_EXPRESSION_LIMIT * 2,
1953
+ 'the term must sit well past the cut'
1954
+ );
1955
+ const expression = `${prefix}"en-US", "${term} of journal entries")`;
1956
+ assert.ok(
1957
+ !summarizeSearchExpression(expression).includes(term),
1958
+ 'the trimmed form must not contain the term, or this proves nothing'
1959
+ );
1960
+
1961
+ const catalog = [
1962
+ {
1963
+ kind: 'measure',
1964
+ parent_name: 'LocalizedLabels',
1965
+ object_name: 'Label 0',
1966
+ description: 'Localized label.',
1967
+ expression,
1968
+ display_folder: '',
1969
+ data_type: '',
1970
+ synonyms: '',
1971
+ },
1972
+ ];
1973
+ const { rows } = trimSearchRows(catalog, catalog);
1974
+
1975
+ assert.equal(rankSemanticSearchRows(catalog, term).length, 1);
1976
+ assert.equal(
1977
+ rankSemanticSearchRows(rows, term).length,
1978
+ 0,
1979
+ 'ranking the trimmed rows finds nothing - so the order is load-bearing'
1980
+ );
1981
+ });
1982
+
1983
+ test('an object findable only through its long expression is still findable after trimming', () => {
1984
+ // Dropping these rows is what an earlier attempt did. It saved more bytes and
1985
+ // made a hundred measures invisible, which is a global regression traded for
1986
+ // a local gain. Ranking therefore still sees the full text.
1987
+ const dir = tempDir();
1988
+ try {
1989
+ const dbPath = join(dir, 'm.sqlite');
1990
+ const prefix = `SWITCH(USERCULTURE(), ${'"padding-locale", "unrelated text", '.repeat(8)}`;
1991
+ const localized = Array.from({ length: 20 }, (_, index) => ({
1992
+ Table: 'LocalizedLabels',
1993
+ Name: `Label ${index}`,
1994
+ Description: 'Localized label. Do not use for calculations.',
1995
+ Expression: `${prefix}"en-US", "reconciliation of journal entries")`,
1996
+ }));
1997
+ buildDatabase(
1998
+ [
1999
+ {
2000
+ table: 'info_view_tables',
2001
+ grain: '',
2002
+ dax: 'INFO.VIEW.TABLES()',
2003
+ rows: [{ Name: 'LocalizedLabels', Description: 'Labels' }],
2004
+ },
2005
+ {
2006
+ table: 'info_view_measures',
2007
+ grain: '',
2008
+ dax: 'INFO.VIEW.MEASURES()',
2009
+ rows: localized,
2010
+ },
2011
+ ],
2012
+ dbPath
2013
+ );
2014
+
2015
+ const out = searchCatalog(dbPath, 'reconciliation', { limit: 50 });
2016
+ assert.equal(out.ok, true);
2017
+ assert.ok(
2018
+ out.totalRows >= 20,
2019
+ 'the rows are still matched by their full expression'
2020
+ );
2021
+
2022
+ const row = out.rows.find(
2023
+ (candidate) => candidate.object_name === 'Label 0'
2024
+ );
2025
+ assert.ok(row, 'the object is still returned');
2026
+ assert.equal(
2027
+ row.description,
2028
+ null,
2029
+ 'the shared boilerplate is not reprinted'
2030
+ );
2031
+ assert.match(row.expression, /substr\(\[Expression\], <offset>, 1000\)/u);
2032
+ assert.ok(
2033
+ !row.expression.includes('reconciliation'),
2034
+ 'the printed expression no longer carries the term it was found by'
2035
+ );
2036
+ assert.ok(out.notes.length >= 1, 'the payload says what it did');
2037
+ } finally {
2038
+ rmSync(dir, { recursive: true, force: true });
2039
+ }
2040
+ });
2041
+
2042
+ test('trimming shrinks the search payload it is meant to shrink', () => {
2043
+ const dir = tempDir();
2044
+ try {
2045
+ const dbPath = join(dir, 'm.sqlite');
2046
+ const rows = Array.from({ length: 30 }, (_, index) => ({
2047
+ Table: 'LocalizedLabels',
2048
+ Name: `Label ${index}`,
2049
+ Description:
2050
+ 'Localized label. Do not use for calculations. It exists only for display.',
2051
+ Expression: `SWITCH(USERCULTURE(), ${Array.from(
2052
+ { length: 30 },
2053
+ (_, locale) => `"l-${locale}", "journal entries"`
2054
+ ).join(', ')})`,
2055
+ }));
2056
+ buildDatabase(
2057
+ [
2058
+ {
2059
+ table: 'info_view_tables',
2060
+ grain: '',
2061
+ dax: 'INFO.VIEW.TABLES()',
2062
+ rows: [{ Name: 'LocalizedLabels' }],
2063
+ },
2064
+ {
2065
+ table: 'info_view_measures',
2066
+ grain: '',
2067
+ dax: 'INFO.VIEW.MEASURES()',
2068
+ rows,
2069
+ },
2070
+ ],
2071
+ dbPath
2072
+ );
2073
+
2074
+ const out = searchCatalog(dbPath, 'journal', { limit: 50 });
2075
+ // Compare like with like: the two fields this trims, before and after.
2076
+ const targetedBytes = (records, expressionKey, descriptionKey) =>
2077
+ records.reduce(
2078
+ (total, record) =>
2079
+ total +
2080
+ Buffer.byteLength(
2081
+ `${record[expressionKey] ?? ''}${record[descriptionKey] ?? ''}`,
2082
+ 'utf8'
2083
+ ),
2084
+ 0
2085
+ );
2086
+ const before = targetedBytes(rows, 'Expression', 'Description');
2087
+ const after = targetedBytes(out.rows, 'expression', 'description');
2088
+ assert.ok(
2089
+ after < before / 4,
2090
+ `expected expression+description to shrink at least 4x: ${after} vs ${before}`
2091
+ );
2092
+ // Every matched row is still returned - smaller, not fewer.
2093
+ assert.equal(out.returnedRows, out.totalRows);
2094
+ } finally {
2095
+ rmSync(dir, { recursive: true, force: true });
2096
+ }
2097
+ });