@microsoft/rayfin-semantic-model-pack 1.0.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG +8 -0
- package/README.md +34 -10
- package/kit/hooks/use-query-factory-target.spec.ts +191 -0
- package/kit/hooks/use-query-factory-target.ts +123 -0
- package/{skills/analytics/kit → kit}/hooks/use-semantic-model-query.spec.ts +59 -23
- package/{skills/analytics/kit → kit}/hooks/use-semantic-model-query.ts +30 -54
- package/kit/lib/query-factory.spec.ts +97 -0
- package/kit/lib/query-factory.ts +68 -0
- package/kit/lib/semantic-model-errors.spec.ts +131 -0
- package/kit/lib/semantic-model-errors.ts +110 -0
- package/kit/lib/to-data-table.spec.ts +444 -0
- package/kit/lib/to-data-table.ts +221 -0
- package/pack.json +141 -0
- package/package.json +3 -1
- package/skills/analytics/SKILL.md +126 -421
- package/skills/dax-authoring/SKILL.md +99 -34
- package/skills/query-design/SKILL.md +22 -29
- package/skills/query-design/references/anti-patterns.md +13 -11
- package/skills/query-design/references/format-strings.md +6 -12
- package/skills/query-design/references/multi-grain-patterns.md +121 -108
- package/skills/schema-discovery/SKILL.md +197 -368
- package/skills/schema-discovery/references/discovery-queries.md +6 -8
- package/skills/visual-interactivity/SKILL.md +68 -0
- package/skills/visual-interactivity/references/advanced-query-shapes.md +155 -0
- package/skills/visual-interactivity/references/filter-aware-grid-totals.md +101 -0
- package/skills/visual-interactivity/references/highlight-queries.md +88 -0
- package/skills/visual-interactivity/references/interaction-manager.md +127 -0
- package/skills/visual-interactivity/references/native-selections.md +20 -0
- package/skills/visual-interactivity/references/query-factories.md +169 -0
- package/skills/analytics/kit/lib/connectors.ts +0 -94
- package/skills/analytics/kit/scripts/rayfin-connector.mjs +0 -265
- package/skills/analytics/kit/scripts/rayfin-connector.test.mjs +0 -243
- package/skills/analytics/kit/scripts/schema-sql-batch.test.mjs +0 -2097
- package/skills/analytics/kit/scripts/schema-sql-cli.test.mjs +0 -488
- package/skills/analytics/kit/scripts/schema-sql.mjs +0 -2650
- package/skills/analytics/kit/scripts/schema-sql.test.mjs +0 -1387
- package/skills/analytics/kit/scripts/schema.mjs +0 -166
- package/skills/analytics/kit/scripts/schema.test.mjs +0 -271
- package/skills/analytics/pack.json +0 -49
- package/skills/query-design/references/filter-strategy.md +0 -68
- package/skills/query-design/references/highlight-queries.md +0 -122
|
@@ -1,2097 +0,0 @@
|
|
|
1
|
-
//-----------------------------------------------------------------------
|
|
2
|
-
// <copyright company="Microsoft Corporation">
|
|
3
|
-
// Copyright (c) Microsoft Corporation. All rights reserved.
|
|
4
|
-
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
|
5
|
-
// </copyright>
|
|
6
|
-
//-----------------------------------------------------------------------
|
|
7
|
-
|
|
8
|
-
// Batching and search-payload trimming.
|
|
9
|
-
//
|
|
10
|
-
// Both exist for the same reason: one invocation costs one model round trip, and
|
|
11
|
-
// a round trip re-sends the whole conversation. Batching removes round trips;
|
|
12
|
-
// trimming shrinks what each one carries forward. The trimming tests care most
|
|
13
|
-
// about what must NOT change - an object that was findable before has to stay
|
|
14
|
-
// findable.
|
|
15
|
-
|
|
16
|
-
import assert from 'node:assert/strict';
|
|
17
|
-
import { createHash } from 'node:crypto';
|
|
18
|
-
import {
|
|
19
|
-
existsSync,
|
|
20
|
-
linkSync,
|
|
21
|
-
mkdirSync,
|
|
22
|
-
mkdtempSync,
|
|
23
|
-
readFileSync,
|
|
24
|
-
renameSync,
|
|
25
|
-
rmSync,
|
|
26
|
-
statSync,
|
|
27
|
-
symlinkSync,
|
|
28
|
-
writeFileSync,
|
|
29
|
-
} from 'node:fs';
|
|
30
|
-
import { tmpdir } from 'node:os';
|
|
31
|
-
import { join, sep } from 'node:path';
|
|
32
|
-
import test from 'node:test';
|
|
33
|
-
|
|
34
|
-
import {
|
|
35
|
-
batchExitCode,
|
|
36
|
-
BOILERPLATE_MIN_ROWS,
|
|
37
|
-
BOILERPLATE_MIN_SHARE,
|
|
38
|
-
boilerplateDescriptions,
|
|
39
|
-
buildDatabase,
|
|
40
|
-
formatBatch,
|
|
41
|
-
formatResult,
|
|
42
|
-
MAX_BATCH_STATEMENTS,
|
|
43
|
-
MAX_RESULT_BYTES,
|
|
44
|
-
matchedSearchTokens,
|
|
45
|
-
parseCliArgs,
|
|
46
|
-
planStatements,
|
|
47
|
-
publishFiles,
|
|
48
|
-
rankSearchWithFallback,
|
|
49
|
-
rankSemanticSearchRows,
|
|
50
|
-
runBatch,
|
|
51
|
-
runSql,
|
|
52
|
-
searchCatalog,
|
|
53
|
-
SEARCH_EXPRESSION_LIMIT,
|
|
54
|
-
summarizeSearchExpression,
|
|
55
|
-
trimSearchRows,
|
|
56
|
-
withCatalogLock,
|
|
57
|
-
} from './schema-sql.mjs';
|
|
58
|
-
|
|
59
|
-
function tempDir() {
|
|
60
|
-
return mkdtempSync(join(tmpdir(), 'schema-sql-batch-'));
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
function runnersFor(dbPath) {
|
|
64
|
-
return {
|
|
65
|
-
sql: (query, options) => runSql(dbPath, query, options),
|
|
66
|
-
search: (query, options) => searchCatalog(dbPath, query, options),
|
|
67
|
-
};
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
// -- Statement planning --
|
|
71
|
-
|
|
72
|
-
test('repeated --search/--sql/--file parse into arrays', () => {
|
|
73
|
-
const { values, positionals } = parseCliArgs([
|
|
74
|
-
'--search',
|
|
75
|
-
'revenue',
|
|
76
|
-
'--search',
|
|
77
|
-
'margin',
|
|
78
|
-
'--sql',
|
|
79
|
-
'SELECT 1',
|
|
80
|
-
'--file',
|
|
81
|
-
'a.sql',
|
|
82
|
-
]);
|
|
83
|
-
assert.deepEqual(values.search, ['revenue', 'margin']);
|
|
84
|
-
assert.deepEqual(values.sql, ['SELECT 1']);
|
|
85
|
-
assert.deepEqual(values.file, ['a.sql']);
|
|
86
|
-
assert.deepEqual(positionals, []);
|
|
87
|
-
});
|
|
88
|
-
|
|
89
|
-
test('--profile stays removed rather than returning with the batch flags', () => {
|
|
90
|
-
// It was deprecated on arrival and ignored by both resolvers. Upstream still
|
|
91
|
-
// parses it; reintroducing it would give the agent an option that does nothing.
|
|
92
|
-
assert.throws(
|
|
93
|
-
() => parseCliArgs(['--profile', 'anything']),
|
|
94
|
-
/unknown option|not (a )?known/iu
|
|
95
|
-
);
|
|
96
|
-
});
|
|
97
|
-
|
|
98
|
-
test('a lone positional statement still plans as a single statement', () => {
|
|
99
|
-
const plan = planStatements({ positionalSql: 'SELECT 1' });
|
|
100
|
-
assert.equal(plan.mode, 'single');
|
|
101
|
-
assert.equal(plan.batch.length, 1);
|
|
102
|
-
assert.equal(plan.batch[0].query, 'SELECT 1');
|
|
103
|
-
});
|
|
104
|
-
|
|
105
|
-
test('searches are planned before SQL so results read in discovery order', () => {
|
|
106
|
-
const plan = planStatements({
|
|
107
|
-
search: ['revenue'],
|
|
108
|
-
sql: ['SELECT 1', 'SELECT 2'],
|
|
109
|
-
});
|
|
110
|
-
assert.equal(plan.mode, 'batch');
|
|
111
|
-
assert.deepEqual(
|
|
112
|
-
plan.batch.map((entry) => entry.kind),
|
|
113
|
-
['search', 'sql', 'sql']
|
|
114
|
-
);
|
|
115
|
-
assert.deepEqual(
|
|
116
|
-
plan.batch.map((entry) => entry.query),
|
|
117
|
-
['revenue', 'SELECT 1', 'SELECT 2']
|
|
118
|
-
);
|
|
119
|
-
});
|
|
120
|
-
|
|
121
|
-
test('--file contributes its contents as a statement and remembers its source', () => {
|
|
122
|
-
const dir = tempDir();
|
|
123
|
-
try {
|
|
124
|
-
const path = join(dir, 'q.sql');
|
|
125
|
-
writeFileSync(path, 'SELECT [Name] FROM info_view_measures', 'utf8');
|
|
126
|
-
const plan = planStatements({ file: [path], root: dir });
|
|
127
|
-
assert.equal(plan.mode, 'batch');
|
|
128
|
-
assert.equal(plan.batch[0].query, 'SELECT [Name] FROM info_view_measures');
|
|
129
|
-
assert.equal(plan.batch[0].source, 'q.sql');
|
|
130
|
-
} finally {
|
|
131
|
-
rmSync(dir, { recursive: true, force: true });
|
|
132
|
-
}
|
|
133
|
-
});
|
|
134
|
-
|
|
135
|
-
test('an unreadable --file is reported rather than thrown', () => {
|
|
136
|
-
const dir = tempDir();
|
|
137
|
-
try {
|
|
138
|
-
const plan = planStatements({
|
|
139
|
-
file: [join(dir, 'missing.sql')],
|
|
140
|
-
root: dir,
|
|
141
|
-
});
|
|
142
|
-
assert.match(plan.error, /Could not read --file/u);
|
|
143
|
-
} finally {
|
|
144
|
-
rmSync(dir, { recursive: true, force: true });
|
|
145
|
-
}
|
|
146
|
-
});
|
|
147
|
-
|
|
148
|
-
test('positional SQL and batch flags cannot be mixed', () => {
|
|
149
|
-
const plan = planStatements({
|
|
150
|
-
positionalSql: 'SELECT 1',
|
|
151
|
-
sql: ['SELECT 2'],
|
|
152
|
-
});
|
|
153
|
-
assert.match(plan.error, /not both/u);
|
|
154
|
-
});
|
|
155
|
-
|
|
156
|
-
test('the batch size is capped so one call cannot become a whole workload', () => {
|
|
157
|
-
const sql = Array.from(
|
|
158
|
-
{ length: MAX_BATCH_STATEMENTS + 1 },
|
|
159
|
-
(_, index) => `SELECT ${index}`
|
|
160
|
-
);
|
|
161
|
-
const plan = planStatements({ sql });
|
|
162
|
-
assert.match(plan.error, new RegExp(`limit is ${MAX_BATCH_STATEMENTS}`, 'u'));
|
|
163
|
-
});
|
|
164
|
-
|
|
165
|
-
test('an empty statement is rejected instead of running as blank SQL', () => {
|
|
166
|
-
assert.match(planStatements({ sql: [' '] }).error, /empty sql statement/u);
|
|
167
|
-
assert.match(
|
|
168
|
-
planStatements({ search: [''] }).error,
|
|
169
|
-
/empty search statement/u
|
|
170
|
-
);
|
|
171
|
-
});
|
|
172
|
-
|
|
173
|
-
test('no statements and no refresh plans nothing', () => {
|
|
174
|
-
assert.equal(planStatements({}).mode, 'none');
|
|
175
|
-
});
|
|
176
|
-
|
|
177
|
-
// -- --file is an input channel, not a new capability --
|
|
178
|
-
//
|
|
179
|
-
// Everything reachable through --file was already reachable through positional
|
|
180
|
-
// SQL. Two things still needed closing, because --file is the only option that
|
|
181
|
-
// reads bytes the caller did not type:
|
|
182
|
-
//
|
|
183
|
-
// 1. the path is constrained to the app, matching the Rayfin CLI's own --file
|
|
184
|
-
// convention (connector-invoke.ts readJsonInputFile), and
|
|
185
|
-
// 2. the printed label names the file rather than quoting its contents.
|
|
186
|
-
//
|
|
187
|
-
// Together those mean a mistyped path cannot pull unrelated file bytes into the
|
|
188
|
-
// agent's context, whether through the label or through a SQLite syntax error
|
|
189
|
-
// that quotes the offending token back.
|
|
190
|
-
|
|
191
|
-
test('a --file outside the app is refused before it is opened', () => {
|
|
192
|
-
const dir = tempDir();
|
|
193
|
-
const outside = tempDir();
|
|
194
|
-
try {
|
|
195
|
-
const path = join(outside, 'secret.txt');
|
|
196
|
-
writeFileSync(path, 'NOT-A-REAL-CREDENTIAL-abcdef1234567890', 'utf8');
|
|
197
|
-
|
|
198
|
-
const plan = planStatements({ file: [path], root: dir });
|
|
199
|
-
assert.match(plan.error, /--file must be inside the app/u);
|
|
200
|
-
assert.ok(
|
|
201
|
-
!plan.error.includes('NOT-A-REAL-CREDENTIAL'),
|
|
202
|
-
'the refusal must not quote what it refused to read'
|
|
203
|
-
);
|
|
204
|
-
|
|
205
|
-
// Traversal out of the app is the same refusal, not a different one.
|
|
206
|
-
assert.match(
|
|
207
|
-
planStatements({ file: ['../secret.txt'], root: dir }).error,
|
|
208
|
-
/--file must be inside the app/u
|
|
209
|
-
);
|
|
210
|
-
} finally {
|
|
211
|
-
rmSync(dir, { recursive: true, force: true });
|
|
212
|
-
rmSync(outside, { recursive: true, force: true });
|
|
213
|
-
}
|
|
214
|
-
});
|
|
215
|
-
|
|
216
|
-
test('a --file statement is labelled by its path, not by the bytes it holds', (t) => {
|
|
217
|
-
// The label is bounded at 512 bytes, so quoting file contents would spill up
|
|
218
|
-
// to 512 bytes of an arbitrary in-app file into every batch payload. Naming
|
|
219
|
-
// the file is both safer and a better label: it is what the caller typed.
|
|
220
|
-
const marker = 'DISTINCTIVE-FILE-CONTENT-MARKER';
|
|
221
|
-
const queries = [
|
|
222
|
-
`SELECT marker FROM t WHERE id = 1 /* ${marker} ${'x'.repeat(4_000)} */`,
|
|
223
|
-
];
|
|
224
|
-
const { labels, payload, executed } = batchFixture(t, queries, {
|
|
225
|
-
via: 'file',
|
|
226
|
-
});
|
|
227
|
-
|
|
228
|
-
assert.equal(labels[0].source, 'statement 0.sql');
|
|
229
|
-
assert.equal(labels[0].query, undefined);
|
|
230
|
-
assert.ok(
|
|
231
|
-
!payload.includes(marker),
|
|
232
|
-
'no part of the file body may appear in the printed label'
|
|
233
|
-
);
|
|
234
|
-
assert.deepEqual(
|
|
235
|
-
executed,
|
|
236
|
-
queries,
|
|
237
|
-
'the full file text still runs - only the label is shortened'
|
|
238
|
-
);
|
|
239
|
-
});
|
|
240
|
-
|
|
241
|
-
test('file-sourced statements still share the one batch byte budget', (t) => {
|
|
242
|
-
const rows = Array.from({ length: 200 }, (_, id) => ({
|
|
243
|
-
id,
|
|
244
|
-
marker: `${id} ${'value '.repeat(50)}`,
|
|
245
|
-
}));
|
|
246
|
-
const queries = Array.from({ length: 8 }, () => 'SELECT * FROM t');
|
|
247
|
-
const { entries, bytes } = batchFixture(t, queries, { via: 'file', rows });
|
|
248
|
-
|
|
249
|
-
assert.ok(bytes <= MAX_RESULT_BYTES, `batch emitted ${bytes} bytes`);
|
|
250
|
-
for (const entry of entries) {
|
|
251
|
-
assert.equal(entry.result.totalRows, rows.length);
|
|
252
|
-
assert.ok(entry.result.returnedRows > 0, 'no statement was starved');
|
|
253
|
-
}
|
|
254
|
-
t.diagnostic(`eight file statements: ${bytes}/${MAX_RESULT_BYTES} bytes`);
|
|
255
|
-
});
|
|
256
|
-
|
|
257
|
-
test('SQL supplied through --file cannot escape the read-only connection', () => {
|
|
258
|
-
const dir = tempDir();
|
|
259
|
-
try {
|
|
260
|
-
const dbPath = join(dir, 'model.sqlite');
|
|
261
|
-
buildDatabase(
|
|
262
|
-
[{ table: 't', grain: '', dax: '', rows: [{ secret: 'customer row' }] }],
|
|
263
|
-
dbPath
|
|
264
|
-
);
|
|
265
|
-
|
|
266
|
-
const copiedPath = join(dir, 'exfiltrated.sqlite');
|
|
267
|
-
const statementPath = join(dir, 'exfiltrate.sql');
|
|
268
|
-
writeFileSync(
|
|
269
|
-
statementPath,
|
|
270
|
-
`VACUUM INTO '${copiedPath.replace(/'/gu, "''")}'`,
|
|
271
|
-
'utf8'
|
|
272
|
-
);
|
|
273
|
-
|
|
274
|
-
const plan = planStatements({ file: [statementPath], root: dir });
|
|
275
|
-
assert.equal(plan.mode, 'batch');
|
|
276
|
-
const [entry] = runBatch(plan.batch, { runners: runnersFor(dbPath) });
|
|
277
|
-
|
|
278
|
-
assert.equal(entry.result.ok, false);
|
|
279
|
-
assert.match(entry.result.error, /read.?only|readonly/iu);
|
|
280
|
-
assert.equal(
|
|
281
|
-
existsSync(copiedPath) ? statSync(copiedPath).size : 0,
|
|
282
|
-
0,
|
|
283
|
-
'no catalog data may escape through a --file statement'
|
|
284
|
-
);
|
|
285
|
-
} finally {
|
|
286
|
-
rmSync(dir, { recursive: true, force: true });
|
|
287
|
-
}
|
|
288
|
-
});
|
|
289
|
-
|
|
290
|
-
test('a query_only bypass cannot carry across two batch statements', () => {
|
|
291
|
-
const dir = tempDir();
|
|
292
|
-
try {
|
|
293
|
-
const dbPath = join(dir, 'model.sqlite');
|
|
294
|
-
buildDatabase(
|
|
295
|
-
[{ table: 't', grain: '', dax: '', rows: [{ secret: 'customer row' }] }],
|
|
296
|
-
dbPath
|
|
297
|
-
);
|
|
298
|
-
|
|
299
|
-
// On one reused connection PRAGMA query_only=OFF re-enables writes and the
|
|
300
|
-
// VACUUM INTO that follows succeeds. Each batch entry opens its own
|
|
301
|
-
// read-only connection, so the PRAGMA cannot carry into the next statement.
|
|
302
|
-
const copiedPath = join(dir, 'exfiltrated.sqlite');
|
|
303
|
-
const plan = planStatements({
|
|
304
|
-
sql: [
|
|
305
|
-
'PRAGMA query_only=OFF',
|
|
306
|
-
`VACUUM INTO '${copiedPath.replace(/'/gu, "''")}'`,
|
|
307
|
-
],
|
|
308
|
-
root: dir,
|
|
309
|
-
});
|
|
310
|
-
assert.equal(plan.mode, 'batch');
|
|
311
|
-
const entries = runBatch(plan.batch, { runners: runnersFor(dbPath) });
|
|
312
|
-
|
|
313
|
-
assert.equal(entries.length, 2);
|
|
314
|
-
assert.equal(entries[1].result.ok, false);
|
|
315
|
-
assert.match(entries[1].result.error, /read.?only|readonly/iu);
|
|
316
|
-
assert.equal(
|
|
317
|
-
existsSync(copiedPath) ? statSync(copiedPath).size : 0,
|
|
318
|
-
0,
|
|
319
|
-
'a PRAGMA in an earlier statement may not unlock a later write'
|
|
320
|
-
);
|
|
321
|
-
} finally {
|
|
322
|
-
rmSync(dir, { recursive: true, force: true });
|
|
323
|
-
}
|
|
324
|
-
});
|
|
325
|
-
|
|
326
|
-
test('an oversized --file is rejected before it is read into memory', () => {
|
|
327
|
-
const dir = tempDir();
|
|
328
|
-
try {
|
|
329
|
-
const path = join(dir, 'huge.sql');
|
|
330
|
-
// Upstream reads --file unboundedly, which turns a wrong path into a memory
|
|
331
|
-
// blowup. The cap is stated in the error so the limit is discoverable.
|
|
332
|
-
writeFileSync(path, `-- ${'x'.repeat(1_000_001)}`, 'utf8');
|
|
333
|
-
const plan = planStatements({ file: [path], root: dir });
|
|
334
|
-
assert.match(plan.error, /Could not read --file/u);
|
|
335
|
-
assert.match(plan.error, /exceeds the 1000000-byte limit/u);
|
|
336
|
-
} finally {
|
|
337
|
-
rmSync(dir, { recursive: true, force: true });
|
|
338
|
-
}
|
|
339
|
-
});
|
|
340
|
-
|
|
341
|
-
test('a --file that is not a regular file is reported, not read', () => {
|
|
342
|
-
const dir = tempDir();
|
|
343
|
-
try {
|
|
344
|
-
const plan = planStatements({ file: [dir], root: dir });
|
|
345
|
-
// Assert the guard's own message rather than just "could not read": a
|
|
346
|
-
// directory would fail inside readFileSync anyway, so a loose match here
|
|
347
|
-
// would be a test that cannot fail.
|
|
348
|
-
assert.match(plan.error, /Could not read --file/u);
|
|
349
|
-
assert.match(plan.error, /not a regular file/u);
|
|
350
|
-
} finally {
|
|
351
|
-
rmSync(dir, { recursive: true, force: true });
|
|
352
|
-
}
|
|
353
|
-
});
|
|
354
|
-
|
|
355
|
-
// -- Batch payload --
|
|
356
|
-
|
|
357
|
-
test('each batch block is byte-identical to the single-statement payload', () => {
|
|
358
|
-
const result = {
|
|
359
|
-
ok: true,
|
|
360
|
-
totalRows: 1,
|
|
361
|
-
returnedRows: 1,
|
|
362
|
-
truncated: false,
|
|
363
|
-
rows: [{ name: 'Revenue' }],
|
|
364
|
-
};
|
|
365
|
-
const out = formatBatch([{ kind: 'sql', query: 'SELECT 1', result }]);
|
|
366
|
-
const lines = out.split('\n');
|
|
367
|
-
assert.deepEqual(JSON.parse(lines[0].slice(2)), {
|
|
368
|
-
ok: true,
|
|
369
|
-
batch: true,
|
|
370
|
-
statements: 1,
|
|
371
|
-
});
|
|
372
|
-
assert.deepEqual(JSON.parse(lines[1].slice(3)), {
|
|
373
|
-
index: 0,
|
|
374
|
-
kind: 'sql',
|
|
375
|
-
query: 'SELECT 1',
|
|
376
|
-
});
|
|
377
|
-
// The body is still TSV - batching must not force results into JSON.
|
|
378
|
-
assert.match(lines[2], /^# \{/u);
|
|
379
|
-
assert.equal(lines[3], 'name');
|
|
380
|
-
assert.equal(lines[4], 'Revenue');
|
|
381
|
-
});
|
|
382
|
-
|
|
383
|
-
test('a failed statement stays in the payload beside the ones that worked', () => {
|
|
384
|
-
const out = formatBatch([
|
|
385
|
-
{
|
|
386
|
-
kind: 'sql',
|
|
387
|
-
query: 'SELECT 1',
|
|
388
|
-
result: {
|
|
389
|
-
ok: true,
|
|
390
|
-
totalRows: 1,
|
|
391
|
-
returnedRows: 1,
|
|
392
|
-
truncated: false,
|
|
393
|
-
rows: [{ n: 1 }],
|
|
394
|
-
},
|
|
395
|
-
},
|
|
396
|
-
{
|
|
397
|
-
kind: 'sql',
|
|
398
|
-
query: 'SELEKT',
|
|
399
|
-
result: { ok: false, error: 'no such function: SELEKT' },
|
|
400
|
-
},
|
|
401
|
-
]);
|
|
402
|
-
assert.match(out, /"statements":2/u);
|
|
403
|
-
assert.match(out, /"ok":false/u);
|
|
404
|
-
assert.match(out, /no such function/u);
|
|
405
|
-
assert.match(out, /"index":1/u);
|
|
406
|
-
});
|
|
407
|
-
|
|
408
|
-
test('the batch exit code is the worst outcome, and failure outranks truncation', () => {
|
|
409
|
-
const ok = { ok: true, truncated: false };
|
|
410
|
-
const cut = { ok: true, truncated: true };
|
|
411
|
-
const bad = { ok: false, error: 'x' };
|
|
412
|
-
const wrap = (results) =>
|
|
413
|
-
results.map((result) => ({ kind: 'sql', query: 'q', result }));
|
|
414
|
-
|
|
415
|
-
assert.equal(batchExitCode(wrap([ok, ok])), 0);
|
|
416
|
-
assert.equal(batchExitCode(wrap([ok, cut])), 2);
|
|
417
|
-
assert.equal(batchExitCode(wrap([cut, bad])), 1);
|
|
418
|
-
assert.equal(batchExitCode(wrap([ok, cut]), { allowTruncated: true }), 0);
|
|
419
|
-
assert.equal(batchExitCode(wrap([])), 0);
|
|
420
|
-
});
|
|
421
|
-
|
|
422
|
-
// -- The whole batch shares one byte budget --
|
|
423
|
-
//
|
|
424
|
-
// The per-statement cap exists so a result cannot bury the session's context.
|
|
425
|
-
// Batching must not multiply it by the number of statements, or it would make
|
|
426
|
-
// the very problem it is meant to reduce strictly worse.
|
|
427
|
-
|
|
428
|
-
test('a full batch of large results still fits the single-statement byte budget', () => {
|
|
429
|
-
const dir = tempDir();
|
|
430
|
-
try {
|
|
431
|
-
const dbPath = join(dir, 'm.sqlite');
|
|
432
|
-
buildDatabase(
|
|
433
|
-
[
|
|
434
|
-
{
|
|
435
|
-
table: 'info_view_measures',
|
|
436
|
-
grain: '',
|
|
437
|
-
dax: 'INFO.VIEW.MEASURES()',
|
|
438
|
-
// Each row alone would blow a meaningful part of the budget.
|
|
439
|
-
rows: Array.from({ length: 200 }, (_, index) => ({
|
|
440
|
-
Table: 'Facts',
|
|
441
|
-
Name: `Measure ${index}`,
|
|
442
|
-
Description: `unique description ${index} ${'padding '.repeat(40)}`,
|
|
443
|
-
})),
|
|
444
|
-
},
|
|
445
|
-
],
|
|
446
|
-
dbPath
|
|
447
|
-
);
|
|
448
|
-
|
|
449
|
-
const batch = Array.from({ length: MAX_BATCH_STATEMENTS }, () => ({
|
|
450
|
-
kind: 'sql',
|
|
451
|
-
query: 'SELECT * FROM info_view_measures',
|
|
452
|
-
}));
|
|
453
|
-
const entries = runBatch(batch, { runners: runnersFor(dbPath) });
|
|
454
|
-
|
|
455
|
-
assert.equal(entries.length, MAX_BATCH_STATEMENTS);
|
|
456
|
-
const payload = Buffer.byteLength(formatBatch(entries), 'utf8');
|
|
457
|
-
assert.ok(
|
|
458
|
-
payload <= MAX_RESULT_BYTES,
|
|
459
|
-
`the whole batch must fit ${MAX_RESULT_BYTES} bytes, got ${payload}`
|
|
460
|
-
);
|
|
461
|
-
|
|
462
|
-
// The property that makes this worth asserting: eight statements did not
|
|
463
|
-
// cost eight budgets. A per-statement cap would have allowed 8x this.
|
|
464
|
-
const single = Buffer.byteLength(
|
|
465
|
-
formatResult(runSql(dbPath, 'SELECT * FROM info_view_measures')),
|
|
466
|
-
'utf8'
|
|
467
|
-
);
|
|
468
|
-
assert.ok(
|
|
469
|
-
payload < single * 2,
|
|
470
|
-
`a batch of ${MAX_BATCH_STATEMENTS} must not approach ${MAX_BATCH_STATEMENTS}x ` +
|
|
471
|
-
`one statement: ${payload} vs ${single} each`
|
|
472
|
-
);
|
|
473
|
-
|
|
474
|
-
// Every statement still answered - bounded, not dropped.
|
|
475
|
-
for (const entry of entries) {
|
|
476
|
-
assert.equal(entry.result.ok, true);
|
|
477
|
-
assert.ok(
|
|
478
|
-
entry.result.returnedRows > 0,
|
|
479
|
-
'no statement was starved to zero rows'
|
|
480
|
-
);
|
|
481
|
-
}
|
|
482
|
-
} finally {
|
|
483
|
-
rmSync(dir, { recursive: true, force: true });
|
|
484
|
-
}
|
|
485
|
-
});
|
|
486
|
-
|
|
487
|
-
test('an unused share rolls forward, so a cheap statement makes a later one roomier', () => {
|
|
488
|
-
const sizes = [];
|
|
489
|
-
const runners = {
|
|
490
|
-
sql: (query, options) => {
|
|
491
|
-
sizes.push(options.maxBytes);
|
|
492
|
-
// The first statement answers in almost nothing.
|
|
493
|
-
return query === 'tiny'
|
|
494
|
-
? {
|
|
495
|
-
ok: true,
|
|
496
|
-
totalRows: 1,
|
|
497
|
-
returnedRows: 1,
|
|
498
|
-
truncated: false,
|
|
499
|
-
rows: [{ a: 1 }],
|
|
500
|
-
}
|
|
501
|
-
: {
|
|
502
|
-
ok: true,
|
|
503
|
-
totalRows: 1,
|
|
504
|
-
returnedRows: 1,
|
|
505
|
-
truncated: false,
|
|
506
|
-
rows: [{ a: 'x'.repeat(200) }],
|
|
507
|
-
};
|
|
508
|
-
},
|
|
509
|
-
search: () => ({ ok: false, error: 'unused' }),
|
|
510
|
-
};
|
|
511
|
-
runBatch(
|
|
512
|
-
[
|
|
513
|
-
{ kind: 'sql', query: 'tiny' },
|
|
514
|
-
{ kind: 'sql', query: 'big' },
|
|
515
|
-
],
|
|
516
|
-
{ runners, maxBytes: 10_000 }
|
|
517
|
-
);
|
|
518
|
-
|
|
519
|
-
assert.ok(
|
|
520
|
-
sizes[1] > sizes[0],
|
|
521
|
-
`expected the second share to grow: ${sizes[0]} -> ${sizes[1]}`
|
|
522
|
-
);
|
|
523
|
-
});
|
|
524
|
-
|
|
525
|
-
function batchFixture(
|
|
526
|
-
t,
|
|
527
|
-
queries,
|
|
528
|
-
{ via = 'sql', rows = [{ id: 1, marker: 'unchanged SQL' }], ...options } = {}
|
|
529
|
-
) {
|
|
530
|
-
const dir = tempDir();
|
|
531
|
-
t.after(() => rmSync(dir, { recursive: true, force: true }));
|
|
532
|
-
const dbPath = join(dir, 'budget.sqlite');
|
|
533
|
-
buildDatabase([{ table: 't', grain: '', dax: '', rows }], dbPath);
|
|
534
|
-
const args = queries.flatMap((query, index) => {
|
|
535
|
-
if (via === 'sql') {
|
|
536
|
-
return ['--sql', query];
|
|
537
|
-
}
|
|
538
|
-
const path = join(dir, `statement ${index}.sql`);
|
|
539
|
-
writeFileSync(path, query, 'utf8');
|
|
540
|
-
return ['--file', path];
|
|
541
|
-
});
|
|
542
|
-
const { values } = parseCliArgs(args);
|
|
543
|
-
const plan = planStatements({ ...values, root: dir });
|
|
544
|
-
assert.equal(plan.mode, 'batch');
|
|
545
|
-
const executed = [];
|
|
546
|
-
const runners = {
|
|
547
|
-
sql: (query, limits) => {
|
|
548
|
-
executed.push(query);
|
|
549
|
-
return runSql(dbPath, query, limits);
|
|
550
|
-
},
|
|
551
|
-
search: (query, limits) => searchCatalog(dbPath, query, limits),
|
|
552
|
-
};
|
|
553
|
-
const entries = runBatch(plan.batch, { runners, ...options });
|
|
554
|
-
assert.deepEqual(
|
|
555
|
-
executed,
|
|
556
|
-
queries,
|
|
557
|
-
'execute the full original statement text, not its printed label'
|
|
558
|
-
);
|
|
559
|
-
const payload = formatBatch(entries);
|
|
560
|
-
const bytes = Buffer.byteLength(payload, 'utf8');
|
|
561
|
-
assert.ok(
|
|
562
|
-
bytes <= (options.maxBytes ?? MAX_RESULT_BYTES),
|
|
563
|
-
`batch emitted ${bytes} bytes`
|
|
564
|
-
);
|
|
565
|
-
const labels = payload
|
|
566
|
-
.split('\n')
|
|
567
|
-
.filter((line) => line.startsWith('## '))
|
|
568
|
-
.map((line) => JSON.parse(line.slice(3)));
|
|
569
|
-
return { entries, payload, bytes, labels, plan, runners, executed, dir };
|
|
570
|
-
}
|
|
571
|
-
|
|
572
|
-
for (const [name, count, padding] of [
|
|
573
|
-
['one oversized statement', 1, 'x'.repeat(30_000)],
|
|
574
|
-
['eight large statements', 8, 'x'.repeat(4_100)],
|
|
575
|
-
['Unicode and JSON escapes', 1, '\u03b1\u{1f642}"\\\t\n'.repeat(4_000)],
|
|
576
|
-
]) {
|
|
577
|
-
test(`batch bounds printed labels from ${name} without altering SQL`, (t) => {
|
|
578
|
-
const queries = Array.from(
|
|
579
|
-
{ length: count },
|
|
580
|
-
(_, index) =>
|
|
581
|
-
`SELECT marker FROM t /* ${index} ${padding} */ WHERE id = 1`
|
|
582
|
-
);
|
|
583
|
-
const { entries, labels, bytes } = batchFixture(t, queries);
|
|
584
|
-
assert.equal(batchExitCode(entries), 0);
|
|
585
|
-
for (const [index, entry] of entries.entries()) {
|
|
586
|
-
assert.deepEqual(
|
|
587
|
-
entry.result.rows.map((row) => ({ ...row })),
|
|
588
|
-
[{ marker: 'unchanged SQL' }]
|
|
589
|
-
);
|
|
590
|
-
assert.equal(labels[index].index, index);
|
|
591
|
-
assert.equal(labels[index].kind, 'sql');
|
|
592
|
-
assert.ok(queries[index].startsWith(labels[index].query));
|
|
593
|
-
assert.equal(labels[index].queryTruncated, true);
|
|
594
|
-
assert.equal(
|
|
595
|
-
labels[index].queryBytes,
|
|
596
|
-
Buffer.byteLength(queries[index], 'utf8')
|
|
597
|
-
);
|
|
598
|
-
assert.equal(
|
|
599
|
-
labels[index].queryHash,
|
|
600
|
-
createHash('sha256').update(queries[index]).digest('hex').slice(0, 16)
|
|
601
|
-
);
|
|
602
|
-
assert.ok(
|
|
603
|
-
!/[\uD800-\uDBFF]$/u.test(labels[index].query),
|
|
604
|
-
'do not split a surrogate pair'
|
|
605
|
-
);
|
|
606
|
-
}
|
|
607
|
-
t.diagnostic(`${name}: ${bytes}/${MAX_RESULT_BYTES} UTF-8 bytes`);
|
|
608
|
-
});
|
|
609
|
-
}
|
|
610
|
-
|
|
611
|
-
test('eight long labels leave fair shares for rows and preserve recovery metadata', (t) => {
|
|
612
|
-
const rows = Array.from({ length: 200 }, (_, id) => ({
|
|
613
|
-
id,
|
|
614
|
-
marker: `${id} ${'value '.repeat(50)}`,
|
|
615
|
-
}));
|
|
616
|
-
const queries = Array.from(
|
|
617
|
-
{ length: 8 },
|
|
618
|
-
() => `SELECT * FROM t /* ${'x'.repeat(4_100)} */`
|
|
619
|
-
);
|
|
620
|
-
const { entries, bytes } = batchFixture(t, queries, { rows });
|
|
621
|
-
assert.equal(batchExitCode(entries), 2);
|
|
622
|
-
for (const entry of entries) {
|
|
623
|
-
assert.equal(entry.result.totalRows, rows.length);
|
|
624
|
-
assert.ok(
|
|
625
|
-
entry.result.returnedRows > 0,
|
|
626
|
-
'labels cannot starve later statements'
|
|
627
|
-
);
|
|
628
|
-
assert.equal(entry.result.truncated, true);
|
|
629
|
-
assert.match(
|
|
630
|
-
entry.result.truncation.requiredAction,
|
|
631
|
-
/narrower WHERE|smaller projection/u
|
|
632
|
-
);
|
|
633
|
-
assert.deepEqual(Object.keys(entry.result.rows[0]), ['id', 'marker']);
|
|
634
|
-
}
|
|
635
|
-
t.diagnostic(`labels and row data: ${bytes}/${MAX_RESULT_BYTES} UTF-8 bytes`);
|
|
636
|
-
});
|
|
637
|
-
|
|
638
|
-
test('large SQLite diagnostics stay indexed errors inside the final budget', (t) => {
|
|
639
|
-
const missing = 'missing_\u03b1\u{1f642}\\"'
|
|
640
|
-
.repeat(4_000)
|
|
641
|
-
.replace(/"/gu, '""');
|
|
642
|
-
const { entries, payload, bytes } = batchFixture(t, [
|
|
643
|
-
'SELECT marker FROM t',
|
|
644
|
-
`SELECT * FROM "${missing}"`,
|
|
645
|
-
'SELEKT broken',
|
|
646
|
-
'SELECT marker FROM t WHERE id = 1',
|
|
647
|
-
]);
|
|
648
|
-
assert.equal(batchExitCode(entries, { allowTruncated: true }), 1);
|
|
649
|
-
assert.equal(entries[0].result.ok, true);
|
|
650
|
-
assert.equal(entries[3].result.ok, true);
|
|
651
|
-
assert.equal(entries[1].result.ok, false);
|
|
652
|
-
assert.match(entries[1].result.error, /no such table/u);
|
|
653
|
-
assert.equal(entries[1].result.errorTruncated, true);
|
|
654
|
-
assert.ok(entries[1].result.errorBytes > MAX_RESULT_BYTES);
|
|
655
|
-
assert.match(entries[2].result.error, /syntax error/u);
|
|
656
|
-
assert.equal(JSON.parse(payload.split('\n')[0].slice(2)).ok, false);
|
|
657
|
-
t.diagnostic(`long error: ${bytes}/${MAX_RESULT_BYTES} UTF-8 bytes`);
|
|
658
|
-
});
|
|
659
|
-
|
|
660
|
-
test('an all-error batch can fill the exact cap without a one-byte header overrun', (t) => {
|
|
661
|
-
const queries = Array.from(
|
|
662
|
-
{ length: 2 },
|
|
663
|
-
(_, index) => `SELECT * FROM "missing_${index}_${'x'.repeat(30_000)}"`
|
|
664
|
-
);
|
|
665
|
-
const { entries, bytes } = batchFixture(t, queries);
|
|
666
|
-
assert.equal(bytes, MAX_RESULT_BYTES);
|
|
667
|
-
assert.equal(batchExitCode(entries), 1);
|
|
668
|
-
assert.ok(
|
|
669
|
-
entries.every(
|
|
670
|
-
(entry) => entry.result.ok === false && entry.result.errorTruncated
|
|
671
|
-
)
|
|
672
|
-
);
|
|
673
|
-
assert.ok(
|
|
674
|
-
entries.every((entry) =>
|
|
675
|
-
entry.result.error.startsWith('no such table: missing_')
|
|
676
|
-
)
|
|
677
|
-
);
|
|
678
|
-
t.diagnostic(`all-error batch: ${bytes}/${MAX_RESULT_BYTES} UTF-8 bytes`);
|
|
679
|
-
});
|
|
680
|
-
|
|
681
|
-
test('short query labels and result blocks retain their original serialized form', (t) => {
|
|
682
|
-
const queries = [
|
|
683
|
-
'SELECT marker FROM t',
|
|
684
|
-
'SELECT marker FROM t WHERE id = 0',
|
|
685
|
-
'SELEKT broken',
|
|
686
|
-
];
|
|
687
|
-
const { entries, payload, labels } = batchFixture(t, queries);
|
|
688
|
-
const expected =
|
|
689
|
-
'# {"ok":false,"batch":true,"statements":3}\n' +
|
|
690
|
-
entries
|
|
691
|
-
.map(
|
|
692
|
-
(entry, index) =>
|
|
693
|
-
`## ${JSON.stringify({ index, kind: 'sql', query: queries[index] })}\n` +
|
|
694
|
-
formatResult(entry.result)
|
|
695
|
-
)
|
|
696
|
-
.join('');
|
|
697
|
-
assert.equal(payload, expected);
|
|
698
|
-
assert.deepEqual(
|
|
699
|
-
labels,
|
|
700
|
-
queries.map((query, index) => ({ index, kind: 'sql', query }))
|
|
701
|
-
);
|
|
702
|
-
assert.equal(entries[1].result.totalRows, 0);
|
|
703
|
-
assert.equal(entries[1].result.truncated, false);
|
|
704
|
-
});
|
|
705
|
-
|
|
706
|
-
test('batch byte boundaries include escaped labels, errors and result metadata', (t) => {
|
|
707
|
-
const queries = [
|
|
708
|
-
'SELECT marker FROM t /* "\u{1f642}\\ */',
|
|
709
|
-
'SELECT * FROM t WHERE id = 0',
|
|
710
|
-
'SELEKT broken',
|
|
711
|
-
];
|
|
712
|
-
for (const maxBytes of [
|
|
713
|
-
2_000,
|
|
714
|
-
2_001,
|
|
715
|
-
4_096,
|
|
716
|
-
MAX_RESULT_BYTES - 1,
|
|
717
|
-
MAX_RESULT_BYTES,
|
|
718
|
-
]) {
|
|
719
|
-
const { entries } = batchFixture(t, queries, { maxBytes });
|
|
720
|
-
assert.equal(batchExitCode(entries), 1);
|
|
721
|
-
assert.equal(entries[1].result.truncated, false);
|
|
722
|
-
}
|
|
723
|
-
});
|
|
724
|
-
|
|
725
|
-
test('budgets too small for batch metadata fail explicitly before execution', () => {
|
|
726
|
-
let executions = 0;
|
|
727
|
-
const runners = {
|
|
728
|
-
sql: () => {
|
|
729
|
-
executions += 1;
|
|
730
|
-
return {
|
|
731
|
-
ok: true,
|
|
732
|
-
totalRows: 0,
|
|
733
|
-
returnedRows: 0,
|
|
734
|
-
truncated: false,
|
|
735
|
-
rows: [],
|
|
736
|
-
};
|
|
737
|
-
},
|
|
738
|
-
};
|
|
739
|
-
for (const maxBytes of [0, 1, 100, -1, NaN, Infinity]) {
|
|
740
|
-
assert.throws(
|
|
741
|
-
() =>
|
|
742
|
-
runBatch([{ kind: 'sql', query: 'SELECT 1' }], { runners, maxBytes }),
|
|
743
|
-
/budget|bytes/iu
|
|
744
|
-
);
|
|
745
|
-
}
|
|
746
|
-
assert.equal(executions, 0);
|
|
747
|
-
});
|
|
748
|
-
|
|
749
|
-
test('label limits use serialized UTF-8 bytes and only summarize above the boundary', (t) => {
|
|
750
|
-
const base = 'SELECT marker FROM t /* "\\\u{1f642} */';
|
|
751
|
-
const labelBytes = Buffer.byteLength(
|
|
752
|
-
`## ${JSON.stringify({ index: 0, kind: 'sql', query: base })}\n`
|
|
753
|
-
);
|
|
754
|
-
for (const offset of [-1, 0, 1]) {
|
|
755
|
-
const query = base + ' '.repeat(512 - labelBytes + offset);
|
|
756
|
-
const { payload, labels } = batchFixture(t, [query]);
|
|
757
|
-
const label = `${payload.split('\n')[1]}\n`;
|
|
758
|
-
assert.ok(Buffer.byteLength(label) <= 512);
|
|
759
|
-
assert.equal(labels[0].queryTruncated, offset > 0 ? true : undefined);
|
|
760
|
-
if (offset <= 0) {
|
|
761
|
-
assert.equal(labels[0].query, query);
|
|
762
|
-
assert.equal(Buffer.byteLength(label), 512 + offset);
|
|
763
|
-
}
|
|
764
|
-
}
|
|
765
|
-
});
|
|
766
|
-
|
|
767
|
-
test('exact final budget includes the failing header and trims whole rows one byte below it', (t) => {
|
|
768
|
-
const fixture = batchFixture(t, ['SELEKT', 'SELECT marker FROM t'], {
|
|
769
|
-
rows: [{ marker: 'x'.repeat(2_000) }],
|
|
770
|
-
});
|
|
771
|
-
const exact = runBatch(fixture.plan.batch, {
|
|
772
|
-
runners: fixture.runners,
|
|
773
|
-
maxBytes: fixture.bytes,
|
|
774
|
-
});
|
|
775
|
-
assert.equal(formatBatch(exact), fixture.payload);
|
|
776
|
-
assert.equal(Buffer.byteLength(formatBatch(exact)), fixture.bytes);
|
|
777
|
-
const below = runBatch(fixture.plan.batch, {
|
|
778
|
-
runners: fixture.runners,
|
|
779
|
-
maxBytes: fixture.bytes - 1,
|
|
780
|
-
});
|
|
781
|
-
assert.ok(Buffer.byteLength(formatBatch(below)) <= fixture.bytes - 1);
|
|
782
|
-
assert.equal(below[0].result.error, exact[0].result.error);
|
|
783
|
-
assert.equal(below[1].result.totalRows, 1);
|
|
784
|
-
assert.equal(below[1].result.returnedRows, 0);
|
|
785
|
-
assert.equal(below[1].result.truncated, true);
|
|
786
|
-
assert.equal(below[1].result.truncation.reason, 'byte_budget');
|
|
787
|
-
assert.equal(batchExitCode(below), 1);
|
|
788
|
-
});
|
|
789
|
-
|
|
790
|
-
test('minimum supported budget reserves all metadata before running any statement', (t) => {
|
|
791
|
-
const fixture = batchFixture(
|
|
792
|
-
t,
|
|
793
|
-
[
|
|
794
|
-
'SELECT marker FROM t WHERE id = 0',
|
|
795
|
-
'SELECT marker FROM t',
|
|
796
|
-
'SELECT marker FROM t',
|
|
797
|
-
],
|
|
798
|
-
{ rows: [{ id: 1, marker: 'x'.repeat(30_000) }] }
|
|
799
|
-
);
|
|
800
|
-
fixture.executed.length = 0;
|
|
801
|
-
let minimum;
|
|
802
|
-
assert.throws(
|
|
803
|
-
() =>
|
|
804
|
-
runBatch(fixture.plan.batch, { runners: fixture.runners, maxBytes: 0 }),
|
|
805
|
-
(error) => {
|
|
806
|
-
minimum = Number(/at least (\d+) bytes/u.exec(error.message)?.[1]);
|
|
807
|
-
return Number.isSafeInteger(minimum) && minimum > 0;
|
|
808
|
-
}
|
|
809
|
-
);
|
|
810
|
-
assert.throws(
|
|
811
|
-
() =>
|
|
812
|
-
runBatch(fixture.plan.batch, {
|
|
813
|
-
runners: fixture.runners,
|
|
814
|
-
maxBytes: minimum - 1,
|
|
815
|
-
}),
|
|
816
|
-
/byte budget/u
|
|
817
|
-
);
|
|
818
|
-
assert.deepEqual(fixture.executed, []);
|
|
819
|
-
for (const maxBytes of [minimum, minimum + 1]) {
|
|
820
|
-
const entries = runBatch(fixture.plan.batch, {
|
|
821
|
-
runners: fixture.runners,
|
|
822
|
-
maxBytes,
|
|
823
|
-
});
|
|
824
|
-
assert.ok(Buffer.byteLength(formatBatch(entries)) <= maxBytes);
|
|
825
|
-
assert.equal(
|
|
826
|
-
entries[0].result.truncated,
|
|
827
|
-
false,
|
|
828
|
-
'empty is complete, not truncated'
|
|
829
|
-
);
|
|
830
|
-
for (const entry of entries.slice(1)) {
|
|
831
|
-
assert.equal(entry.result.totalRows, 1);
|
|
832
|
-
assert.equal(entry.result.returnedRows, 0);
|
|
833
|
-
assert.equal(entry.result.truncated, true);
|
|
834
|
-
assert.match(
|
|
835
|
-
entry.result.truncation.requiredAction,
|
|
836
|
-
/before using this result/u
|
|
837
|
-
);
|
|
838
|
-
}
|
|
839
|
-
}
|
|
840
|
-
t.diagnostic(`minimum supported three-statement budget: ${minimum} bytes`);
|
|
841
|
-
});
|
|
842
|
-
|
|
843
|
-
test('bounded UTF-8 batches retain complete row values or explicit truncation across encodings', (t) => {
|
|
844
|
-
const cells = ['plain', '"\\\u03b1\u{1f642}', 'line1\nline2\t"\\\u{1f642}'];
|
|
845
|
-
for (const [index, cell] of cells.entries()) {
|
|
846
|
-
for (const count of [1, 2, 8]) {
|
|
847
|
-
const rows = Array.from({ length: 30 }, (_, id) => ({
|
|
848
|
-
id,
|
|
849
|
-
marker: cell.repeat(20 + id),
|
|
850
|
-
}));
|
|
851
|
-
const queries = Array.from(
|
|
852
|
-
{ length: count },
|
|
853
|
-
(_, position) =>
|
|
854
|
-
`SELECT * FROM t /* ${position} ${cell.repeat(300 + index)} */`
|
|
855
|
-
);
|
|
856
|
-
const fixture = batchFixture(t, queries, {
|
|
857
|
-
rows,
|
|
858
|
-
maxBytes: 8_000 + index,
|
|
859
|
-
limit: 20,
|
|
860
|
-
});
|
|
861
|
-
for (const entry of fixture.entries) {
|
|
862
|
-
assert.equal(entry.result.totalRows, rows.length);
|
|
863
|
-
assert.equal(entry.result.returnedRows, entry.result.rows.length);
|
|
864
|
-
assert.equal(entry.result.truncated, true);
|
|
865
|
-
assert.deepEqual(
|
|
866
|
-
entry.result.rows.map((row) => ({ ...row })),
|
|
867
|
-
rows.slice(0, entry.result.returnedRows)
|
|
868
|
-
);
|
|
869
|
-
assert.match(entry.result.truncation.reason, /row_limit/u);
|
|
870
|
-
assert.match(entry.result.truncation.requiredAction, /narrower WHERE/u);
|
|
871
|
-
}
|
|
872
|
-
}
|
|
873
|
-
}
|
|
874
|
-
});
|
|
875
|
-
|
|
876
|
-
test('minimum search shares preserve notes and indexed errors beside other statements', (t) => {
|
|
877
|
-
const dir = tempDir();
|
|
878
|
-
t.after(() => rmSync(dir, { recursive: true, force: true }));
|
|
879
|
-
const dbPath = join(dir, 'search.sqlite');
|
|
880
|
-
buildDatabase(
|
|
881
|
-
[
|
|
882
|
-
{
|
|
883
|
-
table: 'info_view_measures',
|
|
884
|
-
grain: '',
|
|
885
|
-
dax: '',
|
|
886
|
-
rows: Array.from({ length: 30 }, (_, index) => ({
|
|
887
|
-
Table: 'Facts',
|
|
888
|
-
Name: `Metric ${index}`,
|
|
889
|
-
Description: 'Shared description',
|
|
890
|
-
Expression: 'measure '.repeat(100),
|
|
891
|
-
})),
|
|
892
|
-
},
|
|
893
|
-
],
|
|
894
|
-
dbPath
|
|
895
|
-
);
|
|
896
|
-
const { values } = parseCliArgs([
|
|
897
|
-
'--search',
|
|
898
|
-
'metric',
|
|
899
|
-
'--search',
|
|
900
|
-
'!!!',
|
|
901
|
-
'--sql',
|
|
902
|
-
'SELECT 1 AS n',
|
|
903
|
-
]);
|
|
904
|
-
const plan = planStatements(values);
|
|
905
|
-
const runners = runnersFor(dbPath);
|
|
906
|
-
let minimum;
|
|
907
|
-
assert.throws(
|
|
908
|
-
() => runBatch(plan.batch, { runners, maxBytes: 0 }),
|
|
909
|
-
(error) => {
|
|
910
|
-
minimum = Number(/at least (\d+) bytes/u.exec(error.message)?.[1]);
|
|
911
|
-
return Number.isSafeInteger(minimum);
|
|
912
|
-
}
|
|
913
|
-
);
|
|
914
|
-
const entries = runBatch(plan.batch, { runners, maxBytes: minimum });
|
|
915
|
-
assert.ok(Buffer.byteLength(formatBatch(entries)) <= minimum);
|
|
916
|
-
assert.equal(entries[0].result.totalRows, 30);
|
|
917
|
-
assert.equal(entries[0].result.truncated, true);
|
|
918
|
-
assert.equal(entries[0].result.notes.length, 2);
|
|
919
|
-
assert.match(entries[0].result.notes[0], /description omitted/u);
|
|
920
|
-
assert.match(entries[0].result.notes[1], /expression shortened/u);
|
|
921
|
-
assert.match(entries[1].result.error, /search requires/u);
|
|
922
|
-
assert.equal(entries[2].result.rows[0].n, 1);
|
|
923
|
-
assert.equal(batchExitCode(entries), 1);
|
|
924
|
-
});
|
|
925
|
-
|
|
926
|
-
// -- The workload batching exists for --
|
|
927
|
-
//
|
|
928
|
-
// schema-discovery/SKILL.md documents a five-query sequence that an agent runs
|
|
929
|
-
// back to back after one --search. That is the shape this must hold at, not a
|
|
930
|
-
// pair of toy SELECTs, and it is the shape where the owned-measure enrichment
|
|
931
|
-
// on info_view_tables competes for the same budget.
|
|
932
|
-
|
|
933
|
-
function realisticModel() {
|
|
934
|
-
const tables = ['Sales', 'Date', 'Product', 'Customer', 'Store', 'Geography'];
|
|
935
|
-
return [
|
|
936
|
-
{
|
|
937
|
-
table: 'info_view_tables',
|
|
938
|
-
grain: '',
|
|
939
|
-
dax: 'INFO.VIEW.TABLES()',
|
|
940
|
-
rows: tables.map((name) => ({
|
|
941
|
-
Name: name,
|
|
942
|
-
Description: `${name} dimension or fact table for the retail model`,
|
|
943
|
-
IsHidden: false,
|
|
944
|
-
})),
|
|
945
|
-
},
|
|
946
|
-
{
|
|
947
|
-
table: 'info_view_measures',
|
|
948
|
-
grain: '',
|
|
949
|
-
dax: 'INFO.VIEW.MEASURES()',
|
|
950
|
-
rows: [
|
|
951
|
-
// Headline business measures, so the --search pass matches the way it
|
|
952
|
-
// would on a real model rather than returning an empty set.
|
|
953
|
-
...Array.from({ length: 4 }, (_, index) => ({
|
|
954
|
-
Table: 'Sales',
|
|
955
|
-
Name: `Revenue by Region ${index}`,
|
|
956
|
-
Expression: `CALCULATE(SUM('Sales'[Amount]), ALLEXCEPT('Geography', 'Geography'[Region]))`,
|
|
957
|
-
Description: 'Revenue comparison across region for the period.',
|
|
958
|
-
FormatString: '$#,0',
|
|
959
|
-
DisplayFolder: 'Sales\\Revenue',
|
|
960
|
-
})),
|
|
961
|
-
...tables.flatMap((table) =>
|
|
962
|
-
Array.from({ length: 12 }, (_, index) => ({
|
|
963
|
-
Table: table,
|
|
964
|
-
Name: `${table} Measure ${index}`,
|
|
965
|
-
Expression: `CALCULATE(SUM('${table}'[Amount]), ALLEXCEPT('${table}', '${table}'[Key]))`,
|
|
966
|
-
Description: `Aggregates ${table} amount for reporting purposes.`,
|
|
967
|
-
FormatString: '$#,0',
|
|
968
|
-
DisplayFolder: `${table}\\Core`,
|
|
969
|
-
}))
|
|
970
|
-
),
|
|
971
|
-
],
|
|
972
|
-
},
|
|
973
|
-
{
|
|
974
|
-
table: 'info_view_columns',
|
|
975
|
-
grain: '',
|
|
976
|
-
dax: 'INFO.VIEW.COLUMNS()',
|
|
977
|
-
rows: tables.flatMap((table) =>
|
|
978
|
-
Array.from({ length: 14 }, (_, index) => ({
|
|
979
|
-
Table: table,
|
|
980
|
-
Name: `${table} Column ${index}`,
|
|
981
|
-
DataType: index % 2 === 0 ? 'String' : 'Decimal',
|
|
982
|
-
Description: `Column ${index} of ${table}.`,
|
|
983
|
-
FormatString: '',
|
|
984
|
-
IsHidden: false,
|
|
985
|
-
}))
|
|
986
|
-
),
|
|
987
|
-
},
|
|
988
|
-
{
|
|
989
|
-
table: 'info_view_relationships',
|
|
990
|
-
grain: '',
|
|
991
|
-
dax: 'INFO.VIEW.RELATIONSHIPS()',
|
|
992
|
-
rows: tables.slice(1).map((table) => ({
|
|
993
|
-
FromTable: 'Sales',
|
|
994
|
-
FromColumn: `${table}Key`,
|
|
995
|
-
ToTable: table,
|
|
996
|
-
ToColumn: `${table}Key`,
|
|
997
|
-
IsActive: true,
|
|
998
|
-
CrossFilteringBehavior: 'OneDirection',
|
|
999
|
-
})),
|
|
1000
|
-
},
|
|
1001
|
-
{
|
|
1002
|
-
table: 'column_statistics',
|
|
1003
|
-
grain: '',
|
|
1004
|
-
dax: 'COLUMNSTATISTICS()',
|
|
1005
|
-
rows: tables.flatMap((table) =>
|
|
1006
|
-
Array.from({ length: 6 }, (_, index) => ({
|
|
1007
|
-
'Table Name': table,
|
|
1008
|
-
'Column Name': `${table} Column ${index}`,
|
|
1009
|
-
Min: '1',
|
|
1010
|
-
Max: '99999',
|
|
1011
|
-
Cardinality: 1000 + index,
|
|
1012
|
-
'Max Length': 64,
|
|
1013
|
-
}))
|
|
1014
|
-
),
|
|
1015
|
-
},
|
|
1016
|
-
];
|
|
1017
|
-
}
|
|
1018
|
-
|
|
1019
|
-
test('the documented six-step discovery sequence answers in one shared budget', (t) => {
|
|
1020
|
-
const dir = tempDir();
|
|
1021
|
-
t.after(() => rmSync(dir, { recursive: true, force: true }));
|
|
1022
|
-
const dbPath = join(dir, 'model.sqlite');
|
|
1023
|
-
buildDatabase(realisticModel(), dbPath);
|
|
1024
|
-
|
|
1025
|
-
// Verbatim shapes from schema-discovery/SKILL.md, narrowed the way that file
|
|
1026
|
-
// instructs, plus the --search pass that precedes them.
|
|
1027
|
-
const { values } = parseCliArgs([
|
|
1028
|
-
'--search',
|
|
1029
|
-
'revenue region comparison',
|
|
1030
|
-
'--sql',
|
|
1031
|
-
"SELECT [Name], [Description], [IsHidden] FROM info_view_tables WHERE lower([Name]) LIKE '%sales%'",
|
|
1032
|
-
'--sql',
|
|
1033
|
-
"SELECT [Table], [Name], [Expression], [Description], [FormatString], [DisplayFolder] FROM info_view_measures WHERE [Table] IN ('Sales', 'Date') ORDER BY [Table], [Name]",
|
|
1034
|
-
'--sql',
|
|
1035
|
-
"SELECT [Table], [Name], [DataType], [Description], [FormatString], [IsHidden] FROM info_view_columns WHERE [Table] IN ('Sales', 'Date') ORDER BY [Table], [Name]",
|
|
1036
|
-
'--sql',
|
|
1037
|
-
"SELECT [FromTable], [FromColumn], [ToTable], [ToColumn], [IsActive], [CrossFilteringBehavior] FROM info_view_relationships WHERE [FromTable] IN ('Sales', 'Date') OR [ToTable] IN ('Sales', 'Date')",
|
|
1038
|
-
'--sql',
|
|
1039
|
-
"SELECT [Table Name], [Column Name], [Min], [Max], [Cardinality], [Max Length] FROM column_statistics WHERE [Table Name] = 'Date'",
|
|
1040
|
-
]);
|
|
1041
|
-
const plan = planStatements(values);
|
|
1042
|
-
assert.equal(plan.batch.length, 6);
|
|
1043
|
-
|
|
1044
|
-
const entries = runBatch(plan.batch, { runners: runnersFor(dbPath) });
|
|
1045
|
-
const bytes = Buffer.byteLength(formatBatch(entries), 'utf8');
|
|
1046
|
-
|
|
1047
|
-
assert.ok(
|
|
1048
|
-
bytes <= MAX_RESULT_BYTES,
|
|
1049
|
-
`six real discovery statements must fit ${MAX_RESULT_BYTES} bytes, got ${bytes}`
|
|
1050
|
-
);
|
|
1051
|
-
for (const [index, entry] of entries.entries()) {
|
|
1052
|
-
assert.equal(entry.result.ok, true, `statement ${index} failed`);
|
|
1053
|
-
assert.ok(
|
|
1054
|
-
entry.result.returnedRows > 0,
|
|
1055
|
-
`statement ${index} was starved to zero rows`
|
|
1056
|
-
);
|
|
1057
|
-
}
|
|
1058
|
-
|
|
1059
|
-
// The info_view_tables statement still carries the owned-measure enrichment,
|
|
1060
|
-
// so P5 and the H2 preview trim coexist inside the same shared budget.
|
|
1061
|
-
const tablesRow = entries[1].result.rows[0];
|
|
1062
|
-
assert.equal(tablesRow.ownedMeasureCount, 16);
|
|
1063
|
-
assert.equal(tablesRow.ownedMeasuresTruncated, true);
|
|
1064
|
-
assert.equal(tablesRow.ownedMeasuresNeedDetail, true);
|
|
1065
|
-
assert.equal(tablesRow.ownedMeasures.length, 10);
|
|
1066
|
-
assert.deepEqual(Object.keys(tablesRow.ownedMeasures[0]), [
|
|
1067
|
-
'name',
|
|
1068
|
-
'displayFolder',
|
|
1069
|
-
]);
|
|
1070
|
-
|
|
1071
|
-
// And the search block reports its own trimming rather than silently
|
|
1072
|
-
// shortening what it printed.
|
|
1073
|
-
assert.ok(entries[0].result.totalRows > 0);
|
|
1074
|
-
|
|
1075
|
-
t.diagnostic(
|
|
1076
|
-
`six documented discovery statements: ${bytes}/${MAX_RESULT_BYTES} UTF-8 bytes`
|
|
1077
|
-
);
|
|
1078
|
-
});
|
|
1079
|
-
|
|
1080
|
-
// -- A zero-result search must not be a silent dead end (F1) --
|
|
1081
|
-
|
|
1082
|
-
test('a term this model has never heard of does not empty the whole result', () => {
|
|
1083
|
-
const dir = tempDir();
|
|
1084
|
-
try {
|
|
1085
|
-
const dbPath = join(dir, 'm.sqlite');
|
|
1086
|
-
buildDatabase(
|
|
1087
|
-
[
|
|
1088
|
-
{
|
|
1089
|
-
table: 'info_view_measures',
|
|
1090
|
-
grain: '',
|
|
1091
|
-
dax: '',
|
|
1092
|
-
rows: [
|
|
1093
|
-
{
|
|
1094
|
-
Table: 'Sales',
|
|
1095
|
-
Name: 'Total Sales Amount',
|
|
1096
|
-
Expression: 'SUM(1)',
|
|
1097
|
-
},
|
|
1098
|
-
{ Table: 'Sales', Name: 'Sales Amount LY', Expression: 'SUM(2)' },
|
|
1099
|
-
],
|
|
1100
|
-
},
|
|
1101
|
-
],
|
|
1102
|
-
dbPath
|
|
1103
|
-
);
|
|
1104
|
-
|
|
1105
|
-
// The strict AND is unchanged and still precise.
|
|
1106
|
-
assert.equal(searchCatalog(dbPath, 'sales amount').totalRows, 2);
|
|
1107
|
-
|
|
1108
|
-
// "by" appears nowhere in this catalog, so upstream returns zero rows for
|
|
1109
|
-
// the user's natural phrasing. The relaxed pass answers instead.
|
|
1110
|
-
const natural = searchCatalog(dbPath, 'sales amount by');
|
|
1111
|
-
assert.equal(natural.ok, true);
|
|
1112
|
-
assert.equal(natural.totalRows, 2);
|
|
1113
|
-
assert.match(natural.notes[0], /no object matched all 3 terms/u);
|
|
1114
|
-
assert.match(natural.notes[0], /"by"/u);
|
|
1115
|
-
assert.match(natural.notes[0], /was dropped/u);
|
|
1116
|
-
} finally {
|
|
1117
|
-
rmSync(dir, { recursive: true, force: true });
|
|
1118
|
-
}
|
|
1119
|
-
});
|
|
1120
|
-
|
|
1121
|
-
test('a search that already found rows is never relaxed', () => {
|
|
1122
|
-
const rows = [
|
|
1123
|
-
{ kind: 'measure', object_name: 'Revenue', parent_name: 'Sales' },
|
|
1124
|
-
{ kind: 'measure', object_name: 'Cost', parent_name: 'Sales' },
|
|
1125
|
-
];
|
|
1126
|
-
const strict = rankSearchWithFallback(rows, 'sales revenue');
|
|
1127
|
-
assert.deepEqual(
|
|
1128
|
-
strict.rows,
|
|
1129
|
-
rankSemanticSearchRows(rows, 'sales revenue'),
|
|
1130
|
-
'the successful path must be byte-identical to plain ranking'
|
|
1131
|
-
);
|
|
1132
|
-
assert.deepEqual(strict.notes, [], 'and must say nothing extra');
|
|
1133
|
-
});
|
|
1134
|
-
|
|
1135
|
-
test('a search that matches nothing at all says how to retry', () => {
|
|
1136
|
-
const rows = [{ kind: 'measure', object_name: 'Revenue', parent_name: 'S' }];
|
|
1137
|
-
const missed = rankSearchWithFallback(rows, 'zzz qqq');
|
|
1138
|
-
assert.deepEqual(missed.rows, []);
|
|
1139
|
-
assert.equal(missed.notes.length, 1);
|
|
1140
|
-
assert.match(missed.notes[0], /retry with fewer and more specific terms/u);
|
|
1141
|
-
});
|
|
1142
|
-
|
|
1143
|
-
test('relaxation only drops terms absent from this model, not a word list', () => {
|
|
1144
|
-
// "by" is a stop word in English and nothing in the other direction: the rule
|
|
1145
|
-
// has to be about the catalog, or it stops working in another locale.
|
|
1146
|
-
const rows = [
|
|
1147
|
-
{ kind: 'column', object_name: 'Byte Size', parent_name: 'Files' },
|
|
1148
|
-
];
|
|
1149
|
-
assert.deepEqual(matchedSearchTokens(rows, ['by']), ['by']);
|
|
1150
|
-
assert.deepEqual(matchedSearchTokens(rows, ['by', 'zzz']), ['by']);
|
|
1151
|
-
// A term the model does contain is kept even when it reads like noise, so a
|
|
1152
|
-
// query of only such terms still answers strictly.
|
|
1153
|
-
assert.equal(rankSearchWithFallback(rows, 'by').rows.length, 1);
|
|
1154
|
-
});
|
|
1155
|
-
|
|
1156
|
-
test('the relaxation note is bounded, so the batch reserve can cover it', (t) => {
|
|
1157
|
-
// The note quotes query terms, and the query is agent-supplied. An unbounded
|
|
1158
|
-
// note would overrun the share reserved for it before any row is printed.
|
|
1159
|
-
const dir = tempDir();
|
|
1160
|
-
t.after(() => rmSync(dir, { recursive: true, force: true }));
|
|
1161
|
-
const dbPath = join(dir, 'm.sqlite');
|
|
1162
|
-
buildDatabase(
|
|
1163
|
-
[
|
|
1164
|
-
{
|
|
1165
|
-
table: 'info_view_measures',
|
|
1166
|
-
grain: '',
|
|
1167
|
-
dax: '',
|
|
1168
|
-
rows: [{ Table: 'Sales', Name: 'Revenue', Expression: 'SUM(1)' }],
|
|
1169
|
-
},
|
|
1170
|
-
],
|
|
1171
|
-
dbPath
|
|
1172
|
-
);
|
|
1173
|
-
|
|
1174
|
-
const absurd = `revenue ${Array.from({ length: 50 }, (_, index) => `${'z'.repeat(200)}${index}`).join(' ')}`;
|
|
1175
|
-
const out = searchCatalog(dbPath, absurd);
|
|
1176
|
-
assert.equal(out.totalRows, 1, 'the one real term still answers');
|
|
1177
|
-
assert.ok(
|
|
1178
|
-
Buffer.byteLength(out.notes.join(''), 'utf8') < 1_000,
|
|
1179
|
-
`the note must stay bounded, got ${out.notes.join('').length} chars`
|
|
1180
|
-
);
|
|
1181
|
-
|
|
1182
|
-
// And the whole thing still fits a batch beside another statement.
|
|
1183
|
-
const entries = runBatch(
|
|
1184
|
-
[
|
|
1185
|
-
{ kind: 'search', query: absurd },
|
|
1186
|
-
{ kind: 'sql', query: 'SELECT 1 AS n' },
|
|
1187
|
-
],
|
|
1188
|
-
{ runners: runnersFor(dbPath) }
|
|
1189
|
-
);
|
|
1190
|
-
assert.equal(entries[0].result.totalRows, 1);
|
|
1191
|
-
assert.equal(entries[1].result.rows[0].n, 1);
|
|
1192
|
-
assert.ok(
|
|
1193
|
-
Buffer.byteLength(formatBatch(entries), 'utf8') <= MAX_RESULT_BYTES
|
|
1194
|
-
);
|
|
1195
|
-
});
|
|
1196
|
-
|
|
1197
|
-
// -- Owned-measure detail is a per-row signal, not a banner (F2) --
|
|
1198
|
-
|
|
1199
|
-
test('a table with no owned measures is not told to go fetch them', () => {
|
|
1200
|
-
const dir = tempDir();
|
|
1201
|
-
try {
|
|
1202
|
-
const dbPath = join(dir, 'm.sqlite');
|
|
1203
|
-
buildDatabase(
|
|
1204
|
-
[
|
|
1205
|
-
{
|
|
1206
|
-
table: 'info_view_tables',
|
|
1207
|
-
grain: '',
|
|
1208
|
-
dax: '',
|
|
1209
|
-
rows: [{ Name: 'Sales' }, { Name: 'Sales Territory' }],
|
|
1210
|
-
},
|
|
1211
|
-
{
|
|
1212
|
-
table: 'info_view_measures',
|
|
1213
|
-
grain: '',
|
|
1214
|
-
dax: '',
|
|
1215
|
-
rows: [{ Table: 'Sales', Name: 'Revenue', DisplayFolder: 'Finance' }],
|
|
1216
|
-
},
|
|
1217
|
-
],
|
|
1218
|
-
dbPath
|
|
1219
|
-
);
|
|
1220
|
-
|
|
1221
|
-
const out = runSql(dbPath, 'SELECT [Name] FROM info_view_tables');
|
|
1222
|
-
const byName = Object.fromEntries(out.rows.map((row) => [row.Name, row]));
|
|
1223
|
-
|
|
1224
|
-
assert.equal(byName.Sales.ownedMeasureCount, 1);
|
|
1225
|
-
assert.equal(byName.Sales.ownedMeasuresNeedDetail, true);
|
|
1226
|
-
|
|
1227
|
-
assert.equal(byName['Sales Territory'].ownedMeasureCount, 0);
|
|
1228
|
-
assert.equal(
|
|
1229
|
-
byName['Sales Territory'].ownedMeasuresNeedDetail,
|
|
1230
|
-
false,
|
|
1231
|
-
'there is nothing to fetch, so do not send the agent to fetch it'
|
|
1232
|
-
);
|
|
1233
|
-
// The key set stays identical across rows either way.
|
|
1234
|
-
assert.deepEqual(
|
|
1235
|
-
Object.keys(byName.Sales),
|
|
1236
|
-
Object.keys(byName['Sales Territory'])
|
|
1237
|
-
);
|
|
1238
|
-
} finally {
|
|
1239
|
-
rmSync(dir, { recursive: true, force: true });
|
|
1240
|
-
}
|
|
1241
|
-
});
|
|
1242
|
-
|
|
1243
|
-
// -- Publication must stay visible, and a failed open is one statement's
|
|
1244
|
-
// problem, not the batch's (A4) --
|
|
1245
|
-
|
|
1246
|
-
test('publication never leaves the destination missing', () => {
|
|
1247
|
-
const dir = tempDir();
|
|
1248
|
-
try {
|
|
1249
|
-
const dbPath = join(dir, 'model.sqlite');
|
|
1250
|
-
const catalogPath = join(dir, 'catalog.md');
|
|
1251
|
-
writeFileSync(dbPath, 'previous database', 'utf8');
|
|
1252
|
-
writeFileSync(catalogPath, 'previous catalog', 'utf8');
|
|
1253
|
-
const temporaryDb = join(dir, 'next.sqlite.tmp');
|
|
1254
|
-
const temporaryCatalog = join(dir, 'next.md.tmp');
|
|
1255
|
-
writeFileSync(temporaryDb, 'next database', 'utf8');
|
|
1256
|
-
writeFileSync(temporaryCatalog, 'next catalog', 'utf8');
|
|
1257
|
-
|
|
1258
|
-
// Observe the live path after every filesystem step. Backing the previous
|
|
1259
|
-
// generation up by renaming it away made this false in the middle, and a
|
|
1260
|
-
// reader landing there got "unable to open database file".
|
|
1261
|
-
const seen = [];
|
|
1262
|
-
const observe =
|
|
1263
|
-
(operation) =>
|
|
1264
|
-
(...args) => {
|
|
1265
|
-
const result = operation(...args);
|
|
1266
|
-
seen.push(existsSync(dbPath));
|
|
1267
|
-
return result;
|
|
1268
|
-
};
|
|
1269
|
-
publishFiles(
|
|
1270
|
-
[
|
|
1271
|
-
{ temporaryPath: temporaryDb, destinationPath: dbPath },
|
|
1272
|
-
{ temporaryPath: temporaryCatalog, destinationPath: catalogPath },
|
|
1273
|
-
],
|
|
1274
|
-
{ rename: observe(renameSync), link: observe(linkSync) }
|
|
1275
|
-
);
|
|
1276
|
-
|
|
1277
|
-
assert.ok(seen.length >= 4, 'the publication really ran');
|
|
1278
|
-
assert.ok(
|
|
1279
|
-
seen.every(Boolean),
|
|
1280
|
-
`the database path must exist at every step, got ${JSON.stringify(seen)}`
|
|
1281
|
-
);
|
|
1282
|
-
assert.equal(readFileSync(dbPath, 'utf8'), 'next database');
|
|
1283
|
-
assert.equal(readFileSync(catalogPath, 'utf8'), 'next catalog');
|
|
1284
|
-
} finally {
|
|
1285
|
-
rmSync(dir, { recursive: true, force: true });
|
|
1286
|
-
}
|
|
1287
|
-
});
|
|
1288
|
-
|
|
1289
|
-
test('a statement that cannot open the catalog does not discard the batch', () => {
|
|
1290
|
-
const dir = tempDir();
|
|
1291
|
-
try {
|
|
1292
|
-
const dbPath = join(dir, 'model.sqlite');
|
|
1293
|
-
buildDatabase(
|
|
1294
|
-
[{ table: 't', grain: '', dax: '', rows: [{ n: 1 }] }],
|
|
1295
|
-
dbPath
|
|
1296
|
-
);
|
|
1297
|
-
|
|
1298
|
-
// Model a reader that lands on the catalog exactly while it is being
|
|
1299
|
-
// replaced: the first statement succeeds, then the file goes away.
|
|
1300
|
-
const runners = {
|
|
1301
|
-
search: (query, options) => searchCatalog(dbPath, query, options),
|
|
1302
|
-
sql: (query, options) => {
|
|
1303
|
-
if (query.includes('second')) {
|
|
1304
|
-
rmSync(dbPath, { force: true });
|
|
1305
|
-
}
|
|
1306
|
-
return runSql(dbPath, query, options);
|
|
1307
|
-
},
|
|
1308
|
-
};
|
|
1309
|
-
const entries = runBatch(
|
|
1310
|
-
[
|
|
1311
|
-
{ kind: 'sql', query: 'SELECT n FROM t' },
|
|
1312
|
-
{ kind: 'sql', query: 'SELECT n FROM t -- second' },
|
|
1313
|
-
{ kind: 'sql', query: 'SELECT n FROM t' },
|
|
1314
|
-
],
|
|
1315
|
-
{ runners }
|
|
1316
|
-
);
|
|
1317
|
-
|
|
1318
|
-
assert.equal(entries.length, 3, 'every statement is still accounted for');
|
|
1319
|
-
assert.equal(entries[0].result.ok, true);
|
|
1320
|
-
assert.equal(
|
|
1321
|
-
entries[0].result.rows[0].n,
|
|
1322
|
-
1,
|
|
1323
|
-
'its work was not thrown away'
|
|
1324
|
-
);
|
|
1325
|
-
assert.equal(entries[1].result.ok, false);
|
|
1326
|
-
assert.match(entries[1].result.error, /unable to open database file/u);
|
|
1327
|
-
assert.equal(batchExitCode(entries), 1);
|
|
1328
|
-
} finally {
|
|
1329
|
-
rmSync(dir, { recursive: true, force: true });
|
|
1330
|
-
}
|
|
1331
|
-
});
|
|
1332
|
-
|
|
1333
|
-
// -- Cuts must not split a surrogate pair (A5) --
|
|
1334
|
-
|
|
1335
|
-
test('an expression cut mid-emoji still emits well-formed text', () => {
|
|
1336
|
-
// 119 ASCII characters then an emoji: the slice lands between the pair, and
|
|
1337
|
-
// there is no space, comma or paren in the head for the boundary rule to
|
|
1338
|
-
// fall back on.
|
|
1339
|
-
const expression = `${'a'.repeat(119)}\u{1F600} trailing text`;
|
|
1340
|
-
const summary = summarizeSearchExpression(expression);
|
|
1341
|
-
|
|
1342
|
-
assert.equal(summary.isWellFormed(), true, 'no lone surrogate may survive');
|
|
1343
|
-
assert.equal(
|
|
1344
|
-
Buffer.from(summary, 'utf8').toString('utf8'),
|
|
1345
|
-
summary,
|
|
1346
|
-
'the summary must round-trip through UTF-8'
|
|
1347
|
-
);
|
|
1348
|
-
assert.ok(!summary.includes('\uFFFD'), 'and must not contain U+FFFD');
|
|
1349
|
-
});
|
|
1350
|
-
|
|
1351
|
-
test('a BMP-only expression is unaffected by the surrogate rule', () => {
|
|
1352
|
-
// Every character here is one UTF-16 unit, so the cut was already safe and
|
|
1353
|
-
// must stay exactly where it was.
|
|
1354
|
-
const expression = '\u6f22\u5b57'.repeat(200);
|
|
1355
|
-
const summary = summarizeSearchExpression(expression);
|
|
1356
|
-
|
|
1357
|
-
assert.equal(summary.isWellFormed(), true);
|
|
1358
|
-
assert.equal(
|
|
1359
|
-
summary.split(' \u2026 ')[0],
|
|
1360
|
-
expression.slice(0, SEARCH_EXPRESSION_LIMIT),
|
|
1361
|
-
'the BMP cut is untouched'
|
|
1362
|
-
);
|
|
1363
|
-
});
|
|
1364
|
-
|
|
1365
|
-
// -- Note budgets are counted in UTF-8 bytes, not UTF-16 length (A1) --
|
|
1366
|
-
|
|
1367
|
-
test('eight non-Latin searches stay inside the reserved note budget', (t) => {
|
|
1368
|
-
const dir = tempDir();
|
|
1369
|
-
t.after(() => rmSync(dir, { recursive: true, force: true }));
|
|
1370
|
-
const dbPath = join(dir, 'm.sqlite');
|
|
1371
|
-
buildDatabase(
|
|
1372
|
-
[
|
|
1373
|
-
{
|
|
1374
|
-
table: 'info_view_measures',
|
|
1375
|
-
grain: '',
|
|
1376
|
-
dax: '',
|
|
1377
|
-
rows: Array.from({ length: 20 }, (_, index) => ({
|
|
1378
|
-
Table: '\u58f2\u4e0a',
|
|
1379
|
-
Name: `\u6307\u6a19 ${index}`,
|
|
1380
|
-
Expression: 'SUM(1)',
|
|
1381
|
-
})),
|
|
1382
|
-
},
|
|
1383
|
-
],
|
|
1384
|
-
dbPath
|
|
1385
|
-
);
|
|
1386
|
-
|
|
1387
|
-
// One term the model has (so the relaxed pass answers and emits the note),
|
|
1388
|
-
// plus terms it does not. Every term is 3 UTF-8 bytes per character, which
|
|
1389
|
-
// is what a UTF-16 length cap under a byte reserve gets wrong.
|
|
1390
|
-
const absent = Array.from(
|
|
1391
|
-
{ length: 6 },
|
|
1392
|
-
(_, index) => `${'\u3042'.repeat(40)}${index}`
|
|
1393
|
-
).join(' ');
|
|
1394
|
-
const batch = Array.from({ length: MAX_BATCH_STATEMENTS }, () => ({
|
|
1395
|
-
kind: 'search',
|
|
1396
|
-
query: `\u6307\u6a19 ${absent}`,
|
|
1397
|
-
}));
|
|
1398
|
-
|
|
1399
|
-
const entries = runBatch(batch, { runners: runnersFor(dbPath) });
|
|
1400
|
-
const bytes = Buffer.byteLength(formatBatch(entries), 'utf8');
|
|
1401
|
-
|
|
1402
|
-
assert.equal(entries.length, MAX_BATCH_STATEMENTS);
|
|
1403
|
-
assert.ok(bytes <= MAX_RESULT_BYTES, `batch emitted ${bytes} bytes`);
|
|
1404
|
-
for (const entry of entries) {
|
|
1405
|
-
assert.equal(entry.result.ok, true);
|
|
1406
|
-
assert.equal(entry.result.totalRows, 20, 'the relaxed pass still answered');
|
|
1407
|
-
assert.match(entry.result.notes[0], /no object matched all/u);
|
|
1408
|
-
assert.ok(
|
|
1409
|
-
Buffer.byteLength(entry.result.notes[0], 'utf8') <= 400,
|
|
1410
|
-
'each note is bounded in bytes, whatever script it quotes'
|
|
1411
|
-
);
|
|
1412
|
-
}
|
|
1413
|
-
t.diagnostic(`eight CJK searches: ${bytes}/${MAX_RESULT_BYTES} bytes`);
|
|
1414
|
-
});
|
|
1415
|
-
|
|
1416
|
-
// -- Containment has to survive a link (A2) --
|
|
1417
|
-
|
|
1418
|
-
test('a --file reached through an in-app link to outside is refused', (t) => {
|
|
1419
|
-
const app = tempDir();
|
|
1420
|
-
const outside = tempDir();
|
|
1421
|
-
t.after(() => {
|
|
1422
|
-
rmSync(app, { recursive: true, force: true });
|
|
1423
|
-
rmSync(outside, { recursive: true, force: true });
|
|
1424
|
-
});
|
|
1425
|
-
writeFileSync(join(outside, 'secret.txt'), 'NOT-A-REAL-CREDENTIAL', 'utf8');
|
|
1426
|
-
|
|
1427
|
-
const linkPath = join(app, 'link');
|
|
1428
|
-
try {
|
|
1429
|
-
symlinkSync(outside, linkPath, 'junction');
|
|
1430
|
-
} catch {
|
|
1431
|
-
t.skip('this filesystem or account cannot create a directory link');
|
|
1432
|
-
return;
|
|
1433
|
-
}
|
|
1434
|
-
|
|
1435
|
-
// Lexically the path is under the app root, so a resolve+startsWith check
|
|
1436
|
-
// passes it. Containment that a link walks around is not containment.
|
|
1437
|
-
const escaped = join(linkPath, 'secret.txt');
|
|
1438
|
-
assert.ok(escaped.startsWith(app + sep), 'the test targets the lexical gap');
|
|
1439
|
-
|
|
1440
|
-
const plan = planStatements({ file: [escaped], root: app });
|
|
1441
|
-
assert.match(plan.error, /--file must be inside the app/u);
|
|
1442
|
-
assert.ok(!plan.error.includes('NOT-A-REAL-CREDENTIAL'));
|
|
1443
|
-
});
|
|
1444
|
-
|
|
1445
|
-
test('an ordinary in-app --file still resolves through a linked app root', (t) => {
|
|
1446
|
-
// tmpdir() is itself a link on some platforms, so the root must be resolved
|
|
1447
|
-
// too or every legitimate read would be refused.
|
|
1448
|
-
const app = tempDir();
|
|
1449
|
-
t.after(() => rmSync(app, { recursive: true, force: true }));
|
|
1450
|
-
writeFileSync(join(app, 'q.sql'), 'SELECT 1', 'utf8');
|
|
1451
|
-
|
|
1452
|
-
const plan = planStatements({ file: ['q.sql'], root: app });
|
|
1453
|
-
assert.equal(plan.mode, 'batch');
|
|
1454
|
-
assert.equal(plan.batch[0].query, 'SELECT 1');
|
|
1455
|
-
assert.equal(plan.batch[0].source, 'q.sql');
|
|
1456
|
-
});
|
|
1457
|
-
|
|
1458
|
-
// -- A reclaimer must not remove a lock it did not judge (A3) --
|
|
1459
|
-
|
|
1460
|
-
test('releasing a lock does not remove one another process now holds', async () => {
|
|
1461
|
-
const dir = tempDir();
|
|
1462
|
-
try {
|
|
1463
|
-
const lockPath = join(dir, '.refresh.lock');
|
|
1464
|
-
await withCatalogLock(dir, async () => {
|
|
1465
|
-
// Another process reclaimed and took the lock while we were working.
|
|
1466
|
-
writeFileSync(
|
|
1467
|
-
join(lockPath, 'owner.json'),
|
|
1468
|
-
JSON.stringify({ pid: process.pid, token: 'someone-else' }),
|
|
1469
|
-
'utf8'
|
|
1470
|
-
);
|
|
1471
|
-
});
|
|
1472
|
-
|
|
1473
|
-
assert.equal(
|
|
1474
|
-
existsSync(lockPath),
|
|
1475
|
-
true,
|
|
1476
|
-
'releasing must not delete a live lock that replaced ours'
|
|
1477
|
-
);
|
|
1478
|
-
assert.equal(
|
|
1479
|
-
JSON.parse(readFileSync(join(lockPath, 'owner.json'), 'utf8')).token,
|
|
1480
|
-
'someone-else'
|
|
1481
|
-
);
|
|
1482
|
-
} finally {
|
|
1483
|
-
rmSync(dir, { recursive: true, force: true });
|
|
1484
|
-
}
|
|
1485
|
-
});
|
|
1486
|
-
|
|
1487
|
-
test('a stale reclaim re-reads before removing, so it cannot steal a new lock', async () => {
|
|
1488
|
-
const dir = tempDir();
|
|
1489
|
-
try {
|
|
1490
|
-
mkdirSync(dir, { recursive: true });
|
|
1491
|
-
const lockPath = join(dir, '.refresh.lock');
|
|
1492
|
-
mkdirSync(lockPath);
|
|
1493
|
-
writeFileSync(
|
|
1494
|
-
join(lockPath, 'owner.json'),
|
|
1495
|
-
JSON.stringify({ pid: 999_999, token: 'abandoned' }),
|
|
1496
|
-
'utf8'
|
|
1497
|
-
);
|
|
1498
|
-
|
|
1499
|
-
let checks = 0;
|
|
1500
|
-
await assert.rejects(
|
|
1501
|
-
withCatalogLock(dir, async () => 'must not run', {
|
|
1502
|
-
timeoutMs: 50,
|
|
1503
|
-
pollMs: 1,
|
|
1504
|
-
processAlive: () => {
|
|
1505
|
-
checks += 1;
|
|
1506
|
-
if (checks === 1) {
|
|
1507
|
-
// Another reclaimer wins the race and installs its own live lock
|
|
1508
|
-
// between our decision and our removal.
|
|
1509
|
-
writeFileSync(
|
|
1510
|
-
join(lockPath, 'owner.json'),
|
|
1511
|
-
JSON.stringify({ pid: process.pid, token: 'winner' }),
|
|
1512
|
-
'utf8'
|
|
1513
|
-
);
|
|
1514
|
-
}
|
|
1515
|
-
return false;
|
|
1516
|
-
},
|
|
1517
|
-
}),
|
|
1518
|
-
/Timed out waiting/u
|
|
1519
|
-
);
|
|
1520
|
-
|
|
1521
|
-
assert.equal(existsSync(lockPath), true, 'the winner keeps its lock');
|
|
1522
|
-
assert.equal(
|
|
1523
|
-
JSON.parse(readFileSync(join(lockPath, 'owner.json'), 'utf8')).token,
|
|
1524
|
-
'winner'
|
|
1525
|
-
);
|
|
1526
|
-
} finally {
|
|
1527
|
-
rmSync(dir, { recursive: true, force: true });
|
|
1528
|
-
}
|
|
1529
|
-
});
|
|
1530
|
-
|
|
1531
|
-
test('publication waits out a reader holding the destination open', () => {
|
|
1532
|
-
// Windows reports a sharing violation under more than one code, so all three
|
|
1533
|
-
// have to be waited out rather than whichever one a local run happened to
|
|
1534
|
-
// produce.
|
|
1535
|
-
for (const code of ['EPERM', 'EBUSY', 'EACCES']) {
|
|
1536
|
-
const dir = tempDir();
|
|
1537
|
-
try {
|
|
1538
|
-
const dbPath = join(dir, 'model.sqlite');
|
|
1539
|
-
const temporaryPath = join(dir, 'next.tmp');
|
|
1540
|
-
writeFileSync(dbPath, 'previous', 'utf8');
|
|
1541
|
-
writeFileSync(temporaryPath, 'next', 'utf8');
|
|
1542
|
-
|
|
1543
|
-
let attempts = 0;
|
|
1544
|
-
const rename = (from, to) => {
|
|
1545
|
-
if (to === dbPath && attempts++ < 3) {
|
|
1546
|
-
const error = new Error(`${code}: rename blocked`);
|
|
1547
|
-
error.code = code;
|
|
1548
|
-
throw error;
|
|
1549
|
-
}
|
|
1550
|
-
renameSync(from, to);
|
|
1551
|
-
};
|
|
1552
|
-
|
|
1553
|
-
publishFiles([{ temporaryPath, destinationPath: dbPath }], {
|
|
1554
|
-
rename,
|
|
1555
|
-
retry: { attempts: 10, delayMs: 0, sleep: () => {} },
|
|
1556
|
-
});
|
|
1557
|
-
|
|
1558
|
-
assert.equal(attempts, 4, `${code} retried rather than giving up`);
|
|
1559
|
-
assert.equal(readFileSync(dbPath, 'utf8'), 'next');
|
|
1560
|
-
} finally {
|
|
1561
|
-
rmSync(dir, { recursive: true, force: true });
|
|
1562
|
-
}
|
|
1563
|
-
}
|
|
1564
|
-
});
|
|
1565
|
-
|
|
1566
|
-
test('publication still gives up once the retry bound is reached', () => {
|
|
1567
|
-
const dir = tempDir();
|
|
1568
|
-
try {
|
|
1569
|
-
const dbPath = join(dir, 'model.sqlite');
|
|
1570
|
-
const temporaryPath = join(dir, 'next.tmp');
|
|
1571
|
-
writeFileSync(dbPath, 'previous', 'utf8');
|
|
1572
|
-
writeFileSync(temporaryPath, 'next', 'utf8');
|
|
1573
|
-
|
|
1574
|
-
const rename = (from, to) => {
|
|
1575
|
-
if (to === dbPath) {
|
|
1576
|
-
const error = new Error('EPERM: operation not permitted, rename');
|
|
1577
|
-
error.code = 'EPERM';
|
|
1578
|
-
throw error;
|
|
1579
|
-
}
|
|
1580
|
-
renameSync(from, to);
|
|
1581
|
-
};
|
|
1582
|
-
|
|
1583
|
-
assert.throws(
|
|
1584
|
-
() =>
|
|
1585
|
-
publishFiles([{ temporaryPath, destinationPath: dbPath }], {
|
|
1586
|
-
rename,
|
|
1587
|
-
retry: { attempts: 3, delayMs: 0, sleep: () => {} },
|
|
1588
|
-
}),
|
|
1589
|
-
/EPERM/u
|
|
1590
|
-
);
|
|
1591
|
-
// The previous generation is restored, not left behind a backup link.
|
|
1592
|
-
assert.equal(readFileSync(dbPath, 'utf8'), 'previous');
|
|
1593
|
-
} finally {
|
|
1594
|
-
rmSync(dir, { recursive: true, force: true });
|
|
1595
|
-
}
|
|
1596
|
-
});
|
|
1597
|
-
|
|
1598
|
-
test('a rename failure that is not a sharing violation is not retried', () => {
|
|
1599
|
-
let attempts = 0;
|
|
1600
|
-
const rename = () => {
|
|
1601
|
-
attempts += 1;
|
|
1602
|
-
const error = new Error('ENOSPC: no space left on device');
|
|
1603
|
-
error.code = 'ENOSPC';
|
|
1604
|
-
throw error;
|
|
1605
|
-
};
|
|
1606
|
-
assert.throws(
|
|
1607
|
-
() =>
|
|
1608
|
-
publishFiles([{ temporaryPath: 'a', destinationPath: 'b' }], {
|
|
1609
|
-
exists: () => false,
|
|
1610
|
-
rename,
|
|
1611
|
-
retry: { attempts: 10, delayMs: 0, sleep: () => {} },
|
|
1612
|
-
}),
|
|
1613
|
-
/ENOSPC/u
|
|
1614
|
-
);
|
|
1615
|
-
assert.equal(attempts, 1, 'only a sharing violation is worth waiting out');
|
|
1616
|
-
});
|
|
1617
|
-
|
|
1618
|
-
// -- An over-budget expression must still be recoverable --
|
|
1619
|
-
|
|
1620
|
-
test('an expression larger than the budget is reconstructible in chunks', () => {
|
|
1621
|
-
const dir = tempDir();
|
|
1622
|
-
try {
|
|
1623
|
-
const dbPath = join(dir, 'm.sqlite');
|
|
1624
|
-
// Non-ASCII, quotes and newlines, so a reconstruction that mangles
|
|
1625
|
-
// encoding or drops the tail cannot pass. Well past MAX_RESULT_BYTES.
|
|
1626
|
-
const unit = `"\u6f22\u5b57" & \u{1F600} & '\u00e9\u00e0'\nRETURN `;
|
|
1627
|
-
const expression = `SWITCH(USERCULTURE(), ${unit.repeat(2_000)})`;
|
|
1628
|
-
assert.ok(
|
|
1629
|
-
Buffer.byteLength(expression, 'utf8') > MAX_RESULT_BYTES * 2,
|
|
1630
|
-
'the fixture has to actually exceed the budget'
|
|
1631
|
-
);
|
|
1632
|
-
buildDatabase(
|
|
1633
|
-
[
|
|
1634
|
-
{
|
|
1635
|
-
table: 'info_view_measures',
|
|
1636
|
-
grain: '',
|
|
1637
|
-
dax: '',
|
|
1638
|
-
rows: [
|
|
1639
|
-
{ Table: 'Sales', Name: 'Localized Label', Expression: expression },
|
|
1640
|
-
],
|
|
1641
|
-
},
|
|
1642
|
-
],
|
|
1643
|
-
dbPath
|
|
1644
|
-
);
|
|
1645
|
-
|
|
1646
|
-
const where =
|
|
1647
|
-
"FROM info_view_measures WHERE [Table] = 'Sales' " +
|
|
1648
|
-
"AND [Name] = 'Localized Label'";
|
|
1649
|
-
|
|
1650
|
-
// The dead end the documented recovery used to walk into: a single cell
|
|
1651
|
-
// has nothing left to narrow, so the row matches but cannot be returned.
|
|
1652
|
-
const whole = runSql(dbPath, `SELECT [Expression] ${where}`);
|
|
1653
|
-
assert.equal(whole.ok, true);
|
|
1654
|
-
assert.equal(whole.totalRows, 1);
|
|
1655
|
-
assert.equal(
|
|
1656
|
-
whole.returnedRows,
|
|
1657
|
-
0,
|
|
1658
|
-
'whole rows are dropped, not shortened'
|
|
1659
|
-
);
|
|
1660
|
-
assert.equal(whole.truncated, true);
|
|
1661
|
-
|
|
1662
|
-
const size = runSql(
|
|
1663
|
-
dbPath,
|
|
1664
|
-
`SELECT length([Expression]) AS chars ${where}`
|
|
1665
|
-
);
|
|
1666
|
-
const chars = size.rows[0].chars;
|
|
1667
|
-
// SQLite counts code points; the size in the search summary comes from
|
|
1668
|
-
// JavaScript and counts UTF-16 units, so an expression with astral
|
|
1669
|
-
// characters reports fewer here. length() is the authoritative bound for
|
|
1670
|
-
// substr() offsets, which are themselves code-point based.
|
|
1671
|
-
assert.ok(chars > 0);
|
|
1672
|
-
assert.equal(
|
|
1673
|
-
chars,
|
|
1674
|
-
[...expression].length,
|
|
1675
|
-
'length() counts code points, not UTF-16 units'
|
|
1676
|
-
);
|
|
1677
|
-
assert.ok(chars < expression.length, 'the fixture exercises that gap');
|
|
1678
|
-
|
|
1679
|
-
const stride = 1000;
|
|
1680
|
-
const parts = [];
|
|
1681
|
-
for (let offset = 1; offset <= chars; offset += stride) {
|
|
1682
|
-
const chunk = runSql(
|
|
1683
|
-
dbPath,
|
|
1684
|
-
`SELECT substr([Expression], ${offset}, ${stride}) AS part ${where}`
|
|
1685
|
-
);
|
|
1686
|
-
assert.equal(chunk.ok, true, `chunk at ${offset} failed`);
|
|
1687
|
-
assert.equal(chunk.truncated, false, `chunk at ${offset} was truncated`);
|
|
1688
|
-
assert.equal(chunk.returnedRows, 1, `chunk at ${offset} was dropped`);
|
|
1689
|
-
parts.push(chunk.rows[0].part);
|
|
1690
|
-
}
|
|
1691
|
-
|
|
1692
|
-
assert.equal(
|
|
1693
|
-
parts.join(''),
|
|
1694
|
-
expression,
|
|
1695
|
-
'the reconstruction must equal the original exactly, tail included'
|
|
1696
|
-
);
|
|
1697
|
-
} finally {
|
|
1698
|
-
rmSync(dir, { recursive: true, force: true });
|
|
1699
|
-
}
|
|
1700
|
-
});
|
|
1701
|
-
|
|
1702
|
-
// -- Search payload trimming --
|
|
1703
|
-
|
|
1704
|
-
test('a short expression survives verbatim apart from whitespace collapsing', () => {
|
|
1705
|
-
assert.equal(
|
|
1706
|
-
summarizeSearchExpression('SUM(Sales[Amount])'),
|
|
1707
|
-
'SUM(Sales[Amount])'
|
|
1708
|
-
);
|
|
1709
|
-
assert.equal(
|
|
1710
|
-
summarizeSearchExpression('SUM(\n Sales[Amount]\n)'),
|
|
1711
|
-
'SUM( Sales[Amount] )'
|
|
1712
|
-
);
|
|
1713
|
-
assert.equal(summarizeSearchExpression(null), '');
|
|
1714
|
-
});
|
|
1715
|
-
|
|
1716
|
-
test('a long expression keeps a readable head and says where the rest is', () => {
|
|
1717
|
-
const long = `SWITCH(USERCULTURE(), ${Array.from(
|
|
1718
|
-
{ length: 40 },
|
|
1719
|
-
(_, index) => `"loc-${index}", "a translated sentence"`
|
|
1720
|
-
).join(', ')})`;
|
|
1721
|
-
const summary = summarizeSearchExpression(long);
|
|
1722
|
-
|
|
1723
|
-
assert.ok(
|
|
1724
|
-
summary.length < long.length / 4,
|
|
1725
|
-
'the summary must actually be smaller'
|
|
1726
|
-
);
|
|
1727
|
-
assert.match(
|
|
1728
|
-
summary,
|
|
1729
|
-
/^SWITCH\(USERCULTURE\(\)/u,
|
|
1730
|
-
'the head identifies what the measure does'
|
|
1731
|
-
);
|
|
1732
|
-
assert.match(
|
|
1733
|
-
summary,
|
|
1734
|
-
new RegExp(`${long.length} chars`, 'u'),
|
|
1735
|
-
'the original size is stated'
|
|
1736
|
-
);
|
|
1737
|
-
assert.match(
|
|
1738
|
-
summary,
|
|
1739
|
-
/substr\(\[Expression\], <offset>, 1000\)/u,
|
|
1740
|
-
'the named next step must be one that can actually succeed'
|
|
1741
|
-
);
|
|
1742
|
-
});
|
|
1743
|
-
|
|
1744
|
-
test('the head is cut at a token boundary when the text offers one', () => {
|
|
1745
|
-
// Only a guarantee when a separator falls in the second half of the head; a
|
|
1746
|
-
// single unbroken token has nowhere to cut and is truncated as-is.
|
|
1747
|
-
const separated = `CALCULATE(${'SUM(Sales[Amount]), '.repeat(30)})`;
|
|
1748
|
-
const head = summarizeSearchExpression(separated).split(' \u2026 ')[0];
|
|
1749
|
-
// The cut lands ON the separator, so the head ends with a whole token and the
|
|
1750
|
-
// next source character is the separator itself - `SUM` rather than `SU`.
|
|
1751
|
-
// Asserting the head's own last character instead would be wrong: a cut at
|
|
1752
|
-
// `(` leaves `SUM`, which ends in no separator at all.
|
|
1753
|
-
assert.ok(separated.startsWith(head), 'the head is a prefix of the source');
|
|
1754
|
-
assert.match(
|
|
1755
|
-
separated[head.length],
|
|
1756
|
-
/[\s,(]/u,
|
|
1757
|
-
'the character the cut landed on is a separator'
|
|
1758
|
-
);
|
|
1759
|
-
assert.ok(head.length <= SEARCH_EXPRESSION_LIMIT);
|
|
1760
|
-
|
|
1761
|
-
const unbroken = 'A'.repeat(400);
|
|
1762
|
-
const unbrokenHead = summarizeSearchExpression(unbroken).split(' \u2026 ')[0];
|
|
1763
|
-
assert.equal(unbrokenHead, 'A'.repeat(SEARCH_EXPRESSION_LIMIT));
|
|
1764
|
-
});
|
|
1765
|
-
|
|
1766
|
-
test('expressions at the limit are untouched, one character over are not', () => {
|
|
1767
|
-
const atLimit = 'A'.repeat(SEARCH_EXPRESSION_LIMIT);
|
|
1768
|
-
assert.equal(summarizeSearchExpression(atLimit), atLimit);
|
|
1769
|
-
assert.notEqual(summarizeSearchExpression(`${atLimit}B`), `${atLimit}B`);
|
|
1770
|
-
});
|
|
1771
|
-
|
|
1772
|
-
test('boilerplate is a description repeated across the model, not merely a common word', () => {
|
|
1773
|
-
const shared = 'Localized label. Do not use for calculations.';
|
|
1774
|
-
const rows = [
|
|
1775
|
-
...Array.from({ length: 20 }, () => ({ description: shared })),
|
|
1776
|
-
{ description: 'Net revenue after returns' },
|
|
1777
|
-
{ description: 'Gross margin percentage' },
|
|
1778
|
-
{ description: '' },
|
|
1779
|
-
];
|
|
1780
|
-
const boilerplate = boilerplateDescriptions(rows);
|
|
1781
|
-
assert.equal(boilerplate.has(shared), true);
|
|
1782
|
-
assert.equal(boilerplate.has('Net revenue after returns'), false);
|
|
1783
|
-
assert.equal(boilerplate.size, 1);
|
|
1784
|
-
});
|
|
1785
|
-
|
|
1786
|
-
test('a description repeated only a handful of times is left alone', () => {
|
|
1787
|
-
const rows = Array.from({ length: 4 }, () => ({
|
|
1788
|
-
description: 'Shared by a few',
|
|
1789
|
-
}));
|
|
1790
|
-
assert.equal(boilerplateDescriptions(rows).size, 0);
|
|
1791
|
-
});
|
|
1792
|
-
|
|
1793
|
-
test('the row floor is exact: one below keeps the description, one at it drops', () => {
|
|
1794
|
-
// Both thresholds are shipped constants, so the boundary is part of the
|
|
1795
|
-
// contract rather than an implementation detail. Asserting it against the
|
|
1796
|
-
// exported value means retuning the constant cannot silently change behaviour
|
|
1797
|
-
// without this test being read.
|
|
1798
|
-
const shared = 'Repeated';
|
|
1799
|
-
const below = Array.from({ length: BOILERPLATE_MIN_ROWS - 1 }, () => ({
|
|
1800
|
-
description: shared,
|
|
1801
|
-
}));
|
|
1802
|
-
const at = Array.from({ length: BOILERPLATE_MIN_ROWS }, () => ({
|
|
1803
|
-
description: shared,
|
|
1804
|
-
}));
|
|
1805
|
-
|
|
1806
|
-
assert.equal(boilerplateDescriptions(below).has(shared), false);
|
|
1807
|
-
assert.equal(boilerplateDescriptions(at).has(shared), true);
|
|
1808
|
-
});
|
|
1809
|
-
|
|
1810
|
-
test('a description on many rows is still kept when it describes a small share of them', () => {
|
|
1811
|
-
// The count alone is not enough. On a large model a genuine description can
|
|
1812
|
-
// easily appear on eight measures while still telling them apart from the
|
|
1813
|
-
// other hundreds, and suppressing it there would remove information rather
|
|
1814
|
-
// than noise.
|
|
1815
|
-
const shared = 'Appears often but not everywhere';
|
|
1816
|
-
const repeats = BOILERPLATE_MIN_ROWS;
|
|
1817
|
-
// Sized so the share lands clearly under the floor rather than adjacent to it.
|
|
1818
|
-
const others = Math.ceil(repeats / BOILERPLATE_MIN_SHARE);
|
|
1819
|
-
const rows = [
|
|
1820
|
-
...Array.from({ length: repeats }, () => ({ description: shared })),
|
|
1821
|
-
...Array.from({ length: others }, (_, index) => ({
|
|
1822
|
-
description: `Distinct ${index}`,
|
|
1823
|
-
})),
|
|
1824
|
-
];
|
|
1825
|
-
|
|
1826
|
-
const share = repeats / (repeats + others);
|
|
1827
|
-
assert.ok(share < BOILERPLATE_MIN_SHARE, `share ${share} is below the floor`);
|
|
1828
|
-
assert.equal(boilerplateDescriptions(rows).has(shared), false);
|
|
1829
|
-
});
|
|
1830
|
-
|
|
1831
|
-
test('the share floor is exact: at it the description drops, just under it survives', () => {
|
|
1832
|
-
// The row floor alone is asserted above. This pins the other half of the
|
|
1833
|
-
// conjunction, so neither threshold can be retuned without a test being read.
|
|
1834
|
-
const shared = 'Repeated';
|
|
1835
|
-
const repeats = BOILERPLATE_MIN_ROWS;
|
|
1836
|
-
const describedAtFloor = Math.floor(repeats / BOILERPLATE_MIN_SHARE);
|
|
1837
|
-
const atFloor = [
|
|
1838
|
-
...Array.from({ length: repeats }, () => ({ description: shared })),
|
|
1839
|
-
...Array.from({ length: describedAtFloor - repeats }, (_, index) => ({
|
|
1840
|
-
description: `Distinct ${index}`,
|
|
1841
|
-
})),
|
|
1842
|
-
];
|
|
1843
|
-
const justUnder = [
|
|
1844
|
-
...atFloor,
|
|
1845
|
-
{ description: `Distinct ${describedAtFloor}` },
|
|
1846
|
-
];
|
|
1847
|
-
|
|
1848
|
-
assert.equal(repeats / describedAtFloor >= BOILERPLATE_MIN_SHARE, true);
|
|
1849
|
-
assert.equal(boilerplateDescriptions(atFloor).has(shared), true);
|
|
1850
|
-
|
|
1851
|
-
assert.equal(repeats / (describedAtFloor + 1) < BOILERPLATE_MIN_SHARE, true);
|
|
1852
|
-
assert.equal(boilerplateDescriptions(justUnder).has(shared), false);
|
|
1853
|
-
});
|
|
1854
|
-
|
|
1855
|
-
// -- Trimming has to be proportional to the model, not tuned to one of them --
|
|
1856
|
-
|
|
1857
|
-
test('trimming is inert on a model whose expressions are already short', () => {
|
|
1858
|
-
// Measured upstream against three bound models by reading INFO.VIEW.MEASURES()
|
|
1859
|
-
// directly:
|
|
1860
|
-
//
|
|
1861
|
-
// model measures median max over the cap chars removed
|
|
1862
|
-
// EYReport_PBIR 2560 400 104916 2366 (92.4%) 81.3%
|
|
1863
|
-
// RetailBenchmark 56 38.5 205 1 (1.8%) 3.5%
|
|
1864
|
-
// HRBenchmark 31 34 111 0 (0.0%) 0.0%
|
|
1865
|
-
//
|
|
1866
|
-
// The cap earns its keep on the model that has bytes to give up and does
|
|
1867
|
-
// nothing on the ones that do not. This encodes the HR shape so a future
|
|
1868
|
-
// retune cannot start charging small models for a problem they do not have.
|
|
1869
|
-
const catalog = Array.from({ length: 31 }, (_, index) => ({
|
|
1870
|
-
object_name: `Measure ${index}`,
|
|
1871
|
-
description: `Distinct description ${index}`,
|
|
1872
|
-
expression: 'A'.repeat(34 + (index % 3)),
|
|
1873
|
-
}));
|
|
1874
|
-
|
|
1875
|
-
const { rows, notes } = trimSearchRows(catalog, catalog);
|
|
1876
|
-
|
|
1877
|
-
assert.deepEqual(rows, catalog, 'nothing should have been rewritten');
|
|
1878
|
-
assert.deepEqual(notes, [], 'and nothing should be reported');
|
|
1879
|
-
});
|
|
1880
|
-
|
|
1881
|
-
test('trimming engages on a model whose expressions are long', () => {
|
|
1882
|
-
// The EY shape: expressions well past the cap, and one description repeated
|
|
1883
|
-
// across enough of the model to discriminate nothing.
|
|
1884
|
-
const shared = 'Localized label. Do not use for calculations.';
|
|
1885
|
-
const catalog = Array.from({ length: 40 }, (_, index) => ({
|
|
1886
|
-
object_name: `Measure ${index}`,
|
|
1887
|
-
description: shared,
|
|
1888
|
-
expression: 'A'.repeat(400),
|
|
1889
|
-
}));
|
|
1890
|
-
|
|
1891
|
-
const { rows, notes } = trimSearchRows(catalog, catalog);
|
|
1892
|
-
|
|
1893
|
-
assert.equal(
|
|
1894
|
-
rows.every((row) => row.description === null),
|
|
1895
|
-
true
|
|
1896
|
-
);
|
|
1897
|
-
assert.equal(
|
|
1898
|
-
rows.every((row) => row.expression.length < 400),
|
|
1899
|
-
true
|
|
1900
|
-
);
|
|
1901
|
-
assert.equal(
|
|
1902
|
-
notes.length,
|
|
1903
|
-
2,
|
|
1904
|
-
'one note for descriptions, one for expressions'
|
|
1905
|
-
);
|
|
1906
|
-
});
|
|
1907
|
-
|
|
1908
|
-
test('trimming nulls boilerplate, keeps real descriptions, and reports itself once', () => {
|
|
1909
|
-
const shared = 'Localized label. Do not use for calculations.';
|
|
1910
|
-
const catalog = [
|
|
1911
|
-
...Array.from({ length: 20 }, (_, index) => ({
|
|
1912
|
-
object_name: `loc-${index}`,
|
|
1913
|
-
description: shared,
|
|
1914
|
-
expression: 'x',
|
|
1915
|
-
})),
|
|
1916
|
-
{
|
|
1917
|
-
object_name: 'Revenue',
|
|
1918
|
-
description: 'Net revenue after returns',
|
|
1919
|
-
expression: 'SUM(Sales[Amount])',
|
|
1920
|
-
},
|
|
1921
|
-
];
|
|
1922
|
-
const { rows, notes } = trimSearchRows(catalog, catalog);
|
|
1923
|
-
|
|
1924
|
-
assert.equal(rows[0].description, null);
|
|
1925
|
-
assert.equal(rows.at(-1).description, 'Net revenue after returns');
|
|
1926
|
-
assert.equal(notes.length, 1, 'no expression was long enough to summarize');
|
|
1927
|
-
assert.match(notes[0], /description omitted on 20 row\(s\)/u);
|
|
1928
|
-
// Every row keeps every key, or the TSV encoding collapses into JSON.
|
|
1929
|
-
assert.deepEqual(Object.keys(rows[0]), Object.keys(rows.at(-1)));
|
|
1930
|
-
});
|
|
1931
|
-
|
|
1932
|
-
test('trimming reports a summarized expression separately from a dropped description', () => {
|
|
1933
|
-
const catalog = [
|
|
1934
|
-
{ object_name: 'm', description: 'unique', expression: 'A'.repeat(500) },
|
|
1935
|
-
];
|
|
1936
|
-
const { rows, notes } = trimSearchRows(catalog, catalog);
|
|
1937
|
-
assert.equal(rows[0].description, 'unique');
|
|
1938
|
-
assert.equal(notes.length, 1);
|
|
1939
|
-
assert.match(notes[0], /expression shortened on 1 row\(s\)/u);
|
|
1940
|
-
});
|
|
1941
|
-
|
|
1942
|
-
// -- The regression this whole design exists to avoid --
|
|
1943
|
-
|
|
1944
|
-
test('ranking sees the full expression, not the summarized one', () => {
|
|
1945
|
-
// The sharp form of the property. The discriminating term appears ONLY past
|
|
1946
|
-
// the cut, so a ranker fed trimmed text could not possibly match it. Placing
|
|
1947
|
-
// the term near the front - as the upstream fixture does - would let this
|
|
1948
|
-
// test pass even if trimming happened before ranking.
|
|
1949
|
-
const term = 'reconciliation';
|
|
1950
|
-
const prefix = `SWITCH(USERCULTURE(), ${'"padding-locale", "unrelated text", '.repeat(8)}`;
|
|
1951
|
-
assert.ok(
|
|
1952
|
-
prefix.length > SEARCH_EXPRESSION_LIMIT * 2,
|
|
1953
|
-
'the term must sit well past the cut'
|
|
1954
|
-
);
|
|
1955
|
-
const expression = `${prefix}"en-US", "${term} of journal entries")`;
|
|
1956
|
-
assert.ok(
|
|
1957
|
-
!summarizeSearchExpression(expression).includes(term),
|
|
1958
|
-
'the trimmed form must not contain the term, or this proves nothing'
|
|
1959
|
-
);
|
|
1960
|
-
|
|
1961
|
-
const catalog = [
|
|
1962
|
-
{
|
|
1963
|
-
kind: 'measure',
|
|
1964
|
-
parent_name: 'LocalizedLabels',
|
|
1965
|
-
object_name: 'Label 0',
|
|
1966
|
-
description: 'Localized label.',
|
|
1967
|
-
expression,
|
|
1968
|
-
display_folder: '',
|
|
1969
|
-
data_type: '',
|
|
1970
|
-
synonyms: '',
|
|
1971
|
-
},
|
|
1972
|
-
];
|
|
1973
|
-
const { rows } = trimSearchRows(catalog, catalog);
|
|
1974
|
-
|
|
1975
|
-
assert.equal(rankSemanticSearchRows(catalog, term).length, 1);
|
|
1976
|
-
assert.equal(
|
|
1977
|
-
rankSemanticSearchRows(rows, term).length,
|
|
1978
|
-
0,
|
|
1979
|
-
'ranking the trimmed rows finds nothing - so the order is load-bearing'
|
|
1980
|
-
);
|
|
1981
|
-
});
|
|
1982
|
-
|
|
1983
|
-
test('an object findable only through its long expression is still findable after trimming', () => {
|
|
1984
|
-
// Dropping these rows is what an earlier attempt did. It saved more bytes and
|
|
1985
|
-
// made a hundred measures invisible, which is a global regression traded for
|
|
1986
|
-
// a local gain. Ranking therefore still sees the full text.
|
|
1987
|
-
const dir = tempDir();
|
|
1988
|
-
try {
|
|
1989
|
-
const dbPath = join(dir, 'm.sqlite');
|
|
1990
|
-
const prefix = `SWITCH(USERCULTURE(), ${'"padding-locale", "unrelated text", '.repeat(8)}`;
|
|
1991
|
-
const localized = Array.from({ length: 20 }, (_, index) => ({
|
|
1992
|
-
Table: 'LocalizedLabels',
|
|
1993
|
-
Name: `Label ${index}`,
|
|
1994
|
-
Description: 'Localized label. Do not use for calculations.',
|
|
1995
|
-
Expression: `${prefix}"en-US", "reconciliation of journal entries")`,
|
|
1996
|
-
}));
|
|
1997
|
-
buildDatabase(
|
|
1998
|
-
[
|
|
1999
|
-
{
|
|
2000
|
-
table: 'info_view_tables',
|
|
2001
|
-
grain: '',
|
|
2002
|
-
dax: 'INFO.VIEW.TABLES()',
|
|
2003
|
-
rows: [{ Name: 'LocalizedLabels', Description: 'Labels' }],
|
|
2004
|
-
},
|
|
2005
|
-
{
|
|
2006
|
-
table: 'info_view_measures',
|
|
2007
|
-
grain: '',
|
|
2008
|
-
dax: 'INFO.VIEW.MEASURES()',
|
|
2009
|
-
rows: localized,
|
|
2010
|
-
},
|
|
2011
|
-
],
|
|
2012
|
-
dbPath
|
|
2013
|
-
);
|
|
2014
|
-
|
|
2015
|
-
const out = searchCatalog(dbPath, 'reconciliation', { limit: 50 });
|
|
2016
|
-
assert.equal(out.ok, true);
|
|
2017
|
-
assert.ok(
|
|
2018
|
-
out.totalRows >= 20,
|
|
2019
|
-
'the rows are still matched by their full expression'
|
|
2020
|
-
);
|
|
2021
|
-
|
|
2022
|
-
const row = out.rows.find(
|
|
2023
|
-
(candidate) => candidate.object_name === 'Label 0'
|
|
2024
|
-
);
|
|
2025
|
-
assert.ok(row, 'the object is still returned');
|
|
2026
|
-
assert.equal(
|
|
2027
|
-
row.description,
|
|
2028
|
-
null,
|
|
2029
|
-
'the shared boilerplate is not reprinted'
|
|
2030
|
-
);
|
|
2031
|
-
assert.match(row.expression, /substr\(\[Expression\], <offset>, 1000\)/u);
|
|
2032
|
-
assert.ok(
|
|
2033
|
-
!row.expression.includes('reconciliation'),
|
|
2034
|
-
'the printed expression no longer carries the term it was found by'
|
|
2035
|
-
);
|
|
2036
|
-
assert.ok(out.notes.length >= 1, 'the payload says what it did');
|
|
2037
|
-
} finally {
|
|
2038
|
-
rmSync(dir, { recursive: true, force: true });
|
|
2039
|
-
}
|
|
2040
|
-
});
|
|
2041
|
-
|
|
2042
|
-
test('trimming shrinks the search payload it is meant to shrink', () => {
|
|
2043
|
-
const dir = tempDir();
|
|
2044
|
-
try {
|
|
2045
|
-
const dbPath = join(dir, 'm.sqlite');
|
|
2046
|
-
const rows = Array.from({ length: 30 }, (_, index) => ({
|
|
2047
|
-
Table: 'LocalizedLabels',
|
|
2048
|
-
Name: `Label ${index}`,
|
|
2049
|
-
Description:
|
|
2050
|
-
'Localized label. Do not use for calculations. It exists only for display.',
|
|
2051
|
-
Expression: `SWITCH(USERCULTURE(), ${Array.from(
|
|
2052
|
-
{ length: 30 },
|
|
2053
|
-
(_, locale) => `"l-${locale}", "journal entries"`
|
|
2054
|
-
).join(', ')})`,
|
|
2055
|
-
}));
|
|
2056
|
-
buildDatabase(
|
|
2057
|
-
[
|
|
2058
|
-
{
|
|
2059
|
-
table: 'info_view_tables',
|
|
2060
|
-
grain: '',
|
|
2061
|
-
dax: 'INFO.VIEW.TABLES()',
|
|
2062
|
-
rows: [{ Name: 'LocalizedLabels' }],
|
|
2063
|
-
},
|
|
2064
|
-
{
|
|
2065
|
-
table: 'info_view_measures',
|
|
2066
|
-
grain: '',
|
|
2067
|
-
dax: 'INFO.VIEW.MEASURES()',
|
|
2068
|
-
rows,
|
|
2069
|
-
},
|
|
2070
|
-
],
|
|
2071
|
-
dbPath
|
|
2072
|
-
);
|
|
2073
|
-
|
|
2074
|
-
const out = searchCatalog(dbPath, 'journal', { limit: 50 });
|
|
2075
|
-
// Compare like with like: the two fields this trims, before and after.
|
|
2076
|
-
const targetedBytes = (records, expressionKey, descriptionKey) =>
|
|
2077
|
-
records.reduce(
|
|
2078
|
-
(total, record) =>
|
|
2079
|
-
total +
|
|
2080
|
-
Buffer.byteLength(
|
|
2081
|
-
`${record[expressionKey] ?? ''}${record[descriptionKey] ?? ''}`,
|
|
2082
|
-
'utf8'
|
|
2083
|
-
),
|
|
2084
|
-
0
|
|
2085
|
-
);
|
|
2086
|
-
const before = targetedBytes(rows, 'Expression', 'Description');
|
|
2087
|
-
const after = targetedBytes(out.rows, 'expression', 'description');
|
|
2088
|
-
assert.ok(
|
|
2089
|
-
after < before / 4,
|
|
2090
|
-
`expected expression+description to shrink at least 4x: ${after} vs ${before}`
|
|
2091
|
-
);
|
|
2092
|
-
// Every matched row is still returned - smaller, not fewer.
|
|
2093
|
-
assert.equal(out.returnedRows, out.totalRows);
|
|
2094
|
-
} finally {
|
|
2095
|
-
rmSync(dir, { recursive: true, force: true });
|
|
2096
|
-
}
|
|
2097
|
-
});
|