@dudousxd/nestjs-catalog 0.13.0 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/catalog.controller.js +66 -6
- package/dist/catalog.csv.d.ts +89 -0
- package/dist/catalog.csv.js +163 -0
- package/dist/catalog.filters.d.ts +174 -0
- package/dist/catalog.filters.js +272 -0
- package/dist/catalog.identifiers.d.ts +161 -0
- package/dist/catalog.identifiers.js +195 -0
- package/dist/catalog.pipeline.d.ts +392 -35
- package/dist/catalog.pipeline.js +175 -25
- package/dist/catalog.query-cache.d.ts +0 -2
- package/dist/catalog.query-cache.js +0 -18
- package/dist/catalog.query.d.ts +43 -0
- package/dist/catalog.query.js +5 -0
- package/dist/catalog.service.d.ts +51 -0
- package/dist/catalog.service.js +126 -3
- package/dist/catalog.store.d.ts +84 -22
- package/dist/catalog.store.js +36 -68
- package/dist/catalog.types.d.ts +64 -0
- package/dist/client.d.ts +70 -2
- package/dist/client.js +66 -1
- package/dist/index.d.ts +6 -4
- package/dist/index.js +24 -3
- package/dist/stores/mikro-orm-read.store.d.ts +8 -2
- package/dist/stores/mikro-orm-read.store.js +77 -11
- package/package.json +1 -1
|
@@ -13,8 +13,9 @@ var __param = (this && this.__param) || function (paramIndex, decorator) {
|
|
|
13
13
|
};
|
|
14
14
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
15
|
exports.createCatalogController = createCatalogController;
|
|
16
|
+
const node_stream_1 = require("node:stream");
|
|
16
17
|
const common_1 = require("@nestjs/common");
|
|
17
|
-
const
|
|
18
|
+
const catalog_csv_1 = require("./catalog.csv");
|
|
18
19
|
const catalog_registry_base_1 = require("./catalog.registry.base");
|
|
19
20
|
const catalog_route_auth_1 = require("./catalog.route-auth");
|
|
20
21
|
const catalog_service_1 = require("./catalog.service");
|
|
@@ -191,7 +192,7 @@ function createCatalogController(path, guards, decorators = []) {
|
|
|
191
192
|
return this.registry.getSnapshot();
|
|
192
193
|
}
|
|
193
194
|
/** One generic read endpoint for every type in the catalog. */
|
|
194
|
-
objects(name, page, size, search, sort, dir, snapshot) {
|
|
195
|
+
objects(name, page, size, search, sort, dir, snapshot, filter) {
|
|
195
196
|
return this.service.readObjects(name, {
|
|
196
197
|
page: page ? Number(page) : undefined,
|
|
197
198
|
size: size ? Number(size) : undefined,
|
|
@@ -199,6 +200,7 @@ function createCatalogController(path, guards, decorators = []) {
|
|
|
199
200
|
sort,
|
|
200
201
|
dir: dir === 'desc' ? 'desc' : 'asc',
|
|
201
202
|
snapshot,
|
|
203
|
+
filters: repeatable(filter),
|
|
202
204
|
});
|
|
203
205
|
}
|
|
204
206
|
/**
|
|
@@ -273,17 +275,60 @@ function createCatalogController(path, guards, decorators = []) {
|
|
|
273
275
|
return this.service.runSavedQuery(id, body?.maxRows);
|
|
274
276
|
}
|
|
275
277
|
/**
|
|
276
|
-
* The same result as CSV.
|
|
278
|
+
* The same result as CSV, written out as the rows arrive.
|
|
277
279
|
*
|
|
278
280
|
* A GET, not a POST, so it can be a plain link — a download that only works
|
|
279
281
|
* from JavaScript cannot be pasted into a mail or a scheduled job.
|
|
282
|
+
*
|
|
283
|
+
* **Nothing here holds the file.** This used to run the query, hold the
|
|
284
|
+
* result, build the whole CSV string and then answer — which is the shape
|
|
285
|
+
* that made a 981,469-row connector load never finish, and it is worse on an
|
|
286
|
+
* export than on a load, because an export has no row cap by design. Now
|
|
287
|
+
* `streamSavedQuery` hands over rows, {@link csvLines} turns each into a
|
|
288
|
+
* line, and the response is written from that.
|
|
289
|
+
*
|
|
290
|
+
* **`@Res({ passthrough: true })` is still right, and the returned value is
|
|
291
|
+
* what had to change.** Passthrough means "I am setting headers, you send
|
|
292
|
+
* the body", and that is exactly what this handler wants. What it must not
|
|
293
|
+
* return any more is a string: the express adapter answers a string body
|
|
294
|
+
* with `res.send()`, which sets `content-length` — and a length header is a
|
|
295
|
+
* promise about a body nobody has counted yet. A `StreamableFile` is the
|
|
296
|
+
* body shape the adapter pipes instead, and `applyStreamHeaders` sets a
|
|
297
|
+
* length only when the `StreamableFile` was given one, which this is not. So
|
|
298
|
+
* the response is chunked and carries no `content-length`. Taking the
|
|
299
|
+
* response over with a bare `@Res()` and pumping it by hand would work too,
|
|
300
|
+
* and would name a platform: this package peer-depends on `@nestjs/common`
|
|
301
|
+
* and `@nestjs/core` and on no adapter, so a hand-written `write`/`drain`
|
|
302
|
+
* loop would be one adapter's API written into a library that deliberately
|
|
303
|
+
* does not have one. A `StreamableFile` leaves the piping to whichever
|
|
304
|
+
* adapter the host mounted.
|
|
305
|
+
*
|
|
306
|
+
* The pipe is also what bounds the memory: `Readable.from` stops pulling
|
|
307
|
+
* from the generator once its buffer is full, the pipe stops reading once
|
|
308
|
+
* the socket says wait, and the generator stops pulling from the store. A
|
|
309
|
+
* client that reads slowly slows the database read down rather than filling
|
|
310
|
+
* this process with the rows it has not collected.
|
|
280
311
|
*/
|
|
281
312
|
async exportSavedQuery(id, response) {
|
|
282
|
-
const { savedQuery,
|
|
313
|
+
const { savedQuery, columns, rows } = await this.service.streamSavedQuery(id);
|
|
283
314
|
const filename = `${savedQuery.name.replace(/[^A-Za-z0-9_-]+/g, '-')}.csv`;
|
|
284
315
|
response.setHeader('content-type', 'text/csv; charset=utf-8');
|
|
285
316
|
response.setHeader('content-disposition', `attachment; filename="${filename}"`);
|
|
286
|
-
|
|
317
|
+
// Byte mode rather than object mode, so the readable's own high-water mark
|
|
318
|
+
// is measured in bytes: the lines coalesce into socket-sized writes on
|
|
319
|
+
// their way out, instead of one write per row for however many rows the
|
|
320
|
+
// table has.
|
|
321
|
+
const file = node_stream_1.Readable.from((0, catalog_csv_1.csvLines)(rows, columns), { objectMode: false });
|
|
322
|
+
// `pipe` unpipes a dead destination and leaves the SOURCE alone, which
|
|
323
|
+
// here means an operator who closes the download tab leaves this generator
|
|
324
|
+
// parked mid-`yield` — and with it, in the shipped MySQL store, an open
|
|
325
|
+
// read-only transaction on a pooled connection, for the life of the
|
|
326
|
+
// process. Destroying the readable is what calls the generator's `return`,
|
|
327
|
+
// which is what runs the `finally` the store put its rollback in. Fires on
|
|
328
|
+
// an ordinary completion too, where destroying a finished stream is a
|
|
329
|
+
// no-op.
|
|
330
|
+
response.on('close', () => file.destroy());
|
|
331
|
+
return new common_1.StreamableFile(file);
|
|
287
332
|
}
|
|
288
333
|
/**
|
|
289
334
|
* Every SQL this query has ever been, newest first.
|
|
@@ -492,8 +537,9 @@ function createCatalogController(path, guards, decorators = []) {
|
|
|
492
537
|
__param(4, (0, common_1.Query)('sort')),
|
|
493
538
|
__param(5, (0, common_1.Query)('dir')),
|
|
494
539
|
__param(6, (0, common_1.Query)('snapshot')),
|
|
540
|
+
__param(7, (0, common_1.Query)('filter')),
|
|
495
541
|
__metadata("design:type", Function),
|
|
496
|
-
__metadata("design:paramtypes", [String, String, String, String, String, String, String]),
|
|
542
|
+
__metadata("design:paramtypes", [String, String, String, String, String, String, String, Object]),
|
|
497
543
|
__metadata("design:returntype", void 0)
|
|
498
544
|
], CatalogController.prototype, "objects", null);
|
|
499
545
|
__decorate([
|
|
@@ -739,6 +785,20 @@ function actorOf(request, claimed) {
|
|
|
739
785
|
return resolved;
|
|
740
786
|
return claimed?.trim() || 'console';
|
|
741
787
|
}
|
|
788
|
+
/**
|
|
789
|
+
* A query parameter that may appear once or many times, as a list either way.
|
|
790
|
+
*
|
|
791
|
+
* Unlike {@link parseOutcomes} below, nothing is split on commas and nothing is
|
|
792
|
+
* dropped. A filter's value is free text that may contain a comma — a
|
|
793
|
+
* description, a date range written by hand — and an unrecognised filter must
|
|
794
|
+
* reach the service so it can be refused by name rather than vanish into an
|
|
795
|
+
* unfiltered page.
|
|
796
|
+
*/
|
|
797
|
+
function repeatable(raw) {
|
|
798
|
+
if (raw === undefined)
|
|
799
|
+
return undefined;
|
|
800
|
+
return Array.isArray(raw) ? raw : [raw];
|
|
801
|
+
}
|
|
742
802
|
/**
|
|
743
803
|
* `?outcome=failed`, `?outcome=failed,incomplete`, `?outcome=a&outcome=b`.
|
|
744
804
|
*
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* CSV, for the export button.
|
|
3
|
+
*
|
|
4
|
+
* Two things here are not the obvious implementation, and both are here because
|
|
5
|
+
* the obvious one was wrong in production.
|
|
6
|
+
*
|
|
7
|
+
* **Rows are written as they arrive.** {@link csvLines} is a generator over an
|
|
8
|
+
* async row source, so the caller can hand each line to a socket and never hold
|
|
9
|
+
* the file. The buffered {@link toCsv} is still exported — it is public API and
|
|
10
|
+
* a caller with a result already in memory has nothing to gain from a stream —
|
|
11
|
+
* but it is now the special case rather than the only shape.
|
|
12
|
+
*
|
|
13
|
+
* **A cell that a spreadsheet would run is neutralised.** See
|
|
14
|
+
* {@link guardFormula}.
|
|
15
|
+
*/
|
|
16
|
+
/** One row of a result, as every store hands it over. */
|
|
17
|
+
export type CsvRow = Record<string, unknown>;
|
|
18
|
+
/**
|
|
19
|
+
* Stop a cell being executed when the file is opened.
|
|
20
|
+
*
|
|
21
|
+
* A CSV is not a document format, it is a program that a spreadsheet is willing
|
|
22
|
+
* to run. A cell whose value begins with one of {@link FORMULA_LEADERS} is
|
|
23
|
+
* evaluated on open, and the values in this file come from whatever the queried
|
|
24
|
+
* source contained — which for this catalog is other people's operational data,
|
|
25
|
+
* loaded by connectors, from systems nobody here controls. So the crafted cell
|
|
26
|
+
* is not a thought experiment: it is one row in a table an operator exports and
|
|
27
|
+
* opens.
|
|
28
|
+
*
|
|
29
|
+
* **The escape is a leading apostrophe, and it is not free.** Excel and Sheets
|
|
30
|
+
* both read `'` as "the rest is literal text" and do not show it in the cell;
|
|
31
|
+
* every other reader on earth — a parser, a `pandas.read_csv`, the next
|
|
32
|
+
* pipeline that ingests this export — sees an apostrophe that was not in the
|
|
33
|
+
* source. That is a real corruption, and it is why the guard is not applied to
|
|
34
|
+
* every cell that merely starts with a leader.
|
|
35
|
+
*
|
|
36
|
+
* **So a value that is plainly a number is left exactly as it was.** `-42` is
|
|
37
|
+
* not an injection vector: a spreadsheet evaluates it to the number -42, which
|
|
38
|
+
* is what it already was. Exempting it is what lets a machine read this file
|
|
39
|
+
* back and still get -42, and it costs no safety, because the population being
|
|
40
|
+
* defended against — `=`, `@`, an operator followed by anything that is not a
|
|
41
|
+
* number — is disjoint from it. Everything outside the exemption gets the
|
|
42
|
+
* apostrophe and reads differently to a parser than it did to the database.
|
|
43
|
+
* That trade is deliberate: a wrong apostrophe is a data-quality bug somebody
|
|
44
|
+
* can see, and a formula is code running on the machine of whoever opened the
|
|
45
|
+
* file.
|
|
46
|
+
*
|
|
47
|
+
* The apostrophe goes at position 0, in front of any leading blank, because
|
|
48
|
+
* that is the only position a spreadsheet honours it in.
|
|
49
|
+
*/
|
|
50
|
+
export declare function guardFormula(text: string): string;
|
|
51
|
+
/**
|
|
52
|
+
* One cell, ready to sit in a row.
|
|
53
|
+
*
|
|
54
|
+
* Order matters and is the second thing people get wrong. The formula guard
|
|
55
|
+
* runs on the raw text, and the CSV quoting runs on the result — so a value that
|
|
56
|
+
* needs both comes out as `"'=1+1,x"`, with the apostrophe INSIDE the quotes and
|
|
57
|
+
* applied once. Doing it the other way round would put the apostrophe outside
|
|
58
|
+
* the quoting and produce a field no CSV reader can parse; doing it twice would
|
|
59
|
+
* put two apostrophes in a cell that needed one.
|
|
60
|
+
*/
|
|
61
|
+
export declare function csvCell(value: unknown): string;
|
|
62
|
+
/**
|
|
63
|
+
* A CSV file, one line at a time, from a source that hands rows over as it has
|
|
64
|
+
* them.
|
|
65
|
+
*
|
|
66
|
+
* Each yield is a complete line including its terminator, so a consumer can
|
|
67
|
+
* write it and forget it. Nothing accumulates here: the generator holds one row
|
|
68
|
+
* and the column list, whatever the source is doing, which is the whole point —
|
|
69
|
+
* an export has no row cap by design, so the only bound available is that no
|
|
70
|
+
* stage keeps more than it is using.
|
|
71
|
+
*
|
|
72
|
+
* `columns` may be omitted, and then the header is taken from the keys of the
|
|
73
|
+
* first row — the same rule the buffered read uses, applied at the only moment a
|
|
74
|
+
* stream can apply it. A source that knows its columns up front should pass
|
|
75
|
+
* them, because it is the only way an empty result gets a header row at all.
|
|
76
|
+
*/
|
|
77
|
+
export declare function csvLines(rows: AsyncIterable<CsvRow>, columns?: readonly string[]): AsyncGenerator<string>;
|
|
78
|
+
/**
|
|
79
|
+
* The whole file as a string.
|
|
80
|
+
*
|
|
81
|
+
* Kept because it is public API and because a caller holding a finished result
|
|
82
|
+
* gains nothing from a stream. It is NOT what the export route uses — see
|
|
83
|
+
* {@link csvLines} — and a caller reaching for it with an unbounded result is
|
|
84
|
+
* choosing to hold the file.
|
|
85
|
+
*/
|
|
86
|
+
export declare function toCsv(result: {
|
|
87
|
+
columns: string[];
|
|
88
|
+
rows: CsvRow[];
|
|
89
|
+
}): string;
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* CSV, for the export button.
|
|
4
|
+
*
|
|
5
|
+
* Two things here are not the obvious implementation, and both are here because
|
|
6
|
+
* the obvious one was wrong in production.
|
|
7
|
+
*
|
|
8
|
+
* **Rows are written as they arrive.** {@link csvLines} is a generator over an
|
|
9
|
+
* async row source, so the caller can hand each line to a socket and never hold
|
|
10
|
+
* the file. The buffered {@link toCsv} is still exported — it is public API and
|
|
11
|
+
* a caller with a result already in memory has nothing to gain from a stream —
|
|
12
|
+
* but it is now the special case rather than the only shape.
|
|
13
|
+
*
|
|
14
|
+
* **A cell that a spreadsheet would run is neutralised.** See
|
|
15
|
+
* {@link guardFormula}.
|
|
16
|
+
*/
|
|
17
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
18
|
+
exports.guardFormula = guardFormula;
|
|
19
|
+
exports.csvCell = csvCell;
|
|
20
|
+
exports.csvLines = csvLines;
|
|
21
|
+
exports.toCsv = toCsv;
|
|
22
|
+
/**
|
|
23
|
+
* The characters a spreadsheet reads as "what follows is a formula".
|
|
24
|
+
*
|
|
25
|
+
* `=` is the obvious one. `+`, `-` and `@` are the ones people forget: Excel and
|
|
26
|
+
* Sheets both accept a leading `+` or `-` as the start of an expression, and `@`
|
|
27
|
+
* introduces a function reference. `-2+3+cmd|' /C calc'!A0` is a working command
|
|
28
|
+
* execution in an unpatched Excel, and it starts with a minus sign.
|
|
29
|
+
*/
|
|
30
|
+
const FORMULA_LEADERS = new Set(['=', '+', '-', '@']);
|
|
31
|
+
/**
|
|
32
|
+
* Blank a spreadsheet strips before it decides whether a cell is a formula.
|
|
33
|
+
*
|
|
34
|
+
* The reason this exists rather than a plain `text[0]` test: importers do not
|
|
35
|
+
* agree on whether leading blank is part of the value. Several strip it and then
|
|
36
|
+
* dispatch on what is left, so ` =1+1` is a formula to them and an innocent
|
|
37
|
+
* string to a guard that only looked at position zero. Guarding the stripped
|
|
38
|
+
* form costs nothing on values that were never formulas.
|
|
39
|
+
*/
|
|
40
|
+
const LEADING_BLANK = /^[\t\n\v\f\r ]+/;
|
|
41
|
+
/**
|
|
42
|
+
* A value a spreadsheet will read back as the number it already is.
|
|
43
|
+
*
|
|
44
|
+
* This is the exemption that keeps the guard from corrupting data — see
|
|
45
|
+
* {@link guardFormula}. Deliberately narrow: an optional sign, digits with at
|
|
46
|
+
* most one point, an optional exponent, and nothing else. `-42` matches.
|
|
47
|
+
* `-42abc` does not, `- 42` does not, and `-2+3+cmd|' /C calc'!A0` does not.
|
|
48
|
+
*/
|
|
49
|
+
const PLAIN_NUMBER = /^[+-]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?$/;
|
|
50
|
+
/**
|
|
51
|
+
* Stop a cell being executed when the file is opened.
|
|
52
|
+
*
|
|
53
|
+
* A CSV is not a document format, it is a program that a spreadsheet is willing
|
|
54
|
+
* to run. A cell whose value begins with one of {@link FORMULA_LEADERS} is
|
|
55
|
+
* evaluated on open, and the values in this file come from whatever the queried
|
|
56
|
+
* source contained — which for this catalog is other people's operational data,
|
|
57
|
+
* loaded by connectors, from systems nobody here controls. So the crafted cell
|
|
58
|
+
* is not a thought experiment: it is one row in a table an operator exports and
|
|
59
|
+
* opens.
|
|
60
|
+
*
|
|
61
|
+
* **The escape is a leading apostrophe, and it is not free.** Excel and Sheets
|
|
62
|
+
* both read `'` as "the rest is literal text" and do not show it in the cell;
|
|
63
|
+
* every other reader on earth — a parser, a `pandas.read_csv`, the next
|
|
64
|
+
* pipeline that ingests this export — sees an apostrophe that was not in the
|
|
65
|
+
* source. That is a real corruption, and it is why the guard is not applied to
|
|
66
|
+
* every cell that merely starts with a leader.
|
|
67
|
+
*
|
|
68
|
+
* **So a value that is plainly a number is left exactly as it was.** `-42` is
|
|
69
|
+
* not an injection vector: a spreadsheet evaluates it to the number -42, which
|
|
70
|
+
* is what it already was. Exempting it is what lets a machine read this file
|
|
71
|
+
* back and still get -42, and it costs no safety, because the population being
|
|
72
|
+
* defended against — `=`, `@`, an operator followed by anything that is not a
|
|
73
|
+
* number — is disjoint from it. Everything outside the exemption gets the
|
|
74
|
+
* apostrophe and reads differently to a parser than it did to the database.
|
|
75
|
+
* That trade is deliberate: a wrong apostrophe is a data-quality bug somebody
|
|
76
|
+
* can see, and a formula is code running on the machine of whoever opened the
|
|
77
|
+
* file.
|
|
78
|
+
*
|
|
79
|
+
* The apostrophe goes at position 0, in front of any leading blank, because
|
|
80
|
+
* that is the only position a spreadsheet honours it in.
|
|
81
|
+
*/
|
|
82
|
+
function guardFormula(text) {
|
|
83
|
+
if (text.length === 0)
|
|
84
|
+
return text;
|
|
85
|
+
// A literal tab or carriage return in first position is itself treated as a
|
|
86
|
+
// leader by some importers, which strip it and evaluate what follows.
|
|
87
|
+
const leadingControl = text[0] === '\t' || text[0] === '\r';
|
|
88
|
+
const body = text.replace(LEADING_BLANK, '');
|
|
89
|
+
if (body.length === 0)
|
|
90
|
+
return text;
|
|
91
|
+
if (!leadingControl && !FORMULA_LEADERS.has(body[0]))
|
|
92
|
+
return text;
|
|
93
|
+
return PLAIN_NUMBER.test(body) ? text : `'${text}`;
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* One cell, ready to sit in a row.
|
|
97
|
+
*
|
|
98
|
+
* Order matters and is the second thing people get wrong. The formula guard
|
|
99
|
+
* runs on the raw text, and the CSV quoting runs on the result — so a value that
|
|
100
|
+
* needs both comes out as `"'=1+1,x"`, with the apostrophe INSIDE the quotes and
|
|
101
|
+
* applied once. Doing it the other way round would put the apostrophe outside
|
|
102
|
+
* the quoting and produce a field no CSV reader can parse; doing it twice would
|
|
103
|
+
* put two apostrophes in a cell that needed one.
|
|
104
|
+
*/
|
|
105
|
+
function csvCell(value) {
|
|
106
|
+
if (value === null || value === undefined)
|
|
107
|
+
return '';
|
|
108
|
+
const text = typeof value === 'object' ? JSON.stringify(value) : String(value);
|
|
109
|
+
const guarded = guardFormula(text);
|
|
110
|
+
// Quote when the value could otherwise break the row apart. Doubling the
|
|
111
|
+
// quote is the CSV escape, not a backslash.
|
|
112
|
+
return /[",\n\r]/.test(guarded) ? `"${guarded.replace(/"/g, '""')}"` : guarded;
|
|
113
|
+
}
|
|
114
|
+
/** CRLF: Excel still treats a bare LF file as one long row in some locales. */
|
|
115
|
+
const EOL = '\r\n';
|
|
116
|
+
function csvLine(cells) {
|
|
117
|
+
return `${cells.join(',')}${EOL}`;
|
|
118
|
+
}
|
|
119
|
+
/**
|
|
120
|
+
* A CSV file, one line at a time, from a source that hands rows over as it has
|
|
121
|
+
* them.
|
|
122
|
+
*
|
|
123
|
+
* Each yield is a complete line including its terminator, so a consumer can
|
|
124
|
+
* write it and forget it. Nothing accumulates here: the generator holds one row
|
|
125
|
+
* and the column list, whatever the source is doing, which is the whole point —
|
|
126
|
+
* an export has no row cap by design, so the only bound available is that no
|
|
127
|
+
* stage keeps more than it is using.
|
|
128
|
+
*
|
|
129
|
+
* `columns` may be omitted, and then the header is taken from the keys of the
|
|
130
|
+
* first row — the same rule the buffered read uses, applied at the only moment a
|
|
131
|
+
* stream can apply it. A source that knows its columns up front should pass
|
|
132
|
+
* them, because it is the only way an empty result gets a header row at all.
|
|
133
|
+
*/
|
|
134
|
+
async function* csvLines(rows, columns) {
|
|
135
|
+
// A declared header goes out before the source is asked for anything, so an
|
|
136
|
+
// empty result is still a file that says which columns were empty rather than
|
|
137
|
+
// zero bytes.
|
|
138
|
+
let header = columns;
|
|
139
|
+
if (header !== undefined)
|
|
140
|
+
yield csvLine(header.map(csvCell));
|
|
141
|
+
for await (const row of rows) {
|
|
142
|
+
if (header === undefined) {
|
|
143
|
+
header = Object.keys(row);
|
|
144
|
+
yield csvLine(header.map(csvCell));
|
|
145
|
+
}
|
|
146
|
+
yield csvLine(header.map((column) => csvCell(row[column])));
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
/**
|
|
150
|
+
* The whole file as a string.
|
|
151
|
+
*
|
|
152
|
+
* Kept because it is public API and because a caller holding a finished result
|
|
153
|
+
* gains nothing from a stream. It is NOT what the export route uses — see
|
|
154
|
+
* {@link csvLines} — and a caller reaching for it with an unbounded result is
|
|
155
|
+
* choosing to hold the file.
|
|
156
|
+
*/
|
|
157
|
+
function toCsv(result) {
|
|
158
|
+
const lines = [csvLine(result.columns.map(csvCell))];
|
|
159
|
+
for (const row of result.rows) {
|
|
160
|
+
lines.push(csvLine(result.columns.map((column) => csvCell(row[column]))));
|
|
161
|
+
}
|
|
162
|
+
return lines.join('');
|
|
163
|
+
}
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
import type { CatalogPropertyDef, ScalarType } from './catalog.types';
|
|
2
|
+
/**
|
|
3
|
+
* Filtering a catalogued type, derived from the type.
|
|
4
|
+
*
|
|
5
|
+
* **Nothing here lists a filterable column, and nothing anywhere else may.** The
|
|
6
|
+
* types this catalog serves are created at runtime — `PUT publish/:type/schema`
|
|
7
|
+
* writes the properties and the store builds the physical columns from them — so
|
|
8
|
+
* a hand-maintained list of what is filterable is a list that goes quiet the day
|
|
9
|
+
* somebody publishes a column nobody edited it for. That is the failure mode this
|
|
10
|
+
* module exists to remove: the operators a column offers are a function of the
|
|
11
|
+
* column, computed by {@link filterOperatorsFor}, and both the server (deciding
|
|
12
|
+
* what it will accept) and the console (deciding what to draw) call that one
|
|
13
|
+
* function. A screen offering a control the server would refuse, or refusing one
|
|
14
|
+
* the server would take, is then not expressible.
|
|
15
|
+
*
|
|
16
|
+
* This is also why `@dudousxd/nestjs-filter` is not what is used here. That
|
|
17
|
+
* library derives its filters from entities and routes known at compile time, and
|
|
18
|
+
* these types do not exist at compile time.
|
|
19
|
+
*
|
|
20
|
+
* **The module is pure and imports nothing but types**, because it ships to the
|
|
21
|
+
* browser through `@dudousxd/nestjs-catalog/client` — the same reason
|
|
22
|
+
* `validateWorkflow` does. A second copy of the derivation living in the console
|
|
23
|
+
* is exactly the drift that entry point exists to prevent.
|
|
24
|
+
*/
|
|
25
|
+
/**
|
|
26
|
+
* Every operator, in the order a control should offer them.
|
|
27
|
+
*
|
|
28
|
+
* One list, so nothing narrows a stored or transmitted operator against a second
|
|
29
|
+
* hand-maintained copy of these names — the same argument
|
|
30
|
+
* `CATALOG_SNAPSHOT_MODES` makes one file over.
|
|
31
|
+
*
|
|
32
|
+
* There is no `between`. A range is `gte` and `lte` on one property, which is two
|
|
33
|
+
* filters that compose with everything else rather than one operator that needs
|
|
34
|
+
* its own wire form, its own second value and its own "what if the ends are the
|
|
35
|
+
* wrong way round" answer.
|
|
36
|
+
*/
|
|
37
|
+
export declare const CATALOG_FILTER_OPERATORS: readonly ["eq", "ne", "contains", "gte", "lte", "gt", "lt", "empty", "notEmpty"];
|
|
38
|
+
export type CatalogFilterOperator = (typeof CATALOG_FILTER_OPERATORS)[number];
|
|
39
|
+
export declare function isCatalogFilterOperator(value: string): value is CatalogFilterOperator;
|
|
40
|
+
/**
|
|
41
|
+
* The operators that take no value.
|
|
42
|
+
*
|
|
43
|
+
* "Has no value" cannot be spelled as a comparison — `= NULL` matches nothing in
|
|
44
|
+
* SQL and `= ''` misses the NULLs — so it is an operator rather than a value a
|
|
45
|
+
* caller types, and a filter carrying one is complete without a `value`.
|
|
46
|
+
*/
|
|
47
|
+
export declare const VALUELESS_FILTER_OPERATORS: readonly ["empty", "notEmpty"];
|
|
48
|
+
export declare function filterOperatorTakesValue(operator: CatalogFilterOperator): boolean;
|
|
49
|
+
/** One filter as it crosses the wire: a property name, an operator, raw text. */
|
|
50
|
+
export interface CatalogObjectFilter {
|
|
51
|
+
/**
|
|
52
|
+
* The property's `name`, which is its identity in the type — never its
|
|
53
|
+
* `columnName`, and never the column the view exposes.
|
|
54
|
+
*
|
|
55
|
+
* Three names can be in play for one field and only this one identifies it: a
|
|
56
|
+
* property called `Asset Id` is stored in a column called `Asset_Id`, is
|
|
57
|
+
* exposed by the SQL console under `Asset_Id`, and may carry any `columnName`
|
|
58
|
+
* its publisher chose. A filter naming any of the others resolves to no
|
|
59
|
+
* property at all. The store is what translates this into a column, which is
|
|
60
|
+
* the only place that translation belongs.
|
|
61
|
+
*/
|
|
62
|
+
property: string;
|
|
63
|
+
op: CatalogFilterOperator;
|
|
64
|
+
/** Absent for {@link VALUELESS_FILTER_OPERATORS}. Text, as it was typed. */
|
|
65
|
+
value?: string;
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* The minimum a column has to say for the rule below to run on it.
|
|
69
|
+
*
|
|
70
|
+
* Structural rather than `CatalogPropertyDef`, because the console holds the
|
|
71
|
+
* *page's* columns rather than the type's — `CatalogObjectPage.columns` — and the
|
|
72
|
+
* whole point is that both sides ask the same function. Both shapes satisfy this.
|
|
73
|
+
*/
|
|
74
|
+
export interface CatalogFilterableColumn {
|
|
75
|
+
name: string;
|
|
76
|
+
type: ScalarType;
|
|
77
|
+
hidden?: boolean;
|
|
78
|
+
classification?: string;
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* What may be filtered on this column, derived from what the column is.
|
|
82
|
+
*
|
|
83
|
+
* An empty list means "not filterable", and there are three ways to get one.
|
|
84
|
+
*
|
|
85
|
+
* **Hidden** is the overlay saying a column is not part of the generic UI, and a
|
|
86
|
+
* filter is generic UI.
|
|
87
|
+
*
|
|
88
|
+
* **Classified** is the one worth stating out loud, because the column is
|
|
89
|
+
* otherwise perfectly filterable and the value is never rendered anyway. It is
|
|
90
|
+
* excluded for the reason `MikroOrmReadStore.buildWhere` already excludes it from
|
|
91
|
+
* SEARCH: a predicate over a classified column leaks it through row membership.
|
|
92
|
+
* Filtering is strictly worse than searching there — `gte`/`lte` let a reader
|
|
93
|
+
* binary-search a value they may not see, in as many requests as it takes.
|
|
94
|
+
*
|
|
95
|
+
* **`json`** because a blob has no useful comparison, and it is already dropped
|
|
96
|
+
* from the readable columns by `CatalogService.visibleColumns`.
|
|
97
|
+
*/
|
|
98
|
+
export declare function filterOperatorsFor(column: CatalogFilterableColumn): CatalogFilterOperator[];
|
|
99
|
+
/** Which operators a column offers, once the store's own limits are applied. */
|
|
100
|
+
export declare function offeredFilterOperators(column: CatalogFilterableColumn, supported: readonly CatalogFilterOperator[]): CatalogFilterOperator[];
|
|
101
|
+
/**
|
|
102
|
+
* `property:op:value`, repeated once per filter.
|
|
103
|
+
*
|
|
104
|
+
* The value is everything after the second colon, so it may contain colons — a
|
|
105
|
+
* timestamp does. A property name containing one cannot be addressed by this
|
|
106
|
+
* form; that is refused by name in {@link resolveObjectFilters} rather than
|
|
107
|
+
* silently mis-parsed, because a filter that quietly does not apply is a screen
|
|
108
|
+
* showing every row as though it had been filtered.
|
|
109
|
+
*/
|
|
110
|
+
export declare function encodeObjectFilter(filter: CatalogObjectFilter): string;
|
|
111
|
+
export declare function parseObjectFilter(raw: string): CatalogObjectFilter | undefined;
|
|
112
|
+
/**
|
|
113
|
+
* A filter with the property it names already resolved to the type's own.
|
|
114
|
+
*
|
|
115
|
+
* The property is carried as the definition rather than as a string, and that is
|
|
116
|
+
* the guard rather than a nicety: a store builds a predicate out of a column
|
|
117
|
+
* name, and the only names it can be handed here are ones that came off the type.
|
|
118
|
+
* A caller's string never reaches SQL — it is matched against the type first, and
|
|
119
|
+
* the store then maps the property to its physical column exactly as `sort` and
|
|
120
|
+
* `search` already do, through the adapter's identifier rule.
|
|
121
|
+
*/
|
|
122
|
+
export interface CatalogResolvedFilter {
|
|
123
|
+
property: CatalogPropertyDef;
|
|
124
|
+
op: CatalogFilterOperator;
|
|
125
|
+
/** Coerced to the property's type. Absent for the valueless operators. */
|
|
126
|
+
value?: string | number | boolean | Date;
|
|
127
|
+
}
|
|
128
|
+
export interface CatalogFilterResolution {
|
|
129
|
+
filters: CatalogResolvedFilter[];
|
|
130
|
+
/** One sentence per filter that could not be honoured. Empty means all were. */
|
|
131
|
+
problems: string[];
|
|
132
|
+
}
|
|
133
|
+
/**
|
|
134
|
+
* How many filters one read may carry.
|
|
135
|
+
*
|
|
136
|
+
* A cap rather than none, because every filter is another conjunct in the two
|
|
137
|
+
* statements a paged read issues, and a caller that can send five hundred can
|
|
138
|
+
* make one request cost whatever it likes. Twenty is far past what a person
|
|
139
|
+
* builds by hand and far short of anything that would trouble the optimiser.
|
|
140
|
+
*/
|
|
141
|
+
export declare const CATALOG_FILTER_LIMIT = 20;
|
|
142
|
+
/**
|
|
143
|
+
* Turn what arrived into what a store may be given, or say why not.
|
|
144
|
+
*
|
|
145
|
+
* **Unhonourable filters are reported, never dropped.** The neighbouring
|
|
146
|
+
* `parseOutcomes` on the controller drops an unrecognised trace outcome on
|
|
147
|
+
* purpose, and this goes the other way for a reason that is worth the two
|
|
148
|
+
* sentences: dropping there costs a filter, so a typo widens the result set and
|
|
149
|
+
* shows more traces than asked for. Dropping HERE would narrow nothing — the read
|
|
150
|
+
* would come back unfiltered and the screen would present the whole table as
|
|
151
|
+
* though it were the matching rows. A filter that silently does not apply is the
|
|
152
|
+
* one failure mode a filtering UI must not have.
|
|
153
|
+
*
|
|
154
|
+
* @param columns the properties this reader may see. Deliberately the visible,
|
|
155
|
+
* non-blob columns rather than `type.properties`: a filter is only ever resolved
|
|
156
|
+
* against what the same request would return.
|
|
157
|
+
*/
|
|
158
|
+
export declare function resolveObjectFilters(columns: readonly CatalogPropertyDef[], raw: readonly string[]): CatalogFilterResolution;
|
|
159
|
+
/**
|
|
160
|
+
* A typed value, or the reason there is none.
|
|
161
|
+
*
|
|
162
|
+
* Refusing rather than passing the text through is the whole of this function's
|
|
163
|
+
* value. MySQL compares a string to a `DOUBLE` by coercing the string — `'abc'`
|
|
164
|
+
* becomes `0` — so `mileage >= abc` is not an error, it is `mileage >= 0`, and it
|
|
165
|
+
* comes back as a full page of rows that look filtered. The same is true of a
|
|
166
|
+
* date that does not parse.
|
|
167
|
+
*/
|
|
168
|
+
export declare function coerceFilterValue(type: ScalarType, value: string): {
|
|
169
|
+
ok: true;
|
|
170
|
+
value: string | number | boolean | Date;
|
|
171
|
+
} | {
|
|
172
|
+
ok: false;
|
|
173
|
+
problem: string;
|
|
174
|
+
};
|