@dudousxd/nestjs-catalog 0.14.0 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/catalog.controller.js +48 -4
- package/dist/catalog.csv.d.ts +89 -0
- package/dist/catalog.csv.js +163 -0
- package/dist/catalog.filters.d.ts +8 -4
- package/dist/catalog.identifiers.d.ts +161 -0
- package/dist/catalog.identifiers.js +195 -0
- package/dist/catalog.pipeline.d.ts +392 -35
- package/dist/catalog.pipeline.js +175 -25
- package/dist/catalog.query-cache.d.ts +0 -2
- package/dist/catalog.query-cache.js +0 -18
- package/dist/catalog.query.d.ts +43 -0
- package/dist/catalog.query.js +5 -0
- package/dist/catalog.service.d.ts +36 -0
- package/dist/catalog.service.js +72 -2
- package/dist/catalog.store.d.ts +22 -21
- package/dist/catalog.store.js +30 -68
- package/dist/catalog.types.d.ts +14 -7
- package/dist/client.d.ts +32 -2
- package/dist/client.js +39 -1
- package/dist/index.d.ts +5 -4
- package/dist/index.js +16 -3
- package/package.json +1 -1
|
@@ -13,8 +13,9 @@ var __param = (this && this.__param) || function (paramIndex, decorator) {
|
|
|
13
13
|
};
|
|
14
14
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
15
|
exports.createCatalogController = createCatalogController;
|
|
16
|
+
const node_stream_1 = require("node:stream");
|
|
16
17
|
const common_1 = require("@nestjs/common");
|
|
17
|
-
const
|
|
18
|
+
const catalog_csv_1 = require("./catalog.csv");
|
|
18
19
|
const catalog_registry_base_1 = require("./catalog.registry.base");
|
|
19
20
|
const catalog_route_auth_1 = require("./catalog.route-auth");
|
|
20
21
|
const catalog_service_1 = require("./catalog.service");
|
|
@@ -274,17 +275,60 @@ function createCatalogController(path, guards, decorators = []) {
|
|
|
274
275
|
return this.service.runSavedQuery(id, body?.maxRows);
|
|
275
276
|
}
|
|
276
277
|
/**
|
|
277
|
-
* The same result as CSV.
|
|
278
|
+
* The same result as CSV, written out as the rows arrive.
|
|
278
279
|
*
|
|
279
280
|
* A GET, not a POST, so it can be a plain link — a download that only works
|
|
280
281
|
* from JavaScript cannot be pasted into a mail or a scheduled job.
|
|
282
|
+
*
|
|
283
|
+
* **Nothing here holds the file.** This used to run the query, hold the
|
|
284
|
+
* result, build the whole CSV string and then answer — which is the shape
|
|
285
|
+
* that made a 981,469-row connector load never finish, and it is worse on an
|
|
286
|
+
* export than on a load, because an export has no row cap by design. Now
|
|
287
|
+
* `streamSavedQuery` hands over rows, {@link csvLines} turns each into a
|
|
288
|
+
* line, and the response is written from that.
|
|
289
|
+
*
|
|
290
|
+
* **`@Res({ passthrough: true })` is still right, and the returned value is
|
|
291
|
+
* what had to change.** Passthrough means "I am setting headers, you send
|
|
292
|
+
* the body", and that is exactly what this handler wants. What it must not
|
|
293
|
+
* return any more is a string: the express adapter answers a string body
|
|
294
|
+
* with `res.send()`, which sets `content-length` — and a length header is a
|
|
295
|
+
* promise about a body nobody has counted yet. A `StreamableFile` is the
|
|
296
|
+
* body shape the adapter pipes instead, and `applyStreamHeaders` sets a
|
|
297
|
+
* length only when the `StreamableFile` was given one, which this is not. So
|
|
298
|
+
* the response is chunked and carries no `content-length`. Taking the
|
|
299
|
+
* response over with a bare `@Res()` and pumping it by hand would work too,
|
|
300
|
+
* and would name a platform: this package peer-depends on `@nestjs/common`
|
|
301
|
+
* and `@nestjs/core` and on no adapter, so a hand-written `write`/`drain`
|
|
302
|
+
* loop would be one adapter's API written into a library that deliberately
|
|
303
|
+
* does not have one. A `StreamableFile` leaves the piping to whichever
|
|
304
|
+
* adapter the host mounted.
|
|
305
|
+
*
|
|
306
|
+
* The pipe is also what bounds the memory: `Readable.from` stops pulling
|
|
307
|
+
* from the generator once its buffer is full, the pipe stops reading once
|
|
308
|
+
* the socket says wait, and the generator stops pulling from the store. A
|
|
309
|
+
* client that reads slowly slows the database read down rather than filling
|
|
310
|
+
* this process with the rows it has not collected.
|
|
281
311
|
*/
|
|
282
312
|
async exportSavedQuery(id, response) {
|
|
283
|
-
const { savedQuery,
|
|
313
|
+
const { savedQuery, columns, rows } = await this.service.streamSavedQuery(id);
|
|
284
314
|
const filename = `${savedQuery.name.replace(/[^A-Za-z0-9_-]+/g, '-')}.csv`;
|
|
285
315
|
response.setHeader('content-type', 'text/csv; charset=utf-8');
|
|
286
316
|
response.setHeader('content-disposition', `attachment; filename="${filename}"`);
|
|
287
|
-
|
|
317
|
+
// Byte mode rather than object mode, so the readable's own high-water mark
|
|
318
|
+
// is measured in bytes: the lines coalesce into socket-sized writes on
|
|
319
|
+
// their way out, instead of one write per row for however many rows the
|
|
320
|
+
// table has.
|
|
321
|
+
const file = node_stream_1.Readable.from((0, catalog_csv_1.csvLines)(rows, columns), { objectMode: false });
|
|
322
|
+
// `pipe` unpipes a dead destination and leaves the SOURCE alone, which
|
|
323
|
+
// here means an operator who closes the download tab leaves this generator
|
|
324
|
+
// parked mid-`yield` — and with it, in the shipped MySQL store, an open
|
|
325
|
+
// read-only transaction on a pooled connection, for the life of the
|
|
326
|
+
// process. Destroying the readable is what calls the generator's `return`,
|
|
327
|
+
// which is what runs the `finally` the store put its rollback in. Fires on
|
|
328
|
+
// an ordinary completion too, where destroying a finished stream is a
|
|
329
|
+
// no-op.
|
|
330
|
+
response.on('close', () => file.destroy());
|
|
331
|
+
return new common_1.StreamableFile(file);
|
|
288
332
|
}
|
|
289
333
|
/**
|
|
290
334
|
* Every SQL this query has ever been, newest first.
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* CSV, for the export button.
|
|
3
|
+
*
|
|
4
|
+
* Two things here are not the obvious implementation, and both are here because
|
|
5
|
+
* the obvious one was wrong in production.
|
|
6
|
+
*
|
|
7
|
+
* **Rows are written as they arrive.** {@link csvLines} is a generator over an
|
|
8
|
+
* async row source, so the caller can hand each line to a socket and never hold
|
|
9
|
+
* the file. The buffered {@link toCsv} is still exported — it is public API and
|
|
10
|
+
* a caller with a result already in memory has nothing to gain from a stream —
|
|
11
|
+
* but it is now the special case rather than the only shape.
|
|
12
|
+
*
|
|
13
|
+
* **A cell that a spreadsheet would run is neutralised.** See
|
|
14
|
+
* {@link guardFormula}.
|
|
15
|
+
*/
|
|
16
|
+
/** One row of a result, as every store hands it over. */
|
|
17
|
+
export type CsvRow = Record<string, unknown>;
|
|
18
|
+
/**
|
|
19
|
+
* Stop a cell being executed when the file is opened.
|
|
20
|
+
*
|
|
21
|
+
* A CSV is not a document format, it is a program that a spreadsheet is willing
|
|
22
|
+
* to run. A cell whose value begins with one of {@link FORMULA_LEADERS} is
|
|
23
|
+
* evaluated on open, and the values in this file come from whatever the queried
|
|
24
|
+
* source contained — which for this catalog is other people's operational data,
|
|
25
|
+
* loaded by connectors, from systems nobody here controls. So the crafted cell
|
|
26
|
+
* is not a thought experiment: it is one row in a table an operator exports and
|
|
27
|
+
* opens.
|
|
28
|
+
*
|
|
29
|
+
* **The escape is a leading apostrophe, and it is not free.** Excel and Sheets
|
|
30
|
+
* both read `'` as "the rest is literal text" and do not show it in the cell;
|
|
31
|
+
* every other reader on earth — a parser, a `pandas.read_csv`, the next
|
|
32
|
+
* pipeline that ingests this export — sees an apostrophe that was not in the
|
|
33
|
+
* source. That is a real corruption, and it is why the guard is not applied to
|
|
34
|
+
* every cell that merely starts with a leader.
|
|
35
|
+
*
|
|
36
|
+
* **So a value that is plainly a number is left exactly as it was.** `-42` is
|
|
37
|
+
* not an injection vector: a spreadsheet evaluates it to the number -42, which
|
|
38
|
+
* is what it already was. Exempting it is what lets a machine read this file
|
|
39
|
+
* back and still get -42, and it costs no safety, because the population being
|
|
40
|
+
* defended against — `=`, `@`, an operator followed by anything that is not a
|
|
41
|
+
* number — is disjoint from it. Everything outside the exemption gets the
|
|
42
|
+
* apostrophe and reads differently to a parser than it did to the database.
|
|
43
|
+
* That trade is deliberate: a wrong apostrophe is a data-quality bug somebody
|
|
44
|
+
* can see, and a formula is code running on the machine of whoever opened the
|
|
45
|
+
* file.
|
|
46
|
+
*
|
|
47
|
+
* The apostrophe goes at position 0, in front of any leading blank, because
|
|
48
|
+
* that is the only position a spreadsheet honours it in.
|
|
49
|
+
*/
|
|
50
|
+
export declare function guardFormula(text: string): string;
|
|
51
|
+
/**
|
|
52
|
+
* One cell, ready to sit in a row.
|
|
53
|
+
*
|
|
54
|
+
* Order matters and is the second thing people get wrong. The formula guard
|
|
55
|
+
* runs on the raw text, and the CSV quoting runs on the result — so a value that
|
|
56
|
+
* needs both comes out as `"'=1+1,x"`, with the apostrophe INSIDE the quotes and
|
|
57
|
+
* applied once. Doing it the other way round would put the apostrophe outside
|
|
58
|
+
* the quoting and produce a field no CSV reader can parse; doing it twice would
|
|
59
|
+
* put two apostrophes in a cell that needed one.
|
|
60
|
+
*/
|
|
61
|
+
export declare function csvCell(value: unknown): string;
|
|
62
|
+
/**
|
|
63
|
+
* A CSV file, one line at a time, from a source that hands rows over as it has
|
|
64
|
+
* them.
|
|
65
|
+
*
|
|
66
|
+
* Each yield is a complete line including its terminator, so a consumer can
|
|
67
|
+
* write it and forget it. Nothing accumulates here: the generator holds one row
|
|
68
|
+
* and the column list, whatever the source is doing, which is the whole point —
|
|
69
|
+
* an export has no row cap by design, so the only bound available is that no
|
|
70
|
+
* stage keeps more than it is using.
|
|
71
|
+
*
|
|
72
|
+
* `columns` may be omitted, and then the header is taken from the keys of the
|
|
73
|
+
* first row — the same rule the buffered read uses, applied at the only moment a
|
|
74
|
+
* stream can apply it. A source that knows its columns up front should pass
|
|
75
|
+
* them, because it is the only way an empty result gets a header row at all.
|
|
76
|
+
*/
|
|
77
|
+
export declare function csvLines(rows: AsyncIterable<CsvRow>, columns?: readonly string[]): AsyncGenerator<string>;
|
|
78
|
+
/**
|
|
79
|
+
* The whole file as a string.
|
|
80
|
+
*
|
|
81
|
+
* Kept because it is public API and because a caller holding a finished result
|
|
82
|
+
* gains nothing from a stream. It is NOT what the export route uses — see
|
|
83
|
+
* {@link csvLines} — and a caller reaching for it with an unbounded result is
|
|
84
|
+
* choosing to hold the file.
|
|
85
|
+
*/
|
|
86
|
+
export declare function toCsv(result: {
|
|
87
|
+
columns: string[];
|
|
88
|
+
rows: CsvRow[];
|
|
89
|
+
}): string;
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* CSV, for the export button.
|
|
4
|
+
*
|
|
5
|
+
* Two things here are not the obvious implementation, and both are here because
|
|
6
|
+
* the obvious one was wrong in production.
|
|
7
|
+
*
|
|
8
|
+
* **Rows are written as they arrive.** {@link csvLines} is a generator over an
|
|
9
|
+
* async row source, so the caller can hand each line to a socket and never hold
|
|
10
|
+
* the file. The buffered {@link toCsv} is still exported — it is public API and
|
|
11
|
+
* a caller with a result already in memory has nothing to gain from a stream —
|
|
12
|
+
* but it is now the special case rather than the only shape.
|
|
13
|
+
*
|
|
14
|
+
* **A cell that a spreadsheet would run is neutralised.** See
|
|
15
|
+
* {@link guardFormula}.
|
|
16
|
+
*/
|
|
17
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
18
|
+
exports.guardFormula = guardFormula;
|
|
19
|
+
exports.csvCell = csvCell;
|
|
20
|
+
exports.csvLines = csvLines;
|
|
21
|
+
exports.toCsv = toCsv;
|
|
22
|
+
/**
|
|
23
|
+
* The characters a spreadsheet reads as "what follows is a formula".
|
|
24
|
+
*
|
|
25
|
+
* `=` is the obvious one. `+`, `-` and `@` are the ones people forget: Excel and
|
|
26
|
+
* Sheets both accept a leading `+` or `-` as the start of an expression, and `@`
|
|
27
|
+
* introduces a function reference. `-2+3+cmd|' /C calc'!A0` is a working command
|
|
28
|
+
* execution in an unpatched Excel, and it starts with a minus sign.
|
|
29
|
+
*/
|
|
30
|
+
const FORMULA_LEADERS = new Set(['=', '+', '-', '@']);
|
|
31
|
+
/**
|
|
32
|
+
* Blank a spreadsheet strips before it decides whether a cell is a formula.
|
|
33
|
+
*
|
|
34
|
+
* The reason this exists rather than a plain `text[0]` test: importers do not
|
|
35
|
+
* agree on whether leading blank is part of the value. Several strip it and then
|
|
36
|
+
* dispatch on what is left, so ` =1+1` is a formula to them and an innocent
|
|
37
|
+
* string to a guard that only looked at position zero. Guarding the stripped
|
|
38
|
+
* form costs nothing on values that were never formulas.
|
|
39
|
+
*/
|
|
40
|
+
const LEADING_BLANK = /^[\t\n\v\f\r ]+/;
|
|
41
|
+
/**
|
|
42
|
+
* A value a spreadsheet will read back as the number it already is.
|
|
43
|
+
*
|
|
44
|
+
* This is the exemption that keeps the guard from corrupting data — see
|
|
45
|
+
* {@link guardFormula}. Deliberately narrow: an optional sign, digits with at
|
|
46
|
+
* most one point, an optional exponent, and nothing else. `-42` matches.
|
|
47
|
+
* `-42abc` does not, `- 42` does not, and `-2+3+cmd|' /C calc'!A0` does not.
|
|
48
|
+
*/
|
|
49
|
+
const PLAIN_NUMBER = /^[+-]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?$/;
|
|
50
|
+
/**
|
|
51
|
+
* Stop a cell being executed when the file is opened.
|
|
52
|
+
*
|
|
53
|
+
* A CSV is not a document format, it is a program that a spreadsheet is willing
|
|
54
|
+
* to run. A cell whose value begins with one of {@link FORMULA_LEADERS} is
|
|
55
|
+
* evaluated on open, and the values in this file come from whatever the queried
|
|
56
|
+
* source contained — which for this catalog is other people's operational data,
|
|
57
|
+
* loaded by connectors, from systems nobody here controls. So the crafted cell
|
|
58
|
+
* is not a thought experiment: it is one row in a table an operator exports and
|
|
59
|
+
* opens.
|
|
60
|
+
*
|
|
61
|
+
* **The escape is a leading apostrophe, and it is not free.** Excel and Sheets
|
|
62
|
+
* both read `'` as "the rest is literal text" and do not show it in the cell;
|
|
63
|
+
* every other reader on earth — a parser, a `pandas.read_csv`, the next
|
|
64
|
+
* pipeline that ingests this export — sees an apostrophe that was not in the
|
|
65
|
+
* source. That is a real corruption, and it is why the guard is not applied to
|
|
66
|
+
* every cell that merely starts with a leader.
|
|
67
|
+
*
|
|
68
|
+
* **So a value that is plainly a number is left exactly as it was.** `-42` is
|
|
69
|
+
* not an injection vector: a spreadsheet evaluates it to the number -42, which
|
|
70
|
+
* is what it already was. Exempting it is what lets a machine read this file
|
|
71
|
+
* back and still get -42, and it costs no safety, because the population being
|
|
72
|
+
* defended against — `=`, `@`, an operator followed by anything that is not a
|
|
73
|
+
* number — is disjoint from it. Everything outside the exemption gets the
|
|
74
|
+
* apostrophe and reads differently to a parser than it did to the database.
|
|
75
|
+
* That trade is deliberate: a wrong apostrophe is a data-quality bug somebody
|
|
76
|
+
* can see, and a formula is code running on the machine of whoever opened the
|
|
77
|
+
* file.
|
|
78
|
+
*
|
|
79
|
+
* The apostrophe goes at position 0, in front of any leading blank, because
|
|
80
|
+
* that is the only position a spreadsheet honours it in.
|
|
81
|
+
*/
|
|
82
|
+
function guardFormula(text) {
|
|
83
|
+
if (text.length === 0)
|
|
84
|
+
return text;
|
|
85
|
+
// A literal tab or carriage return in first position is itself treated as a
|
|
86
|
+
// leader by some importers, which strip it and evaluate what follows.
|
|
87
|
+
const leadingControl = text[0] === '\t' || text[0] === '\r';
|
|
88
|
+
const body = text.replace(LEADING_BLANK, '');
|
|
89
|
+
if (body.length === 0)
|
|
90
|
+
return text;
|
|
91
|
+
if (!leadingControl && !FORMULA_LEADERS.has(body[0]))
|
|
92
|
+
return text;
|
|
93
|
+
return PLAIN_NUMBER.test(body) ? text : `'${text}`;
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* One cell, ready to sit in a row.
|
|
97
|
+
*
|
|
98
|
+
* Order matters and is the second thing people get wrong. The formula guard
|
|
99
|
+
* runs on the raw text, and the CSV quoting runs on the result — so a value that
|
|
100
|
+
* needs both comes out as `"'=1+1,x"`, with the apostrophe INSIDE the quotes and
|
|
101
|
+
* applied once. Doing it the other way round would put the apostrophe outside
|
|
102
|
+
* the quoting and produce a field no CSV reader can parse; doing it twice would
|
|
103
|
+
* put two apostrophes in a cell that needed one.
|
|
104
|
+
*/
|
|
105
|
+
function csvCell(value) {
|
|
106
|
+
if (value === null || value === undefined)
|
|
107
|
+
return '';
|
|
108
|
+
const text = typeof value === 'object' ? JSON.stringify(value) : String(value);
|
|
109
|
+
const guarded = guardFormula(text);
|
|
110
|
+
// Quote when the value could otherwise break the row apart. Doubling the
|
|
111
|
+
// quote is the CSV escape, not a backslash.
|
|
112
|
+
return /[",\n\r]/.test(guarded) ? `"${guarded.replace(/"/g, '""')}"` : guarded;
|
|
113
|
+
}
|
|
114
|
+
/** CRLF: Excel still treats a bare LF file as one long row in some locales. */
|
|
115
|
+
const EOL = '\r\n';
|
|
116
|
+
function csvLine(cells) {
|
|
117
|
+
return `${cells.join(',')}${EOL}`;
|
|
118
|
+
}
|
|
119
|
+
/**
|
|
120
|
+
* A CSV file, one line at a time, from a source that hands rows over as it has
|
|
121
|
+
* them.
|
|
122
|
+
*
|
|
123
|
+
* Each yield is a complete line including its terminator, so a consumer can
|
|
124
|
+
* write it and forget it. Nothing accumulates here: the generator holds one row
|
|
125
|
+
* and the column list, whatever the source is doing, which is the whole point —
|
|
126
|
+
* an export has no row cap by design, so the only bound available is that no
|
|
127
|
+
* stage keeps more than it is using.
|
|
128
|
+
*
|
|
129
|
+
* `columns` may be omitted, and then the header is taken from the keys of the
|
|
130
|
+
* first row — the same rule the buffered read uses, applied at the only moment a
|
|
131
|
+
* stream can apply it. A source that knows its columns up front should pass
|
|
132
|
+
* them, because it is the only way an empty result gets a header row at all.
|
|
133
|
+
*/
|
|
134
|
+
async function* csvLines(rows, columns) {
|
|
135
|
+
// A declared header goes out before the source is asked for anything, so an
|
|
136
|
+
// empty result is still a file that says which columns were empty rather than
|
|
137
|
+
// zero bytes.
|
|
138
|
+
let header = columns;
|
|
139
|
+
if (header !== undefined)
|
|
140
|
+
yield csvLine(header.map(csvCell));
|
|
141
|
+
for await (const row of rows) {
|
|
142
|
+
if (header === undefined) {
|
|
143
|
+
header = Object.keys(row);
|
|
144
|
+
yield csvLine(header.map(csvCell));
|
|
145
|
+
}
|
|
146
|
+
yield csvLine(header.map((column) => csvCell(row[column])));
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
/**
|
|
150
|
+
* The whole file as a string.
|
|
151
|
+
*
|
|
152
|
+
* Kept because it is public API and because a caller holding a finished result
|
|
153
|
+
* gains nothing from a stream. It is NOT what the export route uses — see
|
|
154
|
+
* {@link csvLines} — and a caller reaching for it with an unbounded result is
|
|
155
|
+
* choosing to hold the file.
|
|
156
|
+
*/
|
|
157
|
+
function toCsv(result) {
|
|
158
|
+
const lines = [csvLine(result.columns.map(csvCell))];
|
|
159
|
+
for (const row of result.rows) {
|
|
160
|
+
lines.push(csvLine(result.columns.map((column) => csvCell(row[column]))));
|
|
161
|
+
}
|
|
162
|
+
return lines.join('');
|
|
163
|
+
}
|
|
@@ -50,10 +50,14 @@ export declare function filterOperatorTakesValue(operator: CatalogFilterOperator
|
|
|
50
50
|
export interface CatalogObjectFilter {
|
|
51
51
|
/**
|
|
52
52
|
* The property's `name`, which is its identity in the type — never its
|
|
53
|
-
* `columnName
|
|
54
|
-
*
|
|
55
|
-
*
|
|
56
|
-
* property
|
|
53
|
+
* `columnName`, and never the column the view exposes.
|
|
54
|
+
*
|
|
55
|
+
* Three names can be in play for one field and only this one identifies it: a
|
|
56
|
+
* property called `Asset Id` is stored in a column called `Asset_Id`, is
|
|
57
|
+
* exposed by the SQL console under `Asset_Id`, and may carry any `columnName`
|
|
58
|
+
* its publisher chose. A filter naming any of the others resolves to no
|
|
59
|
+
* property at all. The store is what translates this into a column, which is
|
|
60
|
+
* the only place that translation belongs.
|
|
57
61
|
*/
|
|
58
62
|
property: string;
|
|
59
63
|
op: CatalogFilterOperator;
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* How a name becomes a column, and what it has to look like by the end.
|
|
3
|
+
*
|
|
4
|
+
* Two rules, and they are here together because neither is usable without the
|
|
5
|
+
* other. {@link physicalColumn} is the *cleaning* — the lossy map from a
|
|
6
|
+
* property's name to the column a store creates for it. {@link isSafeIdentifier}
|
|
7
|
+
* is the *character set* the result of that cleaning has to be in. What a
|
|
8
|
+
* publisher is actually promised is the composition: a name may be spelled
|
|
9
|
+
* however the source spells it, and what the cleaning produces has to be
|
|
10
|
+
* something a store can write.
|
|
11
|
+
*
|
|
12
|
+
* Identifiers themselves are *rejected*, never escaped. Every table and column
|
|
13
|
+
* name a store emits arrives from another application over HTTP and ends up in
|
|
14
|
+
* DDL and in SELECT lists, where no placeholder can stand in for it, so anything
|
|
15
|
+
* outside this character set never becomes SQL at all.
|
|
16
|
+
*
|
|
17
|
+
* It is part of what the catalog promises a *publisher*. Refuse a property name
|
|
18
|
+
* and the sentence explaining why is the only statement of the rule most people
|
|
19
|
+
* will ever read, so it belongs to the contract rather than to whichever adapter
|
|
20
|
+
* happens to be mounted.
|
|
21
|
+
*
|
|
22
|
+
* And for one more reason. It used to be two copies — `store-mikro-orm` and
|
|
23
|
+
* `store-clickhouse` each carried this pattern and this sentence, byte for byte
|
|
24
|
+
* — and the publish-time refusal in the pipeline package borrowed the MySQL one
|
|
25
|
+
* so that publish-time and DDL-time could not disagree about the character set,
|
|
26
|
+
* the length or the wording. That bought the guarantee for a MySQL deployment
|
|
27
|
+
* and left a ClickHouse-only one trusting two files to be edited together. One
|
|
28
|
+
* definition is the guarantee; two identical ones are a habit.
|
|
29
|
+
*
|
|
30
|
+
* ---
|
|
31
|
+
*
|
|
32
|
+
* **Why this is its own module, and why it imports nothing.**
|
|
33
|
+
*
|
|
34
|
+
* All of this used to live in `catalog.store.ts`, which is still where every
|
|
35
|
+
* server-side caller reaches it from — that file re-exports all five names, so no
|
|
36
|
+
* import anywhere had to change. What could not stay there is the
|
|
37
|
+
* *reachability*: `catalog.store.ts` imports `BadRequestException` from
|
|
38
|
+
* `@nestjs/common` at module scope, so anything importing a **value** out of it
|
|
39
|
+
* drags NestJS along. That is fine on the server and disqualifying for
|
|
40
|
+
* `/client`, which exists precisely so a browser can share the server's rules
|
|
41
|
+
* without shipping the server.
|
|
42
|
+
*
|
|
43
|
+
* And a browser now has to be able to ask this question. A workflow template
|
|
44
|
+
* that proposes replicating a table has to know, while somebody is still
|
|
45
|
+
* choosing, whether the source's column names could be published as property
|
|
46
|
+
* names — because if they could not, the graph it would draw is refused at
|
|
47
|
+
* publish, or worse, gets "fixed" by a rename that commits nulls and reports
|
|
48
|
+
* success. Answering that from a copy of the pattern is the one thing this
|
|
49
|
+
* module's own history says not to do: the copy is what drifts, and a canvas
|
|
50
|
+
* whose idea of a legal name differs from the store's by one character is a
|
|
51
|
+
* canvas that promises a load the publisher then refuses.
|
|
52
|
+
*
|
|
53
|
+
* That is why {@link physicalColumn} had to come along rather than only
|
|
54
|
+
* {@link isSafeIdentifier}. The question a publisher is refused on is
|
|
55
|
+
* `isSafeIdentifier(physicalColumn(name))`, not `isSafeIdentifier(name)`, and a
|
|
56
|
+
* browser holding only half the composition would answer a different question
|
|
57
|
+
* from the server's — which is the same drift by another route.
|
|
58
|
+
*
|
|
59
|
+
* So the rule moved to a file with no imports, and is exported from both entry
|
|
60
|
+
* points. `validateWorkflow` set the precedent and made the same argument.
|
|
61
|
+
*/
|
|
62
|
+
/**
|
|
63
|
+
* Why a name cannot be written into SQL, in the words a publisher is given.
|
|
64
|
+
*
|
|
65
|
+
* One class for the whole ecosystem rather than one per adapter, so
|
|
66
|
+
* `instanceof` is a usable question across packages. The publish-time check in
|
|
67
|
+
* the pipeline package catches this to tell "that name cannot be an identifier"
|
|
68
|
+
* from "something else failed inside the store", and with a class per adapter
|
|
69
|
+
* that check would re-throw the moment the mounted store was not the one it
|
|
70
|
+
* imported — turning a 400 that names the property into a 500 that names
|
|
71
|
+
* nothing.
|
|
72
|
+
*/
|
|
73
|
+
export declare class UnsafeIdentifierError extends Error {
|
|
74
|
+
constructor(value: string);
|
|
75
|
+
}
|
|
76
|
+
/** Whether a name can be written into SQL as it stands. */
|
|
77
|
+
export declare function isSafeIdentifier(value: string): boolean;
|
|
78
|
+
/**
|
|
79
|
+
* Refuse a name that cannot be a SQL identifier.
|
|
80
|
+
*
|
|
81
|
+
* Throws rather than answering, because the caller's next line writes the value
|
|
82
|
+
* into a statement: a boolean that can be ignored is a boolean that eventually
|
|
83
|
+
* is. {@link isSafeIdentifier} is there for the callers that are asking rather
|
|
84
|
+
* than about to build.
|
|
85
|
+
*/
|
|
86
|
+
export declare function assertSafeIdentifier(value: string): void;
|
|
87
|
+
/**
|
|
88
|
+
* A property's name, cleaned into the column a store can create for it.
|
|
89
|
+
*
|
|
90
|
+
* Here rather than in each adapter for the reason {@link assertSafeIdentifier}
|
|
91
|
+
* is: this is no longer only an adapter's private repair. It decides the column
|
|
92
|
+
* a load's values are written to, the column a filter is applied to, the name a
|
|
93
|
+
* committed view exposes the field under, and — since it does all of that — it
|
|
94
|
+
* decides whether a published name can work at all. The pipeline package refuses
|
|
95
|
+
* a name at publish time by asking whether *this* produces an identifier, so the
|
|
96
|
+
* refusal and the DDL have to be running the same cleaning rather than two
|
|
97
|
+
* copies of it. `store-mikro-orm` and `store-clickhouse` each carried a
|
|
98
|
+
* byte-identical private copy, and `store-mikro-orm` carried two of its own —
|
|
99
|
+
* one in `query.ts` for the view, one in `mysql-warehouse.store.ts` for
|
|
100
|
+
* everything else. Three copies of the function that decides where a column's
|
|
101
|
+
* data lives is three chances for a view to point at a column no load ever
|
|
102
|
+
* wrote.
|
|
103
|
+
*
|
|
104
|
+
* Lossy on purpose, and lossy in a way callers must handle rather than assume
|
|
105
|
+
* away: `Asset Id` and `Asset/Id` both clean to `Asset_Id`, which is what
|
|
106
|
+
* `assertNoColumnCollisions` exists to catch.
|
|
107
|
+
*
|
|
108
|
+
* 60 characters, not the 63 the identifier rule allows, and the three characters
|
|
109
|
+
* of headroom are not decorative — a store that needs to derive a second name
|
|
110
|
+
* from a column has room inside MySQL's 64-character ceiling to do it. Widening
|
|
111
|
+
* this would silently rename the column of every property whose name is 61 to 63
|
|
112
|
+
* characters long, so it stays where it is.
|
|
113
|
+
*
|
|
114
|
+
* Not every output is an identifier: `1 2 3` cleans to `1_2_3`, which no store
|
|
115
|
+
* will quote. That is not this function's business to fix — a suggestion is
|
|
116
|
+
* `toPhysicalName` in an adapter, and a refusal is `assertSafeIdentifier` on the
|
|
117
|
+
* result.
|
|
118
|
+
*/
|
|
119
|
+
export declare function physicalColumn(propertyName: string): string;
|
|
120
|
+
/**
|
|
121
|
+
* The column name a store exposes a property under, in the committed view and
|
|
122
|
+
* in the SELECT list of a read.
|
|
123
|
+
*
|
|
124
|
+
* **Why this is not simply the property's name.** It used to be. Every store
|
|
125
|
+
* wrote `\`Asset_Id\` AS \`Asset Id\`` — the physical column reached by cleaning,
|
|
126
|
+
* the alias taken verbatim — and the alias went through `ident`, which refuses
|
|
127
|
+
* rather than escapes. So a property could only be named something that was
|
|
128
|
+
* already a SQL identifier, which meant a source column genuinely called `Asset
|
|
129
|
+
* Id` could not be published under its own spelling.
|
|
130
|
+
*
|
|
131
|
+
* That mattered far more than it looks. A load matches a source's record to a
|
|
132
|
+
* property by property NAME — the store reads `row[property.name]` — so a
|
|
133
|
+
* publisher forced to rename the property to `Asset_Id`, keeping `Asset Id` in
|
|
134
|
+
* `columnName`, was publishing a type whose every read of that field returned
|
|
135
|
+
* `undefined`. `columnName` is lineage; nothing consults it on the write path.
|
|
136
|
+
* The loads committed, the row counts were right, and the column was NULL in
|
|
137
|
+
* every row. Thirteen types were loaded that way and six of them came out with
|
|
138
|
+
* most of their columns empty — 73 of 84 on the largest, across 313,833 rows.
|
|
139
|
+
* The verbatim alias is what forced the rename, so the alias is what changed.
|
|
140
|
+
*
|
|
141
|
+
* **Why the name is still kept when it is already an identifier.** The obvious
|
|
142
|
+
* fix — always alias to {@link physicalColumn} — would rename the output column
|
|
143
|
+
* of every existing view whose property name is not equal to its own cleaned
|
|
144
|
+
* form: a property called `Asset__Id` (two underscores collapse to one) or one
|
|
145
|
+
* 61 characters long (cut to 60). Those views work today and somebody is
|
|
146
|
+
* selecting from them by name. Renaming a column under a working consumer to
|
|
147
|
+
* tidy up an inconsistency is not a trade worth making, so the rule is the
|
|
148
|
+
* narrower one: **a name that SQL can take as it stands is kept exactly; only a
|
|
149
|
+
* name SQL cannot take is cleaned.** Every view that resolves today keeps every
|
|
150
|
+
* column name it has today.
|
|
151
|
+
*
|
|
152
|
+
* **This introduces no new way for two properties to collide.** If two distinct
|
|
153
|
+
* names produce one alias then they also produce one {@link physicalColumn}, so
|
|
154
|
+
* `assertNoColumnCollisions` already refuses the pair. Both unsafe: equal
|
|
155
|
+
* aliases *are* equal physical columns. Both safe: equal aliases are equal
|
|
156
|
+
* names, and there is only one property per name. One of each — a safe `x` and
|
|
157
|
+
* an unsafe `y` with `physicalColumn(y) === x` — means `x` contains no run of
|
|
158
|
+
* underscores and is at most 60 characters, so `physicalColumn(x) === x ===
|
|
159
|
+
* physicalColumn(y)` and the columns collide too.
|
|
160
|
+
*/
|
|
161
|
+
export declare function outputAlias(propertyName: string): string;
|