@mistriwork/engine 1.0.0-rc.16 → 1.0.0-rc.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -0
- package/dist/adapters/adapter.d.ts +35 -2
- package/dist/adapters/adapter.js +32 -1
- package/dist/adapters/postgres.js +8 -4
- package/dist/adapters/scoped/scopedReads.js +23 -5
- package/dist/adapters/scoped/sqliteTriggers.js +65 -38
- package/dist/adapters/sqlite.js +8 -4
- package/dist/host.d.ts +46 -0
- package/dist/host.js +51 -0
- package/dist/server/columnGrants.d.ts +5 -0
- package/dist/server/connections.d.ts +1 -0
- package/dist/server/connections.js +20 -1
- package/dist/server/http/exportRoute.d.ts +74 -0
- package/dist/server/http/exportRoute.js +206 -169
- package/dist/server/http/firstPaint.d.ts +5 -1
- package/dist/server/http/firstPaint.js +4 -4
- package/dist/server/http/reply.js +6 -0
- package/dist/server/httpConnection.d.ts +4 -2
- package/dist/server/httpConnection.js +10 -3
- package/dist/server/journal.d.ts +35 -6
- package/dist/server/journal.js +79 -8
- package/dist/server/journalStore.d.ts +37 -0
- package/dist/server/journalStore.js +375 -0
- package/dist/server/resourceReadHttp.d.ts +4 -2
- package/dist/server/resourceReadHttp.js +40 -4
- package/dist/server/resourceReadOperations.d.ts +4 -0
- package/dist/server/scaffoldMachine.js +12 -1
- package/dist/shared/appListColumns.d.ts +7 -0
- package/dist/shared/appListColumns.js +13 -0
- package/dist/shared/boardLanes.d.ts +44 -0
- package/dist/shared/boardLanes.js +69 -0
- package/dist/shared/builtInThemes.js +26 -20
- package/dist/shared/dashboardAsks.d.ts +2 -0
- package/dist/shared/filter.d.ts +2 -0
- package/dist/shared/filter.js +3 -2
- package/dist/shared/form.d.ts +24 -1
- package/dist/shared/form.js +53 -0
- package/dist/shared/keysetPage.js +3 -0
- package/dist/shared/listInference.d.ts +91 -0
- package/dist/shared/listInference.js +323 -0
- package/dist/shared/listPresentations.d.ts +3 -0
- package/dist/shared/listQuestion.d.ts +2 -0
- package/dist/shared/listQuestion.js +3 -2
- package/dist/shared/pageAnatomy.d.ts +27 -3
- package/dist/shared/pageAnatomy.js +217 -36
- package/dist/shared/pageModel.d.ts +17 -1
- package/dist/shared/pageModel.js +8 -0
- package/dist/shared/query.d.ts +2 -0
- package/dist/shared/query.js +11 -3
- package/dist/shared/reads.d.ts +9 -0
- package/dist/shared/reads.js +9 -0
- package/dist/shared/recordLabel.d.ts +5 -0
- package/dist/shared/recordLabel.js +22 -2
- package/dist/shared/recordReads.d.ts +30 -0
- package/dist/shared/recordReads.js +101 -19
- package/dist/shared/referenceSubline.d.ts +18 -0
- package/dist/shared/referenceSubline.js +62 -0
- package/dist/shared/resourceHttp.d.ts +5 -0
- package/dist/shared/rowScope.js +52 -6
- package/dist/shared/schema.d.ts +2 -0
- package/dist/shared/servedApp.d.ts +19 -1
- package/dist/shared/servedApp.js +10 -0
- package/dist/shared/themeLooks.js +61 -35
- package/dist/shared/themeRecipeWords.d.ts +5 -0
- package/dist/shared/themeRecipeWords.js +6 -1
- package/dist/shared/themeRecipes.d.ts +5 -0
- package/dist/shared/themeRecipes.js +3 -0
- package/dist/shared/themeType.js +2 -1
- package/dist/shared/view.js +29 -2
- package/dist/shared/viewLabels.js +16 -0
- package/dist/shared/viewPaths.d.ts +9 -0
- package/package.json +6 -1
- package/schema/theme.d.ts +8 -4
|
@@ -133,6 +133,7 @@ export declare class Registry {
|
|
|
133
133
|
private cacheWarnedAt;
|
|
134
134
|
private readonly moduleState;
|
|
135
135
|
private readonly rolesSaid;
|
|
136
|
+
private readonly journalSaid;
|
|
136
137
|
/** A service from the config. Names are unique among services; a kind that is a singleton (the cache, the email service) is configured once. */
|
|
137
138
|
addService(service: ServiceConfig): void;
|
|
138
139
|
/** The entries of kinds this binary does not read (`others: 'keep'`), as written, for a save to put back. */
|
|
@@ -35,6 +35,8 @@ import { listenOf, poolOf, replicaEngine, replicasOf } from './poolConfig.js';
|
|
|
35
35
|
import { slug } from '../shared/slug.js';
|
|
36
36
|
import { sqliteTargetFrom } from './configPaths.js';
|
|
37
37
|
import { product } from './product.js';
|
|
38
|
+
import { flushJournal } from './journal.js';
|
|
39
|
+
import { keepWriteJournal } from './journalStore.js';
|
|
38
40
|
export { sqliteTargetFrom } from './configPaths.js';
|
|
39
41
|
export class Registry {
|
|
40
42
|
/** The environment this registry's references resolve in: an App's own in a suite, else the process's (undefined). */
|
|
@@ -105,6 +107,8 @@ export class Registry {
|
|
|
105
107
|
it again: the same paragraph every time an agent re-read the schema. Keyed by what was said, so a
|
|
106
108
|
role that gained a grant since is still news. */
|
|
107
109
|
rolesSaid = new Set();
|
|
110
|
+
/* What keepWriteJournal said about a connection's journal, once per run, as above. */
|
|
111
|
+
journalSaid = new Set();
|
|
108
112
|
/** A service from the config. Names are unique among services; a kind that is a singleton (the cache, the email service) is configured once. */
|
|
109
113
|
addService(service) {
|
|
110
114
|
if (this.services.some((s) => s.name === service.name))
|
|
@@ -484,6 +488,19 @@ export class Registry {
|
|
|
484
488
|
const schema = entry.config.hidden ? withoutHiddenColumns(read, entry.config.hidden, entry.config.name) : read;
|
|
485
489
|
for (const hook of this.dialedHooks)
|
|
486
490
|
hook(entry.config.id, adapter);
|
|
491
|
+
/* A writable connection keeps its Undo history in its database where the
|
|
492
|
+
store is provisioned (journalStore.ts), read back before the first
|
|
493
|
+
write; elsewhere it stays in memory, as it always has, and in the
|
|
494
|
+
browser a database that could hold the store and does not is told once
|
|
495
|
+
per run: a dial again (a refresh, a reconnect) would say the same
|
|
496
|
+
paragraph. */
|
|
497
|
+
if (entry.config.writable)
|
|
498
|
+
await keepWriteJournal(entry.config.id, adapter, (message) => {
|
|
499
|
+
if (this.journalSaid.has(message))
|
|
500
|
+
return;
|
|
501
|
+
this.journalSaid.add(message);
|
|
502
|
+
this.onWarning(message);
|
|
503
|
+
});
|
|
487
504
|
/* Wrapped before it is stored, so nothing can reach the unwrapped one:
|
|
488
505
|
the boundary is a property of the connection, not a rule each caller
|
|
489
506
|
remembers. Introspection happens first because the catalog is the
|
|
@@ -625,6 +642,8 @@ export class Registry {
|
|
|
625
642
|
const entries = [...this.entries.values()];
|
|
626
643
|
const finish = entries.map((entry) => this.invalidateAdapter(entry));
|
|
627
644
|
try {
|
|
645
|
+
// What the journal wrote behind lands before the handles it is written through close.
|
|
646
|
+
await Promise.all(entries.map((entry) => flushJournal(entry.config.id)));
|
|
628
647
|
await Promise.allSettled(entries.map((entry) => entry.adapter?.close()));
|
|
629
648
|
}
|
|
630
649
|
finally {
|
|
@@ -803,7 +822,7 @@ export async function loadConfig(explicit, sections) {
|
|
|
803
822
|
!['undefined', 'string'].includes(typeof migrations.table))) {
|
|
804
823
|
throw new Error(`${path}: connection "${c.name}" has "migrations": ${JSON.stringify(migrations)}. It is { "dir": "<directory>" }, and "table" when the history table is not ${product()}'s own.`);
|
|
805
824
|
}
|
|
806
|
-
const http = httpConnectionOf(c, path);
|
|
825
|
+
const http = httpConnectionOf(c, path, Object.keys(sections.kinds));
|
|
807
826
|
const pool = poolOf(c, http !== undefined, path);
|
|
808
827
|
const replicas = replicasOf(c, http !== undefined, path);
|
|
809
828
|
const listen = listenOf(c, http !== undefined, path);
|
|
@@ -1,5 +1,56 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* An export, streamed: `POST /api/export { q, format, columns? }` answers
|
|
3
|
+
* with the file itself — CSV, a JSON array or NDJSON — read in batches and
|
|
4
|
+
* written as each arrives, compressed when the request accepts it.
|
|
5
|
+
*
|
|
6
|
+
* "All matching rows" used to be one `/api/run` of up to 100,000 rows: the
|
|
7
|
+
* whole result held as objects, serialized as one JSON string (about 19 MB
|
|
8
|
+
* and 175 ms of a blocked event loop at the ceiling), with a total counted
|
|
9
|
+
* beside it that the file never used. Here a batch is read, written, and let
|
|
10
|
+
* go, so memory stays flat whatever the size, the first byte leaves after
|
|
11
|
+
* the first batch, and nothing is counted. There is no row ceiling: a file
|
|
12
|
+
* holds every row the question matches, or as many as its own `limit` says.
|
|
13
|
+
*
|
|
14
|
+
* The next batch is read only once the last has been handed on: a write the
|
|
15
|
+
* connection cannot take yet waits for it to drain (backpressure), so a slow
|
|
16
|
+
* client holds the server to a batch in flight, not the whole file. A client
|
|
17
|
+
* that goes away stops the export — the statement in flight is cancelled
|
|
18
|
+
* (statementControl.ts) and nothing more is read.
|
|
19
|
+
*
|
|
20
|
+
* Each batch is the ordinary read through this request's own reader —
|
|
21
|
+
* the same grants, fields, row scope, narrowed resources and errors as
|
|
22
|
+
* `/api/run` — so an export can never hold a row the list could not show.
|
|
23
|
+
* Batches page the way the list's own pager does (keysetReadHttp.ts): in the
|
|
24
|
+
* list's order with the key as its tiebreak, each starting after the last row
|
|
25
|
+
* of the one before, as a continuation ANDed into the question's own filter,
|
|
26
|
+
* so a batch costs what the first one did however deep the file goes and no
|
|
27
|
+
* row is skipped or repeated. Where the list pages by offset — no primary
|
|
28
|
+
* key, a union, a sort through a reference or on a value that does not come
|
|
29
|
+
* back as it is stored — so does the file.
|
|
30
|
+
*
|
|
31
|
+
* Inside a row-policy App each batch is a read of the policy's own facade,
|
|
32
|
+
* its row scope ANDed into the batch as into a list's page, and the file
|
|
33
|
+
* holds only what the reader's list could show. A policy reply is held until
|
|
34
|
+
* its context settles (handler.ts): the file's rows go as they are read, each
|
|
35
|
+
* already the reader's, but its last write waits for the context, so a
|
|
36
|
+
* context that fails after the rows were read cuts the connection and the
|
|
37
|
+
* browser never saves a file that looks whole. A tenant's lease streams the
|
|
38
|
+
* same way through `exportFile` (mistri: tenant/tenantHttp.ts).
|
|
39
|
+
*
|
|
40
|
+
* What an App decides of it is `ExportHooks` (mistri: http/appExport.ts): who
|
|
41
|
+
* exports which table, who `me` is in the question, and what an amount kept
|
|
42
|
+
* in minor units says in a CSV. The database browser exports every table it
|
|
43
|
+
* reads, as stored.
|
|
44
|
+
*/
|
|
45
|
+
import type { ServerResponse } from 'node:http';
|
|
46
|
+
import type { Adapter } from '../../adapters/adapter.js';
|
|
1
47
|
import { parseQuery } from '../../shared/query.js';
|
|
48
|
+
import { type DatabaseSchema } from '../../shared/schema.js';
|
|
2
49
|
import type { PostContext } from './context.js';
|
|
50
|
+
/** Rows read per statement, and so the most the server holds of a file at once:
|
|
51
|
+
* large enough that a million-row file is a thousand reads, small enough that
|
|
52
|
+
* one batch is never a memory event. */
|
|
53
|
+
export declare const EXPORT_BATCH = 1000;
|
|
3
54
|
/** Who `me` is in a question, where its words name `me` (parseQuery's `me`). */
|
|
4
55
|
type Person = NonNullable<Parameters<typeof parseQuery>[1]>['me'];
|
|
5
56
|
/** What a product decides of an export. */
|
|
@@ -11,5 +62,28 @@ export interface ExportHooks {
|
|
|
11
62
|
/** A row as a CSV says it: an amount kept in minor units as the figure it is, a value as its component's word. */
|
|
12
63
|
amounts?(post: PostContext, tableId: string): ((row: Record<string, unknown>) => Record<string, unknown>) | undefined;
|
|
13
64
|
}
|
|
65
|
+
/** One export, on what its host reads it through: the routes below, or a tenant's lease (mistri: tenant/tenantHttp.ts). */
|
|
66
|
+
interface ExportAsked {
|
|
67
|
+
res: ServerResponse;
|
|
68
|
+
body: Record<string, unknown>;
|
|
69
|
+
/** The reader every batch is read through: this request's own, with its grants, fields and row scope. */
|
|
70
|
+
adapter: Adapter;
|
|
71
|
+
schema: DatabaseSchema;
|
|
72
|
+
now: () => Date;
|
|
73
|
+
/** Whether a table may be exported (`''`: any table). */
|
|
74
|
+
may: (tableId: string) => boolean;
|
|
75
|
+
person?: (text: string) => Promise<Person | undefined>;
|
|
76
|
+
amounts?: (tableId: string) => ((row: Record<string, unknown>) => Record<string, unknown>) | undefined;
|
|
77
|
+
/** Headers the file carries beside its own: a tenant lease's owner, which its client checks before it keeps the file. */
|
|
78
|
+
headers?: Record<string, string>;
|
|
79
|
+
/** A reply that is not the file (a refusal, the question's errors), sent the host's way. */
|
|
80
|
+
answer?: (status: number, body: unknown) => void;
|
|
81
|
+
}
|
|
14
82
|
export declare function exportRoute(post: PostContext, hooks: ExportHooks): Promise<boolean>;
|
|
83
|
+
/**
|
|
84
|
+
* The export, streamed. Resolves with the file's last write, for the caller
|
|
85
|
+
* to make once whatever the reads ran in has settled; undefined when the
|
|
86
|
+
* answer was not a file (it was sent already) or the file was cut.
|
|
87
|
+
*/
|
|
88
|
+
export declare function exportFile(asked: ExportAsked): Promise<(() => void) | undefined>;
|
|
15
89
|
export {};
|
|
@@ -1,59 +1,82 @@
|
|
|
1
1
|
import { createBrotliCompress, createGzip, constants as zlib } from 'node:zlib';
|
|
2
|
-
import {
|
|
2
|
+
import { withCancellation } from '../../adapters/statementControl.js';
|
|
3
3
|
import { parseQuery } from '../../shared/query.js';
|
|
4
4
|
import { textToView } from '../../shared/viewtext.js';
|
|
5
|
-
import { findTable,
|
|
5
|
+
import { findTable, resolveOrder } from '../../shared/schema.js';
|
|
6
6
|
import { backrefsTo, resolveAggregate, tryResolvePath } from '../../shared/viewPaths.js';
|
|
7
7
|
import { CRLF, csvField, csvRecord, jsonRecord } from '../../shared/csv.js';
|
|
8
8
|
import { readRecords, readView } from '../resourceReadOperations.js';
|
|
9
|
-
import {
|
|
9
|
+
import { recordsPaging, settlePage, viewPaging } from '../keysetReadHttp.js';
|
|
10
|
+
import { acceptedEncoding, httpRead, policyResponses, send } from './reply.js';
|
|
10
11
|
import { rollupReads } from '../../shared/rollups.js';
|
|
11
12
|
const EXPORT_PATH = '/api/export';
|
|
12
|
-
/** Rows read per statement
|
|
13
|
-
*
|
|
14
|
-
|
|
13
|
+
/** Rows read per statement, and so the most the server holds of a file at once:
|
|
14
|
+
* large enough that a million-row file is a thousand reads, small enough that
|
|
15
|
+
* one batch is never a memory event. */
|
|
16
|
+
export const EXPORT_BATCH = 1000;
|
|
15
17
|
const FORMATS = {
|
|
16
18
|
csv: { type: 'text/csv', extension: 'csv' },
|
|
17
19
|
json: { type: 'application/json', extension: 'json' },
|
|
18
20
|
ndjson: { type: 'application/x-ndjson', extension: 'ndjson' },
|
|
19
21
|
};
|
|
20
22
|
export async function exportRoute(post, hooks) {
|
|
21
|
-
|
|
22
|
-
if (path !== EXPORT_PATH)
|
|
23
|
+
if (post.path !== EXPORT_PATH)
|
|
23
24
|
return false;
|
|
25
|
+
const { person, amounts } = hooks;
|
|
26
|
+
const end = await exportFile({
|
|
27
|
+
res: post.res,
|
|
28
|
+
body: post.body,
|
|
29
|
+
adapter: post.adapter,
|
|
30
|
+
schema: post.schema,
|
|
31
|
+
now: post.appNow,
|
|
32
|
+
may: hooks.may(post),
|
|
33
|
+
...(person ? { person: (text) => person(post, text) } : {}),
|
|
34
|
+
...(amounts ? { amounts: (tableId) => amounts(post, tableId) } : {}),
|
|
35
|
+
});
|
|
36
|
+
if (!end)
|
|
37
|
+
return true;
|
|
38
|
+
/* A row policy's reply waits for its context (handler.ts underPolicy): so does the file's last write. */
|
|
39
|
+
const held = policyResponses.get(post.res);
|
|
40
|
+
if (held)
|
|
41
|
+
held.reply = end;
|
|
42
|
+
else
|
|
43
|
+
end();
|
|
44
|
+
return true;
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* The export, streamed. Resolves with the file's last write, for the caller
|
|
48
|
+
* to make once whatever the reads ran in has settled; undefined when the
|
|
49
|
+
* answer was not a file (it was sent already) or the file was cut.
|
|
50
|
+
*/
|
|
51
|
+
export async function exportFile(asked) {
|
|
52
|
+
const { res, body, adapter, schema, may } = asked;
|
|
53
|
+
const answer = asked.answer ?? ((status, value) => send(res, status, value));
|
|
54
|
+
const amountsOf = (tableId) => asked.amounts?.(tableId);
|
|
24
55
|
/* An App that grants by role says who exports which table (appExportable); the file is refused where the Export menu is not offered. */
|
|
25
|
-
const may = hooks.may(post);
|
|
26
56
|
if (!may('')) {
|
|
27
|
-
|
|
28
|
-
return
|
|
29
|
-
}
|
|
30
|
-
// Never reached in one (httpRowPolicyAllows), and refused here as well: a policy reply cannot stream.
|
|
31
|
-
if (post.rowPolicy) {
|
|
32
|
-
send(res, 403, {
|
|
33
|
-
error: 'This operation is unavailable in a row-policy application.',
|
|
34
|
-
});
|
|
35
|
-
return true;
|
|
57
|
+
answer(403, { error: 'Export is not available to this person.' });
|
|
58
|
+
return undefined;
|
|
36
59
|
}
|
|
37
60
|
const format = body.format ?? 'csv';
|
|
38
61
|
if (typeof format !== 'string' || !Object.hasOwn(FORMATS, format)) {
|
|
39
|
-
|
|
40
|
-
return
|
|
62
|
+
answer(400, { error: 'format must be "csv", "json" or "ndjson".' });
|
|
63
|
+
return undefined;
|
|
41
64
|
}
|
|
42
65
|
if (body.columns !== undefined && (!Array.isArray(body.columns) || body.columns.some((column) => typeof column !== 'string'))) {
|
|
43
|
-
|
|
44
|
-
return
|
|
66
|
+
answer(400, { error: 'columns must be a list of column names.' });
|
|
67
|
+
return undefined;
|
|
45
68
|
}
|
|
46
69
|
const text = String(body.q ?? '');
|
|
47
|
-
const me = await
|
|
70
|
+
const me = await asked.person?.(text);
|
|
48
71
|
const person = me === undefined ? {} : { me };
|
|
49
|
-
const parsed = parseQuery(text, { schema, now:
|
|
72
|
+
const parsed = parseQuery(text, { schema, now: asked.now(), ...person });
|
|
50
73
|
/* A question only a view can answer (a path through a reference) exports
|
|
51
74
|
as a view, paged by offset: its rows have no single key to resume from. */
|
|
52
75
|
if ((!parsed.query || parsed.errors.length) && adapter.runView) {
|
|
53
76
|
const asView = textToView(schema, text, {
|
|
54
77
|
id: 'export',
|
|
55
78
|
name: '',
|
|
56
|
-
now:
|
|
79
|
+
now: asked.now(),
|
|
57
80
|
...person,
|
|
58
81
|
});
|
|
59
82
|
if (asView.view && !asView.errors.length) {
|
|
@@ -61,60 +84,72 @@ export async function exportRoute(post, hooks) {
|
|
|
61
84
|
/* Every table the view reaches, not only its base: a column through a reference, an aggregate or a filter on one reads another table's rows into the file. */
|
|
62
85
|
for (const reached of viewTables(schema, view))
|
|
63
86
|
if (!may(reached))
|
|
64
|
-
return refuseTable(
|
|
65
|
-
const
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
87
|
+
return refuseTable(answer, reached === UNKNOWN_TABLE ? view.base : reached);
|
|
88
|
+
const base = findTable(schema, view.base);
|
|
89
|
+
/* A union pages by offset: its rows have no one key order across branches (unionReads.ts), as the list's. */
|
|
90
|
+
const keyed = base && !base.union ? base : undefined;
|
|
91
|
+
let step;
|
|
92
|
+
const read = async (offset, size) => {
|
|
93
|
+
const paging = keyed && viewPaging(schema, keyed, view, step);
|
|
94
|
+
/* Past the first batch with no boundary to resume from, the file goes on by offset in the same total order. */
|
|
95
|
+
const continued = paging && (!step || paging.continuation) ? paging : undefined;
|
|
96
|
+
const { result } = await httpRead(() => readView(adapter, schema, continued?.view ?? view, { limit: size, offset: continued?.continuation ? 0 : offset }, 'none', undefined, continued?.continuation));
|
|
97
|
+
if (!continued)
|
|
98
|
+
return result;
|
|
99
|
+
const settled = settlePage(continued.order, continued.extra, result);
|
|
100
|
+
step = settled.bounds ? { after: settled.bounds.last } : undefined;
|
|
101
|
+
return { columns: settled.columns, rows: settled.rows };
|
|
102
|
+
};
|
|
103
|
+
return stream(res, format, view.base, rowsWanted(view.limit), read, requested(body) ?? undefined, amountsOf(view.base), asked.headers);
|
|
69
104
|
}
|
|
70
105
|
}
|
|
71
|
-
if (!parsed.query) {
|
|
72
|
-
|
|
73
|
-
return
|
|
74
|
-
}
|
|
75
|
-
if (parsed.errors.length) {
|
|
76
|
-
send(res, 200, { errors: parsed.errors });
|
|
77
|
-
return true;
|
|
106
|
+
if (!parsed.query || parsed.errors.length) {
|
|
107
|
+
answer(200, { errors: parsed.errors });
|
|
108
|
+
return undefined;
|
|
78
109
|
}
|
|
79
110
|
const query = parsed.query;
|
|
80
111
|
const table = findTable(schema, query.table);
|
|
81
112
|
if (!table) {
|
|
82
|
-
|
|
83
|
-
return
|
|
113
|
+
answer(404, { error: `Unknown table "${query.table}"` });
|
|
114
|
+
return undefined;
|
|
84
115
|
}
|
|
85
116
|
if (!may(table.id))
|
|
86
|
-
return refuseTable(
|
|
117
|
+
return refuseTable(answer, table.id);
|
|
87
118
|
const wanted = requested(body) ?? (query.columns?.length ? query.columns : undefined);
|
|
88
119
|
for (const reached of queryTables(schema, table, query, wanted))
|
|
89
120
|
if (!may(reached))
|
|
90
|
-
return refuseTable(
|
|
91
|
-
|
|
92
|
-
const order = resolveOrder(table, query.orderBy).order;
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
121
|
+
return refuseTable(answer, reached === UNKNOWN_TABLE ? table.id : reached);
|
|
122
|
+
/* The order the list reads in, made total with the key; an offset batch reads in it too. */
|
|
123
|
+
const order = schema.dialect === 'http' && !query.orderBy?.length ? undefined : resolveOrder(table, query.orderBy).order;
|
|
124
|
+
let step;
|
|
125
|
+
const read = async (offset, size) => {
|
|
126
|
+
/* A union pages by offset: its rows have no one key order across branches (unionReads.ts), as the list's. */
|
|
127
|
+
const paging = table.union ? undefined : recordsPaging(table, schema.dialect, query.orderBy, wanted, query.filter, step);
|
|
128
|
+
/* Past the first batch with no boundary to resume from, the file goes on by offset in the same total order. */
|
|
129
|
+
const continued = paging && (!step || paging.continuation) ? paging : undefined;
|
|
130
|
+
/* An order column the file does not show is read for the boundary and taken out again (settlePage): it never
|
|
131
|
+
reaches the file, whose columns are `wanted`. */
|
|
132
|
+
const columns = continued?.columns ?? wanted;
|
|
99
133
|
const { result } = await httpRead(() => readRecords(adapter, schema, {
|
|
100
134
|
...query,
|
|
101
|
-
...(
|
|
102
|
-
orderBy: order,
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
135
|
+
...(columns ? { columns } : {}),
|
|
136
|
+
...(order ? { orderBy: order } : {}),
|
|
137
|
+
limit: size,
|
|
138
|
+
offset: continued?.continuation ? 0 : offset,
|
|
139
|
+
}, 'none', undefined, continued?.continuation));
|
|
140
|
+
if (!continued)
|
|
141
|
+
return result;
|
|
142
|
+
const settled = settlePage(continued.order, continued.extra, result);
|
|
143
|
+
step = settled.bounds ? { after: settled.bounds.last } : undefined;
|
|
144
|
+
return { columns: settled.columns, rows: settled.rows };
|
|
109
145
|
};
|
|
110
|
-
|
|
111
|
-
return true;
|
|
146
|
+
return stream(res, format, table.id, rowsWanted(query.limit), read, wanted, amountsOf(table.id), asked.headers);
|
|
112
147
|
}
|
|
113
|
-
function refuseTable(
|
|
114
|
-
|
|
148
|
+
function refuseTable(answer, tableId) {
|
|
149
|
+
answer(403, {
|
|
115
150
|
error: `Export of "${tableId}" is not available to this person.`,
|
|
116
151
|
});
|
|
117
|
-
return
|
|
152
|
+
return undefined;
|
|
118
153
|
}
|
|
119
154
|
/** Stands for a table a view reaches that cannot be named (a many-valued condition it cannot resolve): never exportable. */
|
|
120
155
|
const UNKNOWN_TABLE = '\u0000unknown';
|
|
@@ -204,117 +239,119 @@ function pathReach(schema, base, tables) {
|
|
|
204
239
|
function requested(body) {
|
|
205
240
|
return Array.isArray(body.columns) && body.columns.length ? body.columns : undefined;
|
|
206
241
|
}
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
* columns whose value comes back exactly as it is stored: an integer or text
|
|
212
|
-
* on Postgres — a timestamp comes back to the millisecond and a
|
|
213
|
-
* float rounded, and resuming "after" a rounded value repeats or skips rows —
|
|
214
|
-
* and anything on SQLite, which hands back what it holds.
|
|
215
|
-
*/
|
|
216
|
-
function keysetOrder(table, order, dialect) {
|
|
217
|
-
const key = primaryKey(table);
|
|
218
|
-
if (!key.length || order.length !== key.length || order.some((o, i) => o.column !== key[i]))
|
|
219
|
-
return undefined;
|
|
220
|
-
if (dialect === 'http')
|
|
221
|
-
return undefined;
|
|
222
|
-
const exact = order.every((o) => {
|
|
223
|
-
const column = table.columns.find((c) => c.name === o.column);
|
|
224
|
-
if (!column || column.nullable)
|
|
225
|
-
return false;
|
|
226
|
-
return dialect === 'sqlite' || EXACT_TYPE.test(column.type);
|
|
227
|
-
});
|
|
228
|
-
return exact ? order : undefined;
|
|
242
|
+
/** How many rows the file holds: the question's own `limit` where it says one, with no ceiling; every row otherwise. */
|
|
243
|
+
function rowsWanted(limit) {
|
|
244
|
+
const n = typeof limit === 'number' ? limit : typeof limit === 'string' && limit.trim() !== '' ? Number(limit) : NaN;
|
|
245
|
+
return Number.isFinite(n) && n >= 1 ? Math.floor(n) : Infinity;
|
|
229
246
|
}
|
|
230
|
-
/**
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
function beyond(filter, order, last) {
|
|
237
|
-
const steps = order.map((o, i) => [
|
|
238
|
-
...order.slice(0, i).map((earlier) => ({
|
|
239
|
-
column: earlier.column,
|
|
240
|
-
op: '=',
|
|
241
|
-
value: last[earlier.column],
|
|
242
|
-
})),
|
|
243
|
-
{
|
|
244
|
-
column: o.column,
|
|
245
|
-
op: o.direction === 'desc' ? '<' : '>',
|
|
246
|
-
value: last[o.column],
|
|
247
|
-
},
|
|
248
|
-
]);
|
|
249
|
-
const groups = filter?.groups?.length ? filter.groups : [[]];
|
|
250
|
-
return {
|
|
251
|
-
groups: groups.flatMap((group) => steps.map((step) => [...group, ...step])),
|
|
252
|
-
};
|
|
247
|
+
/** Said when the client went away mid-file: nothing reads it, it only stops the loop. */
|
|
248
|
+
class Abandoned extends Error {
|
|
249
|
+
constructor() {
|
|
250
|
+
super('The export was abandoned.');
|
|
251
|
+
this.name = 'Abandoned';
|
|
252
|
+
}
|
|
253
253
|
}
|
|
254
|
-
async function stream(res, format, table, total, read, wanted, amounts) {
|
|
255
|
-
/*
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
'content-type': `${spec.type}; charset=utf-8`,
|
|
264
|
-
'content-disposition': `attachment; filename="${table.replace(/[^\w.-]+/g, '_')}.${spec.extension}"`,
|
|
265
|
-
'cache-control': 'private, no-store',
|
|
266
|
-
'x-content-type-options': 'nosniff',
|
|
267
|
-
vary: 'Accept-Encoding',
|
|
268
|
-
...(encoding ? { 'content-encoding': encoding } : {}),
|
|
269
|
-
});
|
|
270
|
-
const out = encoding === 'br'
|
|
271
|
-
? createBrotliCompress({
|
|
272
|
-
params: {
|
|
273
|
-
[zlib.BROTLI_PARAM_QUALITY]: 4,
|
|
274
|
-
[zlib.BROTLI_PARAM_MODE]: zlib.BROTLI_MODE_TEXT,
|
|
275
|
-
},
|
|
276
|
-
})
|
|
277
|
-
: encoding === 'gzip'
|
|
278
|
-
? createGzip({ level: 6 })
|
|
279
|
-
: res;
|
|
280
|
-
if (out !== res)
|
|
281
|
-
out.pipe(res);
|
|
282
|
-
const write = (chunk) => new Promise((resolve, reject) => {
|
|
283
|
-
if (res.destroyed)
|
|
284
|
-
return reject(new Error('The export was abandoned.'));
|
|
285
|
-
if (out.write(chunk))
|
|
286
|
-
resolve();
|
|
287
|
-
else
|
|
288
|
-
out.once('drain', resolve);
|
|
289
|
-
});
|
|
254
|
+
async function stream(res, format, table, total, read, wanted, amounts, headers = {}) {
|
|
255
|
+
/* A client that leaves stops the read in flight (the adapters watch this signal) and every one after it. */
|
|
256
|
+
const stop = new AbortController();
|
|
257
|
+
const left = () => {
|
|
258
|
+
if (!res.writableFinished)
|
|
259
|
+
stop.abort();
|
|
260
|
+
};
|
|
261
|
+
res.once('close', left);
|
|
262
|
+
const reading = (offset) => withCancellation(stop.signal, () => read(offset, Math.min(EXPORT_BATCH, total - offset)));
|
|
290
263
|
try {
|
|
291
|
-
/*
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
264
|
+
/* The first batch is read before a byte is written: a refusal, a column
|
|
265
|
+
that does not exist, a database that is down all still answer as the
|
|
266
|
+
ordinary error they are, through the handler's own mapping. */
|
|
267
|
+
let batch = await reading(0);
|
|
268
|
+
if (stop.signal.aborted)
|
|
269
|
+
return undefined;
|
|
270
|
+
const columns = wanted ?? batch.columns;
|
|
271
|
+
const spec = FORMATS[format];
|
|
272
|
+
const encoding = acceptedEncoding(res);
|
|
273
|
+
res.writeHead(200, {
|
|
274
|
+
'content-type': `${spec.type}; charset=utf-8`,
|
|
275
|
+
'content-disposition': `attachment; filename="${table.replace(/[^\w.-]+/g, '_')}.${spec.extension}"`,
|
|
276
|
+
'cache-control': 'private, no-store',
|
|
277
|
+
'x-content-type-options': 'nosniff',
|
|
278
|
+
vary: 'Accept-Encoding',
|
|
279
|
+
...(encoding ? { 'content-encoding': encoding } : {}),
|
|
280
|
+
...headers,
|
|
281
|
+
});
|
|
282
|
+
const out = encoding === 'br'
|
|
283
|
+
? createBrotliCompress({
|
|
284
|
+
params: {
|
|
285
|
+
[zlib.BROTLI_PARAM_QUALITY]: 4,
|
|
286
|
+
[zlib.BROTLI_PARAM_MODE]: zlib.BROTLI_MODE_TEXT,
|
|
287
|
+
},
|
|
288
|
+
})
|
|
289
|
+
: encoding === 'gzip'
|
|
290
|
+
? createGzip({ level: 6 })
|
|
291
|
+
: res;
|
|
292
|
+
if (out !== res) {
|
|
293
|
+
out.pipe(res);
|
|
294
|
+
stop.signal.addEventListener('abort', () => out.destroy(), { once: true });
|
|
295
|
+
}
|
|
296
|
+
/* Backpressure: a chunk the connection cannot take yet holds the loop until it drains, so the next batch is
|
|
297
|
+
read only once this one is on its way. A client that leaves meanwhile ends the wait (a drain that never comes
|
|
298
|
+
would hold the loop and its batch for good). */
|
|
299
|
+
const write = (chunk) => new Promise((resolve, reject) => {
|
|
300
|
+
if (stop.signal.aborted || res.destroyed)
|
|
301
|
+
return reject(new Abandoned());
|
|
302
|
+
if (out.write(chunk))
|
|
303
|
+
return resolve();
|
|
304
|
+
const settle = (fn) => () => {
|
|
305
|
+
out.off('drain', drained);
|
|
306
|
+
stop.signal.removeEventListener('abort', gone);
|
|
307
|
+
fn();
|
|
308
|
+
};
|
|
309
|
+
const drained = settle(resolve);
|
|
310
|
+
const gone = settle(() => reject(new Abandoned()));
|
|
311
|
+
out.once('drain', drained);
|
|
312
|
+
stop.signal.addEventListener('abort', gone, { once: true });
|
|
313
|
+
});
|
|
314
|
+
try {
|
|
315
|
+
/* A spreadsheet opens a UTF-8 CSV without a byte-order mark in the local
|
|
316
|
+
codepage; the browser's own export writes one too (export.js). */
|
|
317
|
+
if (format === 'csv')
|
|
318
|
+
await write(`\ufeff${columns.map(csvField).join(',')}${CRLF}`);
|
|
319
|
+
let written = 0;
|
|
320
|
+
for (;;) {
|
|
321
|
+
const lines = batch.rows.map((row, i) => format === 'csv'
|
|
322
|
+
? `${csvRecord(columns, amounts ? amounts(row) : row)}${CRLF}`
|
|
323
|
+
: format === 'ndjson'
|
|
324
|
+
? `${JSON.stringify(jsonRecord(columns, row))}\n`
|
|
325
|
+
: // A JSON array as export.js's toJson writes one: two-space indent, one record per element.
|
|
326
|
+
`${written + i === 0 ? '[\n' : ',\n'}${JSON.stringify(jsonRecord(columns, row), null, 2).replace(/^/gm, ' ')}`);
|
|
327
|
+
if (lines.length)
|
|
328
|
+
await write(lines.join(''));
|
|
329
|
+
const size = Math.min(EXPORT_BATCH, total - written);
|
|
330
|
+
written += batch.rows.length;
|
|
331
|
+
if (batch.rows.length < size || written >= total)
|
|
332
|
+
break;
|
|
333
|
+
batch = await reading(written);
|
|
334
|
+
if (stop.signal.aborted)
|
|
335
|
+
throw new Abandoned();
|
|
336
|
+
}
|
|
337
|
+
if (format === 'json')
|
|
338
|
+
await write(written ? '\n]\n' : '[]\n');
|
|
339
|
+
/* The last write is the caller's to make: a reply held for a context that has not settled yet waits for it. */
|
|
340
|
+
return () => {
|
|
341
|
+
if (!res.destroyed)
|
|
342
|
+
out.end();
|
|
343
|
+
};
|
|
344
|
+
}
|
|
345
|
+
catch {
|
|
346
|
+
/* Past the first byte a status cannot change: the connection is cut, so
|
|
347
|
+
the browser reports a failed download rather than saving half a file
|
|
348
|
+
that looks whole. */
|
|
349
|
+
stop.abort();
|
|
350
|
+
res.destroy();
|
|
351
|
+
return undefined;
|
|
309
352
|
}
|
|
310
|
-
if (format === 'json')
|
|
311
|
-
await write(written ? '\n]\n' : '[]\n');
|
|
312
|
-
out.end();
|
|
313
353
|
}
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
the browser reports a failed download rather than saving half a file
|
|
317
|
-
that looks whole. */
|
|
318
|
-
res.destroy();
|
|
354
|
+
finally {
|
|
355
|
+
res.off('close', left);
|
|
319
356
|
}
|
|
320
357
|
}
|
|
@@ -78,6 +78,8 @@ export interface PaintRooms {
|
|
|
78
78
|
me: Record<string, unknown>;
|
|
79
79
|
runtime: Record<string, unknown>;
|
|
80
80
|
ask: PaintAsk;
|
|
81
|
+
/** The request's Cookie header: how this browser last chose to read a room (an App's board), where it says. */
|
|
82
|
+
cookie?: string;
|
|
81
83
|
}): Promise<{
|
|
82
84
|
app: string;
|
|
83
85
|
plan: RoomPlan;
|
|
@@ -89,7 +91,7 @@ export interface PaintRooms {
|
|
|
89
91
|
* room asks (`rooms`, an App's: firstPaintRooms.ts). Each round's requests
|
|
90
92
|
* are asked together.
|
|
91
93
|
*/
|
|
92
|
-
export declare function firstPaint({ path, search, loop: asked, most, budget, rooms, }: {
|
|
94
|
+
export declare function firstPaint({ path, search, loop: asked, most, budget, rooms, cookie, }: {
|
|
93
95
|
path: string;
|
|
94
96
|
search: string;
|
|
95
97
|
loop: Loop;
|
|
@@ -97,6 +99,8 @@ export declare function firstPaint({ path, search, loop: asked, most, budget, ro
|
|
|
97
99
|
budget?: number;
|
|
98
100
|
/** What the product's page asks past who-am-I and the runtime (PaintRooms), or nothing more. */
|
|
99
101
|
rooms?: PaintRooms;
|
|
102
|
+
/** The request's Cookie header, handed to `rooms`. */
|
|
103
|
+
cookie?: string;
|
|
100
104
|
}): Promise<FirstPaintPage>;
|
|
101
105
|
/**
|
|
102
106
|
* The data block, as the page carries it: JSON with every `<`, `>` and `&`
|