@mistriwork/engine 1.0.0-rc.16 → 1.0.0-rc.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -0
- package/dist/host.d.ts +46 -0
- package/dist/host.js +51 -0
- package/dist/server/connections.js +9 -0
- package/dist/server/http/exportRoute.d.ts +74 -0
- package/dist/server/http/exportRoute.js +206 -169
- package/dist/server/http/reply.js +6 -0
- package/dist/server/journal.d.ts +35 -6
- package/dist/server/journal.js +79 -8
- package/dist/server/journalStore.d.ts +37 -0
- package/dist/server/journalStore.js +352 -0
- package/dist/shared/appListColumns.d.ts +7 -0
- package/dist/shared/appListColumns.js +13 -0
- package/dist/shared/listQuestion.js +2 -2
- package/dist/shared/recordReads.js +29 -11
- package/dist/shared/servedApp.d.ts +1 -1
- package/package.json +3 -1
package/README.md
CHANGED
|
@@ -16,3 +16,8 @@ This package is internal plumbing for `mistri` and `tablewalk`, with no semver
|
|
|
16
16
|
promise to an App's author: its `exports` name each module those two import,
|
|
17
17
|
and any of them may change or go in any release. An App imports `mistri`'s
|
|
18
18
|
names (or `tablewalk`'s), never these.
|
|
19
|
+
|
|
20
|
+
`@mistriwork/engine/host` gathers what the database browser opens a database
|
|
21
|
+
through — connect, the catalog, reads, the recorded writes and their Undo
|
|
22
|
+
journal, and what its agent tools need beside them — into one entry. It is
|
|
23
|
+
Preview like the rest, until the browser has moved onto it.
|
package/dist/host.d.ts
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `@mistriwork/engine/host`: what the database browser opens a database and
|
|
3
|
+
* works with it through, as one entry point instead of the engine's own files.
|
|
4
|
+
*
|
|
5
|
+
* It holds exactly the names the browser's server modules take from the
|
|
6
|
+
* engine for five things, and nothing it does not use:
|
|
7
|
+
*
|
|
8
|
+
* - connect: a connection string becomes an adapter (`connect`), or a set of
|
|
9
|
+
* configured connections a `Registry` opens on demand, with the config file
|
|
10
|
+
* and the secrets it names kept out of what is said;
|
|
11
|
+
* - schema: the catalog the adapter reads, and the helpers that walk it;
|
|
12
|
+
* - reads: a query-language read, a saved view, a count and a breakdown, with
|
|
13
|
+
* the parser and the explainer an agent is shown;
|
|
14
|
+
* - writes: the recorded insert, update and delete the browser's write mode
|
|
15
|
+
* and its agent tools make, what a delete would reach, why a write failed,
|
|
16
|
+
* the write journal behind Undo and revert, and its store's provisioning;
|
|
17
|
+
* - the agent tools' support: cancellation and the statement deadline, the
|
|
18
|
+
* schema in words, and the nearest name to a wrong one.
|
|
19
|
+
*
|
|
20
|
+
* Preview, as the rest of the engine is: it is promoted once the browser has
|
|
21
|
+
* moved onto it, and a name is added here when the browser needs it, not
|
|
22
|
+
* because it is exported below. The serving, the CLI and the HTTP plumbing
|
|
23
|
+
* stay on the engine's deeper paths. test/engine-host-contract.test.ts
|
|
24
|
+
* imports this entry alone and does with it what the browser does.
|
|
25
|
+
*/
|
|
26
|
+
export { connect } from './adapters/connect.js';
|
|
27
|
+
export { UNTRUSTED_NETWORK } from './adapters/net/untrustedDatabase.js';
|
|
28
|
+
export { Registry, redact, saveConfig, scrubSecrets, storeSecret } from './server/connections.js';
|
|
29
|
+
export { resolveConnection } from './server/auth/credentials.js';
|
|
30
|
+
export { Refusal, TransactionOutcomeUnknown, clampLimit, type Adapter, type Filter, type InsertResult, type UpdateRequest, type WriteTransaction, } from './adapters/adapter.js';
|
|
31
|
+
export { findTable, labelPath, primaryKey, referencesFrom, referencesTo, renderDDL, tableNamed, type DatabaseSchema, type DeleteRule, type ForeignKey, type TableMeta, } from './shared/schema.js';
|
|
32
|
+
export { toMarkdown } from './shared/schematext.js';
|
|
33
|
+
export { describeSchema } from './shared/shape.js';
|
|
34
|
+
export { readBreakdown, readCount, readRecords, readView } from './server/resourceReadOperations.js';
|
|
35
|
+
export { explain, parseQuery, quoteValue } from './shared/query.js';
|
|
36
|
+
export { textToView } from './shared/viewtext.js';
|
|
37
|
+
export { looksLikeBreakdown, textToBreakdown } from './shared/breakdowntext.js';
|
|
38
|
+
export type { ViewDef } from './shared/view.js';
|
|
39
|
+
export { changingReferences, recordedDelete, recordedInsert, recordedUpdate } from './server/recordWriteOperations.js';
|
|
40
|
+
export { deleteImpact, inBatches } from './server/impact.js';
|
|
41
|
+
export { explainWriteFailure, writeFailureOutcome } from './server/constraints.js';
|
|
42
|
+
export { attributeAction, attributeWrites } from './adapters/writeAttribution.js';
|
|
43
|
+
export { history, lastRevertible, latestRevertible, recordWrite, revert, summarise } from './server/journal.js';
|
|
44
|
+
export { provisionPostgresWriteJournal, provisionSqliteWriteJournal } from './server/journalStore.js';
|
|
45
|
+
export { STATEMENT_TIMEOUT_MS, StatementStopped, stoppedReason, throwIfCancelled, withCancellation } from './adapters/statementControl.js';
|
|
46
|
+
export { distance } from './shared/suggest.js';
|
package/dist/host.js
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `@mistriwork/engine/host`: what the database browser opens a database and
|
|
3
|
+
* works with it through, as one entry point instead of the engine's own files.
|
|
4
|
+
*
|
|
5
|
+
* It holds exactly the names the browser's server modules take from the
|
|
6
|
+
* engine for five things, and nothing it does not use:
|
|
7
|
+
*
|
|
8
|
+
* - connect: a connection string becomes an adapter (`connect`), or a set of
|
|
9
|
+
* configured connections a `Registry` opens on demand, with the config file
|
|
10
|
+
* and the secrets it names kept out of what is said;
|
|
11
|
+
* - schema: the catalog the adapter reads, and the helpers that walk it;
|
|
12
|
+
* - reads: a query-language read, a saved view, a count and a breakdown, with
|
|
13
|
+
* the parser and the explainer an agent is shown;
|
|
14
|
+
* - writes: the recorded insert, update and delete the browser's write mode
|
|
15
|
+
* and its agent tools make, what a delete would reach, why a write failed,
|
|
16
|
+
* the write journal behind Undo and revert, and its store's provisioning;
|
|
17
|
+
* - the agent tools' support: cancellation and the statement deadline, the
|
|
18
|
+
* schema in words, and the nearest name to a wrong one.
|
|
19
|
+
*
|
|
20
|
+
* Preview, as the rest of the engine is: it is promoted once the browser has
|
|
21
|
+
* moved onto it, and a name is added here when the browser needs it, not
|
|
22
|
+
* because it is exported below. The serving, the CLI and the HTTP plumbing
|
|
23
|
+
* stay on the engine's deeper paths. test/engine-host-contract.test.ts
|
|
24
|
+
* imports this entry alone and does with it what the browser does.
|
|
25
|
+
*/
|
|
26
|
+
/* ---------- connect ---------- */
|
|
27
|
+
export { connect } from './adapters/connect.js';
|
|
28
|
+
export { UNTRUSTED_NETWORK } from './adapters/net/untrustedDatabase.js';
|
|
29
|
+
export { Registry, redact, saveConfig, scrubSecrets, storeSecret } from './server/connections.js';
|
|
30
|
+
export { resolveConnection } from './server/auth/credentials.js';
|
|
31
|
+
/* ---------- the adapter ---------- */
|
|
32
|
+
export { Refusal, TransactionOutcomeUnknown, clampLimit, } from './adapters/adapter.js';
|
|
33
|
+
/* ---------- schema ---------- */
|
|
34
|
+
export { findTable, labelPath, primaryKey, referencesFrom, referencesTo, renderDDL, tableNamed, } from './shared/schema.js';
|
|
35
|
+
export { toMarkdown } from './shared/schematext.js';
|
|
36
|
+
export { describeSchema } from './shared/shape.js';
|
|
37
|
+
/* ---------- reads ---------- */
|
|
38
|
+
export { readBreakdown, readCount, readRecords, readView } from './server/resourceReadOperations.js';
|
|
39
|
+
export { explain, parseQuery, quoteValue } from './shared/query.js';
|
|
40
|
+
export { textToView } from './shared/viewtext.js';
|
|
41
|
+
export { looksLikeBreakdown, textToBreakdown } from './shared/breakdowntext.js';
|
|
42
|
+
/* ---------- writes and the write journal ---------- */
|
|
43
|
+
export { changingReferences, recordedDelete, recordedInsert, recordedUpdate } from './server/recordWriteOperations.js';
|
|
44
|
+
export { deleteImpact, inBatches } from './server/impact.js';
|
|
45
|
+
export { explainWriteFailure, writeFailureOutcome } from './server/constraints.js';
|
|
46
|
+
export { attributeAction, attributeWrites } from './adapters/writeAttribution.js';
|
|
47
|
+
export { history, lastRevertible, latestRevertible, recordWrite, revert, summarise } from './server/journal.js';
|
|
48
|
+
export { provisionPostgresWriteJournal, provisionSqliteWriteJournal } from './server/journalStore.js';
|
|
49
|
+
/* ---------- the agent tools' support ---------- */
|
|
50
|
+
export { STATEMENT_TIMEOUT_MS, StatementStopped, stoppedReason, throwIfCancelled, withCancellation } from './adapters/statementControl.js';
|
|
51
|
+
export { distance } from './shared/suggest.js';
|
|
@@ -35,6 +35,8 @@ import { listenOf, poolOf, replicaEngine, replicasOf } from './poolConfig.js';
|
|
|
35
35
|
import { slug } from '../shared/slug.js';
|
|
36
36
|
import { sqliteTargetFrom } from './configPaths.js';
|
|
37
37
|
import { product } from './product.js';
|
|
38
|
+
import { flushJournal } from './journal.js';
|
|
39
|
+
import { keepWriteJournal } from './journalStore.js';
|
|
38
40
|
export { sqliteTargetFrom } from './configPaths.js';
|
|
39
41
|
export class Registry {
|
|
40
42
|
/** The environment this registry's references resolve in: an App's own in a suite, else the process's (undefined). */
|
|
@@ -484,6 +486,11 @@ export class Registry {
|
|
|
484
486
|
const schema = entry.config.hidden ? withoutHiddenColumns(read, entry.config.hidden, entry.config.name) : read;
|
|
485
487
|
for (const hook of this.dialedHooks)
|
|
486
488
|
hook(entry.config.id, adapter);
|
|
489
|
+
/* A writable connection keeps its Undo history in its database where the
|
|
490
|
+
store is provisioned (journalStore.ts), read back before the first
|
|
491
|
+
write; elsewhere it stays in memory, as it always has. */
|
|
492
|
+
if (entry.config.writable)
|
|
493
|
+
await keepWriteJournal(entry.config.id, adapter, this.onWarning);
|
|
487
494
|
/* Wrapped before it is stored, so nothing can reach the unwrapped one:
|
|
488
495
|
the boundary is a property of the connection, not a rule each caller
|
|
489
496
|
remembers. Introspection happens first because the catalog is the
|
|
@@ -625,6 +632,8 @@ export class Registry {
|
|
|
625
632
|
const entries = [...this.entries.values()];
|
|
626
633
|
const finish = entries.map((entry) => this.invalidateAdapter(entry));
|
|
627
634
|
try {
|
|
635
|
+
// What the journal wrote behind lands before the handles it is written through close.
|
|
636
|
+
await Promise.all(entries.map((entry) => flushJournal(entry.config.id)));
|
|
628
637
|
await Promise.allSettled(entries.map((entry) => entry.adapter?.close()));
|
|
629
638
|
}
|
|
630
639
|
finally {
|
|
@@ -1,5 +1,56 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* An export, streamed: `POST /api/export { q, format, columns? }` answers
|
|
3
|
+
* with the file itself — CSV, a JSON array or NDJSON — read in batches and
|
|
4
|
+
* written as each arrives, compressed when the request accepts it.
|
|
5
|
+
*
|
|
6
|
+
* "All matching rows" used to be one `/api/run` of up to 100,000 rows: the
|
|
7
|
+
* whole result held as objects, serialized as one JSON string (about 19 MB
|
|
8
|
+
* and 175 ms of a blocked event loop at the ceiling), with a total counted
|
|
9
|
+
* beside it that the file never used. Here a batch is read, written, and let
|
|
10
|
+
* go, so memory stays flat whatever the size, the first byte leaves after
|
|
11
|
+
* the first batch, and nothing is counted. There is no row ceiling: a file
|
|
12
|
+
* holds every row the question matches, or as many as its own `limit` says.
|
|
13
|
+
*
|
|
14
|
+
* The next batch is read only once the last has been handed on: a write the
|
|
15
|
+
* connection cannot take yet waits for it to drain (backpressure), so a slow
|
|
16
|
+
* client holds the server to a batch in flight, not the whole file. A client
|
|
17
|
+
* that goes away stops the export — the statement in flight is cancelled
|
|
18
|
+
* (statementControl.ts) and nothing more is read.
|
|
19
|
+
*
|
|
20
|
+
* Each batch is the ordinary read through this request's own reader —
|
|
21
|
+
* the same grants, fields, row scope, narrowed resources and errors as
|
|
22
|
+
* `/api/run` — so an export can never hold a row the list could not show.
|
|
23
|
+
* Batches page the way the list's own pager does (keysetReadHttp.ts): in the
|
|
24
|
+
* list's order with the key as its tiebreak, each starting after the last row
|
|
25
|
+
* of the one before, as a continuation ANDed into the question's own filter,
|
|
26
|
+
* so a batch costs what the first one did however deep the file goes and no
|
|
27
|
+
* row is skipped or repeated. Where the list pages by offset — no primary
|
|
28
|
+
* key, a union, a sort through a reference or on a value that does not come
|
|
29
|
+
* back as it is stored — so does the file.
|
|
30
|
+
*
|
|
31
|
+
* Inside a row-policy App each batch is a read of the policy's own facade,
|
|
32
|
+
* its row scope ANDed into the batch as into a list's page, and the file
|
|
33
|
+
* holds only what the reader's list could show. A policy reply is held until
|
|
34
|
+
* its context settles (handler.ts): the file's rows go as they are read, each
|
|
35
|
+
* already the reader's, but its last write waits for the context, so a
|
|
36
|
+
* context that fails after the rows were read cuts the connection and the
|
|
37
|
+
* browser never saves a file that looks whole. A tenant's lease streams the
|
|
38
|
+
* same way through `exportFile` (mistri: tenant/tenantHttp.ts).
|
|
39
|
+
*
|
|
40
|
+
* What an App decides of it is `ExportHooks` (mistri: http/appExport.ts): who
|
|
41
|
+
* exports which table, who `me` is in the question, and what an amount kept
|
|
42
|
+
* in minor units says in a CSV. The database browser exports every table it
|
|
43
|
+
* reads, as stored.
|
|
44
|
+
*/
|
|
45
|
+
import type { ServerResponse } from 'node:http';
|
|
46
|
+
import type { Adapter } from '../../adapters/adapter.js';
|
|
1
47
|
import { parseQuery } from '../../shared/query.js';
|
|
48
|
+
import { type DatabaseSchema } from '../../shared/schema.js';
|
|
2
49
|
import type { PostContext } from './context.js';
|
|
50
|
+
/** Rows read per statement, and so the most the server holds of a file at once:
|
|
51
|
+
* large enough that a million-row file is a thousand reads, small enough that
|
|
52
|
+
* one batch is never a memory event. */
|
|
53
|
+
export declare const EXPORT_BATCH = 1000;
|
|
3
54
|
/** Who `me` is in a question, where its words name `me` (parseQuery's `me`). */
|
|
4
55
|
type Person = NonNullable<Parameters<typeof parseQuery>[1]>['me'];
|
|
5
56
|
/** What a product decides of an export. */
|
|
@@ -11,5 +62,28 @@ export interface ExportHooks {
|
|
|
11
62
|
/** A row as a CSV says it: an amount kept in minor units as the figure it is, a value as its component's word. */
|
|
12
63
|
amounts?(post: PostContext, tableId: string): ((row: Record<string, unknown>) => Record<string, unknown>) | undefined;
|
|
13
64
|
}
|
|
65
|
+
/** One export, on what its host reads it through: the routes below, or a tenant's lease (mistri: tenant/tenantHttp.ts). */
|
|
66
|
+
interface ExportAsked {
|
|
67
|
+
res: ServerResponse;
|
|
68
|
+
body: Record<string, unknown>;
|
|
69
|
+
/** The reader every batch is read through: this request's own, with its grants, fields and row scope. */
|
|
70
|
+
adapter: Adapter;
|
|
71
|
+
schema: DatabaseSchema;
|
|
72
|
+
now: () => Date;
|
|
73
|
+
/** Whether a table may be exported (`''`: any table). */
|
|
74
|
+
may: (tableId: string) => boolean;
|
|
75
|
+
person?: (text: string) => Promise<Person | undefined>;
|
|
76
|
+
amounts?: (tableId: string) => ((row: Record<string, unknown>) => Record<string, unknown>) | undefined;
|
|
77
|
+
/** Headers the file carries beside its own: a tenant lease's owner, which its client checks before it keeps the file. */
|
|
78
|
+
headers?: Record<string, string>;
|
|
79
|
+
/** A reply that is not the file (a refusal, the question's errors), sent the host's way. */
|
|
80
|
+
answer?: (status: number, body: unknown) => void;
|
|
81
|
+
}
|
|
14
82
|
export declare function exportRoute(post: PostContext, hooks: ExportHooks): Promise<boolean>;
|
|
83
|
+
/**
|
|
84
|
+
* The export, streamed. Resolves with the file's last write, for the caller
|
|
85
|
+
* to make once whatever the reads ran in has settled; undefined when the
|
|
86
|
+
* answer was not a file (it was sent already) or the file was cut.
|
|
87
|
+
*/
|
|
88
|
+
export declare function exportFile(asked: ExportAsked): Promise<(() => void) | undefined>;
|
|
15
89
|
export {};
|