@mnmal-ai/substrate 1.0.0-alpha.11 → 1.0.0-alpha.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -5
- package/dist/{boot-store-C5vNZJ3i.js → boot-store-0BPtJ8ne.js} +167 -71
- package/dist/{bridge-COu0Tg_h.d.ts → bridge-DQyQ6Qhn.d.ts} +1 -1
- package/dist/{cli-BrwW4tjO.js → cli-BvQqNFVn.js} +14 -11
- package/dist/client.d.ts +3 -3
- package/dist/client.js +3 -3
- package/dist/{errors-BOllGhVx.d.ts → errors-xP9Mo7Q1.d.ts} +25 -4
- package/dist/{index-Cp74OcmS.d.ts → index-baF5A5jU.d.ts} +1 -1
- package/dist/pack.d.ts +2 -2
- package/dist/pack.js +1 -1
- package/dist/{pg-edges-store-BLsbcXkk.js → pg-edges-store-B4FSN9yp.js} +72 -9
- package/dist/{plugin-DIYm2cZE.js → plugin-BLm5IRdX.js} +2 -2
- package/dist/proposals-bootstrap.js +2 -2
- package/dist/proposals.d.ts +2 -2
- package/dist/proposals.js +1 -1
- package/dist/recall-bootstrap.js +2 -2
- package/dist/recall.d.ts +10 -10
- package/dist/recall.js +2 -2
- package/dist/{repl-B8gy0YM7.js → repl-Dw2Fb7C1.js} +25 -25
- package/dist/{row-DwaDU4b-.d.ts → row-Dtj3KrPH.d.ts} +9 -2
- package/dist/{run-B18qtGKc.js → run-CgDw2c3X.js} +520 -141
- package/dist/{schema-CijHRfuk.d.ts → schema-oXfQyo0s.d.ts} +272 -27
- package/dist/sdk.d.ts +93 -29
- package/dist/sdk.js +4 -4
- package/dist/{src-D041j6GS.js → src-Bew_JvTo.js} +32 -16
- package/dist/{src-DpPdw2oL.js → src-Bi3NhEsT.js} +1 -1
- package/dist/{src-DptNGNYQ.js → src-BjWVYYoo.js} +175 -91
- package/dist/{src-DK7VEQT6.js → src-C4t1Gy-m.js} +2477 -830
- package/dist/{src-Br6aDUti.js → src-CHND7-Pm.js} +2 -2
- package/dist/src-CSe2tJDf.js +6940 -0
- package/dist/{src-Dbvuel6w.js → src-DWqzrm3O.js} +18 -1
- package/dist/substrate-admin.js +1 -1
- package/dist/substrate.js +1 -1
- package/dist/testing.d.ts +233 -92
- package/dist/testing.js +6 -23
- package/dist/{ws-DC8OhMaV.d.ts → ws-Dx3A2B4U.d.ts} +2 -2
- package/package.json +19 -20
- package/dist/src-gz4I1F29.js +0 -5097
package/README.md
CHANGED
|
@@ -82,7 +82,7 @@ The pieces are exported for use outside the CLI: `createClientFromEnv()` builds
|
|
|
82
82
|
|
|
83
83
|
### `substrate-admin`
|
|
84
84
|
|
|
85
|
-
A separate binary for key management: `bootstrap` (seed the genesis admin key), `doctor` (check a deployment end to end), `onboard`, `list`, `register`, `update`, `rotate`, `revoke`, and `migrate-store` (copy a store into a fresh, empty
|
|
85
|
+
A separate binary for key management and store operations: `bootstrap` (seed the genesis admin key), `doctor` (check a deployment end to end), `onboard`, `list`, `register`, `update`, `rotate`, `revoke`, `promote-fields` (run the promotion sweep now on a running server, giving fields added at runtime their columns), and `migrate-store` (copy a store into a fresh, empty PostgreSQL 18 store on per-type tables and verify it; see [Upgrading to per-type tables](https://github.com/mnmal-ai/substrate/blob/main/docs/upgrading-to-type-tables.md)). `substrate-admin --help` documents each.
|
|
86
86
|
|
|
87
87
|
## Install
|
|
88
88
|
|
|
@@ -126,25 +126,30 @@ The repository's `docker-compose.yml` runs this binary with `substrate-nl` and `
|
|
|
126
126
|
|
|
127
127
|
## Environment variables
|
|
128
128
|
|
|
129
|
-
The main ones; `substrate --help` and the header of `apps/substrate/src/main.ts` list the rest.
|
|
129
|
+
The main ones; `substrate --help` and the header of `apps/substrate/src/main.ts` list the rest. [Operator decisions](https://github.com/mnmal-ai/substrate/blob/main/docs/operator-decisions.md) explains what each choice costs.
|
|
130
130
|
|
|
131
131
|
| Var | Purpose |
|
|
132
132
|
|---|---|
|
|
133
|
-
| `SUBSTRATE_DATABASE_URL` | `memory` (default, ephemeral) or `postgres
|
|
133
|
+
| `SUBSTRATE_DATABASE_URL` | `memory` (default, ephemeral) or `postgres://...`. A Postgres store keeps each type in a table of its own; a store in an old layout (`legacy` or `partitioned`) is refused at boot, naming `substrate-admin migrate-store`, which copies it into a fresh one. `SUBSTRATE_STORAGE_LAYOUT` is no longer read: setting it logs one warning. |
|
|
134
134
|
| `SUBSTRATE_PLUGIN_DIR` | Plugin install root. The loader scans `<dir>/node_modules/{*,@*/*}/package.json` for packages with a `substrate` manifest and merges each plugin's `default-config.json` with `<dir>/<basename>.config.json`. |
|
|
135
|
-
| `PORT` / `HOST` | Listen address (default `3100`
|
|
135
|
+
| `PORT` / `HOST` | Listen address (default `3100`; `HOST` defaults to `0.0.0.0` with auth on and `127.0.0.1` with auth off) |
|
|
136
|
+
| `SUBSTRATE_ALLOW_UNAUTHENTICATED_HOST` | `1` or `true` permits a non-loopback `HOST` with auth off; without it that combination refuses boot. |
|
|
136
137
|
| `SUBSTRATE_SYSTEM_ACTOR` | Who the deployment writes as while auth is off. Required in that case: the server refuses to boot rather than write unattributed rows. |
|
|
137
138
|
| `SUBSTRATE_AUTH_MODE` | `required` or `attribute` (anonymous callers allowed on non-admin routes). Setting it, or `SUBSTRATE_AUTH_KEY_REGISTRY`, enables auth. |
|
|
138
139
|
| `SUBSTRATE_AUTH_KEY_REGISTRY` | Path to a JSON key registry file. Without it, auth reads keys registered in Postgres with `substrate-admin`. |
|
|
139
140
|
| `SUBSTRATE_PUBLIC_ORIGIN` | The origins callers reach this server by, comma-separated (`https://host:3100,https://alias`). A token whose `aud` names none of them is refused. **Required when auth is enabled and `HOST` is not loopback**; on loopback it defaults to the loopback origins. |
|
|
140
141
|
| `SUBSTRATE_STREAM_TICKET_SECRET` | HMAC key signing SSE stream tickets. **Required when auth is enabled**; at least 16 characters. The ticket signature is the only authentication on `GET /substrate/v1/stream` and `GET /substrate/v1/subscribe`, because `EventSource` can send no header or body. Rotating it invalidates every outstanding ticket. |
|
|
141
142
|
| `SUBSTRATE_TLS_KEY_PATH` / `SUBSTRATE_TLS_CERT_PATH` / `SUBSTRATE_TLS_CA_PATH` | Serve HTTPS. Key and cert are required together. |
|
|
142
|
-
| `SUBSTRATE_NAMESPACE` | Default namespace for recall requests that omit one
|
|
143
|
+
| `SUBSTRATE_NAMESPACE` | Default namespace for recall requests that omit one; with neither, recall refuses with `missing_namespace`. |
|
|
144
|
+
| `SUBSTRATE_STATEMENT_TIMEOUT_MS` | Statement timeout for typed edge walk reads (default 30000). A read it cancels is refused `query_timeout`. |
|
|
145
|
+
| `SUBSTRATE_PAGE_LIMIT_DEFAULT` / `SUBSTRATE_PAGE_LIMIT_MAX` | Page size for a remote query that names no `limit` (default 100), and the largest `limit` accepted (default 1000). |
|
|
146
|
+
| `SUBSTRATE_FILTER_WRITE_MAX_MATCH` / `SUBSTRATE_FILTER_WRITE_IDS_CAP` | `updateMany<Type>` / `deleteMany<Type>`: a filter matching more rows than the max is refused `filter_too_broad` (default 10000); the result lists at most the cap of ids (default 1000). |
|
|
143
147
|
| `SUBSTRATE_SWEEP_INTERVAL_MS` | Edge-sweep cadence (default 300000) |
|
|
144
148
|
| `SUBSTRATE_INDEX_DISK_PATH` | The database volume's mount, when the database is on this host: free space there is checked before each background index build. `off` skips the check. |
|
|
145
149
|
| `SUBSTRATE_INDEX_DISK_MARGIN_MB` | Free space a build must leave (default 2048; `0` requires only the estimate). |
|
|
146
150
|
| `SUBSTRATE_INDEX_RETRY_INTERVAL_MS` | How often pending or failed index builds are retried (default 600000; `0` leaves them to the next boot). |
|
|
147
151
|
| `SUBSTRATE_ADVISOR_INTERVAL_MS` | How often the index advisor writes `_system/IndexRecommendation` rows (default 600000; `0` turns it off). |
|
|
152
|
+
| `SUBSTRATE_PROMOTE_INTERVAL_MS` | How often the promotion sweep gives fields added at runtime (`propose_field`) their own column (default 21600000, 6 hours; `0` stops the timer, and `substrate-admin promote-fields` still runs it). |
|
|
148
153
|
| `SUBSTRATE_STORE_PROFILE` | `personal`, `growth` or `scale`: seeds `_system/StoreProfile` once, so changing it later does not change an existing store. |
|
|
149
154
|
| `SUBSTRATE_STORE_WRITE_HEAVY` | `1` or `true`, read on every boot: halves the write rate at which the advisor recommends removing an unused index. |
|
|
150
155
|
| `SUBSTRATE_CORS_ORIGINS` | Comma-separated allowed origins, or `*`. Unset registers no CORS handling. |
|
|
@@ -155,6 +160,8 @@ Natural-language query and the recall embedder are not configured by environment
|
|
|
155
160
|
|
|
156
161
|
- [Getting started](https://github.com/mnmal-ai/substrate/blob/main/docs/getting-started.md)
|
|
157
162
|
- [Design spec](https://github.com/mnmal-ai/substrate/blob/main/docs/substrate-from-first-principles.md)
|
|
163
|
+
- [Operator decisions](https://github.com/mnmal-ai/substrate/blob/main/docs/operator-decisions.md)
|
|
164
|
+
- [Upgrading to per-type tables](https://github.com/mnmal-ai/substrate/blob/main/docs/upgrading-to-type-tables.md)
|
|
158
165
|
- [Plugin author guide](https://github.com/mnmal-ai/substrate/blob/main/docs/plugin-author-guide.md)
|
|
159
166
|
|
|
160
167
|
## License
|
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
import { createRequire as __binCreateRequire } from 'node:module'; const require = __binCreateRequire(import.meta.url);
|
|
2
|
-
import {
|
|
2
|
+
import { Dt as buildContext, Pt as WS_REASON_SERVER_RESTART, Q as effectiveFilterOperators, Tt as HttpBridgeClient, Y as MemoryEdgesStore, _ as AUDIT_EVENT_TYPE, _t as unregisterExtensionOperator, a as composeFromManifests, c as MemoryDataset, g as buildAuditRow, gt as registerExtensionOperator, h as SYSTEM_TYPE_DEFS, ht as listExtensionOperators, i as PluginLoadError, jt as parseAuthConfig, k as isReservedAutoCrudField, mt as getExtensionOperator, n as kernelMigrationTracking, o as DEFAULT_NAMESPACE, r as mountKernelRoute, s as compose, t as composeKernelPlugins, u as isStoreProfileName } from "./src-C4t1Gy-m.js";
|
|
3
3
|
import { a as adminClaimCapabilities, d as resolveDefaultKidPath, l as resolveAdminCredential, n as trimTrailingSlashes, r as ADMIN_CAPABILITIES, s as makeAuthorizationProvider, t as parsePositiveInt, u as resolveDefaultKeyPath } from "./env-utils-CGNazUNt.js";
|
|
4
|
-
import { D as isActiveAdminKeyRow, E as describeGrant, L as encodeList, M as grantEntryError, T as SYSTEM_NAMESPACE, v as grantEntriesConfer } from "./src-
|
|
5
|
-
import "./src-
|
|
6
|
-
import { C as diagnose, S as DIAGNOSE_CHECKS, _ as SKIPPABLE_STEPS, b as planOnboarding, h as CURRENT_MARK, i as tokenAudience, v as STEP_MARKS, x as requiredOutstanding, y as currentStep } from "./src-
|
|
7
|
-
import {
|
|
8
|
-
import { t as PgEdgesStore } from "./pg-edges-store-
|
|
9
|
-
import { a as InMemoryLexicalStore, i as PgClusterStore, n as PgLexicalStore, o as InMemoryEmbeddingsStore, r as PgEmbeddingsStore, s as InMemoryClusterStore } from "./src-
|
|
10
|
-
import {
|
|
4
|
+
import { D as isActiveAdminKeyRow, E as describeGrant, L as encodeList, M as grantEntryError, T as SYSTEM_NAMESPACE, v as grantEntriesConfer } from "./src-Bi3NhEsT.js";
|
|
5
|
+
import { T as DEFAULT_FILTER_WRITE_LIMITS } from "./src-DWqzrm3O.js";
|
|
6
|
+
import { C as diagnose, S as DIAGNOSE_CHECKS, _ as SKIPPABLE_STEPS, b as planOnboarding, h as CURRENT_MARK, i as tokenAudience, v as STEP_MARKS, x as requiredOutstanding, y as currentStep } from "./src-CHND7-Pm.js";
|
|
7
|
+
import { _ as runDerivedMigrations, d as assertNotMigrating, g as pendingMigrations, m as deriveListIndexMigration, p as assertTablesLayout, t as PgDataset, v as runMigrations } from "./src-CSe2tJDf.js";
|
|
8
|
+
import { t as PgEdgesStore } from "./pg-edges-store-B4FSN9yp.js";
|
|
9
|
+
import { a as InMemoryLexicalStore, i as PgClusterStore, n as PgLexicalStore, o as InMemoryEmbeddingsStore, r as PgEmbeddingsStore, s as InMemoryClusterStore } from "./src-BjWVYYoo.js";
|
|
10
|
+
import { et as memberKey } from "./plugin-BLm5IRdX.js";
|
|
11
|
+
import { existsSync, linkSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
|
11
12
|
import { homedir } from "node:os";
|
|
12
13
|
import { basename, dirname, join, resolve } from "node:path";
|
|
13
14
|
import { createHash, createPublicKey, generateKeyPairSync, randomUUID } from "node:crypto";
|
|
@@ -246,7 +247,8 @@ const USAGE = [
|
|
|
246
247
|
" rotate <kid> --out <path> [--encrypt] rotate a Key (new key to <path>)",
|
|
247
248
|
" register <kid> --pubkey <path> --ns <csv> register an existing ed25519 pubkey as an agent Key",
|
|
248
249
|
" update <kid> [--ns <csv>] ... change a live key's namespaces/caps/name in place",
|
|
249
|
-
" migrate-store [options] copy a store into a fresh, empty
|
|
250
|
+
" migrate-store [options] copy a store into a fresh, empty PG 18 store on per-type tables",
|
|
251
|
+
" promote-fields [--url <base>] run the promotion sweep now on a running server",
|
|
250
252
|
"",
|
|
251
253
|
" register transport: drives POST /substrate/v1/_system/keys (credential + directory row,",
|
|
252
254
|
" one transaction, no restart) using --url or SUBSTRATE_URL and an admin identity",
|
|
@@ -280,6 +282,11 @@ const USAGE = [
|
|
|
280
282
|
" credentials at all, whether YOUR admin credential resolves, whether the grant it",
|
|
281
283
|
" resolves to confers admin, and how many admin keys the deployment has left.",
|
|
282
284
|
" A check it could not run prints as `skip`, never as a pass.",
|
|
285
|
+
" promote-fields: gives each field added at runtime (propose_field) its own column now,",
|
|
286
|
+
" instead of at the next SUBSTRATE_PROMOTE_INTERVAL_MS sweep. Goes through the server",
|
|
287
|
+
" (--url or SUBSTRATE_URL) with an admin identity; prints one line per field touched,",
|
|
288
|
+
" and exits 1 when any failed. Also retries a field held back by a unique conflict,",
|
|
289
|
+
" which the timed sweep leaves until its type changes: run it after fixing the data.",
|
|
283
290
|
" --encrypt produces a PKCS#8+PBES2 (aes-256-cbc) encrypted private-key PEM.",
|
|
284
291
|
" Passphrase is read from SUBSTRATE_ADMIN_KEY_PASSPHRASE (required when --encrypt is set).",
|
|
285
292
|
" An encrypted admin key is read with SUBSTRATE_ADMIN_KEY_PASSPHRASE for every command that signs.",
|
|
@@ -289,6 +296,7 @@ const USAGE = [
|
|
|
289
296
|
" only read. The target must be empty and on PostgreSQL 18; it is prepared with the same",
|
|
290
297
|
" migrations and plugins (SUBSTRATE_PLUGIN_DIR) serve uses, copied, indexed and verified.",
|
|
291
298
|
" SUBSTRATE_INDEX_DISK_PATH must be set (the target volume mount, or off).",
|
|
299
|
+
" SUBSTRATE_SYSTEM_ACTOR is not needed: the prepare boot serves no request, so nothing is attributed to it.",
|
|
292
300
|
" --from-schema / --to-schema <s> pin a side to schema <s>",
|
|
293
301
|
" --dry-run preflight only: the plan, every refusal, nothing written",
|
|
294
302
|
" --verify-only build indexes, then verify a finished copy against its source",
|
|
@@ -309,6 +317,7 @@ const SYNOPSIS = {
|
|
|
309
317
|
rotate: "substrate-admin rotate <kid> --out <path> [--encrypt]",
|
|
310
318
|
register: "substrate-admin register <kid> --pubkey <path> --ns <csv> [--name <n>] [--caps <csv>] [--allow-admin]",
|
|
311
319
|
update: "substrate-admin update <kid> [--name <n>] [--ns <csv>] [--caps <csv>] [--widen] [--allow-admin]",
|
|
320
|
+
"promote-fields": "substrate-admin promote-fields [--url <base>]",
|
|
312
321
|
"migrate-store": "substrate-admin migrate-store [--from-env <VAR>] [--to-env <VAR>] [--from-schema <s>] [--to-schema <s>] [--dry-run | --verify-only] [--skip-index-build] [--full-checksum] [--batch-rows <n>] [--skip-table <t>]... [--json]"
|
|
313
322
|
};
|
|
314
323
|
/**
|
|
@@ -423,6 +432,13 @@ const ARGV_SPEC = {
|
|
|
423
432
|
],
|
|
424
433
|
maxPositionals: 1
|
|
425
434
|
},
|
|
435
|
+
"promote-fields": {
|
|
436
|
+
valueFlags: ["--url"],
|
|
437
|
+
boolFlags: [],
|
|
438
|
+
maxPositionals: 0,
|
|
439
|
+
hint: " (promote-fields accepts only --url <base>)",
|
|
440
|
+
excess: (tok) => `promote-fields: unexpected argument '${tok}' (promote-fields accepts only --url <base>)`
|
|
441
|
+
},
|
|
426
442
|
"migrate-store": {
|
|
427
443
|
valueFlags: [
|
|
428
444
|
"--from-env",
|
|
@@ -910,10 +926,12 @@ async function runBootstrap(env, wantsEncrypt = false, openSql = postgres) {
|
|
|
910
926
|
const sql = openSql(storeSpec);
|
|
911
927
|
let dataset;
|
|
912
928
|
try {
|
|
929
|
+
await assertNotMigrating(sql);
|
|
913
930
|
await runMigrations(sql);
|
|
914
931
|
dataset = new PgDataset({
|
|
915
932
|
namespace: SYSTEM_NAMESPACE,
|
|
916
|
-
sql
|
|
933
|
+
sql,
|
|
934
|
+
typeDefLookup: systemTypeDefLookup
|
|
917
935
|
});
|
|
918
936
|
await dataset.ensureTypeStorage("Key");
|
|
919
937
|
return await dataset.withAdvisoryLock("substrate:bootstrap", async () => {
|
|
@@ -927,31 +945,49 @@ async function runBootstrap(env, wantsEncrypt = false, openSql = postgres) {
|
|
|
927
945
|
recursive: true,
|
|
928
946
|
mode: 448
|
|
929
947
|
});
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
948
|
+
const files = [[
|
|
949
|
+
privateKeyPath,
|
|
950
|
+
`${privateKeyPath}.tmp-${randomUUID()}`,
|
|
951
|
+
privateKeyPem
|
|
952
|
+
], [
|
|
953
|
+
kidPath,
|
|
954
|
+
`${kidPath}.tmp-${randomUUID()}`,
|
|
955
|
+
`${kid}\n`
|
|
956
|
+
]];
|
|
957
|
+
const removeTemps = () => {
|
|
958
|
+
for (const [, temp] of files) rmSync(temp, { force: true });
|
|
959
|
+
};
|
|
960
|
+
try {
|
|
961
|
+
for (const [, temp, content] of files) writeFileSync(temp, content, {
|
|
962
|
+
mode: 384,
|
|
963
|
+
flag: "wx"
|
|
964
|
+
});
|
|
965
|
+
const now = /* @__PURE__ */ new Date();
|
|
966
|
+
await dataset.write("Key", [{
|
|
967
|
+
id: kid,
|
|
968
|
+
kid,
|
|
969
|
+
name: "bootstrap-admin",
|
|
970
|
+
algorithm: "EdDSA",
|
|
971
|
+
publicKey: publicKeyPem,
|
|
972
|
+
namespaces: "*",
|
|
973
|
+
capabilities: encodeList(seededCapabilities),
|
|
974
|
+
revokedAt: null
|
|
975
|
+
}], {
|
|
976
|
+
identity: {
|
|
977
|
+
kind: "system",
|
|
978
|
+
id: "bootstrap"
|
|
979
|
+
},
|
|
980
|
+
now
|
|
981
|
+
});
|
|
982
|
+
} catch (err) {
|
|
983
|
+
removeTemps();
|
|
984
|
+
throw err;
|
|
985
|
+
}
|
|
986
|
+
try {
|
|
987
|
+
for (const [final, temp] of files) linkSync(temp, final);
|
|
988
|
+
} finally {
|
|
989
|
+
removeTemps();
|
|
990
|
+
}
|
|
955
991
|
return {
|
|
956
992
|
kid,
|
|
957
993
|
fingerprint,
|
|
@@ -1716,8 +1752,13 @@ async function onboardCommand(argv, env, io, interactive, fetchImpl = fetch) {
|
|
|
1716
1752
|
* `substrate federate enroll`.
|
|
1717
1753
|
*
|
|
1718
1754
|
* It never migrates: only `bootstrap` and the server change the schema. A
|
|
1719
|
-
* store missing a migration this CLI knows is refused.
|
|
1755
|
+
* store in an old layout, or missing a migration this CLI knows, is refused.
|
|
1756
|
+
*/
|
|
1757
|
+
/**
|
|
1758
|
+
* `_system`'s type defs, as serve's boot reconciles them, so a type the CLI
|
|
1759
|
+
* writes first gets its table typed, with `Key`'s live-kid unique index.
|
|
1720
1760
|
*/
|
|
1761
|
+
const systemTypeDefLookup = (type, namespace) => namespace === "_system" && Object.hasOwn(SYSTEM_TYPE_DEFS, type) ? SYSTEM_TYPE_DEFS[type] : void 0;
|
|
1721
1762
|
async function openPgDataset(env, openSql$1 = openSql) {
|
|
1722
1763
|
const storeSpec = env.SUBSTRATE_DATABASE_URL;
|
|
1723
1764
|
if (storeSpec === void 0 || storeSpec.length === 0) throw new AdminCliError("SUBSTRATE_DATABASE_URL is required (must be a postgres:// URL)");
|
|
@@ -1725,6 +1766,8 @@ async function openPgDataset(env, openSql$1 = openSql) {
|
|
|
1725
1766
|
const sql = openSql$1(storeSpec);
|
|
1726
1767
|
let pending;
|
|
1727
1768
|
try {
|
|
1769
|
+
await assertTablesLayout(sql);
|
|
1770
|
+
await assertNotMigrating(sql);
|
|
1728
1771
|
pending = await pendingMigrations(sql);
|
|
1729
1772
|
} catch (err) {
|
|
1730
1773
|
await closeQuietly(sql);
|
|
@@ -1737,7 +1780,8 @@ async function openPgDataset(env, openSql$1 = openSql) {
|
|
|
1737
1780
|
return {
|
|
1738
1781
|
dataset: new PgDataset({
|
|
1739
1782
|
namespace: SYSTEM_NAMESPACE,
|
|
1740
|
-
sql
|
|
1783
|
+
sql,
|
|
1784
|
+
typeDefLookup: systemTypeDefLookup
|
|
1741
1785
|
}),
|
|
1742
1786
|
close: () => closeQuietly(sql)
|
|
1743
1787
|
};
|
|
@@ -1750,7 +1794,7 @@ async function closeQuietly(sql) {
|
|
|
1750
1794
|
|
|
1751
1795
|
//#endregion
|
|
1752
1796
|
//#region package.json
|
|
1753
|
-
var version = "1.0.0-alpha.
|
|
1797
|
+
var version = "1.0.0-alpha.13";
|
|
1754
1798
|
|
|
1755
1799
|
//#endregion
|
|
1756
1800
|
//#region src/plugin-manifest.ts
|
|
@@ -2821,7 +2865,7 @@ const filterRegistry = { register(name, op) {
|
|
|
2821
2865
|
const pendingMcpTools = [];
|
|
2822
2866
|
/** Seconds `close` waits for in-flight queries before it drops their connections. */
|
|
2823
2867
|
const STORE_END_TIMEOUT_S = 2;
|
|
2824
|
-
async function wireStore(storeSpec, fieldDefLookupFor, typeDefLookupFor, diskGuard,
|
|
2868
|
+
async function wireStore(storeSpec, fieldDefLookupFor, typeDefLookupFor, diskGuard, statementTimeoutMs) {
|
|
2825
2869
|
if (storeSpec === "memory") return {
|
|
2826
2870
|
dataset: new MemoryDataset({
|
|
2827
2871
|
namespace: DEFAULT_NAMESPACE,
|
|
@@ -2836,7 +2880,7 @@ async function wireStore(storeSpec, fieldDefLookupFor, typeDefLookupFor, diskGua
|
|
|
2836
2880
|
if (storeSpec.startsWith("postgres://") || storeSpec.startsWith("postgresql://")) {
|
|
2837
2881
|
const sql = openSql(storeSpec);
|
|
2838
2882
|
try {
|
|
2839
|
-
await runMigrations(sql
|
|
2883
|
+
await runMigrations(sql);
|
|
2840
2884
|
} catch (err) {
|
|
2841
2885
|
await sql.end({ timeout: 2 }).catch(() => void 0);
|
|
2842
2886
|
throw err;
|
|
@@ -2849,7 +2893,8 @@ async function wireStore(storeSpec, fieldDefLookupFor, typeDefLookupFor, diskGua
|
|
|
2849
2893
|
edgesStore: edges,
|
|
2850
2894
|
fieldDefLookup: fieldDefLookupFor(DEFAULT_NAMESPACE),
|
|
2851
2895
|
...typeDefLookupFor ? { typeDefLookup: typeDefLookupFor(DEFAULT_NAMESPACE) } : {},
|
|
2852
|
-
...diskGuard !== void 0 ? { diskGuard } : {}
|
|
2896
|
+
...diskGuard !== void 0 ? { diskGuard } : {},
|
|
2897
|
+
...statementTimeoutMs !== void 0 ? { statementTimeoutMs } : {}
|
|
2853
2898
|
}),
|
|
2854
2899
|
edges,
|
|
2855
2900
|
substrateNamespaceDataset: new PgDataset({
|
|
@@ -2858,7 +2903,8 @@ async function wireStore(storeSpec, fieldDefLookupFor, typeDefLookupFor, diskGua
|
|
|
2858
2903
|
edgesStore: edges,
|
|
2859
2904
|
fieldDefLookup: fieldDefLookupFor(SYSTEM_NAMESPACE),
|
|
2860
2905
|
...typeDefLookupFor ? { typeDefLookup: typeDefLookupFor(SYSTEM_NAMESPACE) } : {},
|
|
2861
|
-
...diskGuard !== void 0 ? { diskGuard } : {}
|
|
2906
|
+
...diskGuard !== void 0 ? { diskGuard } : {},
|
|
2907
|
+
...statementTimeoutMs !== void 0 ? { statementTimeoutMs } : {}
|
|
2862
2908
|
}),
|
|
2863
2909
|
sql,
|
|
2864
2910
|
close: () => sql.end({ timeout: 2 })
|
|
@@ -2908,12 +2954,23 @@ function buildBridgeRegistry(specs, creds) {
|
|
|
2908
2954
|
* `engineRef`. `runServe` populates `nsDefs` once plugins are loaded and well
|
|
2909
2955
|
* before the server listens, so every query resolves against a populated map.
|
|
2910
2956
|
*
|
|
2957
|
+
* A field no definition declares (a type or field proposed at runtime) resolves
|
|
2958
|
+
* from the live registry once `registryFor` does, as in
|
|
2959
|
+
* {@link makeTypeDefLookupFactory}; without it a walk over a proposed crossRef
|
|
2960
|
+
* throws and a pending number field sorts as text.
|
|
2961
|
+
*
|
|
2911
2962
|
* A miss (namespace absent, or a hand-crafted `NamespaceDefinition` whose
|
|
2912
2963
|
* `types` is undefined because it bypassed `defineNamespace`) returns
|
|
2913
2964
|
* `undefined`, and the dataset falls back to lexicographic text compare.
|
|
2914
2965
|
*/
|
|
2915
|
-
function makeFieldDefLookupFactory(nsDefs) {
|
|
2916
|
-
return (namespace) => (type, field) =>
|
|
2966
|
+
function makeFieldDefLookupFactory(nsDefs, registryFor = () => void 0) {
|
|
2967
|
+
return (namespace) => (type, field) => {
|
|
2968
|
+
const declared = nsDefs.get(namespace)?.types?.[type]?.fields?.[field];
|
|
2969
|
+
if (declared !== void 0) return declared;
|
|
2970
|
+
const entry = registryFor(namespace)?.get(type);
|
|
2971
|
+
const def = entry?.kind === "type" && Object.hasOwn(entry.fields, field) ? entry.fields[field] : void 0;
|
|
2972
|
+
return def === void 0 || isReservedAutoCrudField(def) ? void 0 : def;
|
|
2973
|
+
};
|
|
2917
2974
|
}
|
|
2918
2975
|
/**
|
|
2919
2976
|
* Build a {@link TypeDefLookup} factory over the same late-bound `nsDefs`
|
|
@@ -2953,7 +3010,7 @@ function makeTypeDefLookupFactory(nsDefs, registryFor = () => void 0) {
|
|
|
2953
3010
|
}
|
|
2954
3011
|
/**
|
|
2955
3012
|
* Prepares storage for every type the namespace definitions declare: its
|
|
2956
|
-
*
|
|
3013
|
+
* table and its unique indexes, through `Dataset.ensureTypeStorage`. Runs
|
|
2957
3014
|
* before any kernel migration, so a migration can rely on a declared unique
|
|
2958
3015
|
* index. Non-unique indexes are left to {@link buildDeferredIndexes}. Errors
|
|
2959
3016
|
* propagate and fail the boot, among them `UniqueIndexDuplicatesError` and
|
|
@@ -2965,7 +3022,10 @@ async function reconcileDeclaredStorage(datasetFor, nsDefs) {
|
|
|
2965
3022
|
for (const [ns, def] of nsDefs) {
|
|
2966
3023
|
const dataset = datasetFor(ns);
|
|
2967
3024
|
if (dataset === void 0) throw new Error(`namespace '${ns}' is declared but has no dataset; its storage cannot be prepared`);
|
|
2968
|
-
for (const type of Object.keys(def.types ?? {})) await dataset.ensureTypeStorage?.(type, {
|
|
3025
|
+
for (const type of Object.keys(def.types ?? {})) await dataset.ensureTypeStorage?.(type, {
|
|
3026
|
+
indexes: "unique",
|
|
3027
|
+
newFields: "column"
|
|
3028
|
+
});
|
|
2969
3029
|
}
|
|
2970
3030
|
}
|
|
2971
3031
|
/** Every type registered in a namespace with a local dataset, with that dataset. */
|
|
@@ -2982,7 +3042,7 @@ function* registeredTypes(engine, datasets) {
|
|
|
2982
3042
|
}
|
|
2983
3043
|
const message = (err) => err instanceof Error ? err.message : String(err);
|
|
2984
3044
|
/**
|
|
2985
|
-
* Prepares storage,
|
|
3045
|
+
* Prepares storage, table and unique indexes, for every type registered in
|
|
2986
3046
|
* every local namespace once compose has run: `_system`, install-time types
|
|
2987
3047
|
* such as the proposals plugin's, and proposed types replayed at start.
|
|
2988
3048
|
* Namespaces without a local dataset (bridged, virtual) are skipped. A type the
|
|
@@ -3094,8 +3154,8 @@ function extractIndexedListFields(nsDefs) {
|
|
|
3094
3154
|
* Pick per-namespace datasets for plugin-contributed namespaces.
|
|
3095
3155
|
*
|
|
3096
3156
|
* When the store is pg (`sharedSql` is non-null) every plugin namespace is
|
|
3097
|
-
* backed by a `PgDataset` pointing at the same connection pool —
|
|
3098
|
-
*
|
|
3157
|
+
* backed by a `PgDataset` pointing at the same connection pool — each
|
|
3158
|
+
* type in a table of its own, listed by namespace in the type catalog, which
|
|
3099
3159
|
* is the core multi-namespace model. Multi-pg (namespace → distinct DB)
|
|
3100
3160
|
* deployments should override `datasets` on the `compose()` call site
|
|
3101
3161
|
* directly; this helper only covers the common single-pg-many-namespaces
|
|
@@ -3109,7 +3169,7 @@ function extractIndexedListFields(nsDefs) {
|
|
|
3109
3169
|
* cast numeric/date/boolean body fields in orderBy + range + cursor compares.
|
|
3110
3170
|
* Memory datasets ignore it.
|
|
3111
3171
|
*/
|
|
3112
|
-
function buildPluginDatasetMap(plugins, defaultDataset, sharedSql, sharedEdges, fieldDefLookupFor, typeDefLookup, diskGuard) {
|
|
3172
|
+
function buildPluginDatasetMap(plugins, defaultDataset, sharedSql, sharedEdges, fieldDefLookupFor, typeDefLookup, diskGuard, statementTimeoutMs) {
|
|
3113
3173
|
const datasets = {};
|
|
3114
3174
|
for (const plugin of plugins) for (const nsDef of plugin.namespaces) {
|
|
3115
3175
|
const ns = nsDef.namespace;
|
|
@@ -3123,9 +3183,9 @@ function buildPluginDatasetMap(plugins, defaultDataset, sharedSql, sharedEdges,
|
|
|
3123
3183
|
sql: sharedSql,
|
|
3124
3184
|
fieldDefLookup: fieldDefLookupFor(ns),
|
|
3125
3185
|
typeDefLookup: typeDefLookup ?? ((type) => nsDef.types?.[type]),
|
|
3126
|
-
...nsDef.promotedColumns !== void 0 ? { promotedColumns: nsDef.promotedColumns } : {},
|
|
3127
3186
|
...sharedEdges !== null ? { edgesStore: sharedEdges } : {},
|
|
3128
|
-
...diskGuard !== void 0 ? { diskGuard } : {}
|
|
3187
|
+
...diskGuard !== void 0 ? { diskGuard } : {},
|
|
3188
|
+
...statementTimeoutMs !== void 0 ? { statementTimeoutMs } : {}
|
|
3129
3189
|
}) : new MemoryDataset({
|
|
3130
3190
|
namespace: ns,
|
|
3131
3191
|
fieldDefLookup: fieldDefLookupFor(ns),
|
|
@@ -3175,6 +3235,7 @@ async function recordMigrationApplied(sql, pluginName, migrationId) {
|
|
|
3175
3235
|
* serve/config.ts — env-var configuration parsing for `substrate serve`.
|
|
3176
3236
|
*/
|
|
3177
3237
|
const DEFAULT_SWEEP_INTERVAL_MS = 3e5;
|
|
3238
|
+
const DEFAULT_STATEMENT_TIMEOUT_MS = 3e4;
|
|
3178
3239
|
const DEFAULT_INVITE_SWEEP_INTERVAL_MS = 36e5;
|
|
3179
3240
|
const DEFAULT_TOMBSTONE_RETENTION_DAYS = 30;
|
|
3180
3241
|
const DEFAULT_SHUTDOWN_GRACE_MS = 5e3;
|
|
@@ -3182,17 +3243,24 @@ const DEFAULT_PARTITION_BUDGET = 500;
|
|
|
3182
3243
|
const DEFAULT_INDEX_DISK_MARGIN_MB = 2048;
|
|
3183
3244
|
const DEFAULT_INDEX_RETRY_INTERVAL_MS = 6e5;
|
|
3184
3245
|
const DEFAULT_ADVISOR_INTERVAL_MS = 6e5;
|
|
3246
|
+
const DEFAULT_PROMOTE_INTERVAL_MS = 216e5;
|
|
3185
3247
|
const MS_PER_DAY = 864e5;
|
|
3186
3248
|
const DEFAULT_PORT = 3100;
|
|
3187
3249
|
var EnvConfigError = class extends Error {
|
|
3188
3250
|
name = "EnvConfigError";
|
|
3189
3251
|
};
|
|
3252
|
+
/** Set once the retired `SUBSTRATE_STORAGE_LAYOUT` has been warned about. */
|
|
3253
|
+
let warnedStorageLayout = false;
|
|
3190
3254
|
/**
|
|
3191
3255
|
* Parses the serve env into a fully resolved `ParsedConfig`. The only I/O is
|
|
3192
3256
|
* reading the auth and TLS PEM / JSON files. Throws `EnvConfigError` on any
|
|
3193
3257
|
* malformed input.
|
|
3194
3258
|
*/
|
|
3195
3259
|
function parseEnv(env) {
|
|
3260
|
+
if (env.SUBSTRATE_STORAGE_LAYOUT !== void 0 && !warnedStorageLayout) {
|
|
3261
|
+
warnedStorageLayout = true;
|
|
3262
|
+
process.stderr.write("warning: SUBSTRATE_STORAGE_LAYOUT is no longer read; every store uses per-type tables\n");
|
|
3263
|
+
}
|
|
3196
3264
|
const knobs = createKnobRecorder();
|
|
3197
3265
|
const storeSpec = env.SUBSTRATE_DATABASE_URL ?? "memory";
|
|
3198
3266
|
knobs.record("storeSpec", "SUBSTRATE_DATABASE_URL", env.SUBSTRATE_DATABASE_URL, redactStoreSpec(storeSpec));
|
|
@@ -3219,10 +3287,13 @@ function parseEnv(env) {
|
|
|
3219
3287
|
const tombstoneRetentionDays = parseTombstoneRetentionDays(env, knobs);
|
|
3220
3288
|
const shutdownGraceMs = parseShutdownGraceMs(env, knobs);
|
|
3221
3289
|
const partitionBudget = parsePositiveIntEnv(env, knobs, "partitionBudget", "SUBSTRATE_PARTITION_BUDGET", DEFAULT_PARTITION_BUDGET, "partitions");
|
|
3290
|
+
const statementTimeoutMs = parsePositiveIntEnv(env, knobs, "statementTimeoutMs", "SUBSTRATE_STATEMENT_TIMEOUT_MS", DEFAULT_STATEMENT_TIMEOUT_MS, "milliseconds");
|
|
3291
|
+
if (statementTimeoutMs > 2147483647) throw new EnvConfigError(`SUBSTRATE_STATEMENT_TIMEOUT_MS must be at most 2147483647 (got '${statementTimeoutMs}')`);
|
|
3222
3292
|
const indexDiskPath = knobs.record("indexDiskPath", "SUBSTRATE_INDEX_DISK_PATH", env.SUBSTRATE_INDEX_DISK_PATH, diskPathOf(nonEmpty(env.SUBSTRATE_INDEX_DISK_PATH?.trim())));
|
|
3223
3293
|
const indexDiskMarginMb = parseNonNegativeIntEnv(env, knobs, "indexDiskMarginMb", "SUBSTRATE_INDEX_DISK_MARGIN_MB", DEFAULT_INDEX_DISK_MARGIN_MB, "megabytes");
|
|
3224
3294
|
const indexRetryIntervalMs = parseNonNegativeIntEnv(env, knobs, "indexRetryIntervalMs", "SUBSTRATE_INDEX_RETRY_INTERVAL_MS", DEFAULT_INDEX_RETRY_INTERVAL_MS, "milliseconds, 0 to disable");
|
|
3225
3295
|
const advisorIntervalMs = parseNonNegativeIntEnv(env, knobs, "advisorIntervalMs", "SUBSTRATE_ADVISOR_INTERVAL_MS", DEFAULT_ADVISOR_INTERVAL_MS, "milliseconds, 0 to disable");
|
|
3296
|
+
const promoteIntervalMs = parseNonNegativeIntEnv(env, knobs, "promoteIntervalMs", "SUBSTRATE_PROMOTE_INTERVAL_MS", DEFAULT_PROMOTE_INTERVAL_MS, "milliseconds, 0 to disable");
|
|
3226
3297
|
const profileRaw = nonEmpty(env.SUBSTRATE_STORE_PROFILE?.trim());
|
|
3227
3298
|
if (profileRaw !== void 0 && !isStoreProfileName(profileRaw)) throw new EnvConfigError(`SUBSTRATE_STORE_PROFILE must be personal, growth or scale, got '${profileRaw}'`);
|
|
3228
3299
|
const storeProfile = knobs.record("storeProfile", "SUBSTRATE_STORE_PROFILE", env.SUBSTRATE_STORE_PROFILE, profileRaw ?? null);
|
|
@@ -3261,6 +3332,7 @@ function parseEnv(env) {
|
|
|
3261
3332
|
const bodyLimit = parseBodyLimit(env, knobs);
|
|
3262
3333
|
const pageLimits = parsePageLimits(env, knobs);
|
|
3263
3334
|
const snapshotLimit = parseSnapshotLimit(env, knobs, pageLimits.maxLimit);
|
|
3335
|
+
const filterWriteLimits = parseFilterWriteLimits(env, knobs);
|
|
3264
3336
|
const keyReprimeIntervalMs = parseKeyReprimeIntervalMs(env, knobs);
|
|
3265
3337
|
knobs.recordSecret("streamTicketSecret", "SUBSTRATE_STREAM_TICKET_SECRET", env.SUBSTRATE_STREAM_TICKET_SECRET);
|
|
3266
3338
|
knobs.record("systemActor", "SUBSTRATE_SYSTEM_ACTOR", env.SUBSTRATE_SYSTEM_ACTOR, nonEmpty(env.SUBSTRATE_SYSTEM_ACTOR?.trim()) ?? null);
|
|
@@ -3277,10 +3349,12 @@ function parseEnv(env) {
|
|
|
3277
3349
|
tombstoneRetentionDays,
|
|
3278
3350
|
shutdownGraceMs,
|
|
3279
3351
|
partitionBudget,
|
|
3352
|
+
statementTimeoutMs,
|
|
3280
3353
|
indexDiskPath,
|
|
3281
3354
|
indexDiskMarginMb,
|
|
3282
3355
|
indexRetryIntervalMs,
|
|
3283
3356
|
advisorIntervalMs,
|
|
3357
|
+
promoteIntervalMs,
|
|
3284
3358
|
storeProfile,
|
|
3285
3359
|
storeWriteHeavy,
|
|
3286
3360
|
idempotency,
|
|
@@ -3291,6 +3365,7 @@ function parseEnv(env) {
|
|
|
3291
3365
|
bodyLimit,
|
|
3292
3366
|
pageLimits,
|
|
3293
3367
|
snapshotLimit,
|
|
3368
|
+
filterWriteLimits,
|
|
3294
3369
|
keyReprimeIntervalMs,
|
|
3295
3370
|
knobs: knobs.report(),
|
|
3296
3371
|
streamTicketSecret,
|
|
@@ -3358,6 +3433,19 @@ function parseSnapshotLimit(env, knobs, fallback) {
|
|
|
3358
3433
|
if (!Number.isInteger(v) || v <= 0) throw new EnvConfigError(`SUBSTRATE_SNAPSHOT_LIMIT: invalid value '${raw}' — expected a positive integer`);
|
|
3359
3434
|
return knobs.record("snapshotLimit", "SUBSTRATE_SNAPSHOT_LIMIT", env.SUBSTRATE_SNAPSHOT_LIMIT, v);
|
|
3360
3435
|
}
|
|
3436
|
+
/** Parse `SUBSTRATE_FILTER_WRITE_MAX_MATCH` (10000) and `SUBSTRATE_FILTER_WRITE_IDS_CAP` (1000). */
|
|
3437
|
+
function parseFilterWriteLimits(env, knobs) {
|
|
3438
|
+
const read = (key, name, fallback) => {
|
|
3439
|
+
const raw = nonEmpty(env[name]);
|
|
3440
|
+
const v = raw === void 0 ? fallback : Number(raw);
|
|
3441
|
+
if (!Number.isSafeInteger(v) || v <= 0) throw new EnvConfigError(`${name}: invalid value '${raw}' — expected a positive integer`);
|
|
3442
|
+
return knobs.record(key, name, env[name], v);
|
|
3443
|
+
};
|
|
3444
|
+
return {
|
|
3445
|
+
maxMatch: read("filterWriteMaxMatch", "SUBSTRATE_FILTER_WRITE_MAX_MATCH", DEFAULT_FILTER_WRITE_LIMITS.maxMatch),
|
|
3446
|
+
idsCap: read("filterWriteIdsCap", "SUBSTRATE_FILTER_WRITE_IDS_CAP", DEFAULT_FILTER_WRITE_LIMITS.idsCap)
|
|
3447
|
+
};
|
|
3448
|
+
}
|
|
3361
3449
|
/**
|
|
3362
3450
|
* Parse `SUBSTRATE_TRUSTED_PROXIES` — CSV of exact peer-IP strings. Entries are
|
|
3363
3451
|
* trimmed; empty entries are dropped. A non-empty value that devolves to an
|
|
@@ -3440,7 +3528,7 @@ function parseInviteSweepIntervalMs(env, knobs) {
|
|
|
3440
3528
|
}
|
|
3441
3529
|
/**
|
|
3442
3530
|
* Parse `SUBSTRATE_TOMBSTONE_RETENTION_DAYS` — retention window before tombstoned
|
|
3443
|
-
*
|
|
3531
|
+
* type-table rows / `edges` are hard-deleted by the sweep loop. Default 30.
|
|
3444
3532
|
*/
|
|
3445
3533
|
function parseTombstoneRetentionDays(env, knobs) {
|
|
3446
3534
|
return parsePositiveIntEnv(env, knobs, "tombstoneRetentionDays", "SUBSTRATE_TOMBSTONE_RETENTION_DAYS", DEFAULT_TOMBSTONE_RETENTION_DAYS, "days");
|
|
@@ -3667,12 +3755,14 @@ async function buildComposeOpts(args) {
|
|
|
3667
3755
|
trustedProxies: config.trustedProxies,
|
|
3668
3756
|
pageLimits: config.pageLimits,
|
|
3669
3757
|
subscriptions: { snapshotLimit: config.snapshotLimit },
|
|
3758
|
+
filterWriteLimits: config.filterWriteLimits,
|
|
3670
3759
|
configReport: config.knobs,
|
|
3671
3760
|
advisor: {
|
|
3672
3761
|
intervalMs: config.advisorIntervalMs,
|
|
3673
3762
|
...config.storeProfile !== null ? { storeProfile: config.storeProfile } : {},
|
|
3674
3763
|
writeHeavy: config.storeWriteHeavy
|
|
3675
|
-
}
|
|
3764
|
+
},
|
|
3765
|
+
promoteFields: { intervalMs: config.promoteIntervalMs }
|
|
3676
3766
|
};
|
|
3677
3767
|
if (dataset.storageReport !== void 0) {
|
|
3678
3768
|
const report = dataset.storageReport.bind(dataset);
|
|
@@ -3680,7 +3770,7 @@ async function buildComposeOpts(args) {
|
|
|
3680
3770
|
}
|
|
3681
3771
|
if (coordinators.length > 0) {
|
|
3682
3772
|
composeOpts.coordinators = coordinators;
|
|
3683
|
-
composeOpts.datasets = buildPluginDatasetMap(coordinators, dataset, storeSql, storeSql !== null ? edges : null, fieldDefLookupFor, typeDefLookup, diskGuard);
|
|
3773
|
+
composeOpts.datasets = buildPluginDatasetMap(coordinators, dataset, storeSql, storeSql !== null ? edges : null, fieldDefLookupFor, typeDefLookup, diskGuard, config.statementTimeoutMs);
|
|
3684
3774
|
}
|
|
3685
3775
|
if (substrateNamespaceDataset !== null) {
|
|
3686
3776
|
if (composeOpts.datasets === void 0) composeOpts.datasets = {};
|
|
@@ -3827,15 +3917,9 @@ async function loadServePlugins(opts) {
|
|
|
3827
3917
|
async function composeKernelPhase(args) {
|
|
3828
3918
|
const { kernel, server, storeSql, coordinators, kernelAuth, trustedProxies } = args;
|
|
3829
3919
|
await reconcileDeclaredStorage(args.datasetFor, args.nsDefs);
|
|
3830
|
-
let layout = null;
|
|
3831
|
-
if (storeSql !== null) {
|
|
3832
|
-
const detected = await detectLayout(storeSql);
|
|
3833
|
-
if (detected === "absent") throw new Error("kernel migrations need `rows`, but it is absent");
|
|
3834
|
-
layout = detected;
|
|
3835
|
-
}
|
|
3836
3920
|
await composeKernelPlugins(kernel, {
|
|
3837
3921
|
sql: storeSql,
|
|
3838
|
-
layout,
|
|
3922
|
+
layout: storeSql === null ? null : "tables",
|
|
3839
3923
|
background: args.background ?? true,
|
|
3840
3924
|
coordinatorNamespaces: coordinators.flatMap((p) => p.namespaces.map((n) => n.namespace)),
|
|
3841
3925
|
coordinatorNamespacesByPlugin: coordinators.flatMap((p) => p.namespaces.map((n) => ({
|
|
@@ -3864,10 +3948,18 @@ async function composeKernelPhase(args) {
|
|
|
3864
3948
|
|
|
3865
3949
|
//#endregion
|
|
3866
3950
|
//#region src/rows-missing-probe.ts
|
|
3867
|
-
/**
|
|
3868
|
-
|
|
3869
|
-
|
|
3870
|
-
|
|
3951
|
+
/**
|
|
3952
|
+
* Host side of substrate-recall's membership decay sweep: "of these cluster
|
|
3953
|
+
* members, which source rows are gone?" Answered with one read per
|
|
3954
|
+
* (namespace, type) group, since only the binary holds the dataset registry.
|
|
3955
|
+
*
|
|
3956
|
+
* POLARITY IS THE SAFETY PROPERTY. This reports rows it can positively
|
|
3957
|
+
* establish are absent. A namespace that will not resolve, a group that never
|
|
3958
|
+
* gets asked, a dataset that answers nothing — all leave their refs OUT of
|
|
3959
|
+
* the result and therefore attached. The sweep's only action is
|
|
3960
|
+
* `markDeparted` and there is no un-depart, so under-reporting is corrected
|
|
3961
|
+
* by the next pass while over-reporting is corrected by nothing.
|
|
3962
|
+
*/
|
|
3871
3963
|
function createRowsMissingProbe(opts) {
|
|
3872
3964
|
return async (refs) => {
|
|
3873
3965
|
const groups = /* @__PURE__ */ new Map();
|
|
@@ -3933,14 +4025,15 @@ async function bootStore(opts) {
|
|
|
3933
4025
|
const background = opts.background ?? true;
|
|
3934
4026
|
const config = background ? opts.config : {
|
|
3935
4027
|
...opts.config,
|
|
3936
|
-
advisorIntervalMs: 0
|
|
4028
|
+
advisorIntervalMs: 0,
|
|
4029
|
+
promoteIntervalMs: 0
|
|
3937
4030
|
};
|
|
3938
4031
|
const nsDefs = /* @__PURE__ */ new Map();
|
|
3939
|
-
const fieldDefLookupFor = makeFieldDefLookupFactory(nsDefs);
|
|
3940
4032
|
const diskGuard = await indexDiskGuard(config);
|
|
3941
4033
|
const refs = createLateRefs(config.auth);
|
|
4034
|
+
const fieldDefLookupFor = makeFieldDefLookupFactory(nsDefs, (ns) => refs.engineRef.current?.registryFor(ns));
|
|
3942
4035
|
const typeDefLookupFor = makeTypeDefLookupFactory(nsDefs, (ns) => refs.engineRef.current?.registryFor(ns));
|
|
3943
|
-
const { dataset, edges, substrateNamespaceDataset, sql: storeSql, close: closeStore } = await wireStore(config.storeSpec, fieldDefLookupFor, typeDefLookupFor, diskGuard,
|
|
4036
|
+
const { dataset, edges, substrateNamespaceDataset, sql: storeSql, close: closeStore } = await wireStore(config.storeSpec, fieldDefLookupFor, typeDefLookupFor, diskGuard, config.statementTimeoutMs);
|
|
3944
4037
|
const opened = [];
|
|
3945
4038
|
if (closeStore !== null) opened.push(closeStore);
|
|
3946
4039
|
try {
|
|
@@ -4012,10 +4105,13 @@ async function bootStore(opts) {
|
|
|
4012
4105
|
nsDefs,
|
|
4013
4106
|
background
|
|
4014
4107
|
});
|
|
4015
|
-
const { engine, keyRegistry, sweepDeps, scheduler, datasets: composedDatasets, stop } = await compose(composeOpts);
|
|
4108
|
+
const { engine, keyRegistry, sweepDeps, scheduler, datasets: composedDatasets, runPromoteFieldsNow, stop } = await compose(composeOpts);
|
|
4016
4109
|
opened.push(() => stop());
|
|
4017
4110
|
populateLateRefs(refs, composedDatasets, engine, keyRegistry);
|
|
4018
4111
|
await reconcileRegisteredStorage(engine, composedDatasets, nsDefs);
|
|
4112
|
+
if (background) runPromoteFieldsNow().catch((err) => {
|
|
4113
|
+
process.stderr.write(`[substrate] boot promotion sweep failed and will run again: ${err instanceof Error ? err.message : String(err)}\n`);
|
|
4114
|
+
});
|
|
4019
4115
|
return {
|
|
4020
4116
|
engine,
|
|
4021
4117
|
datasets: composedDatasets,
|
|
@@ -4062,4 +4158,4 @@ async function release(opened) {
|
|
|
4062
4158
|
}
|
|
4063
4159
|
|
|
4064
4160
|
//#endregion
|
|
4065
|
-
export {
|
|
4161
|
+
export { UPDATE_EXIT_CODES as A, readKeyById as B, grantLines as C, parseBootstrapArgs as D, formatBootstrapOutput as E, CLI_ACTOR_KID as F, validateArgv as G, resolveAdminAuth as H, base64Url as I, argvError as J, RESTART_NOTE as K, forUpdate as L, runKeyPatchViaRoute as M, ACTOR_HINT as N, runBootstrap as O, BootstrapError as P, generateAdminKeypair as R, runRegisterViaRoute as S, formatDoctorReport as T, resolveAdminTarget as U, describeAdminTarget as V, ARGV_SPEC as W, pgNoticeOptions as X, AdminCliError as Y, openPgDataset as _, parseBridgeCreds as a, parseRegisterArgs as b, refuseContractViolation as c, registeredTypes as d, reportUnguardedIndexBuilds as f, version as g, loadPluginsFromDir as h, MS_PER_DAY as i, runKeyListViaRoute as j, REVOKE_EXIT_CODES as k, buildDeferredIndexes as l, startSentinelClient as m, resolvePluginDir as n, parseBridges as o, runDeferredIndexBuilds as p, USAGE as q, BINARY_CONTRACT_VERSION as r, parseEnv as s, bootStore as t, redactStoreSpec as u, onboardCommand as v, doctorCommand as w, runRegister as x, formatRegisterOutput as y, listActiveAdminKeys as z };
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { Mn as Row, f as PackEnvelope, m as Schema, xn as EdgeRef } from "./schema-oXfQyo0s.js";
|
|
2
2
|
//#region ../../packages/substrate-types/src/bridge.d.ts
|
|
3
3
|
/**
|
|
4
4
|
* Per-frame bucket sent to a bridge. Same shape as `QualifiedFrame` in the
|