@powerhousedao/switchboard 6.2.3-dev.44 → 6.2.3-dev.45
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.tsbuild/test/durable-node-fs-pg16.test.d.ts +2 -0
- package/.tsbuild/test/durable-node-fs-pg16.test.d.ts.map +1 -0
- package/.tsbuild/test/pglite-preflight.test.d.ts +2 -0
- package/.tsbuild/test/pglite-preflight.test.d.ts.map +1 -0
- package/.tsbuild/test/snapshot-writer.d.ts +18 -0
- package/.tsbuild/test/snapshot-writer.d.ts.map +1 -0
- package/.tsbuild/test/workflow-reactor-access.test.d.ts +2 -0
- package/.tsbuild/test/workflow-reactor-access.test.d.ts.map +1 -0
- package/.tsbuild/test/workflow-reactor-enforcement.test.d.ts +2 -0
- package/.tsbuild/test/workflow-reactor-enforcement.test.d.ts.map +1 -0
- package/.tsbuild/tsconfig.tsbuildinfo +1 -1
- package/CHANGELOG.md +28 -0
- package/dist/index.mjs +1 -1
- package/dist/{server-Bk_IZzCn.mjs → server-Dvw8_NeM.mjs} +256 -77
- package/dist/server-Dvw8_NeM.mjs.map +1 -0
- package/dist/server.d.mts +3 -1
- package/dist/server.d.mts.map +1 -1
- package/dist/server.mjs +3 -3
- package/package.json +15 -16
- package/test/boot-unsupported-stored-documents.test.ts +5 -5
- package/test/durable-node-fs-pg16.test.ts +80 -0
- package/test/pglite-dialect.test.ts +28 -0
- package/test/pglite-preflight.test.ts +313 -0
- package/test/pglite-version.test.ts +9 -1
- package/test/snapshot-writer.ts +94 -0
- package/test/worker-pool.test.ts +27 -1
- package/test/workflow/resolvers.test.ts +26 -0
- package/test/workflow-reactor-access.test.ts +384 -0
- package/test/workflow-reactor-enforcement.test.ts +805 -0
- package/test/workflow-runtime.test.ts +53 -0
- package/tsconfig.json +0 -3
- package/dist/server-Bk_IZzCn.mjs.map +0 -1
package/dist/server.d.mts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { ILogger } from "document-model";
|
|
2
|
-
import { ChannelScheme, IDocumentModelLoader, IReactorClient, InProcessReactorClientModule, JwtHandler, ReactorBuilder, ReactorClientBuilder, ReactorFeatureFlags, SignerConfig, UnsupportedStoredDocuments, UnsupportedStoredProtocolError } from "@powerhousedao/reactor";
|
|
2
|
+
import { ChannelScheme, IDocumentModelLoader, IReactorClient, InProcessReactorClientModule, JwtHandler, ModelManifestEntry, ReactorBuilder, ReactorClientBuilder, ReactorFeatureFlags, SignerConfig, UnsupportedStoredDocuments, UnsupportedStoredProtocolError } from "@powerhousedao/reactor";
|
|
3
3
|
import { AttachmentReferenceProjectionCapability } from "@powerhousedao/reactor-api";
|
|
4
4
|
import { IAttachmentService } from "@powerhousedao/reactor-attachments";
|
|
5
5
|
import { IRenown } from "@renown/sdk/node";
|
|
@@ -206,6 +206,8 @@ export type SwitchboardReactor = {
|
|
|
206
206
|
workflowTriggers?: WorkflowTriggersCapability;
|
|
207
207
|
/** Whether workflows are on, after the option, PH_WORKFLOWS_ENABLED and the config file. */
|
|
208
208
|
workflowsEnabled: boolean;
|
|
209
|
+
/** The boot list of importable models workers and workflow pieces load; empty when none are resolved. */
|
|
210
|
+
modelManifest(): ModelManifestEntry[];
|
|
209
211
|
/** Present when the privacy add-on runs: its erasure service. */
|
|
210
212
|
privacy?: {
|
|
211
213
|
erasure: IErasureService;
|
package/dist/server.d.mts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"server.d.mts","names":[],"sources":["../src/privacy.mts","../src/workflow-runtime.mts","../src/types.ts","../src/builder-defaults.mts","../src/unsupported-stored-documents.mts","../src/server.mts"],"mappings":";;;;;;;;;;;;KAyBY;;EAEV;;EAEA;;EAEA,SAAS;;;;;;
|
|
1
|
+
{"version":3,"file":"server.d.mts","names":[],"sources":["../src/privacy.mts","../src/workflow-runtime.mts","../src/types.ts","../src/builder-defaults.mts","../src/unsupported-stored-documents.mts","../src/server.mts"],"mappings":";;;;;;;;;;;;KAyBY;;EAEV;;EAEA;;EAEA,SAAS;;;;;;KC8BC;EACN;;EAEA;EACA;;;;;;;;;;YC3CM;;YAKA,wBAAwB;EAClC,eAAe;;YAGL;EACV;EACA;EACA;;YAGU;;EAEV;;;;;EAKA;;EAGA;;YAGU;;;;;;;;;;;;;;;;;;;;;EAqBV,UAAU;;;;;;;;EAQV;;;;;;;EAOA;EACA;EACA;;;;;;EAMA;EACA;EACA;EACA,QAAQ;EACR;EACA;EACA;IAEM;IACA;;;;;;;EASN,WAAW;;EAEX;;;EAGA;;;EAGA;IACE;;;EAGF,UAAU;EACV,kBAAkB;EAClB;EACA;;;;;;EAMA;EACA,SAAS;;;;;;;;;;EAUT;;;;;;;;;;EAUA;;;;;;;;;EASA;IACE;IACA;IACA;;;;;;;;;;;EAWF;IACE;IACA;;;;;;;;;EASF,6BAA6B;;YAGnB;EACV;EACA,SAAS;;EAET,mBAAmB;;EAEnB,+BAA+B;;;EAG/B,mBAAmB;;EAEnB;;EAEA,iBAAiB;;EAEjB;IAAY,SAAS;;;EAErB;;EAEA,QAAQ;;;;;EAKR;;;;;;;;;;;;;EAaA,gBAAgB;;;;KC/MN;EAIV,iBAAiB;;EAEjB,mBAAmB;;EAEnB;;;;;;EAMA,gBAAgB;;EAEhB;;EAEA;IACE;IACA,eAAe,QAAQ;;;EAGzB,sBAAsB;EACtB,SAAS;;;;;;EAMT,SAAS;;;;;EAKT,cAAc,KAAK;;EAEnB,wBAAwB;;wBAsBV,gCACd,gBAAgB,gBAChB,eAAe,sBACf,UAAS;;;;wBC3EK,kCACd,OAAO,wCACP,KAAK,OAAO,aACX;;qBAaU,oCAAoC;WACtC;IAAqB;IAAkB;;WACvC;EAET,YAAY,OAAO;SAcZ,QAAQ,iBAAiB,SAAS;;;;;;;;;wBCoK3B,gBAAgB,eAAe;;wBAoI/B,8BACd,SAAS,KAAK,uDACd,oBACA,QAAQ;EACL;EAAmB,YAAY;;;;;;;;;;;;;;;;;;;qBAm7BvB,mBAAgB,UAClB,uBACR,QAAQ"}
|
package/dist/server.mjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import { a as resolveUnsupportedStoredDocuments, i as StoredDocumentsRefusedError, n as isPortAvailable, o as applySwitchboardReactorDefaults, r as startSwitchboard, t as deriveAttachmentServiceConfig } from "./server-
|
|
2
|
+
import { a as resolveUnsupportedStoredDocuments, i as StoredDocumentsRefusedError, n as isPortAvailable, o as applySwitchboardReactorDefaults, r as startSwitchboard, t as deriveAttachmentServiceConfig } from "./server-Dvw8_NeM.mjs";
|
|
3
3
|
export { StoredDocumentsRefusedError, applySwitchboardReactorDefaults, deriveAttachmentServiceConfig, isPortAvailable, resolveUnsupportedStoredDocuments, startSwitchboard };
|
|
4
|
-
!function(){try{var e="undefined"!=typeof window?window:"undefined"!=typeof global?global:"undefined"!=typeof globalThis?globalThis:"undefined"!=typeof self?self:{},n=(new e.Error).stack;n&&(e._sentryDebugIds=e._sentryDebugIds||{},e._sentryDebugIds[n]="
|
|
5
|
-
//# debugId=
|
|
4
|
+
!function(){try{var e="undefined"!=typeof window?window:"undefined"!=typeof global?global:"undefined"!=typeof globalThis?globalThis:"undefined"!=typeof self?self:{},n=(new e.Error).stack;n&&(e._sentryDebugIds=e._sentryDebugIds||{},e._sentryDebugIds[n]="b5bdc179-e9e6-50c3-a581-a82e554e7130")}catch(e){}}();
|
|
5
|
+
//# debugId=b5bdc179-e9e6-50c3-a581-a82e554e7130
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@powerhousedao/switchboard",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "6.2.3-dev.
|
|
4
|
+
"version": "6.2.3-dev.45",
|
|
5
5
|
"main": "dist/index.mjs",
|
|
6
6
|
"exports": {
|
|
7
7
|
".": {
|
|
@@ -41,7 +41,7 @@
|
|
|
41
41
|
"@openfeature/core": "1.9.1",
|
|
42
42
|
"@openfeature/env-var-provider": "0.3.1",
|
|
43
43
|
"@openfeature/server-sdk": "1.19.0",
|
|
44
|
-
"@powerhousedao/config": "6.2.3-dev.
|
|
44
|
+
"@powerhousedao/config": "6.2.3-dev.45",
|
|
45
45
|
"@opentelemetry/api": "^1.9.0",
|
|
46
46
|
"@opentelemetry/exporter-metrics-otlp-http": "^0.218.0",
|
|
47
47
|
"@opentelemetry/exporter-trace-otlp-http": "^0.218.0",
|
|
@@ -55,23 +55,22 @@
|
|
|
55
55
|
"@opentelemetry/sdk-node": "^0.218.0",
|
|
56
56
|
"@opentelemetry/sdk-trace-base": "^2.7.1",
|
|
57
57
|
"@opentelemetry/semantic-conventions": "^1.41.1",
|
|
58
|
-
"@powerhousedao/opentelemetry-instrumentation-reactor": "6.2.3-dev.
|
|
59
|
-
"@powerhousedao/reactor": "6.2.3-dev.
|
|
60
|
-
"@powerhousedao/shared": "6.2.3-dev.
|
|
61
|
-
"@powerhousedao/vetra": "6.2.3-dev.
|
|
62
|
-
"@powerhousedao/reactor-api": "6.2.3-dev.
|
|
63
|
-
"@powerhousedao/reactor-attachments": "6.2.3-dev.
|
|
64
|
-
"@powerhousedao/reactor-drive": "6.2.3-dev.
|
|
65
|
-
"@powerhousedao/reactor-group": "6.2.3-dev.
|
|
66
|
-
"@powerhousedao/reactor-privacy": "6.2.3-dev.
|
|
67
|
-
"@powerhousedao/reactor-workflow": "6.2.3-dev.
|
|
68
|
-
"@powerhousedao/workflow": "6.2.3-dev.
|
|
69
|
-
"@powerhousedao/pglite-fs": "6.2.3-dev.44",
|
|
58
|
+
"@powerhousedao/opentelemetry-instrumentation-reactor": "6.2.3-dev.45",
|
|
59
|
+
"@powerhousedao/reactor": "6.2.3-dev.45",
|
|
60
|
+
"@powerhousedao/shared": "6.2.3-dev.45",
|
|
61
|
+
"@powerhousedao/vetra": "6.2.3-dev.45",
|
|
62
|
+
"@powerhousedao/reactor-api": "6.2.3-dev.45",
|
|
63
|
+
"@powerhousedao/reactor-attachments": "6.2.3-dev.45",
|
|
64
|
+
"@powerhousedao/reactor-drive": "6.2.3-dev.45",
|
|
65
|
+
"@powerhousedao/reactor-group": "6.2.3-dev.45",
|
|
66
|
+
"@powerhousedao/reactor-privacy": "6.2.3-dev.45",
|
|
67
|
+
"@powerhousedao/reactor-workflow": "6.2.3-dev.45",
|
|
68
|
+
"@powerhousedao/workflow": "6.2.3-dev.45",
|
|
70
69
|
"@pyroscope/nodejs": "^0.4.5",
|
|
71
|
-
"@renown/sdk": "6.2.3-dev.
|
|
70
|
+
"@renown/sdk": "6.2.3-dev.45",
|
|
72
71
|
"@sentry/node": "^10.53.1",
|
|
73
72
|
"@sentry/opentelemetry": "^10.53.1",
|
|
74
|
-
"document-model": "6.2.3-dev.
|
|
73
|
+
"document-model": "6.2.3-dev.45",
|
|
75
74
|
"dotenv": "^16.4.7",
|
|
76
75
|
"express": "^4.22.1",
|
|
77
76
|
"graphql": "^16",
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { PGlite } from "@electric-sql/pglite";
|
|
2
|
-
import {
|
|
2
|
+
import { NodeFS } from "@electric-sql/pglite/nodefs";
|
|
3
3
|
import {
|
|
4
4
|
JobStatus,
|
|
5
5
|
ReactorBuilder,
|
|
@@ -65,9 +65,7 @@ function stubLogger(): StubLogger {
|
|
|
65
65
|
/** A reactor store holding one drive created at base-reducer 7. */
|
|
66
66
|
async function seedStore(dir: string): Promise<void> {
|
|
67
67
|
const db = new Kysely<Database>({
|
|
68
|
-
dialect: new ClosablePGliteDialect(
|
|
69
|
-
new PGlite({ fs: new AtomicNodeFs(dir) }),
|
|
70
|
-
),
|
|
68
|
+
dialect: new ClosablePGliteDialect(new PGlite({ fs: new NodeFS(dir) })),
|
|
71
69
|
});
|
|
72
70
|
const module = await new ReactorBuilder()
|
|
73
71
|
.withKysely(db)
|
|
@@ -89,7 +87,7 @@ async function seedStore(dir: string): Promise<void> {
|
|
|
89
87
|
const { status } = await module.reactor.getJobStatus(created.id);
|
|
90
88
|
return status === JobStatus.READ_READY || status === JobStatus.FAILED;
|
|
91
89
|
},
|
|
92
|
-
//
|
|
90
|
+
// PGlite is several times slower on the Windows runner.
|
|
93
91
|
{ timeout: BOOT_TIMEOUT, interval: 5 },
|
|
94
92
|
);
|
|
95
93
|
expect((await module.reactor.getJobStatus(created.id)).status).toBe(
|
|
@@ -97,6 +95,8 @@ async function seedStore(dir: string): Promise<void> {
|
|
|
97
95
|
);
|
|
98
96
|
await module.reactor.kill().completed;
|
|
99
97
|
await db.destroy();
|
|
98
|
+
// PGlite leaves its lockfile behind after a clean close.
|
|
99
|
+
await rm(join(dir, "postmaster.pid"), { force: true });
|
|
100
100
|
}
|
|
101
101
|
|
|
102
102
|
// PGlite boots are several times slower on Windows runners.
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import { createDurableNodeFs } from "@powerhousedao/reactor-api/pglite-node";
|
|
2
|
+
import nodeFs, { promises as fs } from "node:fs";
|
|
3
|
+
import os from "node:os";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
import { afterEach, describe, expect, it } from "vitest";
|
|
6
|
+
import { loadNodeFsClass, loadPGliteModule } from "../src/pglite-version.js";
|
|
7
|
+
|
|
8
|
+
// A fresh initdb takes ~10 s under suite load; the Windows runner is slower.
|
|
9
|
+
const BOOT = 180_000;
|
|
10
|
+
const COMMITS = 10;
|
|
11
|
+
const RM = { recursive: true, force: true, maxRetries: 10 } as const;
|
|
12
|
+
|
|
13
|
+
describe("durable NodeFS over pglite-legacy-02 (PG16)", () => {
|
|
14
|
+
const tempDirs: string[] = [];
|
|
15
|
+
|
|
16
|
+
afterEach(async () => {
|
|
17
|
+
for (const dir of tempDirs.splice(0)) {
|
|
18
|
+
await fs.rm(dir, RM);
|
|
19
|
+
}
|
|
20
|
+
});
|
|
21
|
+
|
|
22
|
+
it(
|
|
23
|
+
"opens a PG16 dir over pglite-legacy-02 with the durable fs",
|
|
24
|
+
async () => {
|
|
25
|
+
const dir = await fs.mkdtemp(path.join(os.tmpdir(), "durable-pg16-"));
|
|
26
|
+
tempDirs.push(dir);
|
|
27
|
+
const counts = { fsync: 0, fdatasync: 0 };
|
|
28
|
+
const hostFs = {
|
|
29
|
+
fsyncSync: (fd: number) => {
|
|
30
|
+
counts.fsync++;
|
|
31
|
+
nodeFs.fsyncSync(fd);
|
|
32
|
+
},
|
|
33
|
+
fdatasyncSync: (fd: number) => {
|
|
34
|
+
counts.fdatasync++;
|
|
35
|
+
nodeFs.fdatasyncSync(fd);
|
|
36
|
+
},
|
|
37
|
+
};
|
|
38
|
+
const [{ PGlite }, NodeFS] = await Promise.all([
|
|
39
|
+
loadPGliteModule(16),
|
|
40
|
+
loadNodeFsClass(16),
|
|
41
|
+
]);
|
|
42
|
+
const pg = new PGlite({
|
|
43
|
+
fs: createDurableNodeFs(NodeFS, dir, {
|
|
44
|
+
hostFs,
|
|
45
|
+
maintenanceIntervalMs: 0,
|
|
46
|
+
}),
|
|
47
|
+
});
|
|
48
|
+
try {
|
|
49
|
+
await pg.waitReady;
|
|
50
|
+
const version = await pg.query<{ v: string }>(
|
|
51
|
+
"SELECT current_setting('server_version_num') AS v",
|
|
52
|
+
);
|
|
53
|
+
expect(version.rows[0].v.startsWith("16")).toBe(true);
|
|
54
|
+
await pg.exec("CREATE TABLE t (v int)");
|
|
55
|
+
const before = counts.fdatasync;
|
|
56
|
+
for (let i = 0; i < COMMITS; i++) {
|
|
57
|
+
await pg.exec(`INSERT INTO t VALUES (${i})`);
|
|
58
|
+
}
|
|
59
|
+
const perCommit = counts.fdatasync - before;
|
|
60
|
+
const fsyncBefore = counts.fsync;
|
|
61
|
+
await pg.exec("CHECKPOINT");
|
|
62
|
+
const checkpointSyncs = counts.fsync - fsyncBefore;
|
|
63
|
+
console.info(
|
|
64
|
+
`pglite-legacy-02: ${perCommit} host fdatasync over ${COMMITS} commits, ${checkpointSyncs} host fsync on CHECKPOINT, ${fsyncBefore} host fsync since open`,
|
|
65
|
+
);
|
|
66
|
+
// Pins decision 5's outcome on 0.2.17: WAL goes through
|
|
67
|
+
// open_datasync, which never reaches the fdatasync hook, so commits
|
|
68
|
+
// are not synced; the stream-op patch does take (CHECKPOINT syncs).
|
|
69
|
+
expect(perCommit).toBe(0);
|
|
70
|
+
expect(checkpointSyncs).toBeGreaterThan(0);
|
|
71
|
+
} finally {
|
|
72
|
+
await pg.close();
|
|
73
|
+
}
|
|
74
|
+
expect(
|
|
75
|
+
(await fs.readFile(path.join(dir, "PG_VERSION"), "utf8")).trim(),
|
|
76
|
+
).toBe("16");
|
|
77
|
+
},
|
|
78
|
+
BOOT,
|
|
79
|
+
);
|
|
80
|
+
});
|
|
@@ -26,6 +26,34 @@ describe("ClosablePGliteDialect", () => {
|
|
|
26
26
|
expect(pglite.closed).toBe(true);
|
|
27
27
|
});
|
|
28
28
|
|
|
29
|
+
it("tolerates a close that rejects because the runtime aborted", async () => {
|
|
30
|
+
const pglite = new PGlite();
|
|
31
|
+
created.push(pglite);
|
|
32
|
+
const db = new Kysely({ dialect: new ClosablePGliteDialect(pglite) });
|
|
33
|
+
await sql`select 1`.execute(db);
|
|
34
|
+
const realClose = pglite.close.bind(pglite);
|
|
35
|
+
pglite.close = () => {
|
|
36
|
+
pglite.close = realClose;
|
|
37
|
+
return Promise.reject(new Error("PGlite aborted: ENOSPC"));
|
|
38
|
+
};
|
|
39
|
+
|
|
40
|
+
await expect(db.destroy()).resolves.toBeUndefined();
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
it("rethrows a close that fails for any other reason", async () => {
|
|
44
|
+
const pglite = new PGlite();
|
|
45
|
+
created.push(pglite);
|
|
46
|
+
const db = new Kysely({ dialect: new ClosablePGliteDialect(pglite) });
|
|
47
|
+
await sql`select 1`.execute(db);
|
|
48
|
+
const realClose = pglite.close.bind(pglite);
|
|
49
|
+
pglite.close = () => {
|
|
50
|
+
pglite.close = realClose;
|
|
51
|
+
return Promise.reject(new Error("disk gone"));
|
|
52
|
+
};
|
|
53
|
+
|
|
54
|
+
await expect(db.destroy()).rejects.toThrow("disk gone");
|
|
55
|
+
});
|
|
56
|
+
|
|
29
57
|
it("is idempotent if the PGlite is already closed", async () => {
|
|
30
58
|
const pglite = new PGlite();
|
|
31
59
|
created.push(pglite);
|
|
@@ -0,0 +1,313 @@
|
|
|
1
|
+
import { createDurableNodeFs } from "@powerhousedao/reactor-api/pglite-node";
|
|
2
|
+
import type { ILogger } from "document-model";
|
|
3
|
+
import { promises as fs } from "node:fs";
|
|
4
|
+
import os from "node:os";
|
|
5
|
+
import path from "node:path";
|
|
6
|
+
import {
|
|
7
|
+
afterAll,
|
|
8
|
+
afterEach,
|
|
9
|
+
beforeAll,
|
|
10
|
+
beforeEach,
|
|
11
|
+
describe,
|
|
12
|
+
expect,
|
|
13
|
+
it,
|
|
14
|
+
vi,
|
|
15
|
+
} from "vitest";
|
|
16
|
+
import { runPglitePreflight } from "../src/pglite-preflight.js";
|
|
17
|
+
import { loadNodeFsClass, loadPGliteModule } from "../src/pglite-version.js";
|
|
18
|
+
import { writeSnapshotFromDir } from "./snapshot-writer.js";
|
|
19
|
+
|
|
20
|
+
// PGlite initdb and pg_dump are several times slower on the Windows runner.
|
|
21
|
+
const BOOT_TIMEOUT = 120_000;
|
|
22
|
+
const RM = { recursive: true, force: true, maxRetries: 10 } as const;
|
|
23
|
+
const ROWS = 10;
|
|
24
|
+
const CONVERTED_LINE =
|
|
25
|
+
/^Converted PGlite snapshot at (.+): snapshot (\d+) bytes -> (\d+) bytes on disk, (\d+) entries, (\d+) ms$/;
|
|
26
|
+
|
|
27
|
+
type StubLogger = ILogger & {
|
|
28
|
+
debug: ReturnType<typeof vi.fn>;
|
|
29
|
+
info: ReturnType<typeof vi.fn>;
|
|
30
|
+
warn: ReturnType<typeof vi.fn>;
|
|
31
|
+
};
|
|
32
|
+
|
|
33
|
+
function stubLogger(): StubLogger {
|
|
34
|
+
const logger = {
|
|
35
|
+
verbose: vi.fn(),
|
|
36
|
+
debug: vi.fn(),
|
|
37
|
+
info: vi.fn(),
|
|
38
|
+
warn: vi.fn(),
|
|
39
|
+
error: vi.fn(),
|
|
40
|
+
child: vi.fn(),
|
|
41
|
+
};
|
|
42
|
+
logger.child.mockReturnValue(logger);
|
|
43
|
+
return logger as unknown as StubLogger;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
async function exists(p: string): Promise<boolean> {
|
|
47
|
+
try {
|
|
48
|
+
await fs.lstat(p);
|
|
49
|
+
return true;
|
|
50
|
+
} catch {
|
|
51
|
+
return false;
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
async function pgVersion(dir: string): Promise<string> {
|
|
56
|
+
return (await fs.readFile(path.join(dir, "PG_VERSION"), "utf8")).trim();
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/** Opens `dir` the way the server does and counts the seeded rows. */
|
|
60
|
+
async function countRows(dir: string, major: 16 | 17): Promise<number> {
|
|
61
|
+
const [{ PGlite: Pg }, Base] = await Promise.all([
|
|
62
|
+
loadPGliteModule(major),
|
|
63
|
+
loadNodeFsClass(major),
|
|
64
|
+
]);
|
|
65
|
+
const pg = new Pg({
|
|
66
|
+
fs: createDurableNodeFs(Base, dir, {
|
|
67
|
+
fsync: false,
|
|
68
|
+
maintenanceIntervalMs: 0,
|
|
69
|
+
}),
|
|
70
|
+
});
|
|
71
|
+
try {
|
|
72
|
+
await pg.waitReady;
|
|
73
|
+
const { rows } = await pg.query<{ n: number }>(
|
|
74
|
+
'SELECT count(*)::int AS n FROM "Rows"',
|
|
75
|
+
);
|
|
76
|
+
return rows[0].n;
|
|
77
|
+
} finally {
|
|
78
|
+
await pg.close();
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
function convertedLines(logger: StubLogger): string[] {
|
|
83
|
+
return logger.info.mock.calls
|
|
84
|
+
.map((call) => String(call[0]))
|
|
85
|
+
.filter((line) => CONVERTED_LINE.test(line));
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
describe("runPglitePreflight", () => {
|
|
89
|
+
// A pure-snapshot PG16 store: snapshot.bin and nothing else, the layout
|
|
90
|
+
// AtomicNodeFs left for a store it created itself.
|
|
91
|
+
let template: string;
|
|
92
|
+
let root: string;
|
|
93
|
+
let logger: StubLogger;
|
|
94
|
+
|
|
95
|
+
beforeAll(async () => {
|
|
96
|
+
template = await fs.mkdtemp(path.join(os.tmpdir(), "preflight-template-"));
|
|
97
|
+
const loose = path.join(template, "loose");
|
|
98
|
+
const { PGlite: LegacyPGlite } = await loadPGliteModule(16);
|
|
99
|
+
const LegacyNodeFS = await loadNodeFsClass(16);
|
|
100
|
+
const pg = new LegacyPGlite({ fs: new LegacyNodeFS(loose) });
|
|
101
|
+
await pg.waitReady;
|
|
102
|
+
await pg.exec('CREATE TABLE "Rows" (id int PRIMARY KEY, payload text)');
|
|
103
|
+
await pg.exec(
|
|
104
|
+
`INSERT INTO "Rows" SELECT g, repeat('x', 100) FROM generate_series(1, ${ROWS}) g`,
|
|
105
|
+
);
|
|
106
|
+
await pg.close();
|
|
107
|
+
await fs.rm(path.join(loose, "postmaster.pid"), { force: true });
|
|
108
|
+
expect(await pgVersion(loose)).toBe("16");
|
|
109
|
+
|
|
110
|
+
const store = path.join(template, "store");
|
|
111
|
+
await fs.mkdir(store);
|
|
112
|
+
await writeSnapshotFromDir(loose, path.join(store, "snapshot.bin"));
|
|
113
|
+
await fs.rm(loose, RM);
|
|
114
|
+
}, BOOT_TIMEOUT);
|
|
115
|
+
|
|
116
|
+
afterAll(async () => {
|
|
117
|
+
await fs.rm(template, RM);
|
|
118
|
+
}, BOOT_TIMEOUT);
|
|
119
|
+
|
|
120
|
+
beforeEach(async () => {
|
|
121
|
+
root = await fs.mkdtemp(path.join(os.tmpdir(), "preflight-"));
|
|
122
|
+
logger = stubLogger();
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
afterEach(async () => {
|
|
126
|
+
await fs.rm(root, RM);
|
|
127
|
+
}, BOOT_TIMEOUT);
|
|
128
|
+
|
|
129
|
+
async function snapshotDir(name: string): Promise<string> {
|
|
130
|
+
const dir = path.join(root, name);
|
|
131
|
+
await fs.cp(path.join(template, "store"), dir, { recursive: true });
|
|
132
|
+
return dir;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
async function looseDir(name: string, major = 17): Promise<string> {
|
|
136
|
+
const dir = path.join(root, name);
|
|
137
|
+
await fs.mkdir(path.join(dir, "pg_wal"), { recursive: true });
|
|
138
|
+
await fs.writeFile(path.join(dir, "PG_VERSION"), `${major}\n`);
|
|
139
|
+
return dir;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
it(
|
|
143
|
+
"converts before detecting the major, so a PG16 snapshot is migrated",
|
|
144
|
+
async () => {
|
|
145
|
+
const dir = await snapshotDir("reactor-storage");
|
|
146
|
+
|
|
147
|
+
const detected = await runPglitePreflight({
|
|
148
|
+
dirs: [dir],
|
|
149
|
+
migratePglite: true,
|
|
150
|
+
inMemory: false,
|
|
151
|
+
logger,
|
|
152
|
+
});
|
|
153
|
+
|
|
154
|
+
expect(detected.get(dir)).toBe(17);
|
|
155
|
+
expect(await pgVersion(dir)).toBe("17");
|
|
156
|
+
expect(await exists(path.join(dir, "snapshot.bin"))).toBe(false);
|
|
157
|
+
expect(await exists(`${dir}.converting`)).toBe(false);
|
|
158
|
+
expect(await exists(`${dir}.old`)).toBe(false);
|
|
159
|
+
expect(convertedLines(logger)).toHaveLength(1);
|
|
160
|
+
expect(logger.info).toHaveBeenCalledWith(
|
|
161
|
+
expect.stringContaining("Migrating"),
|
|
162
|
+
);
|
|
163
|
+
expect(await countRows(dir, 17)).toBe(ROWS);
|
|
164
|
+
},
|
|
165
|
+
BOOT_TIMEOUT,
|
|
166
|
+
);
|
|
167
|
+
|
|
168
|
+
// Pins the order alone: before this change a pure-snapshot dir had no loose
|
|
169
|
+
// PG_VERSION and the migration logged "No PG_VERSION; skipping". The case
|
|
170
|
+
// above also needs the legacy pg_dump to work.
|
|
171
|
+
it(
|
|
172
|
+
"attempts the migration of a PG16 snapshot instead of skipping it",
|
|
173
|
+
async () => {
|
|
174
|
+
const dir = await snapshotDir("reactor-storage");
|
|
175
|
+
|
|
176
|
+
await runPglitePreflight({
|
|
177
|
+
dirs: [dir],
|
|
178
|
+
migratePglite: true,
|
|
179
|
+
inMemory: false,
|
|
180
|
+
logger,
|
|
181
|
+
}).catch(() => undefined);
|
|
182
|
+
|
|
183
|
+
expect(convertedLines(logger)).toHaveLength(1);
|
|
184
|
+
expect(logger.info).not.toHaveBeenCalledWith(
|
|
185
|
+
expect.stringContaining("No PG_VERSION"),
|
|
186
|
+
);
|
|
187
|
+
expect(logger.info).toHaveBeenCalledWith(
|
|
188
|
+
expect.stringContaining(`Migrating ${dir} from PG16 to PG17`),
|
|
189
|
+
);
|
|
190
|
+
},
|
|
191
|
+
BOOT_TIMEOUT,
|
|
192
|
+
);
|
|
193
|
+
|
|
194
|
+
it(
|
|
195
|
+
"boots a converted dir on the second start without converting",
|
|
196
|
+
async () => {
|
|
197
|
+
const dir = await snapshotDir("reactor-storage");
|
|
198
|
+
const first = { dirs: [dir], inMemory: false, logger };
|
|
199
|
+
await runPglitePreflight(first);
|
|
200
|
+
expect(convertedLines(logger)).toHaveLength(1);
|
|
201
|
+
expect(await pgVersion(dir)).toBe("16");
|
|
202
|
+
|
|
203
|
+
const second = stubLogger();
|
|
204
|
+
const detected = await runPglitePreflight({ ...first, logger: second });
|
|
205
|
+
|
|
206
|
+
expect(detected.get(dir)).toBe(16);
|
|
207
|
+
expect(convertedLines(second)).toHaveLength(0);
|
|
208
|
+
expect(await exists(`${dir}.converting`)).toBe(false);
|
|
209
|
+
expect(await exists(`${dir}.old`)).toBe(false);
|
|
210
|
+
expect(await countRows(dir, 16)).toBe(ROWS);
|
|
211
|
+
},
|
|
212
|
+
BOOT_TIMEOUT,
|
|
213
|
+
);
|
|
214
|
+
|
|
215
|
+
it("removes pg_wal/xlogtemp.*", async () => {
|
|
216
|
+
const dir = await looseDir("read-storage");
|
|
217
|
+
const segment = path.join(dir, "pg_wal", "000000010000000000000001");
|
|
218
|
+
await fs.writeFile(segment, "wal");
|
|
219
|
+
await fs.writeFile(path.join(dir, "pg_wal", "xlogtemp.42"), "");
|
|
220
|
+
await fs.writeFile(path.join(dir, "pg_wal", "xlogtemp.7"), "");
|
|
221
|
+
|
|
222
|
+
const detected = await runPglitePreflight({
|
|
223
|
+
dirs: [dir],
|
|
224
|
+
inMemory: false,
|
|
225
|
+
logger,
|
|
226
|
+
});
|
|
227
|
+
|
|
228
|
+
expect(detected.get(dir)).toBe(17);
|
|
229
|
+
expect(await fs.readdir(path.join(dir, "pg_wal"))).toEqual([
|
|
230
|
+
"000000010000000000000001",
|
|
231
|
+
]);
|
|
232
|
+
expect(await exists(segment)).toBe(true);
|
|
233
|
+
});
|
|
234
|
+
|
|
235
|
+
it("removes postmaster.pid without a warn call", async () => {
|
|
236
|
+
const dir = await looseDir("read-storage");
|
|
237
|
+
const lockfile = path.join(dir, "postmaster.pid");
|
|
238
|
+
await fs.writeFile(lockfile, "42\n");
|
|
239
|
+
|
|
240
|
+
await runPglitePreflight({ dirs: [dir], inMemory: false, logger });
|
|
241
|
+
|
|
242
|
+
expect(await exists(lockfile)).toBe(false);
|
|
243
|
+
expect(logger.warn).not.toHaveBeenCalled();
|
|
244
|
+
expect(logger.debug).toHaveBeenCalledWith(
|
|
245
|
+
expect.stringContaining(lockfile),
|
|
246
|
+
);
|
|
247
|
+
});
|
|
248
|
+
|
|
249
|
+
it(
|
|
250
|
+
"logs one info line per converted dir with sizes and duration",
|
|
251
|
+
async () => {
|
|
252
|
+
const reactor = await snapshotDir("reactor-storage");
|
|
253
|
+
const readModel = await snapshotDir("read-storage");
|
|
254
|
+
|
|
255
|
+
await runPglitePreflight({
|
|
256
|
+
dirs: [reactor, readModel],
|
|
257
|
+
inMemory: false,
|
|
258
|
+
logger,
|
|
259
|
+
});
|
|
260
|
+
|
|
261
|
+
const lines = convertedLines(logger);
|
|
262
|
+
expect(lines).toHaveLength(2);
|
|
263
|
+
const byDir = new Map(
|
|
264
|
+
lines.map((line) => {
|
|
265
|
+
const match = CONVERTED_LINE.exec(line)!;
|
|
266
|
+
return [match[1], match.slice(2).map(Number)] as const;
|
|
267
|
+
}),
|
|
268
|
+
);
|
|
269
|
+
for (const dir of [reactor, readModel]) {
|
|
270
|
+
const [snapshotBytes, convertedBytes, entries] = byDir.get(dir)!;
|
|
271
|
+
expect(snapshotBytes).toBeGreaterThan(0);
|
|
272
|
+
expect(convertedBytes).toBeGreaterThan(0);
|
|
273
|
+
expect(entries).toBeGreaterThan(0);
|
|
274
|
+
}
|
|
275
|
+
},
|
|
276
|
+
BOOT_TIMEOUT,
|
|
277
|
+
);
|
|
278
|
+
|
|
279
|
+
it("touches nothing in memory", async () => {
|
|
280
|
+
const dir = await snapshotDir("reactor-storage");
|
|
281
|
+
|
|
282
|
+
const detected = await runPglitePreflight({
|
|
283
|
+
dirs: [dir],
|
|
284
|
+
inMemory: true,
|
|
285
|
+
logger,
|
|
286
|
+
});
|
|
287
|
+
|
|
288
|
+
expect(detected.size).toBe(0);
|
|
289
|
+
expect(await exists(path.join(dir, "snapshot.bin"))).toBe(true);
|
|
290
|
+
});
|
|
291
|
+
|
|
292
|
+
it("wipes the dirs and their conversion siblings under forcePgVersion", async () => {
|
|
293
|
+
const dir = await snapshotDir("reactor-storage");
|
|
294
|
+
await fs.mkdir(`${dir}.old`);
|
|
295
|
+
await fs.mkdir(`${dir}.converting`);
|
|
296
|
+
|
|
297
|
+
const detected = await runPglitePreflight({
|
|
298
|
+
dirs: [dir],
|
|
299
|
+
forcePgVersion: 17,
|
|
300
|
+
migratePglite: true,
|
|
301
|
+
inMemory: false,
|
|
302
|
+
logger,
|
|
303
|
+
});
|
|
304
|
+
|
|
305
|
+
expect(detected.size).toBe(0);
|
|
306
|
+
expect(await exists(dir)).toBe(false);
|
|
307
|
+
expect(await exists(`${dir}.old`)).toBe(false);
|
|
308
|
+
expect(await exists(`${dir}.converting`)).toBe(false);
|
|
309
|
+
expect(logger.warn).toHaveBeenCalledWith(
|
|
310
|
+
expect.stringContaining("ignoring --migrate-pglite"),
|
|
311
|
+
);
|
|
312
|
+
});
|
|
313
|
+
});
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { describe, expect, it } from "vitest";
|
|
2
|
-
import { parseForcePgVersion } from "../src/pglite-version.js";
|
|
2
|
+
import { openForVerify, parseForcePgVersion } from "../src/pglite-version.js";
|
|
3
3
|
|
|
4
4
|
describe("parseForcePgVersion", () => {
|
|
5
5
|
it("returns null for undefined", () => {
|
|
@@ -35,3 +35,11 @@ describe("parseForcePgVersion", () => {
|
|
|
35
35
|
expect(() => parseForcePgVersion("16.5")).toThrow(/PH_FORCE_PG_VERSION/);
|
|
36
36
|
});
|
|
37
37
|
});
|
|
38
|
+
|
|
39
|
+
describe("openForVerify", () => {
|
|
40
|
+
it("throws on an unsupported major before touching disk", async () => {
|
|
41
|
+
await expect(openForVerify(15, "/nonexistent/pglite-dir")).rejects.toThrow(
|
|
42
|
+
/PG_VERSION=15 is not supported/,
|
|
43
|
+
);
|
|
44
|
+
});
|
|
45
|
+
});
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
// Test-only v1 snapshot.bin writer over a loose on-disk tree.
|
|
2
|
+
import { promises as fs } from "node:fs";
|
|
3
|
+
import type { FileHandle } from "node:fs/promises";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
|
|
6
|
+
const MAGIC = new Uint8Array([0x50, 0x47, 0x4c, 0x41]);
|
|
7
|
+
const FORMAT_VERSION = 1;
|
|
8
|
+
const HEADER_SIZE = 12;
|
|
9
|
+
const ENTRY_PREFIX_SIZE = 9;
|
|
10
|
+
const DEFAULT_SKIP = ["snapshot.bin", "snapshot.bin.tmp"];
|
|
11
|
+
|
|
12
|
+
export interface SnapshotEntry {
|
|
13
|
+
type: 0 | 1;
|
|
14
|
+
mode: number;
|
|
15
|
+
relPath: string;
|
|
16
|
+
data?: Buffer;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export interface WriteSnapshotOptions {
|
|
20
|
+
/** Root-level names to leave out. Defaults to the snapshot files. */
|
|
21
|
+
skip?: string[];
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export async function collectDiskEntries(
|
|
25
|
+
dataDir: string,
|
|
26
|
+
options: WriteSnapshotOptions = {},
|
|
27
|
+
): Promise<SnapshotEntry[]> {
|
|
28
|
+
const skip = new Set(options.skip ?? DEFAULT_SKIP);
|
|
29
|
+
const entries: SnapshotEntry[] = [];
|
|
30
|
+
const walk = async (dir: string, rel: string) => {
|
|
31
|
+
const names = (await fs.readdir(dir)).sort();
|
|
32
|
+
for (const name of names) {
|
|
33
|
+
if (rel === "" && skip.has(name)) continue;
|
|
34
|
+
const full = path.join(dir, name);
|
|
35
|
+
const relPath = rel === "" ? name : `${rel}/${name}`;
|
|
36
|
+
const stat = await fs.lstat(full);
|
|
37
|
+
if (stat.isDirectory()) {
|
|
38
|
+
entries.push({ type: 0, mode: stat.mode & 0o7777, relPath });
|
|
39
|
+
await walk(full, relPath);
|
|
40
|
+
} else if (stat.isFile()) {
|
|
41
|
+
entries.push({
|
|
42
|
+
type: 1,
|
|
43
|
+
mode: stat.mode & 0o7777,
|
|
44
|
+
relPath,
|
|
45
|
+
data: await fs.readFile(full),
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
};
|
|
50
|
+
await walk(dataDir, "");
|
|
51
|
+
return entries;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export function encodeSnapshot(entries: SnapshotEntry[]): Buffer {
|
|
55
|
+
const encoder = new TextEncoder();
|
|
56
|
+
const parts: Buffer[] = [];
|
|
57
|
+
const header = Buffer.alloc(HEADER_SIZE);
|
|
58
|
+
header.set(MAGIC, 0);
|
|
59
|
+
header.writeUInt32LE(FORMAT_VERSION, 4);
|
|
60
|
+
header.writeUInt32LE(entries.length, 8);
|
|
61
|
+
parts.push(header);
|
|
62
|
+
for (const entry of entries) {
|
|
63
|
+
const pathBytes = encoder.encode(entry.relPath);
|
|
64
|
+
const dataLen = entry.data?.byteLength ?? 0;
|
|
65
|
+
const prefix = Buffer.alloc(ENTRY_PREFIX_SIZE + pathBytes.byteLength + 4);
|
|
66
|
+
prefix.writeUInt8(entry.type, 0);
|
|
67
|
+
prefix.writeUInt32LE(entry.mode, 1);
|
|
68
|
+
prefix.writeUInt32LE(pathBytes.byteLength, 5);
|
|
69
|
+
prefix.set(pathBytes, ENTRY_PREFIX_SIZE);
|
|
70
|
+
prefix.writeUInt32LE(dataLen, ENTRY_PREFIX_SIZE + pathBytes.byteLength);
|
|
71
|
+
parts.push(prefix);
|
|
72
|
+
if (entry.data && dataLen > 0) parts.push(entry.data);
|
|
73
|
+
}
|
|
74
|
+
return Buffer.concat(parts);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** Serializes `dataDir` into `snapshotPath`. Returns the entry count and size. */
|
|
78
|
+
export async function writeSnapshotFromDir(
|
|
79
|
+
dataDir: string,
|
|
80
|
+
snapshotPath: string,
|
|
81
|
+
options: WriteSnapshotOptions = {},
|
|
82
|
+
): Promise<{ entries: number; bytes: number }> {
|
|
83
|
+
const entries = await collectDiskEntries(dataDir, options);
|
|
84
|
+
const bytes = encodeSnapshot(entries);
|
|
85
|
+
let fh: FileHandle | undefined;
|
|
86
|
+
try {
|
|
87
|
+
fh = await fs.open(snapshotPath, "w");
|
|
88
|
+
await fh.writeFile(bytes);
|
|
89
|
+
await fh.sync();
|
|
90
|
+
} finally {
|
|
91
|
+
await fh?.close();
|
|
92
|
+
}
|
|
93
|
+
return { entries: entries.length, bytes: bytes.byteLength };
|
|
94
|
+
}
|