@shivaedev/sql 0.25.0 → 0.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/README.md +28 -7
- package/dist/PostgresListenerError.d.ts +10 -0
- package/dist/PostgresListenerError.d.ts.map +1 -0
- package/dist/PostgresListenerError.js +7 -0
- package/dist/PostgresListenerError.js.map +1 -0
- package/dist/SchemaComponent.d.ts +5 -5
- package/dist/SchemaComponent.d.ts.map +1 -1
- package/dist/SchemaComponent.js +1 -2
- package/dist/SchemaComponent.js.map +1 -1
- package/dist/WorkerRelease.d.ts +7 -8
- package/dist/WorkerRelease.d.ts.map +1 -1
- package/dist/WorkerRelease.js +10 -4
- package/dist/WorkerRelease.js.map +1 -1
- package/dist/adapters/makePostgresClient.d.ts +14 -0
- package/dist/adapters/makePostgresClient.d.ts.map +1 -0
- package/dist/adapters/makePostgresClient.js +63 -0
- package/dist/adapters/makePostgresClient.js.map +1 -0
- package/dist/adapters/postgresStream.d.ts +4 -0
- package/dist/adapters/postgresStream.d.ts.map +1 -0
- package/dist/adapters/postgresStream.js +20 -0
- package/dist/adapters/postgresStream.js.map +1 -0
- package/dist/assertPostgresMigrations.d.ts +1 -1
- package/dist/assertPostgresMigrations.d.ts.map +1 -1
- package/dist/assertPostgresMigrations.js.map +1 -1
- package/dist/coordinateWorkerRelease.d.ts +3 -3
- package/dist/coordinateWorkerRelease.d.ts.map +1 -1
- package/dist/coordinateWorkerRelease.js +14 -5
- package/dist/coordinateWorkerRelease.js.map +1 -1
- package/dist/databaseAddress.d.ts +4 -0
- package/dist/databaseAddress.d.ts.map +1 -0
- package/dist/databaseAddress.js +25 -0
- package/dist/databaseAddress.js.map +1 -0
- package/dist/internal/CommandRecovery.d.ts +3 -0
- package/dist/internal/CommandRecovery.d.ts.map +1 -0
- package/dist/internal/CommandRecovery.js +5 -0
- package/dist/internal/CommandRecovery.js.map +1 -0
- package/dist/listenForChanges.d.ts +3 -1
- package/dist/listenForChanges.d.ts.map +1 -1
- package/dist/listenForChanges.js +31 -34
- package/dist/listenForChanges.js.map +1 -1
- package/dist/migrations.d.ts +5 -4
- package/dist/migrations.d.ts.map +1 -1
- package/dist/migrations.js +7 -3
- package/dist/migrations.js.map +1 -1
- package/dist/notifyPayload.d.ts +1 -0
- package/dist/notifyPayload.d.ts.map +1 -1
- package/dist/notifyPayload.js +2 -0
- package/dist/notifyPayload.js.map +1 -1
- package/dist/pendingPostgresMigrations.d.ts +4 -3
- package/dist/pendingPostgresMigrations.d.ts.map +1 -1
- package/dist/pendingPostgresMigrations.js +8 -4
- package/dist/pendingPostgresMigrations.js.map +1 -1
- package/dist/postgresChanges.d.ts +3 -1
- package/dist/postgresChanges.d.ts.map +1 -1
- package/dist/postgresChanges.js +4 -2
- package/dist/postgresChanges.js.map +1 -1
- package/dist/postgresLayer.d.ts +11 -0
- package/dist/postgresLayer.d.ts.map +1 -0
- package/dist/postgresLayer.js +19 -0
- package/dist/postgresLayer.js.map +1 -0
- package/dist/postgresRelease.d.ts +9 -6
- package/dist/postgresRelease.d.ts.map +1 -1
- package/dist/postgresRelease.js +3 -3
- package/dist/postgresRelease.js.map +1 -1
- package/dist/releaseWorkers.d.ts +2 -2
- package/dist/releaseWorkers.d.ts.map +1 -1
- package/dist/releaseWorkers.js +3 -3
- package/dist/releaseWorkers.js.map +1 -1
- package/dist/testing.d.ts +2 -2
- package/dist/testing.d.ts.map +1 -1
- package/dist/testing.js +2 -2
- package/dist/testing.js.map +1 -1
- package/package.json +12 -7
- package/src/PostgresListenerError.ts +9 -0
- package/src/SchemaComponent.ts +6 -6
- package/src/WorkerRelease.ts +11 -9
- package/src/adapters/makePostgresClient.ts +72 -0
- package/src/adapters/postgresStream.ts +26 -0
- package/src/assertPostgresMigrations.ts +2 -2
- package/src/coordinateWorkerRelease.ts +31 -5
- package/src/databaseAddress.ts +25 -0
- package/src/internal/CommandRecovery.ts +5 -0
- package/src/listenForChanges.ts +41 -39
- package/src/migrations.ts +14 -7
- package/src/notifyPayload.ts +2 -0
- package/src/pendingPostgresMigrations.ts +16 -7
- package/src/postgresChanges.ts +10 -2
- package/src/postgresLayer.ts +27 -0
- package/src/postgresRelease.ts +9 -5
- package/src/releaseWorkers.ts +5 -3
- package/src/testing.ts +4 -8
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@shivaedev/sql",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.26.0",
|
|
4
4
|
"description": "Schema-derived repositories over native Effect SQL",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"effect"
|
|
@@ -43,25 +43,30 @@
|
|
|
43
43
|
"README.md"
|
|
44
44
|
],
|
|
45
45
|
"dependencies": {
|
|
46
|
-
"@shivaedev/changes": "0.5.1"
|
|
46
|
+
"@shivaedev/changes": "0.5.1",
|
|
47
|
+
"@shivaedev/service": "0.6.0"
|
|
47
48
|
},
|
|
48
49
|
"devDependencies": {
|
|
49
50
|
"@effect/platform-node": "4.0.6",
|
|
50
51
|
"@effect/sql-pg": "4.0.6",
|
|
51
52
|
"@effect/sql-sqlite-node": "4.0.6",
|
|
52
|
-
"@shivaedev/quality": "0.
|
|
53
|
-
"@shivaedev/
|
|
54
|
-
"@shivaedev/test": "0.
|
|
55
|
-
"@shivaedev/test-sql": "0.11.0",
|
|
56
|
-
"@shivaedev/test-story": "0.24.0",
|
|
53
|
+
"@shivaedev/quality": "0.46.0",
|
|
54
|
+
"@shivaedev/test-sql": "0.12.0",
|
|
55
|
+
"@shivaedev/test-story": "0.25.0",
|
|
57
56
|
"@types/node": "24.19.2",
|
|
58
57
|
"effect": "4.0.6",
|
|
59
58
|
"typescript": "7.0.2",
|
|
60
59
|
"vitest": "5.0.5"
|
|
61
60
|
},
|
|
62
61
|
"peerDependencies": {
|
|
62
|
+
"@effect/sql-pg": "4.0.6",
|
|
63
63
|
"effect": "4.0.6"
|
|
64
64
|
},
|
|
65
|
+
"peerDependenciesMeta": {
|
|
66
|
+
"@effect/sql-pg": {
|
|
67
|
+
"optional": true
|
|
68
|
+
}
|
|
69
|
+
},
|
|
65
70
|
"engines": {
|
|
66
71
|
"node": ">=24"
|
|
67
72
|
},
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { Data } from "effect";
|
|
2
|
+
|
|
3
|
+
export class PostgresListenerError extends Data.TaggedError("PostgresListenerError")<{
|
|
4
|
+
readonly connectionSetting: string;
|
|
5
|
+
}> {
|
|
6
|
+
override get message() {
|
|
7
|
+
return `PostgreSQL change listener heartbeat timed out without receiving its own notification. Set ${this.connectionSetting} to a direct or session-mode connection to the application database.`;
|
|
8
|
+
}
|
|
9
|
+
}
|
package/src/SchemaComponent.ts
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
import { Effect } from "effect";
|
|
2
|
-
import {
|
|
2
|
+
import type { SqlClient } from "effect/sql";
|
|
3
3
|
|
|
4
|
-
export interface SchemaComponent<R = never> {
|
|
5
|
-
readonly apply: Effect.Effect<unknown,
|
|
4
|
+
export interface SchemaComponent<R = never, E = unknown> {
|
|
5
|
+
readonly apply: Effect.Effect<unknown, E, R | SqlClient.SqlClient>;
|
|
6
6
|
readonly name: string;
|
|
7
|
-
readonly pending: Effect.Effect<string | undefined,
|
|
7
|
+
readonly pending: Effect.Effect<string | undefined, E, R | SqlClient.SqlClient>;
|
|
8
8
|
}
|
|
9
9
|
|
|
10
|
-
export function runSchemaComponent<A, R>(name: string, effect: Effect.Effect<A,
|
|
11
|
-
return effect.pipe(Effect.
|
|
10
|
+
export function runSchemaComponent<A, E, R>(name: string, effect: Effect.Effect<A, E, R>) {
|
|
11
|
+
return effect.pipe(Effect.tapCause((cause) => Effect.logError(`Schema component ${name} failed.`, cause)));
|
|
12
12
|
}
|
package/src/WorkerRelease.ts
CHANGED
|
@@ -1,11 +1,6 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
WorkerRelease,
|
|
5
|
-
{
|
|
6
|
-
readonly register: <E>(control: (active: boolean) => Effect.Effect<void, E>) => Effect.Effect<void, E, Scope.Scope>;
|
|
7
|
-
}
|
|
8
|
-
>()("@shivaedev/sql/WorkerRelease") {}
|
|
1
|
+
import { Effect, Semaphore } from "effect";
|
|
2
|
+
import { defineService } from "@shivaedev/service/defineService.ts";
|
|
3
|
+
import { genericMethod } from "@shivaedev/service/genericMethod.ts";
|
|
9
4
|
|
|
10
5
|
export const makeWorkerRelease = Effect.fn("EffectSql.makeWorkerRelease")(function* () {
|
|
11
6
|
const mutex = Semaphore.makeUnsafe(1);
|
|
@@ -23,7 +18,7 @@ export const makeWorkerRelease = Effect.fn("EffectSql.makeWorkerRelease")(functi
|
|
|
23
18
|
return Effect.acquireRelease(locked(attach), () => locked(detach));
|
|
24
19
|
}
|
|
25
20
|
return {
|
|
26
|
-
service:
|
|
21
|
+
service: { register },
|
|
27
22
|
setActive: (next: boolean) =>
|
|
28
23
|
locked(
|
|
29
24
|
Effect.gen(function* () {
|
|
@@ -36,3 +31,10 @@ export const makeWorkerRelease = Effect.fn("EffectSql.makeWorkerRelease")(functi
|
|
|
36
31
|
).pipe(Effect.uninterruptible),
|
|
37
32
|
};
|
|
38
33
|
});
|
|
34
|
+
|
|
35
|
+
export const WorkerRelease = defineService({
|
|
36
|
+
id: "@shivaedev/sql/WorkerRelease",
|
|
37
|
+
initialize: makeWorkerRelease(),
|
|
38
|
+
methods: ({ service }) => ({ register: genericMethod(service.register) }),
|
|
39
|
+
requires: [],
|
|
40
|
+
});
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import * as PgClient from "@effect/sql-pg/PgClient";
|
|
2
|
+
import type * as PgConnection from "@effect/sql-pg/PgConnection";
|
|
3
|
+
import * as PgPool from "@effect/sql-pg/PgPool";
|
|
4
|
+
import { Effect, Predicate, Stream } from "effect";
|
|
5
|
+
import { SqlClient, type SqlConnection, Statement } from "effect/sql";
|
|
6
|
+
import { SqlError, UnknownError } from "effect/sql/SqlError";
|
|
7
|
+
|
|
8
|
+
const json = Statement.custom<PgClient.PgCustom>("PgJson");
|
|
9
|
+
|
|
10
|
+
export const makePostgresClient = Effect.fn("Sql.makePostgresClient")(function* (config: PgClient.PgClientConfig) {
|
|
11
|
+
const pool = yield* PgPool.PgPool;
|
|
12
|
+
const acquirer = Effect.map(pool.get, postgresConnection);
|
|
13
|
+
const client = yield* SqlClient.make({
|
|
14
|
+
acquirer,
|
|
15
|
+
borrower: (use) => pool.use((connection) => use(postgresConnection(connection))),
|
|
16
|
+
commit,
|
|
17
|
+
compiler: PgClient.makeCompiler(config.transformQueryNames),
|
|
18
|
+
driver: "pg",
|
|
19
|
+
prepareTransactionControls: true,
|
|
20
|
+
releaseSavepoint: (name) => `RELEASE SAVEPOINT ${name}`,
|
|
21
|
+
spanAttributes: [
|
|
22
|
+
["db.system.name", "postgresql"],
|
|
23
|
+
["db.namespace", config.database ?? config.username ?? "postgres"],
|
|
24
|
+
["server.address", config.host ?? "localhost"],
|
|
25
|
+
["server.port", config.port ?? 5432],
|
|
26
|
+
],
|
|
27
|
+
transactionAcquirer: Effect.map(pool.reserve, postgresConnection),
|
|
28
|
+
transformRows: config.transformResultNames === undefined ? undefined : Statement.defaultTransforms(config.transformResultNames).array,
|
|
29
|
+
});
|
|
30
|
+
return Object.assign(client, {
|
|
31
|
+
[PgClient.TypeId]: PgClient.TypeId,
|
|
32
|
+
config,
|
|
33
|
+
json: (value: unknown) => Statement.fragment([json(value)]),
|
|
34
|
+
listen: (channel: string) => Effect.flatMap(pool.reserve, (connection) => connection.listen(channel)),
|
|
35
|
+
notify: (channel: string, payload: string) =>
|
|
36
|
+
pool.use((connection) => connection.query("SELECT pg_notify($1, $2)", [channel, payload])).pipe(Effect.asVoid),
|
|
37
|
+
}) satisfies PgClient.PgClient;
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
function postgresConnection(connection: PgConnection.PgConnection): SqlConnection.Connection {
|
|
41
|
+
function execute(sql: string, params: readonly unknown[], transformRows: Parameters<SqlConnection.Connection["execute"]>[2], prepare = true) {
|
|
42
|
+
const rows = Effect.map(connection.query(sql, params, prepare), (result) => result.rows);
|
|
43
|
+
return transformRows === undefined ? rows : Effect.map(rows, transformRows);
|
|
44
|
+
}
|
|
45
|
+
return {
|
|
46
|
+
execute,
|
|
47
|
+
executeRaw: (sql, params) => connection.query(sql, params),
|
|
48
|
+
executeStream: (sql, params, transformRows) => {
|
|
49
|
+
const stream = connection.stream(sql, params);
|
|
50
|
+
return transformRows === undefined ? stream : Stream.mapArray(stream, (rows) => [transformRows(rows)]).pipe(Stream.flattenIterable);
|
|
51
|
+
},
|
|
52
|
+
executeUnprepared: (sql, params, transformRows) => execute(sql, params, transformRows, false),
|
|
53
|
+
executeValues: (sql, params) => connection.queryValues(sql, params),
|
|
54
|
+
executeValuesUnprepared: (sql, params) => connection.queryValues(sql, params, false),
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function commit(connection: SqlConnection.Connection) {
|
|
59
|
+
return Effect.flatMap(connection.executeRaw("COMMIT", []), (result) =>
|
|
60
|
+
Predicate.hasProperty(result, "command") && result.command === "ROLLBACK"
|
|
61
|
+
? Effect.fail(
|
|
62
|
+
new SqlError({
|
|
63
|
+
reason: new UnknownError({
|
|
64
|
+
cause: new Error("COMMIT rolled back an aborted transaction"),
|
|
65
|
+
message: "PgClient: COMMIT rolled back an aborted transaction",
|
|
66
|
+
operation: "commit",
|
|
67
|
+
}),
|
|
68
|
+
}),
|
|
69
|
+
)
|
|
70
|
+
: Effect.void,
|
|
71
|
+
);
|
|
72
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { createConnection } from "node:net";
|
|
2
|
+
import type { PgPoolConfig } from "@effect/sql-pg/PgClient";
|
|
3
|
+
import { Effect, Redacted } from "effect";
|
|
4
|
+
|
|
5
|
+
export const postgresStream = Effect.fn("Sql.postgresStream")(function* (
|
|
6
|
+
url: Redacted.Redacted<string>,
|
|
7
|
+
address: Pick<PgPoolConfig, "host" | "port">,
|
|
8
|
+
) {
|
|
9
|
+
const disconnected = new Set<Promise<void>>();
|
|
10
|
+
// Native pool shutdown sends Terminate without waiting for its sockets to close.
|
|
11
|
+
yield* Effect.addFinalizer(() => Effect.promise(() => Promise.all([...disconnected])));
|
|
12
|
+
return () => {
|
|
13
|
+
const target = new URL(Redacted.value(url));
|
|
14
|
+
const host = address.host ?? target.searchParams.getAll("host").at(-1) ?? (decodeURIComponent(target.hostname) || "localhost");
|
|
15
|
+
const port = address.port ?? 5432;
|
|
16
|
+
const socket = createConnection(host.startsWith("/") ? { path: `${host}/.s.PGSQL.${port}` } : { host, noDelay: true, port });
|
|
17
|
+
const closed = new Promise<void>((resolve) =>
|
|
18
|
+
socket.once("close", () => {
|
|
19
|
+
disconnected.delete(closed);
|
|
20
|
+
resolve();
|
|
21
|
+
}),
|
|
22
|
+
);
|
|
23
|
+
disconnected.add(closed);
|
|
24
|
+
return socket;
|
|
25
|
+
};
|
|
26
|
+
});
|
|
@@ -3,8 +3,8 @@ import { Migrator } from "effect/sql";
|
|
|
3
3
|
import type { PostgresMigrationOptions } from "./migrations.ts";
|
|
4
4
|
import { type PendingPostgresMigrations, pendingPostgresMigrations } from "./pendingPostgresMigrations.ts";
|
|
5
5
|
|
|
6
|
-
export const assertPostgresMigrations = Effect.fn("EffectSql.assertPostgresMigrations")(function* <R>(
|
|
7
|
-
options: Pick<PostgresMigrationOptions<R>, "loader" | "table" | "components" | "release">,
|
|
6
|
+
export const assertPostgresMigrations = Effect.fn("EffectSql.assertPostgresMigrations")(function* <R, E = never>(
|
|
7
|
+
options: Pick<PostgresMigrationOptions<R, E>, "loader" | "table" | "components" | "release">,
|
|
8
8
|
) {
|
|
9
9
|
const pending = yield* pendingPostgresMigrations(options);
|
|
10
10
|
if (pending.migrations.length > 0 || pending.components.length > 0) {
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import { Duration, Effect } from "effect";
|
|
1
|
+
import { Duration, Effect, Exit, Schedule, Scope } from "effect";
|
|
2
|
+
import { isSqlError } from "effect/sql/SqlError";
|
|
2
3
|
import { readPostgresRelease } from "./postgresRelease.ts";
|
|
3
4
|
import { releaseWorkers } from "./releaseWorkers.ts";
|
|
4
5
|
import { makeWorkerRelease } from "./WorkerRelease.ts";
|
|
@@ -9,6 +10,8 @@ export const coordinateWorkerRelease = Effect.fn("EffectSql.coordinateWorkerRele
|
|
|
9
10
|
table = "effect_sql_migrations",
|
|
10
11
|
) {
|
|
11
12
|
const gate = yield* makeWorkerRelease();
|
|
13
|
+
const scope = yield* Scope.fork(yield* Effect.scope);
|
|
14
|
+
const retries = Schedule.spaced(Duration.toMillis(pollInterval) / 4).pipe(Schedule.upTo({ times: 3 }));
|
|
12
15
|
const rows = yield* releaseWorkers(release, Duration.toMillis(pollInterval) * 5, table);
|
|
13
16
|
let claimed = false;
|
|
14
17
|
const poll = Effect.gen(function* () {
|
|
@@ -24,10 +27,33 @@ export const coordinateWorkerRelease = Effect.fn("EffectSql.coordinateWorkerRele
|
|
|
24
27
|
}
|
|
25
28
|
}
|
|
26
29
|
});
|
|
27
|
-
|
|
30
|
+
function tick<A, E, R>(name: string, effect: Effect.Effect<A, E, R>) {
|
|
31
|
+
return effect.pipe(
|
|
32
|
+
Effect.tapCause((cause) => Effect.logWarning(`Worker release ${name} failed`, cause)),
|
|
33
|
+
Effect.retry({ schedule: retries, while: isSqlError }),
|
|
34
|
+
);
|
|
35
|
+
}
|
|
36
|
+
function loop<E, R>(name: string, effect: Effect.Effect<unknown, E, R>) {
|
|
37
|
+
return Effect.forever(Effect.sleep(pollInterval).pipe(Effect.andThen(tick(name, effect)))).pipe(
|
|
38
|
+
Effect.catchCause((cause) =>
|
|
39
|
+
Effect.logError(`Worker release ${name} stopped; closing worker scope`, cause).pipe(
|
|
40
|
+
Effect.andThen(Scope.close(scope, Exit.failCause(cause))),
|
|
41
|
+
),
|
|
42
|
+
),
|
|
43
|
+
Scope.provide(scope),
|
|
44
|
+
Effect.forkIn(scope),
|
|
45
|
+
);
|
|
46
|
+
}
|
|
47
|
+
yield* Scope.addFinalizer(
|
|
48
|
+
scope,
|
|
49
|
+
rows.remove.pipe(
|
|
50
|
+
Effect.tapCause((cause) => Effect.logError("Worker release cleanup failed", cause)),
|
|
51
|
+
Effect.orDie,
|
|
52
|
+
),
|
|
53
|
+
);
|
|
28
54
|
// Heartbeats continue while offWork waits for active jobs to finish.
|
|
29
|
-
yield*
|
|
30
|
-
yield* poll;
|
|
31
|
-
yield*
|
|
55
|
+
yield* loop("heartbeat", rows.heartbeat);
|
|
56
|
+
yield* tick("poll", poll).pipe(Scope.provide(scope));
|
|
57
|
+
yield* loop("poll", poll);
|
|
32
58
|
return gate.service;
|
|
33
59
|
});
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import type { PgPoolConfig } from "@effect/sql-pg/PgClient";
|
|
2
|
+
import { Redacted } from "effect";
|
|
3
|
+
|
|
4
|
+
export function databaseAddress(value: Redacted.Redacted<string>): Pick<PgPoolConfig, "host" | "port"> {
|
|
5
|
+
try {
|
|
6
|
+
const url = new URL(Redacted.value(value));
|
|
7
|
+
if (url.protocol !== "postgres:" && url.protocol !== "postgresql:") {
|
|
8
|
+
return {};
|
|
9
|
+
}
|
|
10
|
+
const hostname = url.hostname.startsWith("[") ? url.hostname.slice(1, -1) : decodeURIComponent(url.hostname);
|
|
11
|
+
// PgConnection applies query parameters in order, after the authority.
|
|
12
|
+
const host = url.searchParams.getAll("host").at(-1) ?? hostname;
|
|
13
|
+
const rawPort = url.searchParams.getAll("port").at(-1) ?? url.port;
|
|
14
|
+
const port = rawPort === "" ? undefined : Number(rawPort);
|
|
15
|
+
if (host.includes(",") || (port !== undefined && (!Number.isInteger(port) || port < 1 || port > 65_535))) {
|
|
16
|
+
return {};
|
|
17
|
+
}
|
|
18
|
+
return {
|
|
19
|
+
...(host === "" && !url.searchParams.has("host") ? {} : { host }),
|
|
20
|
+
...(port === undefined ? {} : { port }),
|
|
21
|
+
};
|
|
22
|
+
} catch {
|
|
23
|
+
return {};
|
|
24
|
+
}
|
|
25
|
+
}
|
package/src/listenForChanges.ts
CHANGED
|
@@ -1,44 +1,44 @@
|
|
|
1
|
-
import { Clock, Deferred, type Duration, Effect,
|
|
1
|
+
import { Clock, Deferred, type Duration, Effect, Queue, Random, Scope } from "effect";
|
|
2
2
|
import * as Reactivity from "effect/reactivity/Reactivity";
|
|
3
|
-
import {
|
|
3
|
+
import type { SqlError } from "effect/sql/SqlError";
|
|
4
|
+
import { decodeKeys, resyncPayload } from "./notifyPayload.ts";
|
|
5
|
+
import { PostgresListenerError } from "./PostgresListenerError.ts";
|
|
4
6
|
import type { PostgresChangesOptions } from "./postgresChanges.ts";
|
|
5
7
|
|
|
6
|
-
function
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
// Bound each finalizer so a stalled UNLISTEN cannot prevent the native connection release that follows it.
|
|
12
|
-
if (scope.state._tag === "Open") {
|
|
13
|
-
const state = scope.state;
|
|
14
|
-
if (state.finalizer !== undefined) {
|
|
15
|
-
state.finalizer = bounded(state.finalizer);
|
|
16
|
-
}
|
|
17
|
-
if (state.finalizers !== undefined) {
|
|
18
|
-
for (const [key, finalizer] of state.finalizers) {
|
|
19
|
-
state.finalizers.set(key, bounded(finalizer));
|
|
20
|
-
}
|
|
21
|
-
}
|
|
22
|
-
}
|
|
23
|
-
return Scope.closeUnsafe(scope, outcome) ?? Effect.void;
|
|
24
|
-
});
|
|
8
|
+
function listenerScope(timeout: Duration.Input) {
|
|
9
|
+
// Release the native connection alongside listener cleanup, even when a listener finalizer stalls.
|
|
10
|
+
return Effect.acquireRelease(Scope.make("parallel"), (scope, exit) =>
|
|
11
|
+
Effect.suspend(() => Scope.closeUnsafe(scope, exit) ?? Effect.void).pipe(Effect.interruptible, Effect.timeoutOption(timeout), Effect.asVoid),
|
|
12
|
+
);
|
|
25
13
|
}
|
|
26
14
|
|
|
27
15
|
export const listenForChanges = Effect.fn("PostgresChanges.listenForChanges")(function* (options: PostgresChangesOptions) {
|
|
28
|
-
const ready = yield* Deferred.make<void>();
|
|
16
|
+
const ready = yield* Deferred.make<void, SqlError | PostgresListenerError>();
|
|
29
17
|
const reactivity = yield* Reactivity.Reactivity;
|
|
30
18
|
const { client, channel, resync } = options;
|
|
31
19
|
const timeout = options.heartbeatTimeout ?? "10 seconds";
|
|
20
|
+
function deadline<A, E, R>(effect: Effect.Effect<A, E, R>) {
|
|
21
|
+
return effect.pipe(
|
|
22
|
+
Effect.timeoutOrElse({
|
|
23
|
+
duration: timeout,
|
|
24
|
+
orElse: () => Effect.fail(new PostgresListenerError({ connectionSetting: options.connectionSetting ?? "client" })),
|
|
25
|
+
}),
|
|
26
|
+
);
|
|
27
|
+
}
|
|
32
28
|
let lostAt: number | undefined;
|
|
33
29
|
const attempt = Effect.scoped(
|
|
34
30
|
Effect.gen(function* () {
|
|
35
|
-
const scope = yield*
|
|
36
|
-
const
|
|
31
|
+
const scope = yield* listenerScope(timeout);
|
|
32
|
+
const notifications = yield* (options.listenerClient ?? client).listen(channel).pipe(Scope.provide(scope), deadline);
|
|
33
|
+
const token = JSON.stringify({ heartbeat: `${yield* Random.nextInt}:${yield* Random.nextInt}` });
|
|
37
34
|
let ack = yield* Deferred.make<void>();
|
|
38
35
|
function receive(payload: string) {
|
|
39
36
|
if (payload === token) {
|
|
40
37
|
return Effect.asVoid(Deferred.succeed(ack, undefined));
|
|
41
38
|
}
|
|
39
|
+
if (payload === resyncPayload) {
|
|
40
|
+
return Effect.asVoid(resync);
|
|
41
|
+
}
|
|
42
42
|
if (!payload.startsWith("[")) {
|
|
43
43
|
return Effect.void;
|
|
44
44
|
}
|
|
@@ -47,17 +47,14 @@ export const listenForChanges = Effect.fn("PostgresChanges.listenForChanges")(fu
|
|
|
47
47
|
Effect.catchCause(() => resync),
|
|
48
48
|
);
|
|
49
49
|
}
|
|
50
|
-
const listening = Effect.
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
),
|
|
56
|
-
);
|
|
57
|
-
});
|
|
50
|
+
const listening = Effect.forever(
|
|
51
|
+
Effect.flatMap(Queue.takeAll(notifications), (batch) =>
|
|
52
|
+
Effect.forEach(batch, (notification) => receive(notification.payload), { discard: true }),
|
|
53
|
+
),
|
|
54
|
+
);
|
|
58
55
|
const send = Effect.asVoid(client`select pg_notify(${channel}, ${token})`);
|
|
59
56
|
const monitor = Effect.gen(function* () {
|
|
60
|
-
yield* Effect.
|
|
57
|
+
yield* Effect.andThen(send, Deferred.await(ack)).pipe(deadline);
|
|
61
58
|
const streamCount = yield* resync;
|
|
62
59
|
if (lostAt !== undefined) {
|
|
63
60
|
const downtimeMs = (yield* Clock.currentTimeMillis) - lostAt;
|
|
@@ -71,23 +68,28 @@ export const listenForChanges = Effect.fn("PostgresChanges.listenForChanges")(fu
|
|
|
71
68
|
Effect.gen(function* () {
|
|
72
69
|
yield* Effect.sleep(options.heartbeatInterval ?? "5 seconds");
|
|
73
70
|
ack = yield* Deferred.make<void>();
|
|
74
|
-
yield* Effect.andThen(send, Deferred.await(ack)).pipe(
|
|
71
|
+
yield* Effect.andThen(send, Deferred.await(ack)).pipe(deadline);
|
|
75
72
|
}),
|
|
76
73
|
);
|
|
77
74
|
});
|
|
78
|
-
yield* Effect.raceFirst(listening, monitor)
|
|
79
|
-
}),
|
|
75
|
+
yield* Effect.raceFirst(listening, monitor);
|
|
76
|
+
}).pipe(Effect.onExit(() => resync)),
|
|
80
77
|
);
|
|
81
78
|
yield* Effect.forever(
|
|
82
79
|
attempt.pipe(
|
|
83
|
-
Effect.catchCause(() =>
|
|
80
|
+
Effect.catchCause((cause) =>
|
|
84
81
|
Effect.gen(function* () {
|
|
82
|
+
if (!(yield* Deferred.isDone(ready))) {
|
|
83
|
+
yield* Effect.logError("PostgreSQL change listener startup failed", cause);
|
|
84
|
+
yield* Deferred.failCause(ready, cause);
|
|
85
|
+
return;
|
|
86
|
+
}
|
|
85
87
|
if (lostAt !== undefined) {
|
|
86
|
-
yield* Effect.logDebug("PostgreSQL change listener reconnect failed; retrying");
|
|
88
|
+
yield* Effect.logDebug("PostgreSQL change listener reconnect failed; retrying", cause);
|
|
87
89
|
return;
|
|
88
90
|
}
|
|
89
91
|
lostAt = yield* Clock.currentTimeMillis;
|
|
90
|
-
yield* Effect.logWarning("PostgreSQL change listener lost; resynchronizing live streams");
|
|
92
|
+
yield* Effect.logWarning("PostgreSQL change listener lost; resynchronizing live streams", cause);
|
|
91
93
|
}),
|
|
92
94
|
),
|
|
93
95
|
Effect.andThen(Effect.sleep(options.retryDelay ?? "1 second")),
|
package/src/migrations.ts
CHANGED
|
@@ -1,13 +1,14 @@
|
|
|
1
|
-
import { Duration, Effect, Option } from "effect";
|
|
1
|
+
import { Duration, Effect, Option, Schema } from "effect";
|
|
2
2
|
import * as Reactivity from "effect/reactivity/Reactivity";
|
|
3
3
|
import { Migrator, SqlClient } from "effect/sql";
|
|
4
|
+
import type { SqlError } from "effect/sql/SqlError";
|
|
4
5
|
import { advisoryXactLock } from "./advisoryXactLock.ts";
|
|
5
6
|
import { postgresRelease } from "./postgresRelease.ts";
|
|
6
7
|
import { runSchemaComponent, type SchemaComponent } from "./SchemaComponent.ts";
|
|
7
8
|
import { transact } from "./transact.ts";
|
|
8
9
|
|
|
9
|
-
export interface PostgresMigrationOptions<R> {
|
|
10
|
-
readonly components?: readonly SchemaComponent<R>[];
|
|
10
|
+
export interface PostgresMigrationOptions<R, E = never> {
|
|
11
|
+
readonly components?: readonly SchemaComponent<R, E>[];
|
|
11
12
|
readonly loader: Migrator.Loader<R>;
|
|
12
13
|
readonly lockTimeout: Duration.Input;
|
|
13
14
|
readonly release?: string;
|
|
@@ -16,13 +17,13 @@ export interface PostgresMigrationOptions<R> {
|
|
|
16
17
|
|
|
17
18
|
const migrate = Migrator.make({});
|
|
18
19
|
|
|
19
|
-
export const migratePostgres = Effect.fn("EffectSql.migratePostgres")(function* <R>({
|
|
20
|
+
export const migratePostgres = Effect.fn("EffectSql.migratePostgres")(function* <R, E = never>({
|
|
20
21
|
loader,
|
|
21
22
|
components = [],
|
|
22
23
|
table = "effect_sql_migrations",
|
|
23
24
|
lockTimeout,
|
|
24
25
|
release,
|
|
25
|
-
}: PostgresMigrationOptions<R>) {
|
|
26
|
+
}: PostgresMigrationOptions<R, E>) {
|
|
26
27
|
const sql = yield* SqlClient.SqlClient;
|
|
27
28
|
const available = yield* Effect.serviceOption(Reactivity.Reactivity);
|
|
28
29
|
const reactivity = Option.isSome(available) ? available.value : yield* Reactivity.make;
|
|
@@ -37,7 +38,9 @@ export const migratePostgres = Effect.fn("EffectSql.migratePostgres")(function*
|
|
|
37
38
|
name text not null
|
|
38
39
|
)`;
|
|
39
40
|
const current = [...(yield* loader)].sort(([left], [right]) => left - right);
|
|
40
|
-
const applied = yield* sql
|
|
41
|
+
const applied = yield* sql`select migration_id as id from ${sql(table)}`.withoutTransform.pipe(
|
|
42
|
+
Effect.flatMap(Schema.decodeUnknownEffect(Schema.Array(Schema.Struct({ id: Schema.Int })))),
|
|
43
|
+
);
|
|
41
44
|
const appliedIds = new Set(applied.map(({ id }) => id));
|
|
42
45
|
const latest = applied.reduce((max, { id }) => Math.max(max, id), 0);
|
|
43
46
|
const missing = current.filter(([id]) => id <= latest && !appliedIds.has(id));
|
|
@@ -51,7 +54,11 @@ export const migratePostgres = Effect.fn("EffectSql.migratePostgres")(function*
|
|
|
51
54
|
}
|
|
52
55
|
return yield* migrate({ loader: Effect.succeed(current), table });
|
|
53
56
|
}).pipe(transact({ onSqlError: (error) => error }), Effect.provideService(Reactivity.Reactivity, reactivity));
|
|
54
|
-
|
|
57
|
+
const schemaComponents: readonly SchemaComponent<R, E | SqlError | Schema.SchemaError>[] = [
|
|
58
|
+
...components,
|
|
59
|
+
...(release === undefined ? [] : [postgresRelease(release, table)]),
|
|
60
|
+
];
|
|
61
|
+
for (const component of schemaComponents) {
|
|
55
62
|
yield* runSchemaComponent(component.name, component.apply);
|
|
56
63
|
}
|
|
57
64
|
return appliedMigrations;
|
package/src/notifyPayload.ts
CHANGED
|
@@ -2,6 +2,8 @@ import { Effect, Schema } from "effect";
|
|
|
2
2
|
import { SqlError, UnknownError } from "effect/sql/SqlError";
|
|
3
3
|
|
|
4
4
|
export const NotificationKeys = Schema.Array(Schema.Union([Schema.String, Schema.Finite]));
|
|
5
|
+
// Older listeners reconcile unknown array entries instead of ignoring them.
|
|
6
|
+
export const resyncPayload = JSON.stringify([{ _tag: "Resync" }]);
|
|
5
7
|
export const decodeKeys = Schema.decodeUnknownEffect(Schema.fromJsonString(NotificationKeys));
|
|
6
8
|
const encoder = new TextEncoder();
|
|
7
9
|
|
|
@@ -1,30 +1,35 @@
|
|
|
1
|
-
import { Effect } from "effect";
|
|
1
|
+
import { Effect, Schema } from "effect";
|
|
2
2
|
import { SqlClient } from "effect/sql";
|
|
3
|
+
import type { SqlError } from "effect/sql/SqlError";
|
|
3
4
|
import type { PostgresMigrationOptions } from "./migrations.ts";
|
|
4
5
|
import { postgresRelease } from "./postgresRelease.ts";
|
|
5
|
-
import { runSchemaComponent } from "./SchemaComponent.ts";
|
|
6
|
+
import { runSchemaComponent, type SchemaComponent } from "./SchemaComponent.ts";
|
|
6
7
|
|
|
7
8
|
export interface PendingPostgresMigrations {
|
|
8
9
|
readonly components: readonly { readonly name: string; readonly detail: string }[];
|
|
9
10
|
readonly migrations: readonly { readonly id: number; readonly name: string }[];
|
|
10
11
|
}
|
|
11
12
|
|
|
12
|
-
export const pendingPostgresMigrations = Effect.fn("EffectSql.pendingPostgresMigrations")(function* <R>({
|
|
13
|
+
export const pendingPostgresMigrations = Effect.fn("EffectSql.pendingPostgresMigrations")(function* <R, E = never>({
|
|
13
14
|
loader,
|
|
14
15
|
components = [],
|
|
15
16
|
table = "effect_sql_migrations",
|
|
16
17
|
release,
|
|
17
|
-
}: Pick<PostgresMigrationOptions<R>, "loader" | "table" | "components" | "release">) {
|
|
18
|
+
}: Pick<PostgresMigrationOptions<R, E>, "loader" | "table" | "components" | "release">) {
|
|
18
19
|
const expected = yield* loader;
|
|
19
20
|
const migrations: { id: number; name: string }[] = [];
|
|
20
21
|
if (expected.length > 0) {
|
|
21
22
|
const sql = yield* SqlClient.SqlClient;
|
|
22
23
|
const identifier = sql`${sql(table)}`.compile()[0];
|
|
23
|
-
const [ledger] = yield* sql
|
|
24
|
+
const [ledger] = yield* sql`select to_regclass(${identifier})::text as name`.withoutTransform.pipe(
|
|
25
|
+
Effect.flatMap(Schema.decodeUnknownEffect(Schema.Array(Schema.Struct({ name: Schema.NullOr(Schema.String) })))),
|
|
26
|
+
);
|
|
24
27
|
const applied =
|
|
25
28
|
ledger === undefined || ledger.name === null
|
|
26
29
|
? []
|
|
27
|
-
: yield* sql
|
|
30
|
+
: yield* sql`select migration_id as id, name from ${sql(table)}`.withoutTransform.pipe(
|
|
31
|
+
Effect.flatMap(Schema.decodeUnknownEffect(Schema.Array(Schema.Struct({ id: Schema.Int, name: Schema.String })))),
|
|
32
|
+
);
|
|
28
33
|
for (const [id, name] of expected) {
|
|
29
34
|
if (!applied.some((row) => row.id === id && row.name === name)) {
|
|
30
35
|
migrations.push({ id, name });
|
|
@@ -32,7 +37,11 @@ export const pendingPostgresMigrations = Effect.fn("EffectSql.pendingPostgresMig
|
|
|
32
37
|
}
|
|
33
38
|
}
|
|
34
39
|
const pendingComponents: { name: string; detail: string }[] = [];
|
|
35
|
-
|
|
40
|
+
const schemaComponents: readonly SchemaComponent<R, E | SqlError | Schema.SchemaError>[] = [
|
|
41
|
+
...components,
|
|
42
|
+
...(release === undefined ? [] : [postgresRelease(release, table)]),
|
|
43
|
+
];
|
|
44
|
+
for (const component of schemaComponents) {
|
|
36
45
|
const detail = yield* runSchemaComponent(component.name, component.pending);
|
|
37
46
|
if (detail !== undefined) {
|
|
38
47
|
pendingComponents.push({ detail, name: component.name });
|
package/src/postgresChanges.ts
CHANGED
|
@@ -2,8 +2,9 @@ import { Context, type Duration, Effect, Layer, type Queue, type Scope } from "e
|
|
|
2
2
|
import * as Reactivity from "effect/reactivity/Reactivity";
|
|
3
3
|
import { SqlClient } from "effect/sql";
|
|
4
4
|
import type { SqlError } from "effect/sql/SqlError";
|
|
5
|
+
import { CommandRecovery } from "#internal/CommandRecovery.ts";
|
|
5
6
|
import { listenForChanges } from "./listenForChanges.ts";
|
|
6
|
-
import { encodeKeys } from "./notifyPayload.ts";
|
|
7
|
+
import { encodeKeys, resyncPayload } from "./notifyPayload.ts";
|
|
7
8
|
import { BeforeCommit, CommittedSink } from "./transact.ts";
|
|
8
9
|
|
|
9
10
|
export interface PostgresNotification {
|
|
@@ -17,8 +18,10 @@ export interface PostgresChangeClient extends SqlClient.SqlClient {
|
|
|
17
18
|
export interface PostgresChangesOptions {
|
|
18
19
|
readonly channel: string;
|
|
19
20
|
readonly client: PostgresChangeClient;
|
|
21
|
+
readonly connectionSetting?: string;
|
|
20
22
|
readonly heartbeatInterval?: Duration.Input;
|
|
21
23
|
readonly heartbeatTimeout?: Duration.Input;
|
|
24
|
+
readonly listenerClient?: Pick<PostgresChangeClient, "listen">;
|
|
22
25
|
readonly resync: Effect.Effect<number>;
|
|
23
26
|
readonly retryDelay?: Duration.Input;
|
|
24
27
|
}
|
|
@@ -42,7 +45,12 @@ export function postgresChanges(options: PostgresChangesOptions) {
|
|
|
42
45
|
sql.transactionService === options.client.transactionService ? Effect.void : Reactivity.invalidate(keys),
|
|
43
46
|
);
|
|
44
47
|
}
|
|
45
|
-
|
|
48
|
+
const recover = options.resync.pipe(
|
|
49
|
+
Effect.andThen(options.client`select pg_notify(${options.channel}, ${resyncPayload})`),
|
|
50
|
+
Effect.catchCause((cause) => Effect.logError("Committed command Resync broadcast failed; other hosts may remain stale", cause)),
|
|
51
|
+
Effect.asVoid,
|
|
52
|
+
);
|
|
53
|
+
return Context.make(BeforeCommit, notify).pipe(Context.add(CommittedSink, committed), Context.add(CommandRecovery, recover));
|
|
46
54
|
}),
|
|
47
55
|
);
|
|
48
56
|
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import * as PgClient from "@effect/sql-pg/PgClient";
|
|
2
|
+
import * as PgPool from "@effect/sql-pg/PgPool";
|
|
3
|
+
import { Context, Effect, Layer, type Redacted } from "effect";
|
|
4
|
+
import * as Reactivity from "effect/reactivity/Reactivity";
|
|
5
|
+
import { SqlClient } from "effect/sql";
|
|
6
|
+
import { makePostgresClient } from "#adapters/makePostgresClient.ts";
|
|
7
|
+
import { postgresStream } from "#adapters/postgresStream.ts";
|
|
8
|
+
import type { SqlNameTransforms } from "#clientOptions.ts";
|
|
9
|
+
import { databaseAddress } from "#databaseAddress.ts";
|
|
10
|
+
|
|
11
|
+
export interface PostgresOptions extends SqlNameTransforms {
|
|
12
|
+
readonly applicationName?: string | undefined;
|
|
13
|
+
readonly maxConnections?: number | undefined;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export function postgresLayer(url: Redacted.Redacted<string>, options?: PostgresOptions) {
|
|
17
|
+
return Layer.effectContext(
|
|
18
|
+
Effect.gen(function* () {
|
|
19
|
+
const address = databaseAddress(url);
|
|
20
|
+
const stream = yield* postgresStream(url, address);
|
|
21
|
+
const config = { ...options, ...address, stream, url };
|
|
22
|
+
const pool = yield* PgPool.make(config);
|
|
23
|
+
const client = yield* makePostgresClient(config).pipe(Effect.provideService(PgPool.PgPool, pool));
|
|
24
|
+
return Context.make(PgPool.PgPool, pool).pipe(Context.add(PgClient.PgClient, client), Context.add(SqlClient.SqlClient, client));
|
|
25
|
+
}),
|
|
26
|
+
).pipe(Layer.provide(Reactivity.layer));
|
|
27
|
+
}
|