@shivaedev/sql 0.32.0 → 0.34.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. package/CHANGELOG.md +34 -0
  2. package/README.md +153 -23
  3. package/dist/MigrationStartedAt.d.ts.map +1 -0
  4. package/dist/{internal/MigrationStartedAt.js → MigrationStartedAt.js} +1 -1
  5. package/dist/MigrationStartedAt.js.map +1 -0
  6. package/dist/batchWrites.d.ts +1 -1
  7. package/dist/batchWrites.d.ts.map +1 -1
  8. package/dist/batchWrites.js.map +1 -1
  9. package/dist/batchedDelete.d.ts +9 -2
  10. package/dist/batchedDelete.d.ts.map +1 -1
  11. package/dist/batchedDelete.js +34 -35
  12. package/dist/batchedDelete.js.map +1 -1
  13. package/dist/changes.d.ts +8 -0
  14. package/dist/changes.d.ts.map +1 -0
  15. package/dist/changes.js +7 -0
  16. package/dist/changes.js.map +1 -0
  17. package/dist/conditionalUpsert.d.ts +4 -3
  18. package/dist/conditionalUpsert.d.ts.map +1 -1
  19. package/dist/conditionalUpsert.js +1 -1
  20. package/dist/conditionalUpsert.js.map +1 -1
  21. package/dist/conflictWrites.d.ts +6 -5
  22. package/dist/conflictWrites.d.ts.map +1 -1
  23. package/dist/conflictWrites.js +1 -1
  24. package/dist/conflictWrites.js.map +1 -1
  25. package/dist/deleteGroupsBatched.d.ts +13 -0
  26. package/dist/deleteGroupsBatched.d.ts.map +1 -0
  27. package/dist/deleteGroupsBatched.js +23 -0
  28. package/dist/deleteGroupsBatched.js.map +1 -0
  29. package/dist/inValues.d.ts +5 -0
  30. package/dist/inValues.d.ts.map +1 -0
  31. package/dist/inValues.js +10 -0
  32. package/dist/inValues.js.map +1 -0
  33. package/dist/insertFromSelect.d.ts +1 -1
  34. package/dist/insertFromSelect.d.ts.map +1 -1
  35. package/dist/insertFromSelect.js.map +1 -1
  36. package/dist/int4Array.d.ts +5 -0
  37. package/dist/int4Array.d.ts.map +1 -0
  38. package/dist/int4Array.js +10 -0
  39. package/dist/int4Array.js.map +1 -0
  40. package/dist/int8.d.ts +5 -0
  41. package/dist/int8.d.ts.map +1 -0
  42. package/dist/int8.js +9 -0
  43. package/dist/int8.js.map +1 -0
  44. package/dist/invalidationKeys.d.ts +8 -0
  45. package/dist/invalidationKeys.d.ts.map +1 -0
  46. package/dist/invalidationKeys.js +15 -0
  47. package/dist/invalidationKeys.js.map +1 -0
  48. package/dist/keysetPage.d.ts +26 -0
  49. package/dist/keysetPage.d.ts.map +1 -0
  50. package/dist/keysetPage.js +23 -0
  51. package/dist/keysetPage.js.map +1 -0
  52. package/dist/listenForChanges.js +2 -2
  53. package/dist/listenForChanges.js.map +1 -1
  54. package/dist/makeConflictClause.d.ts +2 -1
  55. package/dist/makeConflictClause.d.ts.map +1 -1
  56. package/dist/makeConflictClause.js +36 -6
  57. package/dist/makeConflictClause.js.map +1 -1
  58. package/dist/migrationLock.d.ts.map +1 -0
  59. package/dist/migrationLock.js.map +1 -0
  60. package/dist/migrations.d.ts +1 -1
  61. package/dist/migrations.d.ts.map +1 -1
  62. package/dist/migrations.js +18 -9
  63. package/dist/migrations.js.map +1 -1
  64. package/dist/postgresChanges.d.ts.map +1 -1
  65. package/dist/postgresChanges.js +8 -4
  66. package/dist/postgresChanges.js.map +1 -1
  67. package/dist/postgresLayer.d.ts +2 -1
  68. package/dist/postgresLayer.d.ts.map +1 -1
  69. package/dist/postgresLayer.js +1 -1
  70. package/dist/postgresLayer.js.map +1 -1
  71. package/dist/postgresRelease.js +1 -1
  72. package/dist/postgresRelease.js.map +1 -1
  73. package/dist/repositoryChanges.d.ts.map +1 -1
  74. package/dist/repositoryChanges.js +4 -3
  75. package/dist/repositoryChanges.js.map +1 -1
  76. package/dist/transact.d.ts +13 -5
  77. package/dist/transact.d.ts.map +1 -1
  78. package/dist/transact.js +45 -19
  79. package/dist/transact.js.map +1 -1
  80. package/dist/upsertColumns.d.ts +16 -0
  81. package/dist/upsertColumns.d.ts.map +1 -0
  82. package/dist/upsertColumns.js +2 -0
  83. package/dist/upsertColumns.js.map +1 -0
  84. package/dist/versionedStatements.d.ts +1 -1
  85. package/package.json +7 -6
  86. package/src/{internal/MigrationStartedAt.ts → MigrationStartedAt.ts} +1 -1
  87. package/src/batchWrites.ts +2 -1
  88. package/src/batchedDelete.ts +53 -40
  89. package/src/changes.ts +12 -0
  90. package/src/conditionalUpsert.ts +4 -3
  91. package/src/conflictWrites.ts +11 -7
  92. package/src/deleteGroupsBatched.ts +41 -0
  93. package/src/inValues.ts +11 -0
  94. package/src/insertFromSelect.ts +1 -1
  95. package/src/int4Array.ts +14 -0
  96. package/src/int8.ts +14 -0
  97. package/src/invalidationKeys.ts +24 -0
  98. package/src/keysetPage.ts +54 -0
  99. package/src/listenForChanges.ts +2 -2
  100. package/src/makeConflictClause.ts +53 -8
  101. package/src/migrations.ts +21 -10
  102. package/src/postgresChanges.ts +13 -4
  103. package/src/postgresLayer.ts +1 -1
  104. package/src/postgresRelease.ts +1 -1
  105. package/src/repositoryChanges.ts +4 -3
  106. package/src/transact.ts +61 -24
  107. package/src/upsertColumns.ts +16 -0
  108. package/dist/internal/CommandRecovery.d.ts +0 -3
  109. package/dist/internal/CommandRecovery.d.ts.map +0 -1
  110. package/dist/internal/CommandRecovery.js +0 -5
  111. package/dist/internal/CommandRecovery.js.map +0 -1
  112. package/dist/internal/MigrationStartedAt.d.ts.map +0 -1
  113. package/dist/internal/MigrationStartedAt.js.map +0 -1
  114. package/dist/internal/migrationLock.d.ts.map +0 -1
  115. package/dist/internal/migrationLock.js.map +0 -1
  116. package/src/internal/CommandRecovery.ts +0 -5
  117. /package/dist/{internal/MigrationStartedAt.d.ts → MigrationStartedAt.d.ts} +0 -0
  118. /package/dist/{internal/migrationLock.d.ts → migrationLock.d.ts} +0 -0
  119. /package/dist/{internal/migrationLock.js → migrationLock.js} +0 -0
  120. /package/src/{internal/migrationLock.ts → migrationLock.ts} +0 -0
@@ -7,27 +7,62 @@ import type { RepositoryChanges } from "#repositoryChanges.ts";
7
7
  import { type FindManyFilter, makeFilter } from "#repositoryFilter.ts";
8
8
  import { tryPurgeLock } from "#tryPurgeLock.ts";
9
9
 
10
- export interface BatchedDeleteOptions<TModel extends Model.Any> {
10
+ export interface PurgeLimits {
11
11
  readonly batchSize: number;
12
12
  readonly maxRows: number;
13
13
  readonly pause: Duration.Input;
14
- readonly where: FindManyFilter<TModel>;
15
14
  }
16
15
 
17
16
  export class BatchedDeleteUnavailable extends Data.TaggedError("BatchedDeleteUnavailable")<{
18
17
  readonly reason: "transaction" | "dialect";
19
18
  }> {}
20
19
 
20
+ export const PositiveRowCount = Schema.Number.check(Schema.isInt(), Schema.isGreaterThan(0), Schema.isLessThanOrEqualTo(Number.MAX_SAFE_INTEGER));
21
+ const Nonnegative = Schema.Number.check(Schema.isInt(), Schema.isGreaterThanOrEqualTo(0), Schema.isLessThanOrEqualTo(Number.MAX_SAFE_INTEGER));
22
+ const Limits = Schema.Struct({ batchSize: PositiveRowCount, maxRows: Nonnegative });
23
+
24
+ export const runPurgeBatches = Effect.fnUntraced(function* <E, R>(
25
+ tableName: string,
26
+ options: PurgeLimits,
27
+ batch: (limit: number) => Effect.Effect<{ readonly count: number; readonly done: boolean }, E, R>,
28
+ prepare?: Effect.Effect<unknown, E, R>,
29
+ ) {
30
+ const sql = yield* SqlClient.SqlClient;
31
+ if (sql.driver !== "pg") {
32
+ return yield* new BatchedDeleteUnavailable({ reason: "dialect" });
33
+ }
34
+ if (Option.isSome(yield* Effect.serviceOption(sql.transactionService))) {
35
+ return yield* new BatchedDeleteUnavailable({ reason: "transaction" });
36
+ }
37
+ const { batchSize, maxRows } = yield* Schema.decodeUnknownEffect(Limits)(options);
38
+ if (prepare !== undefined) {
39
+ yield* prepare;
40
+ }
41
+ if (maxRows === 0 || !(yield* tryPurgeLock(tableName))) {
42
+ return 0;
43
+ }
44
+ let removed = 0;
45
+ while (removed < maxRows) {
46
+ const { count, done } = yield* batch(Math.min(batchSize, maxRows - removed));
47
+ removed += count;
48
+ if (done || count === 0 || removed === maxRows) {
49
+ break;
50
+ }
51
+ yield* Effect.sleep(options.pause);
52
+ }
53
+ return removed;
54
+ });
55
+
56
+ export interface BatchedDeleteOptions<TModel extends Model.Any> extends PurgeLimits {
57
+ readonly where: FindManyFilter<TModel>;
58
+ }
59
+
21
60
  export interface BatchedDelete<TModel extends Model.Any> {
22
61
  readonly deleteBatched: (
23
62
  options: BatchedDeleteOptions<TModel>,
24
63
  ) => Effect.Effect<number, SqlError | Schema.SchemaError | BatchedDeleteUnavailable, TModel["fields"][keyof TModel["fields"]]["EncodingServices"]>;
25
64
  }
26
65
 
27
- const Positive = Schema.Number.check(Schema.isInt(), Schema.isGreaterThan(0), Schema.isLessThanOrEqualTo(Number.MAX_SAFE_INTEGER));
28
- const Nonnegative = Schema.Number.check(Schema.isInt(), Schema.isGreaterThanOrEqualTo(0), Schema.isLessThanOrEqualTo(Number.MAX_SAFE_INTEGER));
29
- const Limits = Schema.Struct({ batchSize: Positive, maxRows: Nonnegative });
30
-
31
66
  export const makeBatchedDelete = Effect.fnUntraced(function* (
32
67
  model: Model.Any,
33
68
  { tableName, spanPrefix }: { readonly tableName: string; readonly spanPrefix: string },
@@ -35,14 +70,6 @@ export const makeBatchedDelete = Effect.fnUntraced(function* (
35
70
  ) {
36
71
  const sql = yield* SqlClient.SqlClient;
37
72
  const { where } = makeFilter(model, sql);
38
- const available = Effect.gen(function* () {
39
- if (sql.driver !== "pg") {
40
- return yield* new BatchedDeleteUnavailable({ reason: "dialect" });
41
- }
42
- if (Option.isSome(yield* Effect.serviceOption(sql.transactionService))) {
43
- return yield* new BatchedDeleteUnavailable({ reason: "transaction" });
44
- }
45
- });
46
73
  function deleteBatch(predicate: Fragment, limit: number) {
47
74
  const count = SqlSchema.findOne({
48
75
  execute: (batchLimit) => sql`with removed as (
@@ -62,32 +89,18 @@ export const makeBatchedDelete = Effect.fnUntraced(function* (
62
89
  );
63
90
  }
64
91
 
65
- const runBatches = Effect.fnUntraced(function* (options: BatchedDeleteOptions<Model.Any>, predicate: Fragment) {
66
- const { batchSize, maxRows } = options;
67
- let removed = 0;
68
- while (removed < maxRows) {
69
- const limit = Math.min(batchSize, maxRows - removed);
70
- const count = yield* deleteBatch(predicate, limit);
71
- removed += count;
72
- if (count < limit || removed === maxRows) {
73
- break;
74
- }
75
- yield* Effect.sleep(options.pause);
76
- }
77
- return removed;
78
- });
79
-
80
92
  return (options: BatchedDeleteOptions<Model.Any>) =>
81
93
  Effect.gen(function* () {
82
- yield* available;
83
- const { maxRows } = yield* Schema.decodeUnknownEffect(Limits)(options);
84
- const predicate = yield* where(options.where, true, true);
85
- if (maxRows === 0) {
86
- return 0;
87
- }
88
- if (!(yield* tryPurgeLock(tableName).pipe(Effect.provideService(SqlClient.SqlClient, sql)))) {
89
- return 0;
90
- }
91
- return yield* runBatches(options, predicate);
92
- }).pipe(Effect.scoped, Effect.withSpan(`${spanPrefix}.deleteBatched`));
94
+ const predicate = yield* Effect.cached(where(options.where, true, true));
95
+ return yield* runPurgeBatches(
96
+ tableName,
97
+ options,
98
+ (limit) =>
99
+ Effect.gen(function* () {
100
+ const count = yield* deleteBatch(yield* predicate, limit);
101
+ return { count, done: count < limit };
102
+ }),
103
+ predicate,
104
+ );
105
+ }).pipe(Effect.provideService(SqlClient.SqlClient, sql), Effect.scoped, Effect.withSpan(`${spanPrefix}.deleteBatched`));
93
106
  });
package/src/changes.ts ADDED
@@ -0,0 +1,12 @@
1
+ import { type BucketKey, type ItemKey, invalidationKeys } from "@shivaedev/changes/keys.ts";
2
+ import { type AnnouncementOptions, invalidateOnCommit } from "#transact.ts";
3
+
4
+ export interface ChangeDeclaration {
5
+ readonly covers: Readonly<Record<string, readonly (ItemKey | BucketKey)[]>>;
6
+ readonly record: (options?: AnnouncementOptions) => ReturnType<typeof invalidateOnCommit>;
7
+ }
8
+
9
+ export function changes(covers: ChangeDeclaration["covers"]): ChangeDeclaration {
10
+ const keys = invalidationKeys(Object.values(covers).flat());
11
+ return { covers, record: (options) => invalidateOnCommit(keys, options) };
12
+ }
@@ -3,9 +3,10 @@ import type { Model } from "effect/schema";
3
3
  import type { SqlClient } from "effect/sql";
4
4
  import type { SqlError } from "effect/sql/SqlError";
5
5
  import { makeConflictClause } from "#makeConflictClause.ts";
6
+ import type { RepositoryChanges } from "#repositoryChanges.ts";
6
7
  import type { Comparisons, FindManyFilter } from "#repositoryFilter.ts";
7
- import type { RepositoryChanges } from "./repositoryChanges.ts";
8
- import { makeUpsertDecoder, type UpsertRow, upsertReturning } from "./upsertResult.ts";
8
+ import type { UpsertAssignment } from "#upsertColumns.ts";
9
+ import { makeUpsertDecoder, type UpsertRow, upsertReturning } from "#upsertResult.ts";
9
10
 
10
11
  type Fields<TModel extends Model.Any> = Schema.Struct.Type<TModel["fields"]>;
11
12
  interface Incoming<TModel extends Model.Any, TValue> {
@@ -70,7 +71,7 @@ export function makeConditionalUpsert(
70
71
  const clause = makeConflictClause(model, sql, options);
71
72
  return (
72
73
  inputs: readonly unknown[],
73
- settings: { readonly updateColumns: readonly string[]; readonly updateWhere: Readonly<Record<string, unknown>> },
74
+ settings: { readonly updateColumns: readonly UpsertAssignment[]; readonly updateWhere: Readonly<Record<string, unknown>> },
74
75
  ) =>
75
76
  Effect.gen(function* () {
76
77
  const statements = yield* Effect.forEach(inputs, (input) =>
@@ -6,15 +6,16 @@ import { type InsertMany, makeInsertMany } from "#batchWrites.ts";
6
6
  import { type ConditionalUpsert, type ConditionalUpsertMany, makeConditionalUpsert } from "#conditionalUpsert.ts";
7
7
  import { type InsertFromSelect, makeInsertFromSelect } from "#insertFromSelect.ts";
8
8
  import { makeConflictClause } from "#makeConflictClause.ts";
9
- import type { RepositoryChanges } from "./repositoryChanges.ts";
10
- import { makeUpsertDecoder, upsertReturning } from "./upsertResult.ts";
9
+ import type { RepositoryChanges } from "#repositoryChanges.ts";
10
+ import type { UpsertAssignment, UpsertColumn } from "#upsertColumns.ts";
11
+ import { makeUpsertDecoder, upsertReturning } from "#upsertResult.ts";
11
12
 
12
13
  export type InsertColumn<TModel extends Model.Any> = Extract<keyof TModel["insert"]["Type"] & keyof TModel["fields"], string>;
13
14
  export type ConflictColumns<TModel extends Model.Any> = readonly [InsertColumn<TModel>, ...InsertColumn<TModel>[]];
14
15
  interface UpdateOptions<TModel extends Model.Any, TExcluded extends PropertyKey> {
15
16
  readonly updateColumns: readonly [
16
- Exclude<InsertColumn<TModel> & keyof TModel["update"]["Type"], TExcluded>,
17
- ...Exclude<InsertColumn<TModel> & keyof TModel["update"]["Type"], TExcluded>[],
17
+ UpsertColumn<TModel, Exclude<InsertColumn<TModel> & keyof TModel["update"]["Type"], TExcluded>>,
18
+ ...UpsertColumn<TModel, Exclude<InsertColumn<TModel> & keyof TModel["update"]["Type"], TExcluded>>[],
18
19
  ];
19
20
  }
20
21
  export interface ConflictWrites<TModel extends Model.Any, TExcluded extends PropertyKey, TFilterServices = never> {
@@ -58,7 +59,7 @@ export function makeConflictWrites(
58
59
  const record = Schema.decodeUnknownEffect(Schema.Record(Schema.String, Schema.Unknown));
59
60
  const conditional = makeConditionalUpsert(model, sql, options);
60
61
  const clause = makeConflictClause(model, sql, options);
61
- function statement(input: unknown, updateColumns?: readonly string[]) {
62
+ function statement(input: unknown, updateColumns?: readonly UpsertAssignment[]) {
62
63
  return Effect.gen(function* () {
63
64
  const encoded = yield* record(yield* encode(input));
64
65
  const suffix = yield* clause(encoded, updateColumns);
@@ -76,7 +77,10 @@ export function makeConflictWrites(
76
77
  yield* options.changes.record(rows);
77
78
  return rows.length === 0 ? Option.none() : Option.some(yield* decode(rows[0]));
78
79
  }).pipe(options.changes.write, Effect.withSpan(`${spanPrefix}.insertOrIgnore`)),
79
- upsert: (input: unknown, settings: { readonly updateColumns: readonly string[]; readonly updateWhere?: Readonly<Record<string, unknown>> }) =>
80
+ upsert: (
81
+ input: unknown,
82
+ settings: { readonly updateColumns: readonly UpsertAssignment[]; readonly updateWhere?: Readonly<Record<string, unknown>> },
83
+ ) =>
80
84
  settings.updateWhere === undefined
81
85
  ? Effect.gen(function* () {
82
86
  const rows = yield* yield* statement(input, settings.updateColumns);
@@ -86,7 +90,7 @@ export function makeConflictWrites(
86
90
  : conditional([input], { ...settings, updateWhere: settings.updateWhere }).pipe(Effect.map((results) => results[0])),
87
91
  upsertMany: (
88
92
  inputs: readonly unknown[],
89
- settings: { readonly updateColumns: readonly string[]; readonly updateWhere?: Readonly<Record<string, unknown>> },
93
+ settings: { readonly updateColumns: readonly UpsertAssignment[]; readonly updateWhere?: Readonly<Record<string, unknown>> },
90
94
  ) =>
91
95
  settings.updateWhere === undefined
92
96
  ? makeInsertMany(model, sql, options, "upsertMany", (row) => clause(row, settings.updateColumns), true)(inputs)
@@ -0,0 +1,41 @@
1
+ import { Effect, Schema } from "effect";
2
+ import { PositiveRowCount, type PurgeLimits, runPurgeBatches } from "#batchedDelete.ts";
3
+ import { transact } from "#transact.ts";
4
+
5
+ export interface DeleteGroup<T> {
6
+ readonly rowCount: number;
7
+ readonly value: T;
8
+ }
9
+
10
+ export interface DeleteGroupsBatchedOptions<T, TSelectError, TSelectServices, TDeleteError, TDeleteServices> extends PurgeLimits {
11
+ readonly delete: (group: T) => Effect.Effect<unknown, TDeleteError, TDeleteServices>;
12
+ readonly select: (limit: number) => Effect.Effect<readonly DeleteGroup<T>[], TSelectError, TSelectServices>;
13
+ readonly tableName: string;
14
+ }
15
+
16
+ export const deleteGroupsBatched = Effect.fn("Sql.deleteGroupsBatched")(function* <T, TSelectError, TSelectServices, TDeleteError, TDeleteServices>(
17
+ options: DeleteGroupsBatchedOptions<T, TSelectError, TSelectServices, TDeleteError, TDeleteServices>,
18
+ ) {
19
+ return yield* runPurgeBatches(options.tableName, options, (limit) =>
20
+ transact(
21
+ Effect.gen(function* () {
22
+ const candidates = yield* options.select(limit);
23
+ const selected: T[] = [];
24
+ let count = 0;
25
+ for (const candidate of candidates) {
26
+ const size = yield* Schema.decodeUnknownEffect(PositiveRowCount)(candidate.rowCount);
27
+ if (size > limit - count) {
28
+ break;
29
+ }
30
+ selected.push(candidate.value);
31
+ count += size;
32
+ }
33
+ for (const group of selected) {
34
+ yield* options.delete(group);
35
+ }
36
+ return { count, done: candidates.length === 0 };
37
+ }),
38
+ { onSqlError: (error) => error },
39
+ ),
40
+ ).pipe(Effect.scoped);
41
+ });
@@ -0,0 +1,11 @@
1
+ import { Effect } from "effect";
2
+ import { SqlClient } from "effect/sql";
3
+ import type { Fragment } from "effect/sql/Statement";
4
+
5
+ export const inValues = Effect.fnUntraced(function* (column: string | Fragment, values: readonly unknown[]) {
6
+ const sql = yield* SqlClient.SqlClient;
7
+ if (values.length === 0) {
8
+ return sql`1 = 0`;
9
+ }
10
+ return sql`${typeof column === "string" ? sql(column) : column} in ${sql.in(values)}`;
11
+ });
@@ -29,7 +29,7 @@ export function makeInsertFromSelect<TServices = never>(
29
29
  sql: SqlClient.SqlClient,
30
30
  options: { readonly tableName: string; readonly spanPrefix: string; readonly changes: RepositoryChanges },
31
31
  operation: string,
32
- prepare: (columns: readonly string[]) => Effect.Effect<Fragment, Schema.SchemaError, TServices> = () => Effect.succeed(sql``),
32
+ prepare: (columns: readonly string[]) => Effect.Effect<Fragment, SqlError | Schema.SchemaError, TServices> = () => Effect.succeed(sql``),
33
33
  ) {
34
34
  const decode = Schema.decodeUnknownEffect(Schema.Array(model));
35
35
  return (source: RepositorySelect<Schema.Struct.Fields, unknown>, columns: Readonly<Record<string, string>>) =>
@@ -0,0 +1,14 @@
1
+ import * as PgTypes from "@effect/sql-pg/PgTypes";
2
+ import { Result, Schema, SchemaGetter, SchemaParser } from "effect";
3
+
4
+ const Integers = Schema.Array(Schema.Int);
5
+
6
+ export function int4Array<TField extends Schema.Top & { readonly Encoded: readonly number[] }>(field: TField) {
7
+ return Schema.Unknown.pipe(
8
+ Schema.decodeTo(Integers, {
9
+ decode: SchemaGetter.transformEffect((value) => SchemaParser.decodeUnknownEffect(Integers)(PgTypes.isParameter(value) ? value.value : value)),
10
+ encode: SchemaGetter.transform((values) => Result.getOrThrow(PgTypes.array(values, PgTypes.OID.int4))),
11
+ }),
12
+ Schema.decodeTo(field),
13
+ );
14
+ }
package/src/int8.ts ADDED
@@ -0,0 +1,14 @@
1
+ import * as PgTypes from "@effect/sql-pg/PgTypes";
2
+ import { Schema, SchemaGetter, SchemaParser } from "effect";
3
+
4
+ export function int8<TField extends Schema.Top & { readonly Encoded: bigint }>(field: TField) {
5
+ return Schema.Unknown.pipe(
6
+ Schema.decodeTo(Schema.BigInt, {
7
+ decode: SchemaGetter.transformEffect((value) =>
8
+ SchemaParser.decodeUnknownEffect(Schema.BigInt)(PgTypes.isParameter(value) ? value.value : value),
9
+ ),
10
+ encode: SchemaGetter.transform(PgTypes.int8),
11
+ }),
12
+ Schema.decodeTo(field),
13
+ );
14
+ }
@@ -0,0 +1,24 @@
1
+ import { Schema } from "effect";
2
+ import { invalidationKeys, Key, recordKey } from "@shivaedev/changes/keys.ts";
3
+ import type { InvalidationKeys } from "#transact.ts";
4
+
5
+ export interface Invalidation {
6
+ readonly key: unknown;
7
+ readonly publication: "local" | "shared";
8
+ }
9
+
10
+ const isKey = Schema.is(Key);
11
+
12
+ function isList(keys: InvalidationKeys): keys is readonly unknown[] {
13
+ return Array.isArray(keys);
14
+ }
15
+
16
+ export function flatten(keys: InvalidationKeys): readonly unknown[] {
17
+ return isList(keys)
18
+ ? keys.flatMap<unknown>((key) => (isKey(key) ? invalidationKeys([key]) : [key]))
19
+ : Object.entries(keys).flatMap(([name, ids]) => [name, ...ids.map((id) => recordKey(name, id))]);
20
+ }
21
+
22
+ export function keysOf(changes: readonly Invalidation[]) {
23
+ return [...new Set(changes.map((change) => change.key))];
24
+ }
@@ -0,0 +1,54 @@
1
+ import { Effect, Option, Schema } from "effect";
2
+ import { SqlClient } from "effect/sql";
3
+ import type { Fragment } from "effect/sql/Statement";
4
+
5
+ export interface KeysetOrder<TCursor> {
6
+ readonly column: string | Fragment;
7
+ readonly direction: "asc" | "desc";
8
+ readonly value: (cursor: TCursor) => string | number | bigint | boolean;
9
+ }
10
+
11
+ export interface KeysetPageQuery {
12
+ readonly limit: number;
13
+ readonly orderBy: Fragment;
14
+ readonly seek: Fragment;
15
+ }
16
+
17
+ export interface KeysetPage<TRow, TCursor> {
18
+ readonly next: Option.Option<TCursor>;
19
+ readonly rows: readonly TRow[];
20
+ }
21
+
22
+ const PageSize = Schema.Number.check(Schema.isInt(), Schema.isGreaterThan(0), Schema.isLessThan(Number.MAX_SAFE_INTEGER));
23
+
24
+ export const keysetPage = Effect.fnUntraced(function* <TRow, TCursor, E, R>(options: {
25
+ readonly limit: number;
26
+ readonly after?: TCursor | undefined;
27
+ readonly orderBy: readonly KeysetOrder<TCursor>[];
28
+ readonly tieBreaker: KeysetOrder<TCursor>;
29
+ readonly cursorFor: (row: TRow) => TCursor;
30
+ readonly read: (query: KeysetPageQuery) => Effect.Effect<readonly TRow[], E, R>;
31
+ }): Effect.fn.Return<KeysetPage<TRow, TCursor>, E | Schema.SchemaError, R | SqlClient.SqlClient> {
32
+ const limit = yield* Schema.decodeUnknownEffect(PageSize)(options.limit);
33
+ const sql = yield* SqlClient.SqlClient;
34
+ const order = [...options.orderBy, options.tieBreaker];
35
+ const columns = order.map(({ column }) => (typeof column === "string" ? sql`${sql(column)}` : column));
36
+ const after = options.after;
37
+ const values = after === undefined ? undefined : order.map(({ value }) => value(after));
38
+ const seek =
39
+ values === undefined
40
+ ? sql`true`
41
+ : sql.or(
42
+ order.map(({ direction }, index) =>
43
+ sql.and([
44
+ ...columns.slice(0, index).map((column, previous) => sql`${column} = ${values[previous]}`),
45
+ direction === "asc" ? sql`${columns[index]} > ${values[index]}` : sql`${columns[index]} < ${values[index]}`,
46
+ ]),
47
+ ),
48
+ );
49
+ const orderBy = sql.csv(order.map(({ direction }, index) => sql`${columns[index]} ${sql.literal(direction === "asc" ? "asc" : "desc")}`));
50
+ const found = yield* options.read({ limit: limit + 1, orderBy, seek });
51
+ const rows = found.slice(0, limit);
52
+ const last = rows.at(-1);
53
+ return { next: found.length > limit && last !== undefined ? Option.some(options.cursorFor(last)) : Option.none(), rows };
54
+ });
@@ -89,11 +89,11 @@ export const listenForChanges = Effect.fn("PostgresChanges.listenForChanges")(fu
89
89
  return;
90
90
  }
91
91
  if (lostAt !== undefined) {
92
- yield* Effect.logDebug("PostgreSQL change listener reconnect failed; retrying", cause);
92
+ yield* Effect.logDebug("PostgreSQL change listener reconnect failed; retrying");
93
93
  return;
94
94
  }
95
95
  lostAt = yield* Clock.currentTimeMillis;
96
- yield* Effect.logWarning("PostgreSQL change listener lost; resynchronizing live streams", cause);
96
+ yield* Effect.logWarning("PostgreSQL change listener lost; resynchronizing live streams");
97
97
  }).pipe(Effect.andThen(Effect.sleep(options.retryDelay ?? "1 second"))),
98
98
  ),
99
99
  ),
@@ -1,7 +1,8 @@
1
- import { Effect, Schema, SchemaAST, SchemaIssue } from "effect";
1
+ import { Array as Arr, Effect, Schema, SchemaAST, SchemaIssue } from "effect";
2
2
  import type { Model } from "effect/schema";
3
3
  import type { SqlClient } from "effect/sql";
4
4
  import { makeFilter } from "#repositoryFilter.ts";
5
+ import type { UpsertAssignment } from "#upsertColumns.ts";
5
6
 
6
7
  export function makeConflictClause(
7
8
  model: Model.Any,
@@ -33,18 +34,62 @@ export function makeConflictClause(
33
34
  }
34
35
  });
35
36
  }
36
- function clause(encoded: Readonly<Record<string, unknown>>, updateColumns?: readonly string[], updateWhere?: Readonly<Record<string, unknown>>) {
37
+ function assignment(entry: UpsertAssignment) {
38
+ const column = typeof entry === "string" ? entry : entry.column;
39
+ const incoming = sql`excluded.${sql(column)}`;
40
+ const stored = sql`${sql(options.tableName)}.${sql(column)}`;
41
+ let value = incoming;
42
+ if (typeof entry !== "string") {
43
+ value = entry.mode === "add" ? sql`${stored} + ${incoming}` : sql`coalesce(${incoming}, ${stored})`;
44
+ }
45
+ return sql`${sql(column)} = ${value}`;
46
+ }
47
+ function encodePredicate([key, value]: readonly [string, unknown]) {
48
+ return Effect.gen(function* () {
49
+ const field = yield* filter.fieldAt(key);
50
+ const encoded = yield* Schema.encodeEffect(field)(value);
51
+ return { encoded, key, nullish: value === null || encoded === null };
52
+ });
53
+ }
54
+ function postgresPredicate(input: Readonly<Record<string, unknown>>) {
55
+ return Effect.gen(function* () {
56
+ const entries = yield* Effect.forEach(
57
+ Object.entries(input).filter(([, value]) => value !== undefined),
58
+ encodePredicate,
59
+ );
60
+ if (entries.length === 0) {
61
+ return sql.and([]);
62
+ }
63
+ // PostgreSQL must see constants to infer a partial index under generic plans.
64
+ const [quoted] = yield* sql`select ${sql.csv(entries.map(({ encoded }) => sql`quote_nullable(${encoded}::text)`))}`.values;
65
+ const literals = yield* Schema.decodeUnknownEffect(Schema.Array(Schema.String))(quoted);
66
+ return sql.and(
67
+ Arr.zip(entries, literals).map(([{ key, nullish }, literal]) =>
68
+ nullish ? sql`${sql(key)} is null` : sql`${sql(key)} = ${sql.literal(literal)}`,
69
+ ),
70
+ );
71
+ });
72
+ }
73
+ function conflictPredicate(input: Readonly<Record<string, unknown>>) {
74
+ return sql.onDialectOrElse({ orElse: () => filter.where(input), pg: () => postgresPredicate(input) });
75
+ }
76
+ function clause(
77
+ encoded: Readonly<Record<string, unknown>>,
78
+ updateColumns?: readonly UpsertAssignment[],
79
+ updateWhere?: Readonly<Record<string, unknown>>,
80
+ ) {
37
81
  return Effect.gen(function* () {
38
82
  yield* validateColumns(encoded, conflictColumns);
39
83
  if (updateColumns !== undefined) {
40
- yield* validateColumns(encoded, updateColumns, [...idColumns, ...conflictColumns, ...immutable]);
84
+ yield* validateColumns(
85
+ encoded,
86
+ updateColumns.map((entry) => (typeof entry === "string" ? entry : entry.column)),
87
+ [...idColumns, ...conflictColumns, ...immutable],
88
+ );
41
89
  }
42
90
  const conflict = sql.csv(conflictColumns.map((column) => sql`${sql(column)}`));
43
- const where = options.conflictWhere === undefined ? sql`` : sql`where ${yield* filter.where(options.conflictWhere)}`;
44
- const action =
45
- updateColumns === undefined
46
- ? sql`do nothing`
47
- : sql`do update set ${sql.csv(updateColumns.map((column) => sql`${sql(column)} = excluded.${sql(column)}`))}`;
91
+ const where = options.conflictWhere === undefined ? sql`` : sql`where ${yield* conflictPredicate(options.conflictWhere)}`;
92
+ const action = updateColumns === undefined ? sql`do nothing` : sql`do update set ${sql.csv(updateColumns.map(assignment))}`;
48
93
  const condition = updateWhere === undefined ? sql`` : sql`where ${yield* updateFilter.where(updateWhere, true, true)}`;
49
94
  return sql`on conflict (${conflict}) ${where} ${action} ${condition}`;
50
95
  });
package/src/migrations.ts CHANGED
@@ -2,12 +2,12 @@ import { Duration, Effect, Option, Schema } from "effect";
2
2
  import * as Reactivity from "effect/reactivity/Reactivity";
3
3
  import { Migrator, SqlClient } from "effect/sql";
4
4
  import type { SqlError } from "effect/sql/SqlError";
5
- import { MigrationStartedAt } from "#internal/MigrationStartedAt.ts";
6
- import { migrationLock } from "#internal/migrationLock.ts";
7
- import { advisoryXactLock } from "./advisoryXactLock.ts";
8
- import { postgresRelease } from "./postgresRelease.ts";
9
- import { runSchemaComponent, type SchemaComponent } from "./SchemaComponent.ts";
10
- import { transact } from "./transact.ts";
5
+ import { advisoryXactLock } from "#advisoryXactLock.ts";
6
+ import { MigrationStartedAt } from "#MigrationStartedAt.ts";
7
+ import { migrationLock } from "#migrationLock.ts";
8
+ import { postgresRelease } from "#postgresRelease.ts";
9
+ import { runSchemaComponent, type SchemaComponent } from "#SchemaComponent.ts";
10
+ import { transact } from "#transact.ts";
11
11
 
12
12
  export interface PostgresMigrationOptions<R, E = never> {
13
13
  readonly components?: readonly SchemaComponent<R, E>[];
@@ -52,12 +52,12 @@ export const migratePostgres = Effect.fn("EffectSql.migratePostgres")(function*
52
52
  created_at timestamp with time zone not null default now(),
53
53
  name text not null
54
54
  )`;
55
- const applied = yield* sql`select migration_id as id from ${sql(table)}`.withoutTransform.pipe(
56
- Effect.flatMap(Schema.decodeUnknownEffect(Schema.Array(Schema.Struct({ id: Schema.Int })))),
55
+ const applied = yield* sql`select migration_id as id, name from ${sql(table)}`.withoutTransform.pipe(
56
+ Effect.flatMap(Schema.decodeUnknownEffect(Schema.Array(Schema.Struct({ id: Schema.Int, name: Schema.String })))),
57
57
  );
58
- const appliedIds = new Set(applied.map(({ id }) => id));
58
+ const appliedNames = new Map(applied.map(({ id, name }) => [id, name]));
59
59
  const latest = applied.reduce((max, { id }) => Math.max(max, id), 0);
60
- const missing = current.filter(([id]) => id <= latest && !appliedIds.has(id));
60
+ const missing = current.filter(([id]) => id <= latest && !appliedNames.has(id));
61
61
  if (missing.length > 0) {
62
62
  return yield* Effect.fail(
63
63
  new Migrator.MigrationError({
@@ -66,6 +66,17 @@ export const migratePostgres = Effect.fn("EffectSql.migratePostgres")(function*
66
66
  }),
67
67
  );
68
68
  }
69
+ for (const [id, name] of current) {
70
+ const recorded = appliedNames.get(id);
71
+ if (recorded !== undefined && recorded !== name) {
72
+ return yield* Effect.fail(
73
+ new Migrator.MigrationError({
74
+ kind: "BadState",
75
+ message: `Migration ${id} is recorded as ${JSON.stringify(recorded)} but loaded as ${JSON.stringify(name)}`,
76
+ }),
77
+ );
78
+ }
79
+ }
69
80
  return yield* migrate({ loader: Effect.succeed(current), table });
70
81
  }).pipe(transact({ onSqlError: (error) => error }), Effect.provideService(Reactivity.Reactivity, reactivity));
71
82
  const schemaComponents: readonly SchemaComponent<R, E | SqlError | Schema.SchemaError>[] = [
@@ -2,10 +2,10 @@ import { Context, type Duration, Effect, Layer, type Queue, type Scope } from "e
2
2
  import * as Reactivity from "effect/reactivity/Reactivity";
3
3
  import { SqlClient } from "effect/sql";
4
4
  import type { SqlError } from "effect/sql/SqlError";
5
- import { CommandRecovery } from "#internal/CommandRecovery.ts";
5
+ import { CommandPublication, CommandPublicationError, type CommandPublisher } from "@shivaedev/changes/commandPublication.ts";
6
6
  import { listenForChanges } from "./listenForChanges.ts";
7
7
  import { encodeKeys, resyncPayload } from "./notifyPayload.ts";
8
- import { BeforeCommit, CommittedSink } from "./transact.ts";
8
+ import { BeforeCommit, CommittedSink, invalidateOnCommit, transact } from "./transact.ts";
9
9
 
10
10
  export interface PostgresNotification {
11
11
  readonly payload: string;
@@ -48,10 +48,19 @@ export function postgresChanges(options: PostgresChangesOptions) {
48
48
  }
49
49
  const recover = options.resync.pipe(
50
50
  Effect.andThen(options.client`select pg_notify(${options.channel}, ${resyncPayload})`),
51
- Effect.catchCause((cause) => Effect.logError("Committed command Resync broadcast failed; other hosts may remain stale", cause)),
51
+ Effect.catchCause((cause) => Effect.fail(new CommandPublicationError({ cause }))),
52
52
  Effect.asVoid,
53
53
  );
54
- return Context.make(BeforeCommit, notify).pipe(Context.add(CommittedSink, committed), Context.add(CommandRecovery, recover));
54
+ const publication: CommandPublisher = {
55
+ publish: (keys) =>
56
+ invalidateOnCommit(keys).pipe(
57
+ transact({ onSqlError: (error) => error }),
58
+ Effect.provideService(SqlClient.SqlClient, options.client),
59
+ Effect.catchCause((cause) => Effect.fail(new CommandPublicationError({ cause }))),
60
+ ),
61
+ recover,
62
+ };
63
+ return Context.make(BeforeCommit, notify).pipe(Context.add(CommittedSink, committed), Context.add(CommandPublication, publication));
55
64
  }),
56
65
  );
57
66
  }
@@ -26,5 +26,5 @@ export function postgresLayer(url: Redacted.Redacted<string>, options?: Postgres
26
26
  const client = yield* makePostgresClient(config).pipe(Effect.provideService(PgPool.PgPool, pool));
27
27
  return Context.make(PgPool.PgPool, pool).pipe(Context.add(PgClient.PgClient, client), Context.add(SqlClient.SqlClient, client));
28
28
  }),
29
- ).pipe(Layer.provide(Reactivity.layer));
29
+ ).pipe(Layer.provideMerge(Layer.fresh(Reactivity.layer)));
30
30
  }
@@ -1,6 +1,6 @@
1
1
  import { Effect, Schema } from "effect";
2
2
  import { SqlClient } from "effect/sql";
3
- import { MigrationStartedAt } from "#internal/MigrationStartedAt.ts";
3
+ import { MigrationStartedAt } from "#MigrationStartedAt.ts";
4
4
  import type { SchemaComponent } from "./SchemaComponent.ts";
5
5
 
6
6
  export const readPostgresRelease = Effect.fn("EffectSql.readPostgresRelease")(function* (table = "effect_sql_migrations") {
@@ -2,7 +2,7 @@ import { Effect, Option, type Schema } from "effect";
2
2
  import * as Reactivity from "effect/reactivity/Reactivity";
3
3
  import { SqlClient } from "effect/sql";
4
4
  import { declaredBuckets } from "@shivaedev/changes/buckets.ts";
5
- import { compositeId, invalidationKeys, type Key, recordKey } from "@shivaedev/changes/keys.ts";
5
+ import { collectionInvalidationKeys, compositeId, invalidationKeys, type Key, recordKey } from "@shivaedev/changes/keys.ts";
6
6
  import { ChangeDeclarations } from "#ChangeDeclarations.ts";
7
7
  import { invalidateOnCommit, transact } from "#transact.ts";
8
8
 
@@ -71,8 +71,9 @@ export const repositoryChanges = Effect.fnUntraced(function* (
71
71
  return;
72
72
  }
73
73
  const declarations = yield* ChangeDeclarations;
74
- yield* declarations.declare(table, [table], rows);
75
- yield* provide(invalidateOnCommit([table]));
74
+ const keys = collectionInvalidationKeys(table);
75
+ yield* declarations.declare(table, keys, rows);
76
+ yield* provide(invalidateOnCommit(keys));
76
77
  });
77
78
  return { record, recordTable, returning, write };
78
79
  });