@powersync/service-module-postgres 0.16.2 → 0.16.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/dist/api/PostgresRouteAPIAdapter.d.ts +1 -0
- package/dist/api/PostgresRouteAPIAdapter.js +8 -1
- package/dist/api/PostgresRouteAPIAdapter.js.map +1 -1
- package/dist/index.d.ts +0 -1
- package/dist/index.js +0 -1
- package/dist/index.js.map +1 -1
- package/dist/module/PostgresModule.d.ts +1 -0
- package/dist/module/PostgresModule.js +9 -4
- package/dist/module/PostgresModule.js.map +1 -1
- package/dist/replication/ConnectionManagerFactory.d.ts +3 -1
- package/dist/replication/ConnectionManagerFactory.js +4 -2
- package/dist/replication/ConnectionManagerFactory.js.map +1 -1
- package/dist/replication/PgManager.d.ts +8 -2
- package/dist/replication/PgManager.js +5 -4
- package/dist/replication/PgManager.js.map +1 -1
- package/dist/replication/PgRelation.d.ts +1 -0
- package/dist/replication/PgRelation.js +7 -0
- package/dist/replication/PgRelation.js.map +1 -1
- package/dist/replication/WalStream.d.ts +6 -1
- package/dist/replication/WalStream.js +45 -33
- package/dist/replication/WalStream.js.map +1 -1
- package/dist/types/registry.d.ts +69 -0
- package/dist/types/registry.js +196 -0
- package/dist/types/registry.js.map +1 -0
- package/dist/types/resolver.d.ts +47 -0
- package/dist/types/resolver.js +191 -0
- package/dist/types/resolver.js.map +1 -0
- package/dist/types/types.d.ts +4 -1
- package/dist/types/types.js.map +1 -1
- package/dist/utils/postgres_version.d.ts +3 -0
- package/dist/utils/postgres_version.js +7 -0
- package/dist/utils/postgres_version.js.map +1 -0
- package/package.json +10 -10
- package/src/api/PostgresRouteAPIAdapter.ts +9 -1
- package/src/index.ts +0 -2
- package/src/module/PostgresModule.ts +10 -4
- package/src/replication/ConnectionManagerFactory.ts +6 -2
- package/src/replication/PgManager.ts +12 -4
- package/src/replication/PgRelation.ts +9 -0
- package/src/replication/WalStream.ts +49 -30
- package/src/types/registry.ts +278 -0
- package/src/types/resolver.ts +210 -0
- package/src/types/types.ts +5 -1
- package/src/utils/postgres_version.ts +8 -0
- package/test/src/pg_test.test.ts +152 -5
- package/test/src/route_api_adapter.test.ts +60 -0
- package/test/src/schema_changes.test.ts +32 -0
- package/test/src/slow_tests.test.ts +3 -2
- package/test/src/types/registry.test.ts +149 -0
- package/test/src/util.ts +16 -0
- package/test/src/wal_stream.test.ts +24 -0
- package/test/src/wal_stream_utils.ts +2 -1
- package/tsconfig.tsbuildinfo +1 -1
- package/dist/utils/pgwire_utils.d.ts +0 -17
- package/dist/utils/pgwire_utils.js +0 -43
- package/dist/utils/pgwire_utils.js.map +0 -1
- package/src/utils/pgwire_utils.ts +0 -48
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"resolver.js","sourceRoot":"","sources":["../../src/types/resolver.ts"],"names":[],"mappings":"AAAA,OAAO,EAAoC,cAAc,EAAE,MAAM,+BAA+B,CAAC;AACjG,OAAO,KAAK,MAAM,MAAM,4BAA4B,CAAC;AACrD,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,gBAAgB,EAAE,MAAM,8BAA8B,CAAC;AAEhE;;;;GAIG;AACH,MAAM,OAAO,oBAAoB;IAIpB;IACQ;IAJX,aAAa,GAAyB,IAAI,CAAC;IAEnD,YACW,QAA4B,EACpB,IAAqB;QAD7B,aAAQ,GAAR,QAAQ,CAAoB;QACpB,SAAI,GAAJ,IAAI,CAAiB;QAEtC,IAAI,CAAC,QAAQ,GAAG,IAAI,kBAAkB,EAAE,CAAC;IAC3C,CAAC;IAEO,KAAK,CAAC,YAAY;QACxB,IAAI,IAAI,CAAC,aAAa,IAAI,IAAI,EAAE,CAAC;YAC/B,IAAI,CAAC,aAAa,GAAG,CAAC,MAAM,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACpF,CAAC;QAED,OAAO,IAAI,CAAC,aAAc,CAAC;IAC7B,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,mBAAmB;QACvB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAC1C,OAAO,OAAO,CAAC,OAAO,CAAC,oBAAoB,CAAC,uBAAuB,CAAC,IAAI,CAAC,CAAC;IAC5E,CAAC;IAED;;;;;OAKG;IACI,KAAK,CAAC,UAAU,CAAC,IAAc;QACpC,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAE3D,IAAI,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;QAC5D,2FAA2F;QAC3F,MAAM,cAAc,GAAG,yGAAyG,CAAC;QACjI,MAAM,SAAS,GAAG;;;;;;;;;;;;UAYZ,iBAAiB,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE;;;;;CAKhD,CAAC;QAEE,OAAO,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YAC3B,mBAAmB;YACnB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;YAC7F,MAAM,YAAY,GAAa,EAAE,CAAC;YAElC,MAAM,WAAW,GAAG,CAAC,GAAW,EAAE,EAAE;gBAClC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;oBACvF,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACzB,CAAC;YACH,CAAC,CAAC;YAEF,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC3C,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBAC5B,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBAElC,QAAQ,GAAG,CAAC,OAAO,EAAE,CAAC;oBACpB,KAAK,GAAG;wBACN,MAAM,EAAE,YAAY,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC;wBAErC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;4BAC9B,gDAAgD;4BAChD,MAAM,KAAK,GAAG,MAAM,CAAC,YAAY,CAAC,CAAC;4BACnC,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;gCACf,2FAA2F;gCAC3F,WAAW,CAAC,KAAK,CAAC,CAAC;gCACnB,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE;oCACrB,IAAI,EAAE,OAAO;oCACb,OAAO,EAAE,KAAK;oCACd,iBAAiB,EAAG,KAAgB,CAAC,UAAU,CAAC,CAAC,CAAC;oCAClD,UAAU,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,kBAAkB;iCAC5C,CAAC,CAAC;4BACL,CAAC;wBACH,CAAC;wBACD,MAAM;oBACR,KAAK,GAAG;wBACN,wDAAwD;wBACxD,MAAM,QAAQ,GAAuC,EAAE,CAAC;wBACxD,KAAK,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;4BAC3C,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;4BAC5B,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;4BAChC,WAAW,CAAC,MAAM,CAAC,CAAC;wBACtB,CAAC;wBAED,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE;4BACrB,IAAI,EAAE,WAAW;4BACjB,OAAO,EAAE,QAAQ;4BACjB,UAAU,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,kBAAkB;yBAC5C,CAAC,CAAC;wBACH,MAAM;oBACR,KAAK,GAAG;wBACN,wGAAwG;wBACxG,wCAAwC;wBACxC,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;wBAChC,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;wBACxC,WAAW,CAAC,KAAK,CAAC,CAAC;wBACnB,MAAM;oBACR,KAAK,GAAG,CAAC;oBACT,KAAK,GAAG,CAAC,CAAC,CAAC;wBACT,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;wBACjC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE;4BACrB,IAAI,EAAE,GAAG,CAAC,OAAO,IAAI,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,YAAY;4BACjD,OAAO,EAAE,KAAK;4BACd,UAAU,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,kBAAkB;yBAC5C,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;YACH,CAAC;YAED,OAAO,GAAG,YAAY,CAAC;QACzB,CAAC;IACH,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,mBAAmB;QAC9B,MAAM,GAAG,GAAG;;;;;;;;;;KAUX,CAAC;QAEF,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC;QACxD,IAAI,GAAG,GAAa,EAAE,CAAC;QACvB,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YAC3C,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC;QACjC,CAAC;QAED,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC;IAED;;;OAGG;IACH,oBAAoB,CAAC,OAAsD;QACzE,MAAM,OAAO,GAAI,OAAe,CAAC,QAAQ,CAAC;QAE1C,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC3D,OAAO,cAAc,CAAC,MAAM,CAAC,CAAC;IAChC,CAAC;IAED;;;OAGG;IACH,qBAAqB,CAAC,OAAsD;QAC1E,MAAM,OAAO,GAAI,OAAe,CAAC,SAAS,CAAC;QAC3C,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;YACpB,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC3D,OAAO,cAAc,CAAC,MAAM,CAAC,CAAC;IAChC,CAAC;IAED;;;OAGG;IACH,WAAW,CAAC,QAAiC,EAAE,QAA6B;QAC1E,IAAI,MAAM,GAAwB,EAAE,CAAC;QACrC,KAAK,IAAI,UAAU,IAAI,QAAQ,EAAE,CAAC;YAChC,MAAM,MAAM,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC;YACpC,MAAM,OAAO,GAAI,QAAgB,CAAC,aAAa,CAAC,SAAS,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YAC1E,IAAI,OAAO,MAAM,IAAI,QAAQ,IAAI,OAAO,EAAE,CAAC;gBACzC,MAAM,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,mBAAmB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YAC1E,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,UAAU,CAAC,GAAG,MAAM,CAAC;YAC9B,CAAC;QACH,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,MAAM,CAAC,uBAAuB,GAAkB,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAE,CAAC"}
|
package/dist/types/types.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import * as lib_postgres from '@powersync/lib-service-postgres';
|
|
2
2
|
import * as service_types from '@powersync/service-types';
|
|
3
3
|
import * as t from 'ts-codec';
|
|
4
|
+
import { CustomTypeRegistry } from './registry.js';
|
|
4
5
|
export declare const validatePort: typeof lib_postgres.validatePort;
|
|
5
6
|
export declare const baseUri: typeof lib_postgres.baseUri;
|
|
6
7
|
export type NormalizedPostgresConnectionConfig = lib_postgres.NormalizedBasePostgresConnectionConfig;
|
|
@@ -59,7 +60,9 @@ export type PostgresConnectionConfig = t.Decoded<typeof PostgresConnectionConfig
|
|
|
59
60
|
/**
|
|
60
61
|
* Resolved version of {@link PostgresConnectionConfig}
|
|
61
62
|
*/
|
|
62
|
-
export type ResolvedConnectionConfig = PostgresConnectionConfig & NormalizedPostgresConnectionConfig
|
|
63
|
+
export type ResolvedConnectionConfig = PostgresConnectionConfig & NormalizedPostgresConnectionConfig & {
|
|
64
|
+
typeRegistry: CustomTypeRegistry;
|
|
65
|
+
};
|
|
63
66
|
export declare function isPostgresConfig(config: service_types.configFile.DataSourceConfig): config is PostgresConnectionConfig;
|
|
64
67
|
/**
|
|
65
68
|
* Validate and normalize connection options.
|
package/dist/types/types.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/types/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,YAAY,MAAM,iCAAiC,CAAC;AAChE,OAAO,KAAK,aAAa,MAAM,0BAA0B,CAAC;AAC1D,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/types/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,YAAY,MAAM,iCAAiC,CAAC;AAChE,OAAO,KAAK,aAAa,MAAM,0BAA0B,CAAC;AAC1D,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAG9B,sDAAsD;AACtD,MAAM,CAAC,MAAM,YAAY,GAAG,YAAY,CAAC,YAAY,CAAC;AACtD,MAAM,CAAC,MAAM,OAAO,GAAG,YAAY,CAAC,OAAO,CAAC;AAE5C,MAAM,CAAC,MAAM,wBAAwB,GAAG,YAAY,CAAC,wBAAwB,CAAC;AAE9E,MAAM,CAAC,MAAM,wBAAwB,GAAG,aAAa,CAAC,UAAU,CAAC,gBAAgB,CAAC,GAAG,CACnF,YAAY,CAAC,4BAA4B,CAC1C,CAAC,GAAG,CACH,CAAC,CAAC,MAAM,CAAC;AACP,gEAAgE;CACjE,CAAC,CACH,CAAC;AAeF,MAAM,UAAU,gBAAgB,CAC9B,MAAiD;IAEjD,OAAO,MAAM,CAAC,IAAI,IAAI,YAAY,CAAC,wBAAwB,CAAC;AAC9D,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,yBAAyB,CAAC,OAAiC;IACzE,OAAO;QACL,GAAG,YAAY,CAAC,yBAAyB,CAAC,OAAO,CAAC;KACN,CAAC;AACjD,CAAC"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import semver from 'semver';
|
|
2
|
+
export async function getServerVersion(db) {
|
|
3
|
+
const result = await db.query(`SHOW server_version;`);
|
|
4
|
+
// The result is usually of the form "16.2 (Debian 16.2-1.pgdg120+2)"
|
|
5
|
+
return semver.coerce(result.rows[0][0].split(' ')[0]);
|
|
6
|
+
}
|
|
7
|
+
//# sourceMappingURL=postgres_version.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"postgres_version.js","sourceRoot":"","sources":["../../src/utils/postgres_version.ts"],"names":[],"mappings":"AACA,OAAO,MAAuB,MAAM,QAAQ,CAAC;AAE7C,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,EAAmB;IACxD,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;IACtD,qEAAqE;IACrE,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACxD,CAAC"}
|
package/package.json
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
"publishConfig": {
|
|
6
6
|
"access": "public"
|
|
7
7
|
},
|
|
8
|
-
"version": "0.16.
|
|
8
|
+
"version": "0.16.3",
|
|
9
9
|
"main": "dist/index.js",
|
|
10
10
|
"license": "FSL-1.1-ALv2",
|
|
11
11
|
"type": "module",
|
|
@@ -28,20 +28,20 @@
|
|
|
28
28
|
"ts-codec": "^1.3.0",
|
|
29
29
|
"uri-js": "^4.4.1",
|
|
30
30
|
"uuid": "^11.1.0",
|
|
31
|
-
"@powersync/lib-service-postgres": "0.4.
|
|
32
|
-
"@powersync/lib-services-framework": "0.7.
|
|
33
|
-
"@powersync/service-core": "1.15.
|
|
34
|
-
"@powersync/service-jpgwire": "0.
|
|
31
|
+
"@powersync/lib-service-postgres": "0.4.8",
|
|
32
|
+
"@powersync/lib-services-framework": "0.7.4",
|
|
33
|
+
"@powersync/service-core": "1.15.3",
|
|
34
|
+
"@powersync/service-jpgwire": "0.21.0",
|
|
35
35
|
"@powersync/service-jsonbig": "0.17.11",
|
|
36
|
-
"@powersync/service-sync-rules": "0.29.
|
|
36
|
+
"@powersync/service-sync-rules": "0.29.1",
|
|
37
37
|
"@powersync/service-types": "0.13.0"
|
|
38
38
|
},
|
|
39
39
|
"devDependencies": {
|
|
40
40
|
"@types/semver": "^7.5.4",
|
|
41
|
-
"@powersync/service-core-tests": "0.12.
|
|
42
|
-
"@powersync/service-module-mongodb-storage": "0.12.
|
|
43
|
-
"@powersync/lib-service-postgres": "0.4.
|
|
44
|
-
"@powersync/service-module-postgres-storage": "0.10.
|
|
41
|
+
"@powersync/service-core-tests": "0.12.3",
|
|
42
|
+
"@powersync/service-module-mongodb-storage": "0.12.3",
|
|
43
|
+
"@powersync/lib-service-postgres": "0.4.8",
|
|
44
|
+
"@powersync/service-module-postgres-storage": "0.10.3"
|
|
45
45
|
},
|
|
46
46
|
"scripts": {
|
|
47
47
|
"build": "tsc -b",
|
|
@@ -9,8 +9,11 @@ import { getDebugTableInfo } from '../replication/replication-utils.js';
|
|
|
9
9
|
import { KEEPALIVE_STATEMENT, PUBLICATION_NAME } from '../replication/WalStream.js';
|
|
10
10
|
import * as types from '../types/types.js';
|
|
11
11
|
import { getApplicationName } from '../utils/application-name.js';
|
|
12
|
+
import { CustomTypeRegistry } from '../types/registry.js';
|
|
13
|
+
import { PostgresTypeResolver } from '../types/resolver.js';
|
|
12
14
|
|
|
13
15
|
export class PostgresRouteAPIAdapter implements api.RouteAPI {
|
|
16
|
+
private typeCache: PostgresTypeResolver;
|
|
14
17
|
connectionTag: string;
|
|
15
18
|
// TODO this should probably be configurable one day
|
|
16
19
|
publicationName = PUBLICATION_NAME;
|
|
@@ -31,6 +34,7 @@ export class PostgresRouteAPIAdapter implements api.RouteAPI {
|
|
|
31
34
|
connectionTag?: string,
|
|
32
35
|
private config?: types.ResolvedConnectionConfig
|
|
33
36
|
) {
|
|
37
|
+
this.typeCache = new PostgresTypeResolver(config?.typeRegistry ?? new CustomTypeRegistry(), pool);
|
|
34
38
|
this.connectionTag = connectionTag ?? sync_rules.DEFAULT_TAG;
|
|
35
39
|
}
|
|
36
40
|
|
|
@@ -297,6 +301,7 @@ LEFT JOIN (
|
|
|
297
301
|
SELECT
|
|
298
302
|
attrelid,
|
|
299
303
|
attname,
|
|
304
|
+
atttypid,
|
|
300
305
|
format_type(atttypid, atttypmod) as data_type,
|
|
301
306
|
(SELECT typname FROM pg_catalog.pg_type WHERE oid = atttypid) as pg_type,
|
|
302
307
|
attnum,
|
|
@@ -311,6 +316,7 @@ LEFT JOIN (
|
|
|
311
316
|
)
|
|
312
317
|
GROUP BY schemaname, tablename, quoted_name`
|
|
313
318
|
);
|
|
319
|
+
await this.typeCache.fetchTypesForSchema();
|
|
314
320
|
const rows = pgwire.pgwireRows(results);
|
|
315
321
|
|
|
316
322
|
let schemas: Record<string, service_types.DatabaseSchema> = {};
|
|
@@ -332,9 +338,11 @@ GROUP BY schemaname, tablename, quoted_name`
|
|
|
332
338
|
if (pg_type.startsWith('_')) {
|
|
333
339
|
pg_type = `${pg_type.substring(1)}[]`;
|
|
334
340
|
}
|
|
341
|
+
|
|
342
|
+
const knownType = this.typeCache.registry.lookupType(Number(column.atttypid));
|
|
335
343
|
table.columns.push({
|
|
336
344
|
name: column.attname,
|
|
337
|
-
sqlite_type: sync_rules.
|
|
345
|
+
sqlite_type: sync_rules.ExpressionType.fromTypeText(knownType.sqliteType()).typeFlags,
|
|
338
346
|
type: column.data_type,
|
|
339
347
|
internal_type: column.data_type,
|
|
340
348
|
pg_type: pg_type
|
package/src/index.ts
CHANGED
|
@@ -19,8 +19,11 @@ import { WalStreamReplicator } from '../replication/WalStreamReplicator.js';
|
|
|
19
19
|
import * as types from '../types/types.js';
|
|
20
20
|
import { PostgresConnectionConfig } from '../types/types.js';
|
|
21
21
|
import { getApplicationName } from '../utils/application-name.js';
|
|
22
|
+
import { CustomTypeRegistry } from '../types/registry.js';
|
|
22
23
|
|
|
23
24
|
export class PostgresModule extends replication.ReplicationModule<types.PostgresConnectionConfig> {
|
|
25
|
+
private customTypes: CustomTypeRegistry = new CustomTypeRegistry();
|
|
26
|
+
|
|
24
27
|
constructor() {
|
|
25
28
|
super({
|
|
26
29
|
name: 'Postgres',
|
|
@@ -48,7 +51,7 @@ export class PostgresModule extends replication.ReplicationModule<types.Postgres
|
|
|
48
51
|
protected createReplicator(context: system.ServiceContext): replication.AbstractReplicator {
|
|
49
52
|
const normalisedConfig = this.resolveConfig(this.decodedConfig!);
|
|
50
53
|
const syncRuleProvider = new ConfigurationFileSyncRulesProvider(context.configuration.sync_rules);
|
|
51
|
-
const connectionFactory = new ConnectionManagerFactory(normalisedConfig);
|
|
54
|
+
const connectionFactory = new ConnectionManagerFactory(normalisedConfig, this.customTypes);
|
|
52
55
|
|
|
53
56
|
return new WalStreamReplicator({
|
|
54
57
|
id: this.getDefaultId(normalisedConfig.database),
|
|
@@ -66,7 +69,8 @@ export class PostgresModule extends replication.ReplicationModule<types.Postgres
|
|
|
66
69
|
private resolveConfig(config: types.PostgresConnectionConfig): types.ResolvedConnectionConfig {
|
|
67
70
|
return {
|
|
68
71
|
...config,
|
|
69
|
-
...types.normalizeConnectionConfig(config)
|
|
72
|
+
...types.normalizeConnectionConfig(config),
|
|
73
|
+
typeRegistry: this.customTypes
|
|
70
74
|
};
|
|
71
75
|
}
|
|
72
76
|
|
|
@@ -75,7 +79,8 @@ export class PostgresModule extends replication.ReplicationModule<types.Postgres
|
|
|
75
79
|
const connectionManager = new PgManager(normalisedConfig, {
|
|
76
80
|
idleTimeout: 30_000,
|
|
77
81
|
maxSize: 1,
|
|
78
|
-
applicationName: getApplicationName()
|
|
82
|
+
applicationName: getApplicationName(),
|
|
83
|
+
registry: this.customTypes
|
|
79
84
|
});
|
|
80
85
|
|
|
81
86
|
try {
|
|
@@ -106,7 +111,8 @@ export class PostgresModule extends replication.ReplicationModule<types.Postgres
|
|
|
106
111
|
const connectionManager = new PgManager(normalizedConfig, {
|
|
107
112
|
idleTimeout: 30_000,
|
|
108
113
|
maxSize: 1,
|
|
109
|
-
applicationName: getApplicationName()
|
|
114
|
+
applicationName: getApplicationName(),
|
|
115
|
+
registry: new CustomTypeRegistry()
|
|
110
116
|
});
|
|
111
117
|
const connection = await connectionManager.snapshotConnection();
|
|
112
118
|
try {
|
|
@@ -2,18 +2,22 @@ import { PgManager } from './PgManager.js';
|
|
|
2
2
|
import { NormalizedPostgresConnectionConfig } from '../types/types.js';
|
|
3
3
|
import { PgPoolOptions } from '@powersync/service-jpgwire';
|
|
4
4
|
import { logger } from '@powersync/lib-services-framework';
|
|
5
|
+
import { CustomTypeRegistry } from '../types/registry.js';
|
|
5
6
|
|
|
6
7
|
export class ConnectionManagerFactory {
|
|
7
8
|
private readonly connectionManagers: PgManager[];
|
|
8
9
|
public readonly dbConnectionConfig: NormalizedPostgresConnectionConfig;
|
|
9
10
|
|
|
10
|
-
constructor(
|
|
11
|
+
constructor(
|
|
12
|
+
dbConnectionConfig: NormalizedPostgresConnectionConfig,
|
|
13
|
+
private readonly registry: CustomTypeRegistry
|
|
14
|
+
) {
|
|
11
15
|
this.dbConnectionConfig = dbConnectionConfig;
|
|
12
16
|
this.connectionManagers = [];
|
|
13
17
|
}
|
|
14
18
|
|
|
15
19
|
create(poolOptions: PgPoolOptions) {
|
|
16
|
-
const manager = new PgManager(this.dbConnectionConfig, poolOptions);
|
|
20
|
+
const manager = new PgManager(this.dbConnectionConfig, { ...poolOptions, registry: this.registry });
|
|
17
21
|
this.connectionManagers.push(manager);
|
|
18
22
|
return manager;
|
|
19
23
|
}
|
|
@@ -2,6 +2,13 @@ import * as pgwire from '@powersync/service-jpgwire';
|
|
|
2
2
|
import semver from 'semver';
|
|
3
3
|
import { NormalizedPostgresConnectionConfig } from '../types/types.js';
|
|
4
4
|
import { getApplicationName } from '../utils/application-name.js';
|
|
5
|
+
import { PostgresTypeResolver } from '../types/resolver.js';
|
|
6
|
+
import { getServerVersion } from '../utils/postgres_version.js';
|
|
7
|
+
import { CustomTypeRegistry } from '../types/registry.js';
|
|
8
|
+
|
|
9
|
+
export interface PgManagerOptions extends pgwire.PgPoolOptions {
|
|
10
|
+
registry: CustomTypeRegistry;
|
|
11
|
+
}
|
|
5
12
|
|
|
6
13
|
/**
|
|
7
14
|
* Shorter timeout for snapshot connections than for replication connections.
|
|
@@ -14,14 +21,17 @@ export class PgManager {
|
|
|
14
21
|
*/
|
|
15
22
|
public readonly pool: pgwire.PgClient;
|
|
16
23
|
|
|
24
|
+
public readonly types: PostgresTypeResolver;
|
|
25
|
+
|
|
17
26
|
private connectionPromises: Promise<pgwire.PgConnection>[] = [];
|
|
18
27
|
|
|
19
28
|
constructor(
|
|
20
29
|
public options: NormalizedPostgresConnectionConfig,
|
|
21
|
-
public poolOptions:
|
|
30
|
+
public poolOptions: PgManagerOptions
|
|
22
31
|
) {
|
|
23
32
|
// The pool is lazy - no connections are opened until a query is performed.
|
|
24
33
|
this.pool = pgwire.connectPgWirePool(this.options, poolOptions);
|
|
34
|
+
this.types = new PostgresTypeResolver(poolOptions.registry, this.pool);
|
|
25
35
|
}
|
|
26
36
|
|
|
27
37
|
public get connectionTag() {
|
|
@@ -41,9 +51,7 @@ export class PgManager {
|
|
|
41
51
|
* @returns The Postgres server version in a parsed Semver instance
|
|
42
52
|
*/
|
|
43
53
|
async getServerVersion(): Promise<semver.SemVer | null> {
|
|
44
|
-
|
|
45
|
-
// The result is usually of the form "16.2 (Debian 16.2-1.pgdg120+2)"
|
|
46
|
-
return semver.coerce(result.rows[0][0].split(' ')[0]);
|
|
54
|
+
return await getServerVersion(this.pool);
|
|
47
55
|
}
|
|
48
56
|
|
|
49
57
|
/**
|
|
@@ -30,3 +30,12 @@ export function getPgOutputRelation(source: PgoutputRelation): storage.SourceEnt
|
|
|
30
30
|
replicaIdColumns: getReplicaIdColumns(source)
|
|
31
31
|
} satisfies storage.SourceEntityDescriptor;
|
|
32
32
|
}
|
|
33
|
+
|
|
34
|
+
export function referencedColumnTypeIds(source: PgoutputRelation): number[] {
|
|
35
|
+
const oids = new Set<number>();
|
|
36
|
+
for (const column of source.columns) {
|
|
37
|
+
oids.add(column.typeOid);
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
return [...oids];
|
|
41
|
+
}
|
|
@@ -29,10 +29,9 @@ import {
|
|
|
29
29
|
TablePattern,
|
|
30
30
|
toSyncRulesRow
|
|
31
31
|
} from '@powersync/service-sync-rules';
|
|
32
|
-
import * as pg_utils from '../utils/pgwire_utils.js';
|
|
33
32
|
|
|
34
33
|
import { PgManager } from './PgManager.js';
|
|
35
|
-
import { getPgOutputRelation, getRelId } from './PgRelation.js';
|
|
34
|
+
import { getPgOutputRelation, getRelId, referencedColumnTypeIds } from './PgRelation.js';
|
|
36
35
|
import { checkSourceConfiguration, checkTableRls, getReplicationIdentityColumns } from './replication-utils.js';
|
|
37
36
|
import { ReplicationMetric } from '@powersync/service-types';
|
|
38
37
|
import {
|
|
@@ -189,28 +188,30 @@ export class WalStream {
|
|
|
189
188
|
|
|
190
189
|
let tableRows: any[];
|
|
191
190
|
const prefix = tablePattern.isWildcard ? tablePattern.tablePrefix : undefined;
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
191
|
+
|
|
192
|
+
{
|
|
193
|
+
let query = `
|
|
194
|
+
SELECT
|
|
195
|
+
c.oid AS relid,
|
|
196
|
+
c.relname AS table_name,
|
|
197
|
+
(SELECT
|
|
198
|
+
json_agg(DISTINCT a.atttypid)
|
|
199
|
+
FROM pg_attribute a
|
|
200
|
+
WHERE a.attnum > 0 AND NOT a.attisdropped AND a.attrelid = c.oid)
|
|
201
|
+
AS column_types
|
|
195
202
|
FROM pg_class c
|
|
196
203
|
JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
197
204
|
WHERE n.nspname = $1
|
|
198
|
-
AND c.relkind = 'r'
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
}
|
|
205
|
-
|
|
206
|
-
} else {
|
|
205
|
+
AND c.relkind = 'r'`;
|
|
206
|
+
|
|
207
|
+
if (tablePattern.isWildcard) {
|
|
208
|
+
query += ' AND c.relname LIKE $2';
|
|
209
|
+
} else {
|
|
210
|
+
query += ' AND c.relname = $2';
|
|
211
|
+
}
|
|
212
|
+
|
|
207
213
|
const result = await db.query({
|
|
208
|
-
statement:
|
|
209
|
-
FROM pg_class c
|
|
210
|
-
JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
211
|
-
WHERE n.nspname = $1
|
|
212
|
-
AND c.relkind = 'r'
|
|
213
|
-
AND c.relname = $2`,
|
|
214
|
+
statement: query,
|
|
214
215
|
params: [
|
|
215
216
|
{ type: 'varchar', value: schema },
|
|
216
217
|
{ type: 'varchar', value: tablePattern.tablePattern }
|
|
@@ -219,6 +220,7 @@ export class WalStream {
|
|
|
219
220
|
|
|
220
221
|
tableRows = pgwire.pgwireRows(result);
|
|
221
222
|
}
|
|
223
|
+
|
|
222
224
|
let result: storage.SourceTable[] = [];
|
|
223
225
|
|
|
224
226
|
for (let row of tableRows) {
|
|
@@ -258,16 +260,18 @@ export class WalStream {
|
|
|
258
260
|
|
|
259
261
|
const cresult = await getReplicationIdentityColumns(db, relid);
|
|
260
262
|
|
|
261
|
-
const
|
|
263
|
+
const columnTypes = (JSON.parse(row.column_types) as string[]).map((e) => Number(e));
|
|
264
|
+
const table = await this.handleRelation({
|
|
262
265
|
batch,
|
|
263
|
-
{
|
|
266
|
+
descriptor: {
|
|
264
267
|
name,
|
|
265
268
|
schema,
|
|
266
269
|
objectId: relid,
|
|
267
270
|
replicaIdColumns: cresult.replicationColumns
|
|
268
271
|
} as SourceEntityDescriptor,
|
|
269
|
-
false
|
|
270
|
-
|
|
272
|
+
snapshot: false,
|
|
273
|
+
referencedTypeIds: columnTypes
|
|
274
|
+
});
|
|
271
275
|
|
|
272
276
|
result.push(table);
|
|
273
277
|
}
|
|
@@ -683,7 +687,14 @@ WHERE oid = $1::regclass`,
|
|
|
683
687
|
}
|
|
684
688
|
}
|
|
685
689
|
|
|
686
|
-
async handleRelation(
|
|
690
|
+
async handleRelation(options: {
|
|
691
|
+
batch: storage.BucketStorageBatch;
|
|
692
|
+
descriptor: SourceEntityDescriptor;
|
|
693
|
+
snapshot: boolean;
|
|
694
|
+
referencedTypeIds: number[];
|
|
695
|
+
}) {
|
|
696
|
+
const { batch, descriptor, snapshot, referencedTypeIds } = options;
|
|
697
|
+
|
|
687
698
|
if (!descriptor.objectId && typeof descriptor.objectId != 'number') {
|
|
688
699
|
throw new ReplicationAssertionError(`objectId expected, got ${typeof descriptor.objectId}`);
|
|
689
700
|
}
|
|
@@ -699,6 +710,9 @@ WHERE oid = $1::regclass`,
|
|
|
699
710
|
// Drop conflicting tables. This includes for example renamed tables.
|
|
700
711
|
await batch.drop(result.dropTables);
|
|
701
712
|
|
|
713
|
+
// Ensure we have a description for custom types referenced in the table.
|
|
714
|
+
await this.connections.types.fetchTypes(referencedTypeIds);
|
|
715
|
+
|
|
702
716
|
// Snapshot if:
|
|
703
717
|
// 1. Snapshot is requested (false for initial snapshot, since that process handles it elsewhere)
|
|
704
718
|
// 2. Snapshot is not already done, AND:
|
|
@@ -789,7 +803,7 @@ WHERE oid = $1::regclass`,
|
|
|
789
803
|
|
|
790
804
|
if (msg.tag == 'insert') {
|
|
791
805
|
this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED).add(1);
|
|
792
|
-
const baseRecord =
|
|
806
|
+
const baseRecord = this.connections.types.constructAfterRecord(msg);
|
|
793
807
|
return await batch.save({
|
|
794
808
|
tag: storage.SaveOperationTag.INSERT,
|
|
795
809
|
sourceTable: table,
|
|
@@ -802,8 +816,8 @@ WHERE oid = $1::regclass`,
|
|
|
802
816
|
this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED).add(1);
|
|
803
817
|
// "before" may be null if the replica id columns are unchanged
|
|
804
818
|
// It's fine to treat that the same as an insert.
|
|
805
|
-
const before =
|
|
806
|
-
const after =
|
|
819
|
+
const before = this.connections.types.constructBeforeRecord(msg);
|
|
820
|
+
const after = this.connections.types.constructAfterRecord(msg);
|
|
807
821
|
return await batch.save({
|
|
808
822
|
tag: storage.SaveOperationTag.UPDATE,
|
|
809
823
|
sourceTable: table,
|
|
@@ -814,7 +828,7 @@ WHERE oid = $1::regclass`,
|
|
|
814
828
|
});
|
|
815
829
|
} else if (msg.tag == 'delete') {
|
|
816
830
|
this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED).add(1);
|
|
817
|
-
const before =
|
|
831
|
+
const before = this.connections.types.constructBeforeRecord(msg)!;
|
|
818
832
|
|
|
819
833
|
return await batch.save({
|
|
820
834
|
tag: storage.SaveOperationTag.DELETE,
|
|
@@ -955,7 +969,12 @@ WHERE oid = $1::regclass`,
|
|
|
955
969
|
|
|
956
970
|
for (const msg of messages) {
|
|
957
971
|
if (msg.tag == 'relation') {
|
|
958
|
-
await this.handleRelation(
|
|
972
|
+
await this.handleRelation({
|
|
973
|
+
batch,
|
|
974
|
+
descriptor: getPgOutputRelation(msg),
|
|
975
|
+
snapshot: true,
|
|
976
|
+
referencedTypeIds: referencedColumnTypeIds(msg)
|
|
977
|
+
});
|
|
959
978
|
} else if (msg.tag == 'begin') {
|
|
960
979
|
// This may span multiple transactions in the same chunk, or even across chunks.
|
|
961
980
|
skipKeepalive = true;
|