@wbce/projen-d9 0.0.27

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. package/.jsii +552 -0
  2. package/API.md +8472 -0
  3. package/CHANGELOG.md +189 -0
  4. package/LICENSE +674 -0
  5. package/README.md +82 -0
  6. package/lib/cli/commands/init.d.ts +1 -0
  7. package/lib/cli/commands/init.js +8 -0
  8. package/lib/cli/commands/save/apply-snapshot.d.ts +7 -0
  9. package/lib/cli/commands/save/apply-snapshot.js +278 -0
  10. package/lib/cli/commands/save/cli.d.ts +8 -0
  11. package/lib/cli/commands/save/cli.js +33 -0
  12. package/lib/cli/commands/save/draft.d.ts +0 -0
  13. package/lib/cli/commands/save/draft.js +2 -0
  14. package/lib/cli/commands/save/utils.d.ts +9 -0
  15. package/lib/cli/commands/save/utils.js +123 -0
  16. package/lib/cli/commands/sync.d.ts +5 -0
  17. package/lib/cli/commands/sync.js +13 -0
  18. package/lib/cli/index.d.ts +2 -0
  19. package/lib/cli/index.js +44 -0
  20. package/lib/index.d.ts +59 -0
  21. package/lib/index.js +322 -0
  22. package/node_modules/@types/node/LICENSE +21 -0
  23. package/node_modules/@types/node/README.md +16 -0
  24. package/node_modules/@types/node/assert.d.ts +117 -0
  25. package/node_modules/@types/node/async_hooks.d.ts +246 -0
  26. package/node_modules/@types/node/buffer.d.ts +22 -0
  27. package/node_modules/@types/node/child_process.d.ts +507 -0
  28. package/node_modules/@types/node/cluster.d.ts +260 -0
  29. package/node_modules/@types/node/console.d.ts +3 -0
  30. package/node_modules/@types/node/constants.d.ts +448 -0
  31. package/node_modules/@types/node/crypto.d.ts +1183 -0
  32. package/node_modules/@types/node/dgram.d.ts +118 -0
  33. package/node_modules/@types/node/dns.d.ts +372 -0
  34. package/node_modules/@types/node/domain.d.ts +16 -0
  35. package/node_modules/@types/node/events.d.ts +47 -0
  36. package/node_modules/@types/node/fs.d.ts +2567 -0
  37. package/node_modules/@types/node/globals.d.ts +1351 -0
  38. package/node_modules/@types/node/globals.global.d.ts +1 -0
  39. package/node_modules/@types/node/http.d.ts +456 -0
  40. package/node_modules/@types/node/http2.d.ts +952 -0
  41. package/node_modules/@types/node/https.d.ts +156 -0
  42. package/node_modules/@types/node/index.d.ts +91 -0
  43. package/node_modules/@types/node/inspector.d.ts +3040 -0
  44. package/node_modules/@types/node/module.d.ts +3 -0
  45. package/node_modules/@types/node/net.d.ts +290 -0
  46. package/node_modules/@types/node/os.d.ts +263 -0
  47. package/node_modules/@types/node/package.json +195 -0
  48. package/node_modules/@types/node/path.d.ts +159 -0
  49. package/node_modules/@types/node/perf_hooks.d.ts +259 -0
  50. package/node_modules/@types/node/process.d.ts +15 -0
  51. package/node_modules/@types/node/punycode.d.ts +75 -0
  52. package/node_modules/@types/node/querystring.d.ts +29 -0
  53. package/node_modules/@types/node/readline.d.ts +158 -0
  54. package/node_modules/@types/node/repl.d.ts +390 -0
  55. package/node_modules/@types/node/stream.d.ts +334 -0
  56. package/node_modules/@types/node/string_decoder.d.ts +7 -0
  57. package/node_modules/@types/node/timers.d.ts +16 -0
  58. package/node_modules/@types/node/tls.d.ts +713 -0
  59. package/node_modules/@types/node/trace_events.d.ts +61 -0
  60. package/node_modules/@types/node/tty.d.ts +66 -0
  61. package/node_modules/@types/node/url.d.ts +147 -0
  62. package/node_modules/@types/node/util.d.ts +200 -0
  63. package/node_modules/@types/node/v8.d.ts +197 -0
  64. package/node_modules/@types/node/vm.d.ts +112 -0
  65. package/node_modules/@types/node/wasi.d.ts +55 -0
  66. package/node_modules/@types/node/worker_threads.d.ts +207 -0
  67. package/node_modules/@types/node/zlib.d.ts +356 -0
  68. package/node_modules/@types/pg/LICENSE +21 -0
  69. package/node_modules/@types/pg/README.md +15 -0
  70. package/node_modules/@types/pg/index.d.mts +56 -0
  71. package/node_modules/@types/pg/index.d.ts +360 -0
  72. package/node_modules/@types/pg/lib/connection-parameters.d.ts +58 -0
  73. package/node_modules/@types/pg/lib/type-overrides.d.ts +77 -0
  74. package/node_modules/@types/pg/package.json +38 -0
  75. package/node_modules/@types/pg-copy-streams/LICENSE +21 -0
  76. package/node_modules/@types/pg-copy-streams/README.md +38 -0
  77. package/node_modules/@types/pg-copy-streams/index.d.ts +19 -0
  78. package/node_modules/@types/pg-copy-streams/package.json +28 -0
  79. package/node_modules/commander/LICENSE +22 -0
  80. package/node_modules/commander/Readme.md +1176 -0
  81. package/node_modules/commander/esm.mjs +16 -0
  82. package/node_modules/commander/index.js +24 -0
  83. package/node_modules/commander/lib/argument.js +150 -0
  84. package/node_modules/commander/lib/command.js +2777 -0
  85. package/node_modules/commander/lib/error.js +39 -0
  86. package/node_modules/commander/lib/help.js +747 -0
  87. package/node_modules/commander/lib/option.js +380 -0
  88. package/node_modules/commander/lib/suggestSimilar.js +101 -0
  89. package/node_modules/commander/package-support.json +19 -0
  90. package/node_modules/commander/package.json +82 -0
  91. package/node_modules/commander/typings/esm.d.mts +3 -0
  92. package/node_modules/commander/typings/index.d.ts +1113 -0
  93. package/node_modules/pg/LICENSE +21 -0
  94. package/node_modules/pg/README.md +95 -0
  95. package/node_modules/pg/esm/index.mjs +20 -0
  96. package/node_modules/pg/lib/client.js +743 -0
  97. package/node_modules/pg/lib/connection-parameters.js +171 -0
  98. package/node_modules/pg/lib/connection.js +221 -0
  99. package/node_modules/pg/lib/crypto/cert-signatures.js +122 -0
  100. package/node_modules/pg/lib/crypto/sasl.js +212 -0
  101. package/node_modules/pg/lib/crypto/utils-legacy.js +43 -0
  102. package/node_modules/pg/lib/crypto/utils-webcrypto.js +89 -0
  103. package/node_modules/pg/lib/crypto/utils.js +9 -0
  104. package/node_modules/pg/lib/defaults.js +91 -0
  105. package/node_modules/pg/lib/index.js +73 -0
  106. package/node_modules/pg/lib/native/client.js +323 -0
  107. package/node_modules/pg/lib/native/index.js +2 -0
  108. package/node_modules/pg/lib/native/query.js +165 -0
  109. package/node_modules/pg/lib/query.js +252 -0
  110. package/node_modules/pg/lib/result.js +109 -0
  111. package/node_modules/pg/lib/stream.js +83 -0
  112. package/node_modules/pg/lib/type-overrides.js +35 -0
  113. package/node_modules/pg/lib/utils.js +217 -0
  114. package/node_modules/pg/package.json +76 -0
  115. package/node_modules/pg-copy-streams/Makefile +19 -0
  116. package/node_modules/pg-copy-streams/README.md +315 -0
  117. package/node_modules/pg-copy-streams/copy-both.js +267 -0
  118. package/node_modules/pg-copy-streams/copy-from.js +173 -0
  119. package/node_modules/pg-copy-streams/copy-to.js +177 -0
  120. package/node_modules/pg-copy-streams/eslint.config.js +55 -0
  121. package/node_modules/pg-copy-streams/index.js +17 -0
  122. package/node_modules/pg-copy-streams/message-formats.js +27 -0
  123. package/node_modules/pg-copy-streams/obuf.js +376 -0
  124. package/node_modules/pg-copy-streams/package.json +51 -0
  125. package/package.json +136 -0
@@ -0,0 +1,76 @@
1
+ {
2
+ "name": "pg",
3
+ "version": "8.20.0",
4
+ "description": "PostgreSQL client - pure javascript & libpq with the same API",
5
+ "keywords": [
6
+ "database",
7
+ "libpq",
8
+ "pg",
9
+ "postgre",
10
+ "postgres",
11
+ "postgresql",
12
+ "rdbms"
13
+ ],
14
+ "homepage": "https://github.com/brianc/node-postgres",
15
+ "repository": {
16
+ "type": "git",
17
+ "url": "git://github.com/brianc/node-postgres.git",
18
+ "directory": "packages/pg"
19
+ },
20
+ "author": "Brian Carlson <brian.m.carlson@gmail.com>",
21
+ "main": "./lib",
22
+ "exports": {
23
+ ".": {
24
+ "import": "./esm/index.mjs",
25
+ "require": "./lib/index.js",
26
+ "default": "./lib/index.js"
27
+ },
28
+ "./package.json": {
29
+ "default": "./package.json"
30
+ },
31
+ "./lib/*": "./lib/*.js",
32
+ "./lib/*.js": "./lib/*.js"
33
+ },
34
+ "dependencies": {
35
+ "pg-connection-string": "^2.12.0",
36
+ "pg-pool": "^3.13.0",
37
+ "pg-protocol": "^1.13.0",
38
+ "pg-types": "2.2.0",
39
+ "pgpass": "1.0.5"
40
+ },
41
+ "devDependencies": {
42
+ "@cloudflare/vitest-pool-workers": "0.8.23",
43
+ "@cloudflare/workers-types": "^4.20230404.0",
44
+ "async": "2.6.4",
45
+ "bluebird": "3.7.2",
46
+ "co": "4.6.0",
47
+ "pg-copy-streams": "0.3.0",
48
+ "typescript": "^4.0.3",
49
+ "vitest": "~3.0.9",
50
+ "wrangler": "^3.x"
51
+ },
52
+ "optionalDependencies": {
53
+ "pg-cloudflare": "^1.3.0"
54
+ },
55
+ "peerDependencies": {
56
+ "pg-native": ">=3.0.1"
57
+ },
58
+ "peerDependenciesMeta": {
59
+ "pg-native": {
60
+ "optional": true
61
+ }
62
+ },
63
+ "scripts": {
64
+ "test": "make test-all"
65
+ },
66
+ "files": [
67
+ "lib",
68
+ "esm",
69
+ "SPONSORS.md"
70
+ ],
71
+ "license": "MIT",
72
+ "engines": {
73
+ "node": ">= 16.0.0"
74
+ },
75
+ "gitHead": "c9070cc8d526fca65780cedc25c1966b57cf7532"
76
+ }
@@ -0,0 +1,19 @@
1
+ .PHONY: publish-patch test
2
+
3
+ test:
4
+ npm test
5
+
6
+ patch: test
7
+ npm version patch -m "Bump version"
8
+ git push origin master --tags
9
+ npm publish
10
+
11
+ minor: test
12
+ npm version minor -m "Bump version"
13
+ git push origin master --tags
14
+ npm publish
15
+
16
+ major: test
17
+ npm version major -m "Bump version"
18
+ git push origin master --tags
19
+ npm publish
@@ -0,0 +1,315 @@
1
+ ## pg-copy-streams
2
+
3
+ [![Build Status](https://travis-ci.org/brianc/node-pg-copy-streams.svg)](https://travis-ci.org/brianc/node-pg-copy-streams)
4
+
5
+ COPY FROM / COPY TO for node-postgres. Stream from one database to another, and stuff.
6
+
7
+ ## how? what? huh?
8
+
9
+ Did you know that PostgreSQL supports streaming data directly into and out of a table?
10
+ This means you can take your favorite CSV or TSV file and pipe it directly into an existing PostgreSQL table.
11
+
12
+ PostgreSQL supports text, csv/tsv and binary data. If you have data in another format (say for example JSON) convert it to one of the supported format and pipe it directly into an existing PostgreSQL table !
13
+
14
+ You can also take a table and pipe it directly to a file, another database, stdout, even to `/dev/null` if you're crazy!
15
+
16
+ What this module gives you is a [Readable](http://nodejs.org/api/stream.html#stream_class_stream_readable) or [Writable](http://nodejs.org/api/stream.html#stream_class_stream_writable) stream directly into/out of a table in your database.
17
+ This mode of interfacing with your table is _very fast_ and _very brittle_. You are responsible for properly encoding and ordering all your columns. If anything is out of place PostgreSQL will send you back an error. The stream works within a transaction so you wont leave things in a 1/2 borked state, but it's still good to be aware of.
18
+
19
+ If you're not familiar with the feature (I wasn't either) you can read this for some good helps: https://www.postgresql.org/docs/current/sql-copy.html
20
+
21
+ ## examples
22
+
23
+ ### pipe from a table to stdout (copyOut - copy-to)
24
+
25
+ ```js
26
+ var { Pool } = require('pg')
27
+ var { to as copyTo } = require('pg-copy-streams')
28
+
29
+ var pool = new Pool()
30
+
31
+ pool.connect(function (err, client, done) {
32
+ var stream = client.query(copyTo('COPY my_table TO STDOUT'))
33
+ stream.pipe(process.stdout)
34
+ stream.on('end', done)
35
+ stream.on('error', done)
36
+ })
37
+
38
+
39
+ // async/await
40
+ import { pipeline } from 'node:stream/promises'
41
+ import pg from 'pg'
42
+ import { to as copyTo } from 'pg-copy-streams'
43
+
44
+ const pool = new pg.Pool()
45
+ const client = await pool.connect()
46
+ try {
47
+ const stream = client.query(copyTo('COPY my_table TO STDOUT'))
48
+ await pipeline(stream, process.stdout)
49
+ } finally {
50
+ client.release()
51
+ }
52
+ await pool.end()
53
+ ```
54
+
55
+ _Important_: When copying data out of postgresql, postgresql will chunk the data on 64kB boundaries. You should expect rows to be cut across the boundaries of these chunks (the end of a chunk will not always match the end of a row). If you are piping the csv output of postgres into a file, this might not be a problem. But if you are trying to analyse the csv output on-the-fly, you need to make sure that you correctly discover the lines of the csv output across the chunk boundaries. We are not recommending any specific streaming csv parser but `csv-parser` and `csv-parse` seem to correctly handle this.
56
+
57
+ ### pipe from a file to table (copyIn - copy-from)
58
+
59
+ ```js
60
+ var fs = require('node:fs')
61
+ var { Pool } = require('pg')
62
+ var { from as copyFrom } = require('pg-copy-streams')
63
+
64
+ var pool = new Pool()
65
+
66
+ pool.connect(function (err, client, done) {
67
+ var stream = client.query(copyFrom('COPY my_table FROM STDIN'))
68
+ var fileStream = fs.createReadStream('some_file.tsv')
69
+ fileStream.on('error', done)
70
+ stream.on('error', done)
71
+ stream.on('finish', done)
72
+ fileStream.pipe(stream)
73
+ })
74
+
75
+
76
+ // async/await
77
+ import { pipeline } from 'node:stream/promises'
78
+ import fs from 'node:fs'
79
+ import pg from 'pg'
80
+ import { from as copyFrom } from 'pg-copy-streams'
81
+
82
+ const pool = new pg.Pool()
83
+ const client = await pool.connect()
84
+ try {
85
+ const ingestStream = client.query(copyFrom('COPY my_table FROM STDIN'))
86
+ const sourceStream = fs.createReadStream('some_file.tsv')
87
+ await pipeline(sourceStream, ingestStream)
88
+ } finally {
89
+ client.release()
90
+ }
91
+ await pool.end()
92
+ ```
93
+
94
+ _Note_: In version prior to 4.0.0, when copying data into postgresql, it was necessary to wait for the 'end' event of `pg-copy-streams.from` to correctly detect the end of the COPY operation. This was necessary due to the internals of the module but non-standard. This is not true for versions including and after 4.0.0. The end of the COPY operation must now be detected via the standard 'finish' event. **Users of 4.0.0+ should not wait for the 'end' event because it is not fired anymore.**
95
+
96
+ In version 6.0.0+, If you have not yet finished ingesting data into a copyFrom stream and you want to ask postgresql to abort the process, you can call `destroy()` on the stream (or let `pipeline` do it for you if it detects an error in the pipeline). This will send a CopyFail message to the backend that will rollback the operation. Please take into account that this will not revert the operation if the CopyDone message has already been sent and is being processed by the backend.
97
+
98
+ ### duplex stream for replication / logical decoding scenarios (copyBoth - copy-both)
99
+
100
+ This is a more advanded topic.
101
+ Check the test/copy-both.js file for an example of how this can be used.
102
+
103
+ _Note regarding logical decoding_: Parsers for logical decoding scenarios are easier to write when copy-both.js pushes chunks that are aligned on the copyData protocol frames. This is not the default mode of operation of copy-both.js in order to increase the streaming performance. If you need the pushed chunks to be aligned on copyData frames, use the `alignOnCopyDataFrame: true` option.
104
+
105
+
106
+ ## install
107
+
108
+ ```sh
109
+ $ npm install pg-copy-streams
110
+ ```
111
+
112
+ ## notice
113
+
114
+ This module **only** works with the pure JavaScript bindings. If you're using `require('pg').native` please make sure to use normal `require('pg')` or `require('pg.js')` when you're using copy streams.
115
+
116
+ Before you set out on this magical piping journey, you _really_ should read this: http://www.postgresql.org/docs/current/static/sql-copy.html, and you might want to take a look at the [tests](https://github.com/brianc/node-pg-copy-streams/tree/master/test) to get an idea of how things work.
117
+
118
+ Take note of the following warning in the PostgreSQL documentation:
119
+
120
+ > COPY stops operation at the first error. This should not lead to problems in the event of a COPY TO, but the target table will already have received earlier rows in a COPY FROM. These rows will not be visible or accessible, but they still occupy disk space. This might amount to a considerable amount of wasted disk space if the failure happened well into a large copy operation. You might wish to invoke VACUUM to recover the wasted space.
121
+
122
+ ## benchmarks
123
+
124
+ The COPY command is commonly used to move huge sets of data. This can put some pressure on the node.js loop, the amount of CPU or the amount of memory used.
125
+ There is a bench/ directory in the repository where benchmark scripts are stored. If you have performance issues with `pg-copy-stream` do not hesitate to write a new benchmark that highlights your issue. Please avoid to commit huge files (PR won't be accepted) and find other ways to generate huge datasets.
126
+
127
+ If you have a local instance of postgres on your machine, you can start a benchmark for example with
128
+
129
+ ```sh
130
+ $ cd bench
131
+ $ PGPORT=5432 PGDATABASE=postgres node copy-from.js
132
+ ```
133
+
134
+ ## tests
135
+
136
+ In order to launch the test suite, you need to have a local instance of postgres running on your machine.
137
+
138
+ Since version 5.1.0 and the implementation of copy-both.js for logical decoding scenarios, your local postgres instance will need to be configured to accept replication scenarios :
139
+
140
+ ```
141
+ postgresql.conf
142
+ wal_level = logical
143
+ max_wal_senders > 0
144
+ max_replication_slots > 0
145
+
146
+ pg_hba.conf
147
+ make sure your user can connect using the replication mode
148
+ ```
149
+
150
+ ```sh
151
+ $ PGPORT=5432 PGDATABASE=postgres make test
152
+ ```
153
+
154
+ ## contributing
155
+
156
+ Instead of adding a bunch more code to the already bloated [node-postgres](https://github.com/brianc/node-postgres) I am trying to make the internals extensible and work on adding edge-case features as 3rd party modules.
157
+ This is one of those.
158
+
159
+ Please, if you have any issues with this, open an issue.
160
+
161
+ Better yet, submit a pull request. I _love_ pull requests.
162
+
163
+ Generally how I work is if you submit a few pull requests and you're interested I'll make you a contributor and give you full access to everything.
164
+
165
+ Since this isn't a module with tons of installs and dependent modules I hope we can work together on this to iterate faster here and make something really useful.
166
+
167
+ ## changelog
168
+
169
+ ### version 7.0.0 - published 2025-05-27
170
+
171
+ No new feature. This is a maintenance release.
172
+ The major version was decided because the 'obuf' dependency was internalized and slightly modified to pass prettier, pass eslint and replace deprecated `new Buffer(size)` calls by `Buffer.alloc(size)` which while not beeing a revolution, is worth a major version after (previous version was 2 years ago)
173
+
174
+ - "Forked" the obuf project into the repo in order to keep maintain it internally after no response from the maintainer
175
+ https://github.com/indutny/offset-buffer - License MIT - By Fedor Indutny, 2015
176
+ open issue asking for transfer of npm ownership of the original obuf project - https://github.com/indutny/offset-buffer/issues/8#issuecomment-2817245503
177
+ This library has an API that is elegant for protocol/buffer handling, which is why it was chosen originally
178
+ Original tests were also added to the test suite of `pg-copy-stream`
179
+ - updated dev dependencies: mocha, eslint, cvs-parse, cvs-parser, pg
180
+
181
+ ### version 6.0.6 - published 2023-07-17
182
+
183
+ - copy-to: fix rowCount in BINARY mode. The file trailer was incorrectly counted as a row
184
+
185
+ ### version 6.0.5 - published 2023-03-07
186
+
187
+ - improve ejs/cjs Named exports compatibility for easier async/await usage
188
+
189
+ ### version 6.0.4 - published 2022-09-05
190
+ ### version 6.0.3 - published 2022-09-05
191
+
192
+ - copy-from: fix issue #136 when the _writev mechanism was triggered with a very large number of chunks
193
+
194
+ ### version 6.0.2 - published 2021-09-13
195
+
196
+ - copy-from : fix interaction with `pg` optional timeout mechanism
197
+
198
+ ### version 6.0.1 - published 2021-08-23
199
+
200
+ - Bugfix for node 14+. The order of _destroy / _final calls are different before and after node 14 which caused an issue with the COPY FROM _destroy implementation that appeared in version 6.0.0.
201
+
202
+ ### version 6.0.0 - published 2021-08-20
203
+
204
+ - Implement _destroy in COPY FROM operations. `pipeline` will automatically send a CopyFail message to the backend is a source triggers an error. cf #115
205
+
206
+ This version is a major change because some users of the library may have been using other techniques in order to ask the backend to rollback the current operation.
207
+
208
+ ### version 5.1.1 - published 2020-07-21
209
+
210
+ Bugfix release handling a corner case when an empty stream is piped into copy-from
211
+
212
+ - fix copy-from.js handling of an empty source
213
+
214
+ ### version 5.1.0 - published 2020-06-07
215
+
216
+ This version adds a Duplex stream implementation of the PostgreSQL copyBoth mode described on https://www.postgresql.org/docs/9.6/protocol-flow.html. This mode opens the possibility of dealing with replication and logical decoding scenarios.
217
+
218
+ - implement copy-both.js
219
+
220
+ ### version 5.0.0 - published 2020-05-14
221
+
222
+ This version's major change is a modification in the COPY TO implementation. The new implementation now extends `Readable` while previous version where extending `Transform`. This should not have an effect on how users use the module but was considered to justify a major version number because even if the test suite coverage is wide, it could have an impact on the streaming dynamics in certain edge cases that are not yet captured by the tests.
223
+
224
+ - Rewrite copy-to in order to have it extend `Readable` instead of `Transform`
225
+
226
+
227
+ ### version 4.0.0 - published 2020-05-11
228
+
229
+ This version's major change is a modification in the COPY FROM implementation. In previous version, copy-from was internally designed as a `Transform` duplex stream. The user-facing API was writable, and the readable side of the `Transform` was piped into the postgres connection stream to copy the data inside the database.
230
+ This led to an issue because `Transform` was emitting its 'finish' too early after the writable side was ended. Postgres had not yet read all the data on the readable side and had not confirmed that the COPY operation was finished. The recommendation was to wait for the 'end' event on the readable side which correctly detected the end of the COPY operation and the fact that the pg connection was ready for new queries.
231
+ This recommendation worked ok but this way of detecting the end of a writable is not standard and was leading to different issues (interaction with the `finished` and `pipeline` API for example)
232
+ The new copy-from implementation extends writable and now emits 'finish' with the correct timing : after the COPY operation and after the postgres connection has reached the readyForQuery state.
233
+ Another big change in this version is that copy-to now shortcuts the core `pg` parsing during the COPY operation. This avoids double-parsing and avoids the fact that `pg` buffers whole postgres protocol messages.
234
+
235
+ - Rewrite copy-from in order to have it extend `Writable` instead of `Transform`
236
+ - Modify copy-to to shortcut the pg protocol parser during the COPY operation
237
+ - Add Stream compliance tests for copy-to and copy-from
238
+
239
+
240
+ ### version 3.0.0 - published 2020-05-02
241
+
242
+ This version's major change is a modification in the COPY TO implementation. In the previous versions, a row could be pushed downstream only after the full row was gathered in memory. In many cases, rows are small and this is not an issue. But there are some use cases where rows can grow bigger (think of a row containing a 1MB raw image in a BYTEA field. cf issue #91). In these cases, the library was constantly trying to allocate very big buffers and this could lead to severe performance issues.
243
+ In the new implementation, all the data payload received from a postgres chunk is sent downstream without waiting for full row boundaries.
244
+
245
+ Some users may in the past have relied on the fact the the copy-to chunk boundaries exactly matched row boundaries. A major difference in the 3.x version is that the module does not offer any guarantee that its chunk boundaries match row boundaries. A row data could (and you have to realize that this will happen) be split across 2 or more chunks depending on the size of the rows and on postgres's own chunking decisions.
246
+
247
+ As a consequence, when the copy-to stream is piped into a pipeline that does row/CSV parsing, you need to make sure that this pipeline correctly handles rows than span across chunk boundaries. For its tests, this module uses the [csv-parser](https://github.com/mafintosh/csv-parser) module
248
+
249
+ - Add `prettier` configuration following discussion on brianc/node-postgres#2172
250
+ - Rewrite the copy-to implementation in order to avoid fetching whole rows in memory
251
+ - Use mocha for tests
252
+ - Add new tests for copy-to.js focusing on chunk boundaries
253
+ - Add integration tests for two streaming csv parsers: csv-parser and csv-parse
254
+ - Add eslint
255
+ - Add test for quick&dirty bytea binary extraction
256
+ - Add benchmark for copy-to in bench/copy-to.js
257
+
258
+ ### version 2.2.2 - published 2019-07-22
259
+
260
+ - Bugfix copy-to could pause the client connection, preventing re-use
261
+
262
+ ### version 2.2.1 - published 2019-07-22
263
+
264
+ - Bugfix copy-from was not correctly unpiped from the the connection stream
265
+
266
+ ### version 2.2.0 - published 2019-03-21
267
+
268
+ - Small refactor in copy-from passing from 3 push to 2 push in every chunk transform loop
269
+ - Add bench/ directory for benchmarks
270
+ - Add benchmark to compare performance of pg-copy-stream wrt psql during copy-from
271
+ - Add benchmark to measure memory usage of copy-from
272
+
273
+ ### version 2.1.0 - published 2019-03-19
274
+
275
+ - Change README to stop using the pg pool singleton (removed after pg 7.0)
276
+ - Do not register copy-to.pushBufferIfNeeded on the instance itself (avoid dangling method on the object)
277
+ - Fix copy-to test wrt intermittent unhandled promise bug
278
+ - Add tests regarding client re-use
279
+
280
+ ### version 2.0.0 - published 2019-03-14
281
+
282
+ This version's major change is a modification in the COPY TO implementation. In the previous version, when a chunk was received from the database, it was analyzed and every row contained within that chunk was pushed individually down the stream pipeline. Small rows could lead to a "one chunk" / "thousands of row pushed" performance issue in node. Thanks to @rafatower & CartoDB for the patch.
283
+ This is considered to be a major change since some people could be relying on the fact that each outgoing chunk is an individual row.
284
+
285
+ Other changes in this version
286
+
287
+ - Use Strict
288
+ - Travis deprecation of old node version (0.12, 0.4). Support LTS 6, 8, 10 and Current 11
289
+ - Update dev dependencies (pg, lodash)
290
+ - Stop using deprecated Buffer constructor
291
+ - Add package-lock.json
292
+
293
+ ## license
294
+
295
+ The MIT License (MIT)
296
+
297
+ Copyright (c) 2013 Brian M. Carlson
298
+
299
+ Permission is hereby granted, free of charge, to any person obtaining a copy
300
+ of this software and associated documentation files (the "Software"), to deal
301
+ in the Software without restriction, including without limitation the rights
302
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
303
+ copies of the Software, and to permit persons to whom the Software is
304
+ furnished to do so, subject to the following conditions:
305
+
306
+ The above copyright notice and this permission notice shall be included in
307
+ all copies or substantial portions of the Software.
308
+
309
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
310
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
311
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
312
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
313
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
314
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
315
+ THE SOFTWARE.
@@ -0,0 +1,267 @@
1
+ 'use strict'
2
+
3
+ module.exports = function (txt, options) {
4
+ return new CopyStreamQuery(txt, options)
5
+ }
6
+
7
+ const { Duplex } = require('stream')
8
+ const assert = require('assert')
9
+ const BufferList = require('./obuf')
10
+ const code = require('./message-formats')
11
+
12
+ // Readable decoder states
13
+ const PG_CODE = 0
14
+ const PG_LENGTH = 1
15
+ const PG_MESSAGE = 2
16
+
17
+ class CopyStreamQuery extends Duplex {
18
+ constructor(text, options) {
19
+ super(options)
20
+ this.text = text
21
+
22
+ // Readable side
23
+ this._state = PG_CODE
24
+ this._buffer = new BufferList()
25
+ this._unreadMessageContentLength = 0
26
+ this._copyDataChunks = new BufferList()
27
+ this._pgDataHandler = null
28
+ this._drained = false
29
+ this._forwarding = false
30
+ this._onReadableEvent = this._onReadable.bind(this)
31
+ this.alignOnCopyDataFrame = options ? options.alignOnCopyDataFrame === true : false
32
+
33
+ // Writable side
34
+ this._gotCopyInResponse = false
35
+ this.chunks = []
36
+ this.cb = null
37
+ this.cork()
38
+ }
39
+
40
+ submit(connection) {
41
+ this.connection = connection
42
+ this._attach()
43
+ connection.query(this.text)
44
+ }
45
+
46
+ /* Readable implementation */
47
+ _attach() {
48
+ const connectionStream = this.connection.stream
49
+ const pgDataListeners = connectionStream.listeners('data')
50
+ assert(pgDataListeners.length == 1)
51
+ this._pgDataHandler = pgDataListeners.pop()
52
+ connectionStream.removeListener('data', this._pgDataHandler)
53
+ connectionStream.pause()
54
+ this._forward()
55
+ connectionStream.on('readable', this._onReadableEvent)
56
+ }
57
+
58
+ _detach() {
59
+ const connectionStream = this.connection.stream
60
+ const unreadBuffer = this._buffer.take(this._buffer.size)
61
+ connectionStream.removeListener('readable', this._onReadableEvent)
62
+ connectionStream.addListener('data', this._pgDataHandler)
63
+ this._pgDataHandler(unreadBuffer)
64
+
65
+ // unpipe can pause the stream but also underlying onData event can potentially pause the stream because of hitting
66
+ // the highWaterMark and pausing the stream, so we resume the stream in the next tick after the underlying onData
67
+ // event has finished
68
+ process.nextTick(function () {
69
+ connectionStream.resume()
70
+ })
71
+ }
72
+
73
+ _cleanup() {
74
+ this._buffer = null
75
+ this._copyDataChunks = null
76
+ this._pgDataHandler = null
77
+ this._onReadableEvent = null
78
+ }
79
+
80
+ _onReadable() {
81
+ this._forward()
82
+ }
83
+
84
+ _read() {
85
+ this._drained = true
86
+ this._forward()
87
+ }
88
+
89
+ _forward() {
90
+ if (this._forwarding || !this._drained || !this.connection) return
91
+ this._forwarding = true
92
+ const connectionStream = this.connection.stream
93
+ let chunk
94
+ while (this._drained && (chunk = connectionStream.read()) !== null) {
95
+ this._drained = this._parse(chunk)
96
+ }
97
+ this._forwarding = false
98
+ }
99
+
100
+ _parse(chunk) {
101
+ let done = false
102
+ let drained = true
103
+ this._buffer.push(chunk)
104
+
105
+ while (!done && this._buffer.size > 0) {
106
+ if (PG_CODE === this._state) {
107
+ if (!this._buffer.has(1)) break
108
+ this._code = this._buffer.peekUInt8()
109
+ if (this._code === code.ErrorResponse) {
110
+ // ErrorResponse Interception
111
+ // We must let pg parse future messages and handle their consequences on
112
+ // the ActiveQuery
113
+ this._detach()
114
+ return
115
+ }
116
+ this._buffer.readUInt8()
117
+ this._state = PG_LENGTH
118
+ }
119
+
120
+ if (PG_LENGTH === this._state) {
121
+ if (!this._buffer.has(4)) break
122
+ this._unreadMessageContentLength = this._buffer.readUInt32BE() - 4
123
+ this._state = PG_MESSAGE
124
+ }
125
+
126
+ if (PG_MESSAGE === this._state) {
127
+ if (this._unreadMessageContentLength > 0 && this._buffer.size > 0) {
128
+ const n = Math.min(this._buffer.size, this._unreadMessageContentLength)
129
+ const messageContentChunk = this._buffer.take(n)
130
+ this._unreadMessageContentLength -= n
131
+ if (this._code === code.CopyData) {
132
+ this._copyDataChunks.push(messageContentChunk)
133
+ }
134
+ }
135
+
136
+ if (this._unreadMessageContentLength === 0) {
137
+ // a full message has been captured
138
+ switch (this._code) {
139
+ case code.CopyBothResponse:
140
+ this._startCopyIn()
141
+ break
142
+ case code.CopyData:
143
+ if (this.alignOnCopyDataFrame) {
144
+ drained = this._flushCopyData()
145
+ }
146
+ break
147
+ // standard interspersed messages.
148
+ // see https://www.postgresql.org/docs/9.6/protocol-flow.html#PROTOCOL-COPY
149
+ case code.ParameterStatus:
150
+ case code.NoticeResponse:
151
+ case code.NotificationResponse:
152
+ break
153
+ case code.CopyDone:
154
+ default:
155
+ done = true
156
+ break
157
+ }
158
+ this._state = PG_CODE
159
+ }
160
+ }
161
+ }
162
+
163
+ // When we are not in alignOnCopyDataFrame, copyData payload is not buffered
164
+ // Forward payload bytes as they arrive
165
+ if (!this.alignOnCopyDataFrame) {
166
+ drained = this._flushCopyData()
167
+ }
168
+
169
+ if (done) {
170
+ this._detach()
171
+ this.push(null)
172
+ this._cleanup()
173
+ }
174
+
175
+ return drained
176
+ }
177
+
178
+ _flushCopyData() {
179
+ let drained = true
180
+ const len = this._copyDataChunks.size
181
+ if (len > 0) {
182
+ drained = this.push(this._copyDataChunks.take(len))
183
+ }
184
+ return drained
185
+ }
186
+
187
+ /* Writable implementation */
188
+ _write(chunk, enc, cb) {
189
+ this.chunks.push({ chunk: chunk, encoding: enc })
190
+ if (this._gotCopyInResponse) {
191
+ return this.flush(cb)
192
+ }
193
+ this.cb = cb
194
+ }
195
+
196
+ _writev(chunks, cb) {
197
+ this.chunks.push(...chunks)
198
+ if (this._gotCopyInResponse) {
199
+ return this.flush(cb)
200
+ }
201
+ this.cb = cb
202
+ }
203
+
204
+ _final(cb) {
205
+ this.flush()
206
+ const Int32Len = 4
207
+ const finBuffer = Buffer.from([code.CopyDone, 0, 0, 0, Int32Len])
208
+ this.connection.stream.write(finBuffer)
209
+ this.cb_flush = cb
210
+ }
211
+
212
+ flush(callback) {
213
+ let chunk
214
+ let ok = true
215
+ while (ok && (chunk = this.chunks.shift())) {
216
+ ok = this.flushChunk(chunk.chunk)
217
+ }
218
+ if (callback) {
219
+ if (ok) {
220
+ callback()
221
+ } else {
222
+ if (this.chunks.length) {
223
+ this.connection.stream.once('drain', this.flush.bind(this, callback))
224
+ } else {
225
+ this.connection.stream.once('drain', callback)
226
+ }
227
+ }
228
+ }
229
+ }
230
+
231
+ flushChunk(chunk) {
232
+ const Int32Len = 4
233
+ const lenBuffer = Buffer.from([code.CopyData, 0, 0, 0, 0])
234
+ lenBuffer.writeUInt32BE(chunk.length + Int32Len, 1)
235
+ this.connection.stream.write(lenBuffer)
236
+ return this.connection.stream.write(chunk)
237
+ }
238
+
239
+ _startCopyIn() {
240
+ this._gotCopyInResponse = true
241
+ this.uncork()
242
+ this.flush()
243
+ if (this.cb) {
244
+ const { cb } = this
245
+ this.cb = null
246
+ cb()
247
+ }
248
+ }
249
+
250
+ handleError(err) {
251
+ this.emit('error', err)
252
+ this._cleanup()
253
+ }
254
+
255
+ handleCopyData(chunk) {
256
+ // an out of band copyData message
257
+ // is received after copyDone
258
+ // this is currently discarded
259
+ }
260
+
261
+ handleCommandComplete() {}
262
+
263
+ handleReadyForQuery() {
264
+ this.connection = null
265
+ this.cb_flush()
266
+ }
267
+ }