@naturalcycles/datastore-lib 3.22.0 → 3.23.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (7) hide show
  1. package/dist/DatastoreStreamReadable.js +4 -0
  2. package/dist/datastore.db.js +1 -1
  3. package/dist/datastore.model.d.ts +12 -1
  4. package/package.json +2 -2
  5. package/src/DatastoreStreamReadable.ts +3 -0
  6. package/src/datastore.db.ts +1 -1
  7. package/src/datastore.model.ts +13 -1
package/dist/DatastoreStreamReadable.js CHANGED
@@ -54,6 +54,10 @@ class DatastoreStreamReadable extends stream_1.Readable {
54
54
  this.push(null);
55
55
  this.done = true;
56
56
  }
57
+ else if (this.opt.singleBatchBuffer) {
58
+ // here we don't start next query until we're asked (via next _read call)
59
+ // do, let's do nothing
60
+ }
57
61
  else if (this.opt.rssLimitMB) {
58
62
  const rssMB = Math.round(process.memoryUsage().rss / 1024 / 1024);
59
63
  if (rssMB <= this.opt.rssLimitMB) {
package/dist/datastore.db.js CHANGED
@@ -10,7 +10,7 @@ const DatastoreStreamReadable_1 = require("./DatastoreStreamReadable");
10
10
  const query_util_1 = require("./query.util");
11
11
  // Datastore (also Firestore and other Google APIs) supports max 500 of items when saving/deleting, etc.
12
12
  const MAX_ITEMS = 500;
13
- const RETRY_ON = ['GOAWAY', 'UNAVAILABLE', 'UNKNOWN', 'much contention'];
13
+ const RETRY_ON = ['GOAWAY', 'UNAVAILABLE', 'UNKNOWN', 'DEADLINE_EXCEEDED', 'much contention'];
14
14
  // Examples of errors:
15
15
  // UNKNOWN: Stream removed
16
16
  const methodMap = {
package/dist/datastore.model.d.ts CHANGED
@@ -60,7 +60,8 @@ export interface DatastoreDBStreamOptions extends DatastoreDBOptions {
60
60
  */
61
61
  experimentalCursorStream?: boolean;
62
62
  /**
63
- * Applicable to `experimentalCursorStream`
63
+ * Applicable to `experimentalCursorStream`.
64
+ * Defines the size (limit) of each individual query.
64
65
  *
65
66
  * @default 1000
66
67
  */
@@ -78,6 +79,16 @@ export interface DatastoreDBStreamOptions extends DatastoreDBOptions {
78
79
  * @default 1000
79
80
  */
80
81
  rssLimitMB?: number;
82
+ /**
83
+ * Applicable to `experimentalCursorStream`
84
+ * Default false.
85
+ * If true, stream will pause until consumer requests more data (via _read).
86
+ * It means it'll run slower, as buffer will be equal to batchSize (1000) at max.
87
+ * There will be gaps in time between "last query loaded" and "next query requested".
88
+ * This mode is useful e.g for DB migrations, where you want to avoid "stale data".
89
+ * So, it minimizes the time between "item loaded" and "item saved" during DB migration.
90
+ */
91
+ singleBatchBuffer?: boolean;
81
92
  /**
82
93
  * Set to `true` to log additional debug info, when using experimentalCursorStream.
83
94
  *
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@naturalcycles/datastore-lib",
3
- "version": "3.22.0",
3
+ "version": "3.23.1",
4
4
  "description": "Opinionated library to work with Google Datastore",
5
5
  "scripts": {
6
6
  "prepare": "husky install"
@@ -15,7 +15,7 @@
15
15
  "devDependencies": {
16
16
  "@naturalcycles/dev-lib": "^13.0.0",
17
17
  "@types/node": "^18.7.2",
18
- "jest": "^28.1.0"
18
+ "jest": "^29.0.3"
19
19
  },
20
20
  "resolutions": {
21
21
  "long": "^4.0.0"
package/src/DatastoreStreamReadable.ts CHANGED
@@ -79,6 +79,9 @@ export class DatastoreStreamReadable<T = any> extends Readable implements Readab
79
79
  )
80
80
  this.push(null)
81
81
  this.done = true
82
+ } else if (this.opt.singleBatchBuffer) {
83
+ // here we don't start next query until we're asked (via next _read call)
84
+ // do, let's do nothing
82
85
  } else if (this.opt.rssLimitMB) {
83
86
  const rssMB = Math.round(process.memoryUsage().rss / 1024 / 1024)
84
87
 
package/src/datastore.db.ts CHANGED
@@ -45,7 +45,7 @@ import { dbQueryToDatastoreQuery } from './query.util'
45
45
  // Datastore (also Firestore and other Google APIs) supports max 500 of items when saving/deleting, etc.
46
46
  const MAX_ITEMS = 500
47
47
 
48
- const RETRY_ON = ['GOAWAY', 'UNAVAILABLE', 'UNKNOWN', 'much contention']
48
+ const RETRY_ON = ['GOAWAY', 'UNAVAILABLE', 'UNKNOWN', 'DEADLINE_EXCEEDED', 'much contention']
49
49
  // Examples of errors:
50
50
  // UNKNOWN: Stream removed
51
51
 
package/src/datastore.model.ts CHANGED
@@ -71,7 +71,8 @@ export interface DatastoreDBStreamOptions extends DatastoreDBOptions {
71
71
  experimentalCursorStream?: boolean
72
72
 
73
73
  /**
74
- * Applicable to `experimentalCursorStream`
74
+ * Applicable to `experimentalCursorStream`.
75
+ * Defines the size (limit) of each individual query.
75
76
  *
76
77
  * @default 1000
77
78
  */
@@ -91,6 +92,17 @@ export interface DatastoreDBStreamOptions extends DatastoreDBOptions {
91
92
  */
92
93
  rssLimitMB?: number
93
94
 
95
+ /**
96
+ * Applicable to `experimentalCursorStream`
97
+ * Default false.
98
+ * If true, stream will pause until consumer requests more data (via _read).
99
+ * It means it'll run slower, as buffer will be equal to batchSize (1000) at max.
100
+ * There will be gaps in time between "last query loaded" and "next query requested".
101
+ * This mode is useful e.g for DB migrations, where you want to avoid "stale data".
102
+ * So, it minimizes the time between "item loaded" and "item saved" during DB migration.
103
+ */
104
+ singleBatchBuffer?: boolean
105
+
94
106
  /**
95
107
  * Set to `true` to log additional debug info, when using experimentalCursorStream.
96
108
  *