@pipedream/google_drive 4.0.0 → 5.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/actions/add-comment/add-comment.mjs +1 -1
  2. package/actions/add-file-sharing-preference/add-file-sharing-preference.mjs +1 -1
  3. package/actions/copy-file/copy-file.mjs +1 -1
  4. package/actions/create-file-from-template/create-file-from-template.mjs +1 -1
  5. package/actions/create-file-from-text/create-file-from-text.mjs +1 -1
  6. package/actions/create-folder/create-folder.mjs +1 -1
  7. package/actions/create-shared-drive/create-shared-drive.mjs +1 -1
  8. package/actions/create-text-file/create-text-file.mjs +1 -1
  9. package/actions/delete-comment/delete-comment.mjs +1 -1
  10. package/actions/delete-file/delete-file.mjs +1 -1
  11. package/actions/delete-reply/delete-reply.mjs +1 -1
  12. package/actions/delete-shared-drive/delete-shared-drive.mjs +1 -1
  13. package/actions/download-file/download-file.mjs +26 -7
  14. package/actions/find-file/find-file.mjs +1 -1
  15. package/actions/find-folder/find-folder.mjs +1 -1
  16. package/actions/find-forms/find-forms.mjs +1 -1
  17. package/actions/find-spreadsheets/find-spreadsheets.mjs +1 -1
  18. package/actions/get-comment/get-comment.mjs +1 -1
  19. package/actions/get-current-user/get-current-user.mjs +1 -1
  20. package/actions/get-file/get-file.mjs +1 -1
  21. package/actions/get-file-by-id/get-file-by-id.mjs +1 -1
  22. package/actions/get-folder-id-for-path/get-folder-id-for-path.mjs +1 -1
  23. package/actions/get-reply/get-reply.mjs +1 -1
  24. package/actions/get-shared-drive/get-shared-drive.mjs +1 -1
  25. package/actions/get-user-details/get-user-details.mjs +1 -1
  26. package/actions/is-folder-ancestor/is-folder-ancestor.mjs +1 -1
  27. package/actions/list-access-proposals/list-access-proposals.mjs +1 -1
  28. package/actions/list-comments/list-comments.mjs +1 -1
  29. package/actions/list-files/list-files.mjs +1 -1
  30. package/actions/list-mime-type-options/list-mime-type-options.mjs +1 -1
  31. package/actions/list-permissions/list-permissions.mjs +1 -1
  32. package/actions/list-replies/list-replies.mjs +1 -1
  33. package/actions/list-shared-drives/list-shared-drives.mjs +1 -1
  34. package/actions/list-theme-id-options/list-theme-id-options.mjs +1 -1
  35. package/actions/move-file/move-file.mjs +1 -1
  36. package/actions/move-file-to-trash/move-file-to-trash.mjs +1 -1
  37. package/actions/remove-file-sharing-permission/remove-file-sharing-permission.mjs +1 -1
  38. package/actions/remove-sharing/remove-sharing.mjs +1 -1
  39. package/actions/reply-to-comment/reply-to-comment.mjs +1 -1
  40. package/actions/resolve-access-proposal/resolve-access-proposal.mjs +1 -1
  41. package/actions/resolve-comment/resolve-comment.mjs +1 -1
  42. package/actions/search-files/search-files.mjs +1 -1
  43. package/actions/search-shared-drives/search-shared-drives.mjs +1 -1
  44. package/actions/share-file/share-file.mjs +1 -1
  45. package/actions/trash-file/trash-file.mjs +1 -1
  46. package/actions/update-comment/update-comment.mjs +1 -1
  47. package/actions/update-file/update-file.mjs +1 -1
  48. package/actions/update-reply/update-reply.mjs +1 -1
  49. package/actions/update-shared-drive/update-shared-drive.mjs +1 -1
  50. package/actions/upload-file/upload-file.mjs +1 -1
  51. package/common/constants.mjs +22 -12
  52. package/common/utils.mjs +32 -0
  53. package/google_drive.app.mjs +90 -11
  54. package/package.json +1 -1
  55. package/sources/changes-to-files-in-drive/changes-to-files-in-drive.mjs +9 -6
  56. package/sources/changes-to-specific-files/changes-to-specific-files.mjs +1 -1
  57. package/sources/changes-to-specific-files-shared-drive/changes-to-specific-files-shared-drive.mjs +9 -6
  58. package/sources/common-dedupe-changes.mjs +22 -11
  59. package/sources/common-webhook.mjs +104 -40
  60. package/sources/new-access-proposal/new-access-proposal.mjs +1 -1
  61. package/sources/new-files-instant/new-files-instant.mjs +4 -1
  62. package/sources/new-files-instant-polling/new-files-instant-polling.mjs +38 -47
  63. package/sources/new-files-shared-drive/new-files-shared-drive.mjs +2 -16
  64. package/sources/new-or-modified-comments/new-or-modified-comments.mjs +10 -7
  65. package/sources/new-or-modified-comments-polling/new-or-modified-comments-polling.mjs +1 -1
  66. package/sources/new-or-modified-files/new-or-modified-files.mjs +20 -1
  67. package/sources/new-or-modified-files-polling/new-or-modified-files-polling.mjs +37 -36
  68. package/sources/new-or-modified-folders/new-or-modified-folders.mjs +52 -50
  69. package/sources/new-or-modified-folders-polling/new-or-modified-folders-polling.mjs +55 -63
  70. package/sources/new-shared-drive/new-shared-drive.mjs +1 -1
  71. package/sources/new-spreadsheet/new-spreadsheet.mjs +1 -1
  72. package/sources/new-spreadsheet-polling/new-spreadsheet-polling.mjs +38 -47
@@ -122,17 +122,10 @@ const LEGACY_MY_DRIVE_VALUE = "myDrive";
122
122
  */
123
123
  const WEBHOOK_SUBSCRIPTION_EXPIRATION_TIME_MILLISECONDS = 24 * 60 * 60 * 1000;
124
124
 
125
- /**
126
- * The default time interval between webhook subscription renewals. Since
127
- * subscriptions expire after 24 hours at most, we set this time to 95% of this
128
- * time window by default to make sure the event sources don't miss any events
129
- * due to an expired subscription not being renewed on time.
130
- *
131
- * More information can be found in the API docs:
132
- * https://developers.google.com/drive/api/v3/push#optional-properties
133
- */
134
- const WEBHOOK_SUBSCRIPTION_RENEWAL_SECONDS =
135
- (WEBHOOK_SUBSCRIPTION_EXPIRATION_TIME_MILLISECONDS * 0.95) / 1000;
125
+ // The timer also resumes capped backlogs, not just renewals
126
+ const WEBHOOK_TIMER_INTERVAL_SECONDS = 10 * 60;
127
+
128
+ const WEBHOOK_RENEWAL_WINDOW_MILLISECONDS = 60 * 60 * 1000;
136
129
 
137
130
  /**
138
131
  * The maximum number of path segments to include in an option label for a prop whose value is a
@@ -273,6 +266,16 @@ const RATE_LIMIT_ERROR_REASONS = [
273
266
 
274
267
  const CHANGED_FILE_FIELDS = "kind,id,name,mimeType,parents,createdTime,modifiedTime,trashed,version,size,md5Checksum,webViewLink,lastModifyingUser";
275
268
 
269
+ const CHANGE_FILTER_FILE_FIELDS = "id,name,mimeType,parents,createdTime,modifiedTime";
270
+
271
+ const MAX_CHANGES_PAGES_PER_RUN = 10;
272
+ const DEFAULT_MAX_EMITS_PER_RUN = 1000;
273
+ const MAX_EMITS_PER_RUN = 5000;
274
+
275
+ const MAX_FILE_INTERVAL_ENTRIES = 2000;
276
+
277
+ const MASS_CHANGE_MAX_AGE_MILLISECONDS = 60 * 60 * 1000;
278
+
276
279
  /** Google Workspace types that `stashFile` can export to PDF. */
277
280
  const PDF_EXPORTABLE_MIME_TYPES = [
278
281
  "application/vnd.google-apps.document",
@@ -294,7 +297,8 @@ export {
294
297
  MY_DRIVE_VALUE,
295
298
  LEGACY_MY_DRIVE_VALUE,
296
299
  WEBHOOK_SUBSCRIPTION_EXPIRATION_TIME_MILLISECONDS,
297
- WEBHOOK_SUBSCRIPTION_RENEWAL_SECONDS,
300
+ WEBHOOK_TIMER_INTERVAL_SECONDS,
301
+ WEBHOOK_RENEWAL_WINDOW_MILLISECONDS,
298
302
  MAX_FILE_OPTION_PATH_SEGMENTS,
299
303
  PAGINATION_TOKEN_FIELD,
300
304
  GOOGLE_DRIVE_MIME_TYPE_PREFIX,
@@ -328,5 +332,11 @@ export {
328
332
  RETRYABLE_STATUS_CODES,
329
333
  RATE_LIMIT_ERROR_REASONS,
330
334
  CHANGED_FILE_FIELDS,
335
+ CHANGE_FILTER_FILE_FIELDS,
336
+ MAX_CHANGES_PAGES_PER_RUN,
337
+ DEFAULT_MAX_EMITS_PER_RUN,
338
+ MAX_EMITS_PER_RUN,
339
+ MAX_FILE_INTERVAL_ENTRIES,
340
+ MASS_CHANGE_MAX_AGE_MILLISECONDS,
331
341
  PDF_EXPORTABLE_MIME_TYPES,
332
342
  };
package/common/utils.mjs CHANGED
@@ -349,6 +349,37 @@ function sanitizeFileName(name) {
349
349
  return safe;
350
350
  }
351
351
 
352
+ /**
353
+ * Minimal inline worker pool. Runs `worker` over `items` with at most `limit`
354
+ * calls in flight at any moment and returns the results in input order.
355
+ *
356
+ * @param {Array} items the items to process
357
+ * @param {Number} limit the max number of concurrent `worker` calls
358
+ * @param {Function} worker async function invoked as `worker(item, index)`
359
+ * @returns {Promise<Array>} the results, in the same order as `items`
360
+ */
361
+ async function mapWithConcurrency(items, limit, worker) {
362
+ const results = new Array(items.length);
363
+ let cursor = 0;
364
+
365
+ const runNext = async () => {
366
+ while (cursor < items.length) {
367
+ const index = cursor;
368
+ cursor += 1;
369
+ results[index] = await worker(items[index], index);
370
+ }
371
+ };
372
+
373
+ const runners = [];
374
+ const poolSize = Math.min(limit, items.length);
375
+ for (let i = 0; i < poolSize; i++) {
376
+ runners.push(runNext());
377
+ }
378
+ await Promise.allSettled(runners);
379
+
380
+ return results;
381
+ }
382
+
352
383
  async function stashFile(item, googleDrive, dir) {
353
384
  const fileMetadata = await googleDrive.getFile(item.id, {
354
385
  fields: "name,mimeType",
@@ -444,6 +475,7 @@ export {
444
475
  getFilePaths,
445
476
  getListFilesOpts,
446
477
  isMyDrive,
478
+ mapWithConcurrency,
447
479
  MY_DRIVE_VALUE,
448
480
  omitEmptyStringValues,
449
481
  parseObjectEntries,
@@ -7,11 +7,14 @@ const mimeTypes = Object.keys(mimeDb);
7
7
 
8
8
  import googleMimeTypes from "./actions/google-mime-types.mjs";
9
9
  import {
10
+ DEFAULT_MAX_EMITS_PER_RUN,
10
11
  GOOGLE_DRIVE_FOLDER_MIME_TYPE,
11
12
  GOOGLE_DRIVE_GRANTEE_TYPES,
12
13
  GOOGLE_DRIVE_UPDATE_TYPE_OPTIONS,
13
14
  GOOGLE_DRIVE_UPDATE_TYPES,
14
15
  GOOGLE_DRIVE_UPLOAD_TYPE_OPTIONS,
16
+ MAX_CHANGES_PAGES_PER_RUN,
17
+ MAX_EMITS_PER_RUN,
15
18
  MY_DRIVE_VALUE,
16
19
  RATE_LIMIT_ERROR_REASONS,
17
20
  RETRYABLE_STATUS_CODES,
@@ -161,10 +164,19 @@ export default {
161
164
  changesPageSize: {
162
165
  type: "integer",
163
166
  label: "Changes Page Size",
164
- description: "Maximum number of changes to fetch per API call (1-1000). Lower values reduce memory usage and the risk of execution timeouts or out-of-memory errors on active drives.",
167
+ description: "Maximum number of changes to fetch per API call (max 1000). Lower values mean more API calls per run. Example: `500`",
165
168
  min: 1,
166
169
  max: 1000,
167
- default: 100,
170
+ default: 1000,
171
+ optional: true,
172
+ },
173
+ maxEmitsPerRun: {
174
+ type: "integer",
175
+ label: "Max Events Per Run",
176
+ description: `Stop after about this many events and resume on the next run (max ${MAX_EMITS_PER_RUN}). A single page of changes larger than this is still processed in full. Example: \`1000\``,
177
+ min: 1,
178
+ max: MAX_EMITS_PER_RUN,
179
+ default: DEFAULT_MAX_EMITS_PER_RUN,
168
180
  optional: true,
169
181
  },
170
182
  filePath: {
@@ -449,17 +461,22 @@ export default {
449
461
  * per page
450
462
  * @param {string} [fileFields] - the file fields to return for each change,
451
463
  * e.g. `id,name,parents`. Defaults to the API's minimal file fields
464
+ * @param {object} [options] - `stopOnRateLimit` ends the stream instead of
465
+ * throwing; `changeFilter` drops changes before they are yielded
452
466
  * @yields
453
467
  * @type {ChangesPage}
454
468
  */
455
- async *listChanges(pageToken, driveId, pageSize = 1000, fileFields) {
469
+ async *listChanges(pageToken, driveId, pageSize = 1000, fileFields, options = {}) {
470
+ const {
471
+ stopOnRateLimit = false, changeFilter,
472
+ } = options;
456
473
  const drive = this.drive();
457
474
  let changeRequest = {
458
475
  pageToken,
459
476
  pageSize,
460
477
  };
461
478
  if (fileFields) {
462
- changeRequest.fields = `nextPageToken,newStartPageToken,changes(file(${fileFields}))`;
479
+ changeRequest.fields = `nextPageToken,newStartPageToken,changes(time,file(${fileFields}))`;
463
480
  }
464
481
 
465
482
  // As with many of the methods for Google Drive, we must
@@ -475,9 +492,18 @@ export default {
475
492
  }
476
493
 
477
494
  while (true) {
478
- const { data } = await this.retryWithExponentialBackoff(
479
- () => drive.changes.list(changeRequest),
480
- );
495
+ let data;
496
+ try {
497
+ ({ data } = await this.retryWithExponentialBackoff(
498
+ () => drive.changes.list(changeRequest),
499
+ ));
500
+ } catch (error) {
501
+ if (!stopOnRateLimit || !this.isRateLimitError(error)) {
502
+ throw error;
503
+ }
504
+ console.log(`Rate limited by Google Drive (${error.message}). Stopping; remaining changes resume on the next run.`);
505
+ return;
506
+ }
481
507
  const {
482
508
  changes = [],
483
509
  newStartPageToken,
@@ -486,13 +512,22 @@ export default {
486
512
 
487
513
  // Some changes do not include an associated file object. Return only
488
514
  // those that do
489
- const changedFiles = changes
490
- .map((change) => change.file)
491
- .filter((f) => typeof f === "object");
515
+ const withFiles = changes.filter((change) => typeof change.file === "object");
516
+ const kept = [];
517
+ for (const change of withFiles) {
518
+ if (!changeFilter || await changeFilter(change)) {
519
+ kept.push(change);
520
+ }
521
+ }
522
+ if (kept.length < withFiles.length) {
523
+ console.log(`Skipped ${withFiles.length - kept.length} changes filtered by the source`);
524
+ }
525
+ const changedFiles = kept.map((change) => change.file);
492
526
 
493
527
  yield {
494
528
  changedFiles,
495
529
  nextPageToken: nextPageToken || newStartPageToken,
530
+ isLastPage: Boolean(newStartPageToken),
496
531
  };
497
532
 
498
533
  if (newStartPageToken) {
@@ -504,6 +539,46 @@ export default {
504
539
  changeRequest.pageToken = nextPageToken;
505
540
  }
506
541
  },
542
+ // Returns true once the feed is drained; stops early at the emit or page cap
543
+ async processChangesPages({
544
+ pageToken,
545
+ driveId,
546
+ pageSize,
547
+ fileFields,
548
+ maxEmits,
549
+ processPage,
550
+ savePageToken,
551
+ changeFilter,
552
+ }) {
553
+ const changedFilesStream = this.listChanges(pageToken, driveId, pageSize, fileFields, {
554
+ stopOnRateLimit: true,
555
+ changeFilter,
556
+ });
557
+ const cap = maxEmits ?? DEFAULT_MAX_EMITS_PER_RUN;
558
+ let pages = 0;
559
+ let emitted = 0;
560
+ for await (const {
561
+ changedFiles, nextPageToken, isLastPage,
562
+ } of changedFilesStream) {
563
+ if (emitted && emitted + changedFiles.length > cap) {
564
+ console.log(`Emitted ${emitted} events; remaining changes resume on the next run.`);
565
+ return false;
566
+ }
567
+
568
+ emitted += (await processPage(changedFiles)) ?? changedFiles.length;
569
+ savePageToken(nextPageToken);
570
+
571
+ if (isLastPage) {
572
+ return true;
573
+ }
574
+ if (emitted >= cap || ++pages >= MAX_CHANGES_PAGES_PER_RUN) {
575
+ console.log(`Emitted ${emitted} events; remaining changes resume on the next run.`);
576
+ return false;
577
+ }
578
+ }
579
+ // Rate limited
580
+ return false;
581
+ },
507
582
  async getPageToken(driveId) {
508
583
  const drive = this.drive();
509
584
  const request = driveId
@@ -1703,7 +1778,11 @@ export default {
1703
1778
  return (await drive.accessproposals.resolve(opts)).data;
1704
1779
  },
1705
1780
  isRetryableError(error, statusCode) {
1706
- if (RETRYABLE_STATUS_CODES.includes(statusCode)) {
1781
+ return RETRYABLE_STATUS_CODES.includes(statusCode)
1782
+ || this.isRateLimitError(error, statusCode);
1783
+ },
1784
+ isRateLimitError(error, statusCode = error.status || error.response?.status) {
1785
+ if (statusCode === 429) {
1707
1786
  return true;
1708
1787
  }
1709
1788
  const errors = error.errors ?? error.response?.data?.error?.errors ?? [];
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@pipedream/google_drive",
3
- "version": "4.0.0",
3
+ "version": "5.0.0",
4
4
  "description": "Pipedream Google_drive Components",
5
5
  "main": "google_drive.app.mjs",
6
6
  "keywords": [
@@ -14,7 +14,7 @@ export default {
14
14
  key: "google_drive-changes-to-files-in-drive",
15
15
  name: "Changes to Files in Drive",
16
16
  description: "Emit new event when a change is made to one of the specified files. [See the documentation](https://developers.google.com/drive/api/v3/reference/changes/watch)",
17
- version: "0.0.16",
17
+ version: "0.1.0",
18
18
  type: "source",
19
19
  dedupe: "unique",
20
20
  props: {
@@ -69,6 +69,7 @@ export default {
69
69
  const { files } = await this.googleDrive.listFilesInPage(null, args);
70
70
 
71
71
  await this.processChanges(files);
72
+ this.flushFileIntervals();
72
73
  },
73
74
  ...common.hooks,
74
75
  },
@@ -103,6 +104,9 @@ export default {
103
104
  isFileRelevant(file) {
104
105
  return this.files.includes(file.id);
105
106
  },
107
+ isRelevantChange({ file }) {
108
+ return this.isFileRelevant(file);
109
+ },
106
110
  getChanges(headers) {
107
111
  if (!headers) {
108
112
  return {
@@ -133,14 +137,13 @@ export default {
133
137
  async processChanges(changedFiles, headers) {
134
138
  console.log(`Processing ${changedFiles.length} changed files`);
135
139
 
136
- const filteredFiles = this.checkMinimumInterval(changedFiles);
140
+ // Filter first so unwatched files aren't recorded
141
+ const relevantFiles = changedFiles.filter((file) => this.isFileRelevant(file));
142
+ const filteredFiles = this.checkMinimumInterval(relevantFiles);
137
143
  for (const file of filteredFiles) {
138
- if (!this.isFileRelevant(file)) {
139
- console.log(`Skipping event for irrelevant file ${file.id}`);
140
- continue;
141
- }
142
144
  await this.processChange(file, headers);
143
145
  }
146
+ return filteredFiles.length;
144
147
  },
145
148
  },
146
149
  };
@@ -16,7 +16,7 @@ export default {
16
16
  key: "google_drive-changes-to-specific-files",
17
17
  name: "Changes to Specific Files",
18
18
  description: "Watches for changes to specific files, emitting an event when a change is made to one of those files. To watch for changes to [shared drive](https://support.google.com/a/users/answer/9310351) files, use the **Changes to Specific Files (Shared Drive)** source instead.",
19
- version: "0.3.14",
19
+ version: "0.3.16",
20
20
  type: "source",
21
21
  // Dedupe events based on the "x-goog-message-number" header for the target channel:
22
22
  // https://developers.google.com/drive/api/v3/push#making-watch-requests
@@ -29,7 +29,7 @@ export default {
29
29
  key: "google_drive-changes-to-specific-files-shared-drive",
30
30
  name: "Changes to Specific Files (Shared Drive)",
31
31
  description: "Watches for changes to specific files in a shared drive, emitting an event when a change is made to one of those files",
32
- version: "0.3.16",
32
+ version: "0.4.0",
33
33
  type: "source",
34
34
  // Dedupe events based on the "x-goog-message-number" header for the target channel:
35
35
  // https://developers.google.com/drive/api/v3/push#making-watch-requests
@@ -74,6 +74,7 @@ export default {
74
74
  const { files } = await this.googleDrive.listFilesInPage(null, args);
75
75
 
76
76
  await this.processChanges(files);
77
+ this.flushFileIntervals();
77
78
  },
78
79
  ...common.hooks,
79
80
  },
@@ -108,6 +109,9 @@ export default {
108
109
  isFileRelevant(file) {
109
110
  return this.files.includes(file.id);
110
111
  },
112
+ isRelevantChange({ file }) {
113
+ return this.isFileRelevant(file);
114
+ },
111
115
  getChanges(headers) {
112
116
  if (!headers) {
113
117
  return {
@@ -138,14 +142,13 @@ export default {
138
142
  async processChanges(changedFiles, headers) {
139
143
  console.log(`Processing ${changedFiles.length} changed files`);
140
144
 
141
- const filteredFiles = this.checkMinimumInterval(changedFiles);
145
+ // Filter first so unwatched files aren't recorded
146
+ const relevantFiles = changedFiles.filter((file) => this.isFileRelevant(file));
147
+ const filteredFiles = this.checkMinimumInterval(relevantFiles);
142
148
  for (const file of filteredFiles) {
143
- if (!this.isFileRelevant(file)) {
144
- console.log(`Skipping event for irrelevant file ${file.id}`);
145
- continue;
146
- }
147
149
  await this.processChange(file, headers);
148
150
  }
151
+ return filteredFiles.length;
149
152
  },
150
153
  },
151
154
  sampleEmit,
@@ -1,3 +1,5 @@
1
+ import { MAX_FILE_INTERVAL_ENTRIES } from "../common/constants.mjs";
2
+
1
3
  export default {
2
4
  props: {
3
5
  intervalAlert: {
@@ -19,7 +21,10 @@ You can change or disable this minimum interval using the prop \`Minimum Interva
19
21
  },
20
22
  methods: {
21
23
  _getFileIntervals() {
22
- return this.db.get("fileIntervals") ?? {};
24
+ if (!this._fileIntervals) {
25
+ this._fileIntervals = this.db.get("fileIntervals") ?? {};
26
+ }
27
+ return this._fileIntervals;
23
28
  },
24
29
  _setFileIntervals(value) {
25
30
  this.db.set("fileIntervals", value);
@@ -40,20 +45,26 @@ You can change or disable this minimum interval using the prop \`Minimum Interva
40
45
  }
41
46
 
42
47
  const now = Date.now();
43
- const minTimestamp = now - (this.perFileInterval * 1000 * 60);
44
48
  const savedData = this._getFileIntervals();
45
- Object.entries(savedData).forEach(([
46
- key,
47
- value,
48
- ]) => {
49
- if (value < minTimestamp) {
50
- delete savedData[key];
51
- }
52
- });
53
49
  fileIds.forEach((id) => {
54
50
  savedData[id] = now;
55
51
  });
56
- this._setFileIntervals(savedData);
52
+ this._fileIntervalsChanged = true;
53
+ },
54
+ flushFileIntervals() {
55
+ if (this._fileIntervalsChanged) {
56
+ const minTimestamp = Date.now() - (this.perFileInterval * 1000 * 60);
57
+ const entries = Object.entries(this._getFileIntervals())
58
+ .filter((entry) => entry[1] >= minTimestamp)
59
+ .sort((a, b) => b[1] - a[1])
60
+ .slice(0, MAX_FILE_INTERVAL_ENTRIES);
61
+ this._setFileIntervals(Object.fromEntries(entries));
62
+ }
63
+ this.resetFileIntervals();
64
+ },
65
+ resetFileIntervals() {
66
+ this._fileIntervals = null;
67
+ this._fileIntervalsChanged = false;
57
68
  },
58
69
  checkMinimumInterval(files) {
59
70
  const filteredFiles = this.filterByMinimumInterval(files);
@@ -2,7 +2,11 @@ import includes from "lodash/includes.js";
2
2
  import { v4 as uuid } from "uuid";
3
3
 
4
4
  import googleDrive from "../google_drive.app.mjs";
5
- import { WEBHOOK_SUBSCRIPTION_RENEWAL_SECONDS } from "../common/constants.mjs";
5
+ import {
6
+ MY_DRIVE_VALUE,
7
+ WEBHOOK_RENEWAL_WINDOW_MILLISECONDS,
8
+ WEBHOOK_TIMER_INTERVAL_SECONDS,
9
+ } from "../common/constants.mjs";
6
10
  import { getListFilesOpts } from "../common/utils.mjs";
7
11
  import commonDedupeChanges from "./common-dedupe-changes.mjs";
8
12
 
@@ -19,8 +23,9 @@ export default {
19
23
  googleDrive,
20
24
  "watchedDrive",
21
25
  ],
22
- description: "Defaults to **All Drives** (My Drive and all Shared Drives you have access to) when no selection is made. To limit to your personal drive or a [Shared Drive](https://support.google.com/a/users/answer/9310351), select it from this list.",
26
+ description: "The drive to watch. Defaults to **My Drive**. To watch a [Shared Drive](https://support.google.com/a/users/answer/9310351), select it from this list.",
23
27
  optional: false,
28
+ default: MY_DRIVE_VALUE,
24
29
  },
25
30
  timer: {
26
31
  label: "Push notification renewal schedule",
@@ -28,16 +33,30 @@ export default {
28
33
  "The Google Drive API requires occasional renewal of push notification subscriptions. **This runs in the background, so you should not need to modify this schedule**.",
29
34
  type: "$.interface.timer",
30
35
  static: {
31
- intervalSeconds: WEBHOOK_SUBSCRIPTION_RENEWAL_SECONDS,
36
+ intervalSeconds: WEBHOOK_TIMER_INTERVAL_SECONDS,
32
37
  },
33
38
  hidden: true,
34
39
  },
40
+ // Unused; kept so older deployments can still save
41
+ updateTypes: {
42
+ type: "string[]",
43
+ label: "Types of updates",
44
+ description: "Deprecated and ignored.",
45
+ optional: true,
46
+ hidden: true,
47
+ },
35
48
  changesPageSize: {
36
49
  propDefinition: [
37
50
  googleDrive,
38
51
  "changesPageSize",
39
52
  ],
40
53
  },
54
+ maxEmitsPerRun: {
55
+ propDefinition: [
56
+ googleDrive,
57
+ "maxEmitsPerRun",
58
+ ],
59
+ },
41
60
  },
42
61
  hooks: {
43
62
  async activate() {
@@ -97,6 +116,38 @@ export default {
97
116
  _setPageToken(pageToken) {
98
117
  this.db.set("pageToken", pageToken);
99
118
  },
119
+ _getBacklogHeaders() {
120
+ return this.db.get("backlogHeaders");
121
+ },
122
+ _setBacklogHeaders(headers) {
123
+ this.db.set("backlogHeaders", headers);
124
+ },
125
+ isSubscriptionExpiring(subscription) {
126
+ const expiration = Number(subscription?.expiration);
127
+ return !expiration || expiration - Date.now() < WEBHOOK_RENEWAL_WINDOW_MILLISECONDS;
128
+ },
129
+ async renewSubscription(subscription, channelID, pageToken) {
130
+ const {
131
+ newChannelID,
132
+ newPageToken,
133
+ expiration,
134
+ resourceId,
135
+ } = await this.googleDrive.renewSubscription(
136
+ this.drive,
137
+ subscription,
138
+ this.http.endpoint,
139
+ channelID,
140
+ pageToken,
141
+ );
142
+
143
+ this._setSubscription({
144
+ expiration,
145
+ resourceId,
146
+ });
147
+ this._setChannelID(newChannelID);
148
+ this._setPageToken(newPageToken);
149
+ return newPageToken;
150
+ },
100
151
  isMyDrive(drive = this.drive) {
101
152
  return googleDrive.methods.isMyDrive(drive);
102
153
  },
@@ -128,6 +179,16 @@ export default {
128
179
  getChangesFileFields() {
129
180
  return undefined;
130
181
  },
182
+ isRelevantChange() {
183
+ return true;
184
+ },
185
+ async getChangeFilter() {
186
+ return (change) => this.isRelevantChange(change);
187
+ },
188
+ // false: processChanges() ignores changedFiles, so run it once, not per page
189
+ consumesChangedFiles() {
190
+ return true;
191
+ },
131
192
  /**
132
193
  * This method is responsible for processing a list of changed files
133
194
  * according to the event source's purpose. As an abstract method, it must
@@ -137,10 +198,36 @@ export default {
137
198
  * by the API](https://bit.ly/3h7WeUa)
138
199
  * @param {object} [headers] - an object containing the request headers of
139
200
  * the webhook call made by Google Drive
201
+ * @returns {number} the number of events emitted
140
202
  */
141
203
  processChanges() {
142
204
  throw new Error("processChanges is not implemented");
143
205
  },
206
+ async processChangesFrom(pageToken, headers) {
207
+ this.resetFileIntervals();
208
+ // Thrown runs keep db writes, so flush intervals with the token
209
+ try {
210
+ const caughtUp = await this.googleDrive.processChangesPages({
211
+ pageToken,
212
+ driveId: this.getDriveId(),
213
+ pageSize: this.changesPageSize,
214
+ fileFields: this.getChangesFileFields(),
215
+ maxEmits: this.maxEmitsPerRun,
216
+ changeFilter: await this.getChangeFilter(),
217
+ processPage: (changedFiles) => this.processChanges(changedFiles, headers),
218
+ savePageToken: (nextPageToken) => this._setPageToken(nextPageToken),
219
+ });
220
+ this._setBacklogHeaders(caughtUp
221
+ ? null
222
+ : {
223
+ "x-goog-resource-state": headers["x-goog-resource-state"],
224
+ "x-goog-resource-uri": headers["x-goog-resource-uri"],
225
+ "x-goog-changed": headers["x-goog-changed"],
226
+ });
227
+ } finally {
228
+ this.flushFileIntervals();
229
+ }
230
+ },
144
231
  },
145
232
  async run(event) {
146
233
  // This function is polymorphic: it can be triggered as a cron job, to make
@@ -152,25 +239,15 @@ export default {
152
239
 
153
240
  // Component was invoked by timer
154
241
  if (event.timestamp) {
155
- const {
156
- newChannelID,
157
- newPageToken,
158
- expiration,
159
- resourceId,
160
- } = await this.googleDrive.renewSubscription(
161
- this.drive,
162
- subscription,
163
- this.http.endpoint,
164
- channelID,
165
- pageToken,
166
- );
242
+ const currentPageToken = this.isSubscriptionExpiring(subscription)
243
+ ? await this.renewSubscription(subscription, channelID, pageToken)
244
+ : pageToken;
167
245
 
168
- this._setSubscription({
169
- expiration,
170
- resourceId,
171
- });
172
- this._setChannelID(newChannelID);
173
- this._setPageToken(newPageToken);
246
+ const backlogHeaders = this._getBacklogHeaders();
247
+ if (backlogHeaders && this.consumesChangedFiles()) {
248
+ console.log("Resuming changes left by a capped run");
249
+ await this.processChangesFrom(currentPageToken, backlogHeaders);
250
+ }
174
251
  return;
175
252
  } else {
176
253
  this.http.respond({
@@ -206,25 +283,12 @@ export default {
206
283
  }
207
284
 
208
285
  const driveId = this.getDriveId();
209
- const changedFilesStream =
210
- this.googleDrive.listChanges(
211
- pageToken,
212
- driveId,
213
- this.changesPageSize,
214
- this.getChangesFileFields(),
215
- );
216
- for await (const changedFilesPage of changedFilesStream) {
217
- const {
218
- changedFiles,
219
- nextPageToken,
220
- } = changedFilesPage;
221
-
222
- // Process all the changed files retrieved from the current page
223
- await this.processChanges(changedFiles, headers);
224
-
225
- // After successfully processing the changed files, we store the page
226
- // token of the next page
227
- this._setPageToken(nextPageToken);
286
+ if (!this.consumesChangedFiles()) {
287
+ await this.processChanges([], headers);
288
+ this._setPageToken(await this.googleDrive.getPageToken(driveId));
289
+ return;
228
290
  }
291
+
292
+ await this.processChangesFrom(pageToken, headers);
229
293
  },
230
294
  };
@@ -6,7 +6,7 @@ export default {
6
6
  key: "google_drive-new-access-proposal",
7
7
  name: "New Access Proposal",
8
8
  description: "Emit new event when a new access proposal is requested in Google Drive",
9
- version: "0.0.19",
9
+ version: "0.0.21",
10
10
  type: "source",
11
11
  dedupe: "unique",
12
12
  props: {
@@ -11,7 +11,7 @@ export default {
11
11
  key: "google_drive-new-files-instant",
12
12
  name: "New Files (Instant)",
13
13
  description: "Emit new event when a new file is added in your linked Google Drive",
14
- version: "0.2.16",
14
+ version: "0.3.0",
15
15
  type: "source",
16
16
  dedupe: "unique",
17
17
  props: {
@@ -131,6 +131,9 @@ export default {
131
131
  GOOGLE_DRIVE_NOTIFICATION_CHANGE,
132
132
  ];
133
133
  },
134
+ consumesChangedFiles() {
135
+ return false;
136
+ },
134
137
  async emitFiles(files) {
135
138
  for (const file of files) {
136
139
  if (!(await this.shouldProcess(file))) {