@pipedream/google_drive 4.0.1 → 5.0.0

package/google_drive.app.mjs

@@ -7,11 +7,14 @@ const mimeTypes = Object.keys(mimeDb);
 
 import googleMimeTypes from "./actions/google-mime-types.mjs";
 import {
+  DEFAULT_MAX_EMITS_PER_RUN,
   GOOGLE_DRIVE_FOLDER_MIME_TYPE,
   GOOGLE_DRIVE_GRANTEE_TYPES,
   GOOGLE_DRIVE_UPDATE_TYPE_OPTIONS,
   GOOGLE_DRIVE_UPDATE_TYPES,
   GOOGLE_DRIVE_UPLOAD_TYPE_OPTIONS,
+  MAX_CHANGES_PAGES_PER_RUN,
+  MAX_EMITS_PER_RUN,
   MY_DRIVE_VALUE,
   RATE_LIMIT_ERROR_REASONS,
   RETRYABLE_STATUS_CODES,
@@ -161,10 +164,19 @@ export default {
     changesPageSize: {
       type: "integer",
       label: "Changes Page Size",
-      description: "Maximum number of changes to fetch per API call (1-1000). Lower values reduce memory usage and the risk of execution timeouts or out-of-memory errors on active drives.",
+      description: "Maximum number of changes to fetch per API call (max 1000). Lower values mean more API calls per run. Example: `500`",
       min: 1,
       max: 1000,
-      default: 100,
+      default: 1000,
+      optional: true,
+    },
+    maxEmitsPerRun: {
+      type: "integer",
+      label: "Max Events Per Run",
+      description: `Stop after about this many events and resume on the next run (max ${MAX_EMITS_PER_RUN}). A single page of changes larger than this is still processed in full. Example: \`1000\``,
+      min: 1,
+      max: MAX_EMITS_PER_RUN,
+      default: DEFAULT_MAX_EMITS_PER_RUN,
       optional: true,
     },
     filePath: {
@@ -449,17 +461,22 @@ export default {
      * per page
      * @param {string} [fileFields] - the file fields to return for each change,
      * e.g. `id,name,parents`. Defaults to the API's minimal file fields
+     * @param {object} [options] - `stopOnRateLimit` ends the stream instead of
+     * throwing; `changeFilter` drops changes before they are yielded
      * @yields
      * @type {ChangesPage}
      */
-    async *listChanges(pageToken, driveId, pageSize = 1000, fileFields) {
+    async *listChanges(pageToken, driveId, pageSize = 1000, fileFields, options = {}) {
+      const {
+        stopOnRateLimit = false, changeFilter,
+      } = options;
       const drive = this.drive();
       let changeRequest = {
         pageToken,
         pageSize,
       };
       if (fileFields) {
-        changeRequest.fields = `nextPageToken,newStartPageToken,changes(file(${fileFields}))`;
+        changeRequest.fields = `nextPageToken,newStartPageToken,changes(time,file(${fileFields}))`;
       }
 
       // As with many of the methods for Google Drive, we must
@@ -475,9 +492,18 @@ export default {
       }
 
       while (true) {
-        const { data } = await this.retryWithExponentialBackoff(
-          () => drive.changes.list(changeRequest),
-        );
+        let data;
+        try {
+          ({ data } = await this.retryWithExponentialBackoff(
+            () => drive.changes.list(changeRequest),
+          ));
+        } catch (error) {
+          if (!stopOnRateLimit || !this.isRateLimitError(error)) {
+            throw error;
+          }
+          console.log(`Rate limited by Google Drive (${error.message}). Stopping; remaining changes resume on the next run.`);
+          return;
+        }
         const {
           changes = [],
           newStartPageToken,
@@ -486,13 +512,22 @@ export default {
 
         // Some changes do not include an associated file object. Return only
         // those that do
-        const changedFiles = changes
-          .map((change) => change.file)
-          .filter((f) => typeof f === "object");
+        const withFiles = changes.filter((change) => typeof change.file === "object");
+        const kept = [];
+        for (const change of withFiles) {
+          if (!changeFilter || await changeFilter(change)) {
+            kept.push(change);
+          }
+        }
+        if (kept.length < withFiles.length) {
+          console.log(`Skipped ${withFiles.length - kept.length} changes filtered by the source`);
+        }
+        const changedFiles = kept.map((change) => change.file);
 
         yield {
           changedFiles,
           nextPageToken: nextPageToken || newStartPageToken,
+          isLastPage: Boolean(newStartPageToken),
         };
 
         if (newStartPageToken) {
@@ -504,6 +539,46 @@ export default {
         changeRequest.pageToken = nextPageToken;
       }
     },
+    // Returns true once the feed is drained; stops early at the emit or page cap
+    async processChangesPages({
+      pageToken,
+      driveId,
+      pageSize,
+      fileFields,
+      maxEmits,
+      processPage,
+      savePageToken,
+      changeFilter,
+    }) {
+      const changedFilesStream = this.listChanges(pageToken, driveId, pageSize, fileFields, {
+        stopOnRateLimit: true,
+        changeFilter,
+      });
+      const cap = maxEmits ?? DEFAULT_MAX_EMITS_PER_RUN;
+      let pages = 0;
+      let emitted = 0;
+      for await (const {
+        changedFiles, nextPageToken, isLastPage,
+      } of changedFilesStream) {
+        if (emitted && emitted + changedFiles.length > cap) {
+          console.log(`Emitted ${emitted} events; remaining changes resume on the next run.`);
+          return false;
+        }
+
+        emitted += (await processPage(changedFiles)) ?? changedFiles.length;
+        savePageToken(nextPageToken);
+
+        if (isLastPage) {
+          return true;
+        }
+        if (emitted >= cap || ++pages >= MAX_CHANGES_PAGES_PER_RUN) {
+          console.log(`Emitted ${emitted} events; remaining changes resume on the next run.`);
+          return false;
+        }
+      }
+      // Rate limited
+      return false;
+    },
     async getPageToken(driveId) {
       const drive = this.drive();
       const request = driveId
@@ -1703,7 +1778,11 @@ export default {
       return (await drive.accessproposals.resolve(opts)).data;
     },
     isRetryableError(error, statusCode) {
-      if (RETRYABLE_STATUS_CODES.includes(statusCode)) {
+      return RETRYABLE_STATUS_CODES.includes(statusCode)
+        || this.isRateLimitError(error, statusCode);
+    },
+    isRateLimitError(error, statusCode = error.status || error.response?.status) {
+      if (statusCode === 429) {
         return true;
       }
       const errors = error.errors ?? error.response?.data?.error?.errors ?? [];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pipedream/google_drive",
-  "version": "4.0.1",
+  "version": "5.0.0",
   "description": "Pipedream Google_drive Components",
   "main": "google_drive.app.mjs",
   "keywords": [

package/sources/changes-to-files-in-drive/changes-to-files-in-drive.mjs

@@ -14,7 +14,7 @@ export default {
   key: "google_drive-changes-to-files-in-drive",
   name: "Changes to Files in Drive",
   description: "Emit new event when a change is made to one of the specified files. [See the documentation](https://developers.google.com/drive/api/v3/reference/changes/watch)",
-  version: "0.0.17",
+  version: "0.1.0",
   type: "source",
   dedupe: "unique",
   props: {
@@ -69,6 +69,7 @@ export default {
       const { files } = await this.googleDrive.listFilesInPage(null, args);
 
       await this.processChanges(files);
+      this.flushFileIntervals();
     },
     ...common.hooks,
   },
@@ -103,6 +104,9 @@ export default {
     isFileRelevant(file) {
       return this.files.includes(file.id);
     },
+    isRelevantChange({ file }) {
+      return this.isFileRelevant(file);
+    },
     getChanges(headers) {
       if (!headers) {
         return {
@@ -133,14 +137,13 @@ export default {
     async processChanges(changedFiles, headers) {
       console.log(`Processing ${changedFiles.length} changed files`);
 
-      const filteredFiles = this.checkMinimumInterval(changedFiles);
+      // Filter first so unwatched files aren't recorded
+      const relevantFiles = changedFiles.filter((file) => this.isFileRelevant(file));
+      const filteredFiles = this.checkMinimumInterval(relevantFiles);
       for (const file of filteredFiles) {
-        if (!this.isFileRelevant(file)) {
-          console.log(`Skipping event for irrelevant file ${file.id}`);
-          continue;
-        }
         await this.processChange(file, headers);
       }
+      return filteredFiles.length;
     },
   },
 };

package/sources/changes-to-specific-files/changes-to-specific-files.mjs

@@ -16,7 +16,7 @@ export default {
   key: "google_drive-changes-to-specific-files",
   name: "Changes to Specific Files",
   description: "Watches for changes to specific files, emitting an event when a change is made to one of those files. To watch for changes to [shared drive](https://support.google.com/a/users/answer/9310351) files, use the **Changes to Specific Files (Shared Drive)** source instead.",
-  version: "0.3.15",
+  version: "0.3.16",
   type: "source",
   // Dedupe events based on the "x-goog-message-number" header for the target channel:
   // https://developers.google.com/drive/api/v3/push#making-watch-requests

package/sources/changes-to-specific-files-shared-drive/changes-to-specific-files-shared-drive.mjs

@@ -29,7 +29,7 @@ export default {
   key: "google_drive-changes-to-specific-files-shared-drive",
   name: "Changes to Specific Files (Shared Drive)",
   description: "Watches for changes to specific files in a shared drive, emitting an event when a change is made to one of those files",
-  version: "0.3.17",
+  version: "0.4.0",
   type: "source",
   // Dedupe events based on the "x-goog-message-number" header for the target channel:
   // https://developers.google.com/drive/api/v3/push#making-watch-requests
@@ -74,6 +74,7 @@ export default {
       const { files } = await this.googleDrive.listFilesInPage(null, args);
 
       await this.processChanges(files);
+      this.flushFileIntervals();
     },
     ...common.hooks,
   },
@@ -108,6 +109,9 @@ export default {
     isFileRelevant(file) {
       return this.files.includes(file.id);
     },
+    isRelevantChange({ file }) {
+      return this.isFileRelevant(file);
+    },
     getChanges(headers) {
       if (!headers) {
         return {
@@ -138,14 +142,13 @@ export default {
     async processChanges(changedFiles, headers) {
       console.log(`Processing ${changedFiles.length} changed files`);
 
-      const filteredFiles = this.checkMinimumInterval(changedFiles);
+      // Filter first so unwatched files aren't recorded
+      const relevantFiles = changedFiles.filter((file) => this.isFileRelevant(file));
+      const filteredFiles = this.checkMinimumInterval(relevantFiles);
       for (const file of filteredFiles) {
-        if (!this.isFileRelevant(file)) {
-          console.log(`Skipping event for irrelevant file ${file.id}`);
-          continue;
-        }
         await this.processChange(file, headers);
       }
+      return filteredFiles.length;
     },
   },
   sampleEmit,

package/sources/common-dedupe-changes.mjs

@@ -1,3 +1,5 @@
+import { MAX_FILE_INTERVAL_ENTRIES } from "../common/constants.mjs";
+
 export default {
   props: {
     intervalAlert: {
@@ -19,7 +21,10 @@ You can change or disable this minimum interval using the prop \`Minimum Interva
   },
   methods: {
     _getFileIntervals() {
-      return this.db.get("fileIntervals") ?? {};
+      if (!this._fileIntervals) {
+        this._fileIntervals = this.db.get("fileIntervals") ?? {};
+      }
+      return this._fileIntervals;
     },
     _setFileIntervals(value) {
       this.db.set("fileIntervals", value);
@@ -40,20 +45,26 @@ You can change or disable this minimum interval using the prop \`Minimum Interva
       }
 
       const now = Date.now();
-      const minTimestamp = now - (this.perFileInterval * 1000 * 60);
       const savedData = this._getFileIntervals();
-      Object.entries(savedData).forEach(([
-        key,
-        value,
-      ]) => {
-        if (value < minTimestamp) {
-          delete savedData[key];
-        }
-      });
       fileIds.forEach((id) => {
         savedData[id] = now;
       });
-      this._setFileIntervals(savedData);
+      this._fileIntervalsChanged = true;
+    },
+    flushFileIntervals() {
+      if (this._fileIntervalsChanged) {
+        const minTimestamp = Date.now() - (this.perFileInterval * 1000 * 60);
+        const entries = Object.entries(this._getFileIntervals())
+          .filter((entry) => entry[1] >= minTimestamp)
+          .sort((a, b) => b[1] - a[1])
+          .slice(0, MAX_FILE_INTERVAL_ENTRIES);
+        this._setFileIntervals(Object.fromEntries(entries));
+      }
+      this.resetFileIntervals();
+    },
+    resetFileIntervals() {
+      this._fileIntervals = null;
+      this._fileIntervalsChanged = false;
     },
     checkMinimumInterval(files) {
       const filteredFiles = this.filterByMinimumInterval(files);

package/sources/common-webhook.mjs

@@ -2,7 +2,11 @@ import includes from "lodash/includes.js";
 import { v4 as uuid } from "uuid";
 
 import googleDrive from "../google_drive.app.mjs";
-import { WEBHOOK_SUBSCRIPTION_RENEWAL_SECONDS } from "../common/constants.mjs";
+import {
+  MY_DRIVE_VALUE,
+  WEBHOOK_RENEWAL_WINDOW_MILLISECONDS,
+  WEBHOOK_TIMER_INTERVAL_SECONDS,
+} from "../common/constants.mjs";
 import { getListFilesOpts } from "../common/utils.mjs";
 import commonDedupeChanges from "./common-dedupe-changes.mjs";
 
@@ -19,8 +23,9 @@ export default {
         googleDrive,
         "watchedDrive",
       ],
-      description: "Defaults to **All Drives** (My Drive and all Shared Drives you have access to) when no selection is made. To limit to your personal drive or a [Shared Drive](https://support.google.com/a/users/answer/9310351), select it from this list.",
+      description: "The drive to watch. Defaults to **My Drive**. To watch a [Shared Drive](https://support.google.com/a/users/answer/9310351), select it from this list.",
       optional: false,
+      default: MY_DRIVE_VALUE,
     },
     timer: {
       label: "Push notification renewal schedule",
@@ -28,16 +33,30 @@ export default {
         "The Google Drive API requires occasional renewal of push notification subscriptions. **This runs in the background, so you should not need to modify this schedule**.",
       type: "$.interface.timer",
       static: {
-        intervalSeconds: WEBHOOK_SUBSCRIPTION_RENEWAL_SECONDS,
+        intervalSeconds: WEBHOOK_TIMER_INTERVAL_SECONDS,
       },
       hidden: true,
     },
+    // Unused; kept so older deployments can still save
+    updateTypes: {
+      type: "string[]",
+      label: "Types of updates",
+      description: "Deprecated and ignored.",
+      optional: true,
+      hidden: true,
+    },
     changesPageSize: {
       propDefinition: [
         googleDrive,
         "changesPageSize",
       ],
     },
+    maxEmitsPerRun: {
+      propDefinition: [
+        googleDrive,
+        "maxEmitsPerRun",
+      ],
+    },
   },
   hooks: {
     async activate() {
@@ -97,6 +116,38 @@ export default {
     _setPageToken(pageToken) {
       this.db.set("pageToken", pageToken);
     },
+    _getBacklogHeaders() {
+      return this.db.get("backlogHeaders");
+    },
+    _setBacklogHeaders(headers) {
+      this.db.set("backlogHeaders", headers);
+    },
+    isSubscriptionExpiring(subscription) {
+      const expiration = Number(subscription?.expiration);
+      return !expiration || expiration - Date.now() < WEBHOOK_RENEWAL_WINDOW_MILLISECONDS;
+    },
+    async renewSubscription(subscription, channelID, pageToken) {
+      const {
+        newChannelID,
+        newPageToken,
+        expiration,
+        resourceId,
+      } = await this.googleDrive.renewSubscription(
+        this.drive,
+        subscription,
+        this.http.endpoint,
+        channelID,
+        pageToken,
+      );
+
+      this._setSubscription({
+        expiration,
+        resourceId,
+      });
+      this._setChannelID(newChannelID);
+      this._setPageToken(newPageToken);
+      return newPageToken;
+    },
     isMyDrive(drive = this.drive) {
       return googleDrive.methods.isMyDrive(drive);
     },
@@ -128,6 +179,16 @@ export default {
     getChangesFileFields() {
       return undefined;
     },
+    isRelevantChange() {
+      return true;
+    },
+    async getChangeFilter() {
+      return (change) => this.isRelevantChange(change);
+    },
+    // false: processChanges() ignores changedFiles, so run it once, not per page
+    consumesChangedFiles() {
+      return true;
+    },
     /**
      * This method is responsible for processing a list of changed files
      * according to the event source's purpose. As an abstract method, it must
@@ -137,10 +198,36 @@ export default {
      * by the API](https://bit.ly/3h7WeUa)
      * @param {object} [headers] - an object containing the request headers of
      * the webhook call made by Google Drive
+     * @returns {number} the number of events emitted
      */
     processChanges() {
       throw new Error("processChanges is not implemented");
     },
+    async processChangesFrom(pageToken, headers) {
+      this.resetFileIntervals();
+      // Thrown runs keep db writes, so flush intervals with the token
+      try {
+        const caughtUp = await this.googleDrive.processChangesPages({
+          pageToken,
+          driveId: this.getDriveId(),
+          pageSize: this.changesPageSize,
+          fileFields: this.getChangesFileFields(),
+          maxEmits: this.maxEmitsPerRun,
+          changeFilter: await this.getChangeFilter(),
+          processPage: (changedFiles) => this.processChanges(changedFiles, headers),
+          savePageToken: (nextPageToken) => this._setPageToken(nextPageToken),
+        });
+        this._setBacklogHeaders(caughtUp
+          ? null
+          : {
+            "x-goog-resource-state": headers["x-goog-resource-state"],
+            "x-goog-resource-uri": headers["x-goog-resource-uri"],
+            "x-goog-changed": headers["x-goog-changed"],
+          });
+      } finally {
+        this.flushFileIntervals();
+      }
+    },
   },
   async run(event) {
     // This function is polymorphic: it can be triggered as a cron job, to make
@@ -152,25 +239,15 @@ export default {
 
     // Component was invoked by timer
     if (event.timestamp) {
-      const {
-        newChannelID,
-        newPageToken,
-        expiration,
-        resourceId,
-      } = await this.googleDrive.renewSubscription(
-        this.drive,
-        subscription,
-        this.http.endpoint,
-        channelID,
-        pageToken,
-      );
+      const currentPageToken = this.isSubscriptionExpiring(subscription)
+        ? await this.renewSubscription(subscription, channelID, pageToken)
+        : pageToken;
 
-      this._setSubscription({
-        expiration,
-        resourceId,
-      });
-      this._setChannelID(newChannelID);
-      this._setPageToken(newPageToken);
+      const backlogHeaders = this._getBacklogHeaders();
+      if (backlogHeaders && this.consumesChangedFiles()) {
+        console.log("Resuming changes left by a capped run");
+        await this.processChangesFrom(currentPageToken, backlogHeaders);
+      }
       return;
     } else {
       this.http.respond({
@@ -206,25 +283,12 @@ export default {
     }
 
     const driveId = this.getDriveId();
-    const changedFilesStream =
-      this.googleDrive.listChanges(
-        pageToken,
-        driveId,
-        this.changesPageSize,
-        this.getChangesFileFields(),
-      );
-    for await (const changedFilesPage of changedFilesStream) {
-      const {
-        changedFiles,
-        nextPageToken,
-      } = changedFilesPage;
-
-      // Process all the changed files retrieved from the current page
-      await this.processChanges(changedFiles, headers);
-
-      // After successfully processing the changed files, we store the page
-      // token of the next page
-      this._setPageToken(nextPageToken);
+    if (!this.consumesChangedFiles()) {
+      await this.processChanges([], headers);
+      this._setPageToken(await this.googleDrive.getPageToken(driveId));
+      return;
     }
+
+    await this.processChangesFrom(pageToken, headers);
   },
 };

package/sources/new-access-proposal/new-access-proposal.mjs

@@ -6,7 +6,7 @@ export default {
   key: "google_drive-new-access-proposal",
   name: "New Access Proposal",
   description: "Emit new event when a new access proposal is requested in Google Drive",
-  version: "0.0.20",
+  version: "0.0.21",
   type: "source",
   dedupe: "unique",
   props: {

package/sources/new-files-instant/new-files-instant.mjs

@@ -11,7 +11,7 @@ export default {
   key: "google_drive-new-files-instant",
   name: "New Files (Instant)",
   description: "Emit new event when a new file is added in your linked Google Drive",
-  version: "0.2.17",
+  version: "0.3.0",
   type: "source",
   dedupe: "unique",
   props: {
@@ -131,6 +131,9 @@ export default {
         GOOGLE_DRIVE_NOTIFICATION_CHANGE,
       ];
     },
+    consumesChangedFiles() {
+      return false;
+    },
     async emitFiles(files) {
       for (const file of files) {
         if (!(await this.shouldProcess(file))) {