@pipedream/google_drive 4.0.0 → 5.0.0

package/sources/new-files-instant-polling/new-files-instant-polling.mjs

@@ -1,14 +1,17 @@
 import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform";
 import googleDrive from "../../google_drive.app.mjs";
 import { getListFilesOpts } from "../../common/utils.mjs";
-import { GOOGLE_DRIVE_FOLDER_MIME_TYPE } from "../../common/constants.mjs";
+import {
+  CHANGE_FILTER_FILE_FIELDS,
+  GOOGLE_DRIVE_FOLDER_MIME_TYPE,
+} from "../../common/constants.mjs";
 import sampleEmit from "./test-event.mjs";
 
 export default {
   key: "google_drive-new-files-instant-polling",
   name: "New Files (Polling)",
   description: "Emit new event when a new file is added in your linked Google Drive",
-  version: "0.0.14",
+  version: "0.1.0",
   type: "source",
   dedupe: "unique",
   props: {
@@ -51,6 +54,12 @@ export default {
         "changesPageSize",
       ],
     },
+    maxEmitsPerRun: {
+      propDefinition: [
+        googleDrive,
+        "maxEmitsPerRun",
+      ],
+    },
   },
   hooks: {
     async deploy() {
@@ -143,54 +152,36 @@ export default {
     const currentRunTimestamp = Date.now();
     const lastRunTimestamp = this._getLastRunTimestamp();
 
-    const pageToken = this._getPageToken();
-    const driveId = this.getDriveId();
-
-    const changedFilesStream =
-      this.googleDrive.listChanges(pageToken, driveId, this.changesPageSize);
-
-    for await (const changedFilesPage of changedFilesStream) {
-      const {
-        changedFiles,
-        nextPageToken,
-      } = changedFilesPage;
-
-      console.log(changedFiles.length
-        ? `Processing ${changedFiles.length} changed files`
-        : "No changed files since last run");
-
-      for (const file of changedFiles) {
-        // Skip folders
-        if (file.mimeType === GOOGLE_DRIVE_FOLDER_MIME_TYPE) {
-          continue;
+    const caughtUp = await this.googleDrive.processChangesPages({
+      pageToken: this._getPageToken(),
+      driveId: this.getDriveId(),
+      pageSize: this.changesPageSize,
+      fileFields: CHANGE_FILTER_FILE_FIELDS,
+      maxEmits: this.maxEmitsPerRun,
+      changeFilter: ({ file }) => Date.parse(file.createdTime) > lastRunTimestamp
+        && this.shouldProcess(file),
+      processPage: async (changedFiles) => {
+        console.log(changedFiles.length
+          ? `Processing ${changedFiles.length} new files`
+          : "No new files since last run");
+
+        let emitted = 0;
+        for (const file of changedFiles) {
+          const fullFile = await this.googleDrive.getFile(file.id, {
+            fields: "*",
+          });
+          await this.emitFile(fullFile);
+          emitted++;
         }
+        return emitted;
+      },
+      savePageToken: (nextPageToken) => this._setPageToken(nextPageToken),
+    });
 
-        // Get full file metadata including parents
-        const fullFile = await this.googleDrive.getFile(file.id, {
-          fields: "*",
-        });
-
-        // Check if it's a new file (created after last run)
-        const fileCreatedTime = Date.parse(fullFile.createdTime);
-        if (fileCreatedTime <= lastRunTimestamp) {
-          console.log(`Skipping existing file ${fullFile.name || fullFile.id}`);
-          continue;
-        }
-
-        if (!this.shouldProcess(fullFile)) {
-          console.log(`Skipping file ${fullFile.name || fullFile.id}`);
-          continue;
-        }
-
-        await this.emitFile(fullFile);
-      }
-
-      // Save the next page token after successfully processing
-      this._setPageToken(nextPageToken);
+    // Hold until caught up so backlog files aren't skipped
+    if (caughtUp) {
+      this._setLastRunTimestamp(currentRunTimestamp);
     }
-
-    // Update the last run timestamp after processing all changes
-    this._setLastRunTimestamp(currentRunTimestamp);
   },
   sampleEmit,
 };

package/sources/new-files-shared-drive/new-files-shared-drive.mjs

@@ -7,7 +7,7 @@ export default {
   key: "google_drive-new-files-shared-drive",
   name: "New Files (Shared Drive)",
   description: "Emit new event when a new file is added in your shared Google Drive",
-  version: "0.1.16",
+  version: "0.1.18",
   type: "source",
   dedupe: "unique",
   props: {
@@ -69,21 +69,7 @@ export default {
   },
   methods: sourceComponent.methods,
   async run() {
-    const pageToken = this._getPageToken();
-
-    const driveId = this.getDriveId();
-    const changedFilesStream =
-      this.googleDrive.listChanges(pageToken, driveId, this.changesPageSize);
-    for await (const changedFilesPage of changedFilesStream) {
-      const { nextPageToken } = changedFilesPage;
-
-      // Process all the changed files retrieved from the current page
-      await this.processChanges();
-
-      // After successfully processing the changed files, we store the page
-      // token of the next page
-      this._setPageToken(nextPageToken);
-    }
+    await this.processChanges();
   },
   sampleEmit,
 };

package/sources/new-or-modified-comments/new-or-modified-comments.mjs

@@ -18,10 +18,8 @@ export default {
   name: "New or Modified Comments (Instant)",
   description:
     "Emit new event when a comment is created or modified in the selected file",
-  version: "1.0.25",
+  version: "1.1.0",
   type: "source",
-  // Dedupe events based on the "x-goog-message-number" header for the target channel:
-  // https://developers.google.com/drive/api/v3/push#making-watch-requests
   dedupe: "unique",
   props: {
     ...common.props,
@@ -72,17 +70,19 @@ export default {
         GOOGLE_DRIVE_NOTIFICATION_CHANGE,
       ];
     },
-    generateMeta(data, headers) {
+    isRelevantChange({ file }) {
+      return file.id === this.fileId;
+    },
+    generateMeta(data) {
       const {
         id: commentId,
         content: summary,
         modifiedTime: tsString,
       } = data;
       const ts = Date.parse(tsString);
-      const eventId = headers && headers["x-goog-message-number"];
 
       return {
-        id: md5(`${commentId}-${eventId || ts}`),
+        id: md5(`${commentId}-${ts}`),
         summary,
         ts,
       };
@@ -109,6 +109,7 @@ export default {
         changedFiles = changedFiles.filter(({ id }) => id === this.fileId);
       }
 
+      let emitted = 0;
       for (const file of changedFiles) {
         const lastCommentTimeForFile = this._getLastCommentTimeForFile(file.id);
         let maxModifiedTime = lastCommentTimeForFile;
@@ -127,13 +128,15 @@ export default {
             comment,
             ...changes,
           };
-          const meta = this.generateMeta(comment, headers);
+          const meta = this.generateMeta(comment);
           this.$emit(eventToEmit, meta);
+          emitted++;
 
           maxModifiedTime = Math.max(maxModifiedTime, commentTime);
           this._updateLastCommentTimeForFile(file.id, maxModifiedTime);
         }
       }
+      return emitted;
     },
   },
 };

package/sources/new-or-modified-comments-polling/new-or-modified-comments-polling.mjs

@@ -7,7 +7,7 @@ export default {
   key: "google_drive-new-or-modified-comments-polling",
   name: "New or Modified Comments (Polling)",
   description: "Emit new event when a comment is created or modified in the selected file",
-  version: "0.0.14",
+  version: "0.0.16",
   type: "source",
   dedupe: "unique",
   props: {

package/sources/new-or-modified-files/new-or-modified-files.mjs

@@ -14,6 +14,7 @@ import {
   GOOGLE_DRIVE_NOTIFICATION_ADD,
   GOOGLE_DRIVE_NOTIFICATION_CHANGE,
   GOOGLE_DRIVE_NOTIFICATION_UPDATE,
+  MASS_CHANGE_MAX_AGE_MILLISECONDS,
   PDF_EXPORTABLE_MIME_TYPES,
 } from "../../common/constants.mjs";
 import commonDedupeChanges from "../common-dedupe-changes.mjs";
@@ -28,7 +29,7 @@ export default {
   key: "google_drive-new-or-modified-files",
   name: "New or Modified Files (Instant)",
   description: "Emit new event when a file in the selected Drive is created, modified or trashed.",
-  version: "1.0.0",
+  version: "2.0.0",
   type: "source",
   dedupe: "unique",
   props: {
@@ -92,6 +93,7 @@ export default {
       const { files } = await this.googleDrive.listFilesInPage(null, args);
 
       await this.processChanges(files);
+      this.flushFileIntervals();
     },
     ...common.hooks,
   },
@@ -116,6 +118,22 @@ export default {
     getChangesFileFields() {
       return CHANGED_FILE_FIELDS;
     },
+    // Skip bulk moves/sharing of old files; createdTime keeps uploads with an old modifiedTime
+    isRelevantChange({
+      time, file,
+    }) {
+      if (!this.shouldProcess(file)) {
+        return false;
+      }
+      if (this.watchForPropertiesChanges || file.trashed || !time) {
+        return true;
+      }
+      const changedAt = Date.parse(time);
+      return [
+        file.modifiedTime,
+        file.createdTime,
+      ].some((t) => !t || changedAt - Date.parse(t) <= MASS_CHANGE_MAX_AGE_MILLISECONDS);
+    },
     generateMeta({
       id, name, modifiedTime, trashed,
     }) {
@@ -187,6 +205,7 @@ export default {
       } finally {
         this.recordFileEmits(emittedFileIds);
       }
+      return emittedFileIds.length;
     },
   },
   sampleEmit,

package/sources/new-or-modified-files-polling/new-or-modified-files-polling.mjs

@@ -2,14 +2,17 @@ import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform";
 import googleDrive from "../../google_drive.app.mjs";
 import { getListFilesOpts } from "../../common/utils.mjs";
 import sampleEmit from "./test-event.mjs";
-import { GOOGLE_DRIVE_FOLDER_MIME_TYPE } from "../../common/constants.mjs";
+import {
+  CHANGE_FILTER_FILE_FIELDS,
+  GOOGLE_DRIVE_FOLDER_MIME_TYPE,
+} from "../../common/constants.mjs";
 import md5 from "md5";
 
 export default {
   key: "google_drive-new-or-modified-files-polling",
   name: "New or Modified Files (Polling)",
   description: "Emit new event when a file in the selected Drive is created, modified or trashed. [See the documentation](https://developers.google.com/drive/api/v3/reference/changes/list)",
-  version: "0.0.16",
+  version: "0.1.0",
   type: "source",
   dedupe: "unique",
   props: {
@@ -72,6 +75,12 @@ export default {
         "changesPageSize",
       ],
     },
+    maxEmitsPerRun: {
+      propDefinition: [
+        googleDrive,
+        "maxEmitsPerRun",
+      ],
+    },
   },
   hooks: {
     async deploy() {
@@ -177,46 +186,38 @@ export default {
     },
   },
   async run() {
-    // Store the current run timestamp at the start
     const currentRunTimestamp = Date.now();
     const lastRunTimestamp = this._getLastRunTimestamp();
 
-    const pageToken = this._getPageToken();
-    const driveId = this.getDriveId();
-
-    const changedFilesStream =
-      this.googleDrive.listChanges(pageToken, driveId, this.changesPageSize);
-
-    for await (const changedFilesPage of changedFilesStream) {
-      const {
-        changedFiles,
-        nextPageToken,
-      } = changedFilesPage;
-
-      console.log(changedFiles.length
-        ? `Processing ${changedFiles.length} changed files`
-        : "No changed files since last run");
-
-      for (const file of changedFiles) {
-        // Get full file metadata including parents
-        const fullFile = await this.googleDrive.getFile(file.id, {
-          fields: "*",
-        });
-
-        if (!this.shouldProcess(fullFile, lastRunTimestamp)) {
-          console.log(`Skipping file ${fullFile.name || fullFile.id}`);
-          continue;
+    const caughtUp = await this.googleDrive.processChangesPages({
+      pageToken: this._getPageToken(),
+      driveId: this.getDriveId(),
+      pageSize: this.changesPageSize,
+      fileFields: CHANGE_FILTER_FILE_FIELDS,
+      maxEmits: this.maxEmitsPerRun,
+      changeFilter: ({ file }) => this.shouldProcess(file, lastRunTimestamp),
+      processPage: async (changedFiles) => {
+        console.log(changedFiles.length
+          ? `Processing ${changedFiles.length} changed files`
+          : "No changed files since last run");
+
+        let emitted = 0;
+        for (const file of changedFiles) {
+          const fullFile = await this.googleDrive.getFile(file.id, {
+            fields: "*",
+          });
+          await this.emitFile(fullFile);
+          emitted++;
         }
+        return emitted;
+      },
+      savePageToken: (nextPageToken) => this._setPageToken(nextPageToken),
+    });
 
-        await this.emitFile(fullFile);
-      }
-
-      // Save the next page token after successfully processing
-      this._setPageToken(nextPageToken);
+    // Hold until caught up so backlog files aren't skipped
+    if (caughtUp) {
+      this._setLastRunTimestamp(currentRunTimestamp);
     }
-
-    // Update the last run timestamp after processing all changes
-    this._setLastRunTimestamp(currentRunTimestamp);
   },
   sampleEmit,
 };

package/sources/new-or-modified-folders/new-or-modified-folders.mjs

@@ -9,6 +9,8 @@
 // 2) A timer that runs on regular intervals, renewing the notification channel as needed
 
 import {
+  CHANGE_FILTER_FILE_FIELDS,
+  GOOGLE_DRIVE_FOLDER_MIME_TYPE,
   GOOGLE_DRIVE_NOTIFICATION_ADD,
   GOOGLE_DRIVE_NOTIFICATION_CHANGE,
   GOOGLE_DRIVE_NOTIFICATION_UPDATE,
@@ -21,7 +23,7 @@ export default {
   key: "google_drive-new-or-modified-folders",
   name: "New or Modified Folders (Instant)",
   description: "Emit new event when a folder is created or modified in the selected Drive",
-  version: "0.2.19",
+  version: "1.0.0",
   type: "source",
   // Dedupe events based on the "x-goog-message-number" header for the target channel:
   // https://developers.google.com/drive/api/v3/push#making-watch-requests
@@ -55,12 +57,21 @@ export default {
 
       const args = this.getListFilesOpts({
         q: `mimeType = "application/vnd.google-apps.folder" and modifiedTime > "${timeString}" and trashed = false`,
-        fields: "files(id, mimeType)",
+        fields: `files(${CHANGE_FILTER_FILE_FIELDS})`,
       });
 
       const { files } = await this.googleDrive.listFilesInPage(null, args);
 
-      await this.processChanges(files, null, 5);
+      const changeFilter = await this.getChangeFilter();
+      const folders = [];
+      for (const file of files) {
+        if (folders.length < 5 && await changeFilter({
+          file,
+        })) {
+          folders.push(file);
+        }
+      }
+      await this.processChanges(folders);
     },
     ...common.hooks,
   },
@@ -79,20 +90,24 @@ export default {
         GOOGLE_DRIVE_NOTIFICATION_UPDATE,
       ];
     },
-    async getAllParents(folderId) {
+    getChangesFileFields() {
+      return CHANGE_FILTER_FILE_FIELDS;
+    },
+    async getAllParents(folderId, parentsCache = new Map()) {
       const allParents = [];
       let currentId = folderId;
 
       while (currentId) {
-        const folder = await this.googleDrive.getFile(currentId, {
-          fields: "parents",
-        });
-        const parents = folder.parents;
-
-        if (parents && parents.length > 0) {
-          allParents.push(parents[0]);
+        if (!parentsCache.has(currentId)) {
+          const { parents } = await this.googleDrive.getFile(currentId, {
+            fields: "parents",
+          });
+          parentsCache.set(currentId, parents?.[0]);
+        }
+        currentId = parentsCache.get(currentId);
+        if (currentId) {
+          allParents.push(currentId);
         }
-        currentId = parents?.[0];
       }
 
       return allParents;
@@ -108,16 +123,13 @@ export default {
         ts,
       };
     },
-    async getChanges(headers) {
+    getChanges(headers) {
       if (!headers) {
         return {
           change: { },
         };
       }
-      const resourceUri = headers["x-goog-resource-uri"];
-      const metadata = await this.googleDrive.getFileMetadata(`${resourceUri}&fields=*`);
       return {
-        ...metadata,
         change: {
           state: headers["x-goog-resource-state"],
           resourceURI: headers["x-goog-resource-uri"],
@@ -125,45 +137,34 @@ export default {
         },
       };
     },
-    async processChanges(changedFiles, headers, maxResults) {
-      const files = changedFiles.filter(
-        // API docs that define Google Drive folders:
-        // https://developers.google.com/drive/api/v3/folder
-        (file) => file.mimeType === "application/vnd.google-apps.folder",
-      );
-
-      const filteredFiles = [];
-      for (const file of files) {
+    async getChangeFilter() {
+      let targetId = this.folderId;
+      const parentsCache = new Map();
+      return async ({ file }) => {
         // The changelog is updated each time a folder is opened. Check the
         // folder's `modifiedTime` to see if the folder has been modified.
-        const fileInfo = await this.googleDrive.getFile(file.id);
-        const root = await this.googleDrive.getFile(this.drive === "My Drive"
-          ? "root"
-          : this.drive);
-
-        const allParents = [];
-        if (this.includeSubfolders) {
-          allParents.push(...(await this.getAllParents(file.id)));
-        } else if (fileInfo.parents) {
-          allParents.push(fileInfo.parents[0]);
+        if (file.mimeType !== GOOGLE_DRIVE_FOLDER_MIME_TYPE
+          || this._getLastModifiedTimeForFile(file.id) == Date.parse(file.modifiedTime)) {
+          return false;
         }
-
-        if (!allParents.includes(this.folderId || root.id)) {
-          continue;
+        if (!targetId) {
+          ({ id: targetId } = await this.googleDrive.getFile(this.isMyDrive()
+            ? "root"
+            : this.drive, {
+            fields: "id",
+          }));
         }
-
-        filteredFiles.push(fileInfo);
-      }
-
-      if (maxResults && filteredFiles.length >= maxResults) {
-        filteredFiles.length = maxResults;
-      }
-      for (const file of filteredFiles) {
-        const lastModifiedTimeForFile = this._getLastModifiedTimeForFile(file.id);
+        const allParents = this.includeSubfolders
+          ? await this.getAllParents(file.id, parentsCache)
+          : (file.parents ?? []).slice(0, 1);
+        return allParents.includes(targetId);
+      };
+    },
+    async processChanges(changedFiles, headers) {
+      const changes = this.getChanges(headers);
+      for (const { id } of changedFiles) {
+        const file = await this.googleDrive.getFile(id);
         const modifiedTime = Date.parse(file.modifiedTime);
-        if (lastModifiedTimeForFile == modifiedTime) continue;
-
-        const changes = await this.getChanges(headers);
 
         const eventToEmit = {
           file,
@@ -175,6 +176,7 @@ export default {
 
         this._setModifiedTimeForFile(file.id, modifiedTime);
       }
+      return changedFiles.length;
     },
   },
 };