@pipedream/google_drive 4.0.1 → 5.0.0

package/sources/new-or-modified-folders-polling/new-or-modified-folders-polling.mjs

@@ -4,7 +4,10 @@ import {
   getListFilesOpts,
   isMyDrive,
 } from "../../common/utils.mjs";
-import { GOOGLE_DRIVE_FOLDER_MIME_TYPE } from "../../common/constants.mjs";
+import {
+  CHANGE_FILTER_FILE_FIELDS,
+  GOOGLE_DRIVE_FOLDER_MIME_TYPE,
+} from "../../common/constants.mjs";
 import sampleEmit from "./test-event.mjs";
 import md5 from "md5";
 
@@ -12,7 +15,7 @@ export default {
   key: "google_drive-new-or-modified-folders-polling",
   name: "New or Modified Folders (Polling)",
   description: "Emit new event when a folder is created or modified in the selected Drive",
-  version: "0.0.16",
+  version: "0.1.0",
   type: "source",
   dedupe: "unique",
   props: {
@@ -56,6 +59,12 @@ export default {
         "changesPageSize",
       ],
     },
+    maxEmitsPerRun: {
+      propDefinition: [
+        googleDrive,
+        "maxEmitsPerRun",
+      ],
+    },
   },
   hooks: {
     async deploy() {
@@ -115,20 +124,21 @@ export default {
         ...args,
       });
     },
-    async getAllParents(folderId) {
+    async getAllParents(folderId, parentsCache = new Map()) {
       const allParents = [];
       let currentId = folderId;
 
       while (currentId) {
-        const folder = await this.googleDrive.getFile(currentId, {
-          fields: "parents",
-        });
-        const parents = folder.parents;
-
-        if (parents && parents.length > 0) {
-          allParents.push(parents[0]);
+        if (!parentsCache.has(currentId)) {
+          const { parents } = await this.googleDrive.getFile(currentId, {
+            fields: "parents",
+          });
+          parentsCache.set(currentId, parents?.[0]);
+        }
+        currentId = parentsCache.get(currentId);
+        if (currentId) {
+          allParents.push(currentId);
         }
-        currentId = parents?.[0];
       }
 
       return allParents;
@@ -141,7 +151,7 @@ export default {
       );
       return root.id;
     },
-    async shouldProcess(file, rootId) {
+    async shouldProcess(file, rootId, parentsCache) {
       // Skip if not a folder
       if (file.mimeType !== GOOGLE_DRIVE_FOLDER_MIME_TYPE) {
         return false;
@@ -154,7 +164,7 @@ export default {
 
       const allParents = [];
       if (this.includeSubfolders) {
-        allParents.push(...(await this.getAllParents(file.id)));
+        allParents.push(...(await this.getAllParents(file.id, parentsCache)));
       } else if (file.parents) {
         allParents.push(file.parents[0]);
       }
@@ -182,60 +192,42 @@ export default {
   },
   async run() {
     const currentRunTimestamp = Date.now();
-
-    const pageToken = this._getPageToken();
-    const driveId = this.getDriveId();
     const rootId = await this.getRootId();
-
-    const changedFilesStream =
-      this.googleDrive.listChanges(pageToken, driveId, this.changesPageSize);
-
-    for await (const changedFilesPage of changedFilesStream) {
-      const {
-        changedFiles,
-        nextPageToken,
-      } = changedFilesPage;
-
-      console.log(changedFiles.length
-        ? `Processing ${changedFiles.length} changed files`
-        : "No changed files since last run");
-
-      for (const file of changedFiles) {
-        // Skip if not a folder
-        if (file.mimeType !== GOOGLE_DRIVE_FOLDER_MIME_TYPE) {
-          continue;
+    const parentsCache = new Map();
+
+    const caughtUp = await this.googleDrive.processChangesPages({
+      pageToken: this._getPageToken(),
+      driveId: this.getDriveId(),
+      pageSize: this.changesPageSize,
+      fileFields: CHANGE_FILTER_FILE_FIELDS,
+      maxEmits: this.maxEmitsPerRun,
+      changeFilter: async ({ file }) => file.mimeType === GOOGLE_DRIVE_FOLDER_MIME_TYPE
+        && this._getLastModifiedTimeForFile(file.id) !== Date.parse(file.modifiedTime)
+        && await this.shouldProcess(file, rootId, parentsCache),
+      processPage: async (changedFiles) => {
+        console.log(changedFiles.length
+          ? `Processing ${changedFiles.length} changed folders`
+          : "No changed folders since last run");
+
+        let emitted = 0;
+        for (const file of changedFiles) {
+          const modifiedTime = Date.parse(file.modifiedTime);
+          const fullFile = await this.googleDrive.getFile(file.id, {
+            fields: "*",
+          });
+          await this.emitFolder(fullFile);
+          emitted++;
+
+          this._setModifiedTimeForFile(fullFile.id, modifiedTime);
         }
+        return emitted;
+      },
+      savePageToken: (nextPageToken) => this._setPageToken(nextPageToken),
+    });
 
-        // Get full file metadata
-        const fullFile = await this.googleDrive.getFile(file.id, {
-          fields: "*",
-        });
-
-        const modifiedTime = Date.parse(fullFile.modifiedTime);
-        const lastModifiedTimeForFile = this._getLastModifiedTimeForFile(fullFile.id);
-
-        // Skip if not modified since last check
-        if (lastModifiedTimeForFile === modifiedTime) {
-          console.log(`Skipping unmodified folder ${fullFile.name || fullFile.id}`);
-          continue;
-        }
-
-        if (!await this.shouldProcess(fullFile, rootId)) {
-          console.log(`Skipping folder ${fullFile.name || fullFile.id}`);
-          continue;
-        }
-
-        await this.emitFolder(fullFile);
-
-        this._setModifiedTimeForFile(fullFile.id, modifiedTime);
-      }
-
-      // Save the next page token after successfully processing
-      this._setPageToken(nextPageToken);
+    if (caughtUp) {
+      this._setLastRunTimestamp(currentRunTimestamp);
     }
-
-    // Update the last run timestamp after processing all changes
-    this._setLastRunTimestamp(currentRunTimestamp);
   },
   sampleEmit,
 };

package/sources/new-shared-drive/new-shared-drive.mjs

@@ -5,7 +5,7 @@ export default {
   key: "google_drive-new-shared-drive",
   name: "New Shared Drive",
   description: "Emits a new event any time a shared drive is created.",
-  version: "0.1.27",
+  version: "0.1.28",
   type: "source",
   dedupe: "unique",
   props: {

package/sources/new-spreadsheet/new-spreadsheet.mjs

@@ -6,7 +6,7 @@ export default {
   type: "source",
   name: "New Spreadsheet (Instant)",
   description: "Emit new event when a new spreadsheet is created in a drive.",
-  version: "0.1.30",
+  version: "0.2.0",
   props: {
     googleDrive: newFilesInstant.props.googleDrive,
     db: newFilesInstant.props.db,

package/sources/new-spreadsheet-polling/new-spreadsheet-polling.mjs

@@ -1,14 +1,17 @@
 import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform";
 import googleDrive from "../../google_drive.app.mjs";
 import { getListFilesOpts } from "../../common/utils.mjs";
-import { GOOGLE_DRIVE_FOLDER_MIME_TYPE } from "../../common/constants.mjs";
+import {
+  CHANGE_FILTER_FILE_FIELDS,
+  GOOGLE_DRIVE_FOLDER_MIME_TYPE,
+} from "../../common/constants.mjs";
 import sampleEmit from "./test-event.mjs";
 
 export default {
   key: "google_drive-new-spreadsheet-polling",
   name: "New Spreadsheet (Polling)",
   description: "Emit new event when a new spreadsheet is created in a drive.",
-  version: "0.0.15",
+  version: "0.1.0",
   type: "source",
   dedupe: "unique",
   props: {
@@ -50,6 +53,12 @@ export default {
         "changesPageSize",
       ],
     },
+    maxEmitsPerRun: {
+      propDefinition: [
+        googleDrive,
+        "maxEmitsPerRun",
+      ],
+    },
   },
   hooks: {
     async deploy() {
@@ -141,54 +150,36 @@ export default {
     const currentRunTimestamp = Date.now();
     const lastRunTimestamp = this._getLastRunTimestamp();
 
-    const pageToken = this._getPageToken();
-    const driveId = this.getDriveId();
-
-    const changedFilesStream =
-      this.googleDrive.listChanges(pageToken, driveId, this.changesPageSize);
-
-    for await (const changedFilesPage of changedFilesStream) {
-      const {
-        changedFiles,
-        nextPageToken,
-      } = changedFilesPage;
-
-      console.log(changedFiles.length
-        ? `Processing ${changedFiles.length} changed files`
-        : "No changed files since last run");
-
-      for (const file of changedFiles) {
-        // Skip if not a spreadsheet
-        if (!file.mimeType || !file.mimeType.includes("spreadsheet")) {
-          continue;
+    const caughtUp = await this.googleDrive.processChangesPages({
+      pageToken: this._getPageToken(),
+      driveId: this.getDriveId(),
+      pageSize: this.changesPageSize,
+      fileFields: CHANGE_FILTER_FILE_FIELDS,
+      maxEmits: this.maxEmitsPerRun,
+      changeFilter: ({ file }) => Date.parse(file.createdTime) > lastRunTimestamp
+        && this.shouldProcess(file),
+      processPage: async (changedFiles) => {
+        console.log(changedFiles.length
+          ? `Processing ${changedFiles.length} new spreadsheets`
+          : "No new spreadsheets since last run");
+
+        let emitted = 0;
+        for (const file of changedFiles) {
+          const fullFile = await this.googleDrive.getFile(file.id, {
+            fields: "*",
+          });
+          await this.emitSpreadsheet(fullFile);
+          emitted++;
         }
+        return emitted;
+      },
+      savePageToken: (nextPageToken) => this._setPageToken(nextPageToken),
+    });
 
-        // Get full file metadata including parents
-        const fullFile = await this.googleDrive.getFile(file.id, {
-          fields: "*",
-        });
-
-        // Check if it's a new spreadsheet (created after last run)
-        const fileCreatedTime = Date.parse(fullFile.createdTime);
-        if (fileCreatedTime <= lastRunTimestamp) {
-          console.log(`Skipping existing spreadsheet ${fullFile.name || fullFile.id}`);
-          continue;
-        }
-
-        if (!this.shouldProcess(fullFile)) {
-          console.log(`Skipping spreadsheet ${fullFile.name || fullFile.id}`);
-          continue;
-        }
-
-        await this.emitSpreadsheet(fullFile);
-      }
-
-      // Save the next page token after successfully processing
-      this._setPageToken(nextPageToken);
+    // Hold until caught up so backlog files aren't skipped
+    if (caughtUp) {
+      this._setLastRunTimestamp(currentRunTimestamp);
     }
-
-    // Update the last run timestamp after processing all changes
-    this._setLastRunTimestamp(currentRunTimestamp);
   },
   sampleEmit,
 };