@pipedream/google_drive 0.3.3 → 0.4.0

package/sources/changes-to-specific-files-shared-drive/changes-to-specific-files-shared-drive.mjs

@@ -0,0 +1,110 @@
+// This source processes changes to specific files in a user's Google Drive,
+// implementing strategy enumerated in the Push Notifications API docs:
+// https://developers.google.com/drive/api/v3/push .
+//
+// This source has two interfaces:
+//
+// 1) The HTTP requests tied to changes in files in the user's Google Drive
+// 2) A timer that runs on regular intervals, renewing the notification channel as needed
+
+import common from "../common-webhook.mjs";
+
+import {
+  GOOGLE_DRIVE_NOTIFICATION_CHANGE,
+  GOOGLE_DRIVE_NOTIFICATION_ADD,
+  GOOGLE_DRIVE_NOTIFICATION_UPDATE,
+} from "../../constants.mjs";
+
+/**
+ * This source uses the Google Drive API's
+ * {@link https://developers.google.com/drive/api/v3/reference/changes/watch changes: watch}
+ * endpoint to subscribe to changes to the user's drive or a shard drive.
+ */
+export default {
+  ...common,
+  key: "google_drive-changes-to-specific-files-shared-drive",
+  name: "Changes to Specific Files (Shared Drive)",
+  description: "Watches for changes to specific files in a shared drive, emitting an event any time a change is made to one of those files",
+  version: "0.0.1",
+  type: "source",
+  // Dedupe events based on the "x-goog-message-number" header for the target channel:
+  // https://developers.google.com/drive/api/v3/push#making-watch-requests
+  dedupe: "unique",
+  props: {
+    ...common.props,
+    files: {
+      type: "string[]",
+      label: "Files",
+      description: "The files you want to watch for changes.",
+      optional: true,
+      default: [],
+      options({ prevContext }) {
+        const { nextPageToken } = prevContext;
+        const baseOpts = {};
+        const opts = this.isMyDrive()
+          ? baseOpts
+          : {
+            ...baseOpts,
+            corpora: "drive",
+            driveId: this.getDriveId(),
+            includeItemsFromAllDrives: true,
+            supportsAllDrives: true,
+          };
+        return this.googleDrive.listFilesOptions(nextPageToken, opts);
+      },
+    },
+  },
+  methods: {
+    ...common.methods,
+    getUpdateTypes() {
+      return [
+        GOOGLE_DRIVE_NOTIFICATION_ADD,
+        GOOGLE_DRIVE_NOTIFICATION_CHANGE,
+        GOOGLE_DRIVE_NOTIFICATION_UPDATE,
+      ];
+    },
+    generateMeta(data, headers) {
+      const {
+        id: fileId,
+        name: fileName,
+        modifiedTime: tsString,
+      } = data;
+      const {
+        "x-goog-message-number": eventId,
+        "x-goog-resource-state": resourceState,
+      } = headers;
+
+      return {
+        id: `${fileId}-${eventId}`,
+        summary: `${resourceState.toUpperCase()} - ${
+          fileName || "Untitled"
+        }`,
+        ts: Date.parse(tsString),
+      };
+    },
+    isFileRelevant(file) {
+      return this.files.includes(file.id);
+    },
+    async processChange(file, headers) {
+      const eventToEmit = {
+        file,
+        change: {
+          state: headers["x-goog-resource-state"],
+          resourceURI: headers["x-goog-resource-uri"],
+          changed: headers["x-goog-changed"], // "Additional details about the changes. Possible values: content, parents, children, permissions"
+        },
+      };
+      const meta = this.generateMeta(file, headers);
+      this.$emit(eventToEmit, meta);
+    },
+    async processChanges(changedFiles, headers) {
+      for (const file of changedFiles) {
+        if (!this.isFileRelevant(file)) {
+          console.log(`Skipping event for irrelevant file ${file.id}`);
+          continue;
+        }
+        this.processChange(file, headers);
+      }
+    },
+  },
+};

package/sources/common-webhook.mjs

@@ -0,0 +1,201 @@
+import includes from "lodash/includes.js";
+import { v4 as uuid } from "uuid";
+
+import googleDrive from "../google_drive.app.mjs";
+import { WEBHOOK_SUBSCRIPTION_RENEWAL_SECONDS } from "../constants.mjs";
+
+export default {
+  props: {
+    googleDrive,
+    db: "$.service.db",
+    http: "$.interface.http",
+    drive: {
+      propDefinition: [
+        googleDrive,
+        "watchedDrive",
+      ],
+      description: "Defaults to My Drive. To select a [Shared Drive](https://support.google.com/a/users/answer/9310351) instead, select it from this list.",
+      optional: false,
+    },
+    watchForPropertiesChanges: {
+      propDefinition: [
+        googleDrive,
+        "watchForPropertiesChanges",
+      ],
+    },
+    timer: {
+      label: "Push notification renewal schedule",
+      description:
+        "The Google Drive API requires occasional renewal of push notification subscriptions. **This runs in the background, so you should not need to modify this schedule**.",
+      type: "$.interface.timer",
+      static: {
+        intervalSeconds: WEBHOOK_SUBSCRIPTION_RENEWAL_SECONDS,
+      },
+    },
+  },
+  hooks: {
+    async activate() {
+      // Called when a component is created or updated. Handles all the logic
+      // for starting and stopping watch notifications tied to the desired
+      // files.
+      const channelID = uuid();
+      const {
+        startPageToken,
+        expiration,
+        resourceId,
+      } = await this.googleDrive.activateHook(
+        channelID,
+        this.http.endpoint,
+        this.getDriveId(),
+      );
+
+      // We use and increment the pageToken as new changes arrive, in run()
+      this._setPageToken(startPageToken);
+
+      // Save metadata on the subscription so we can stop / renew later
+      // Subscriptions are tied to Google's resourceID, "an opaque value that
+      // identifies the watched resource". This value is included in request headers
+      this._setSubscription({
+        resourceId,
+        expiration,
+      });
+      this._setChannelID(channelID);
+    },
+    async deactivate() {
+      const channelID = this._getChannelID();
+      const { resourceId } = this._getSubscription();
+      await this.googleDrive.deactivateHook(channelID, resourceId);
+
+      this._setSubscription(null);
+      this._setChannelID(null);
+      this._setPageToken(null);
+    },
+  },
+  methods: {
+    _getSubscription() {
+      return this.db.get("subscription");
+    },
+    _setSubscription(subscription) {
+      this.db.set("subscription", subscription);
+    },
+    _getChannelID() {
+      return this.db.get("channelID");
+    },
+    _setChannelID(channelID) {
+      this.db.set("channelID", channelID);
+    },
+    _getPageToken() {
+      return this.db.get("pageToken");
+    },
+    _setPageToken(pageToken) {
+      this.db.set("pageToken", pageToken);
+    },
+    isMyDrive(drive = this.drive) {
+      return googleDrive.methods.isMyDrive(drive);
+    },
+    getDriveId(drive = this.drive) {
+      return googleDrive.methods.getDriveId(drive);
+    },
+    /**
+     * This method returns the types of updates/events from Google Drive that
+     * the event source should listen to. This base implementation returns an
+     * empty list, which means that any event source that extends this module
+     * and that does not refine this implementation will essentially ignore
+     * every incoming event from Google Drive.
+     *
+     * @returns
+     * @type {UpdateType[]}
+     */
+    getUpdateTypes() {
+      return [];
+    },
+    /**
+     * This method is responsible for processing a list of changed files
+     * according to the event source's purpose. As an abstract method, it must
+     * be implemented by every event source that extends this module.
+     *
+     * @param {object[]} [changedFiles] - the list of file changes, as [defined
+     * by the API](https://bit.ly/3h7WeUa)
+     * @param {object} [headers] - an object containing the request headers of
+     * the webhook call made by Google Drive
+     */
+    processChanges() {
+      throw new Error("processChanges is not implemented");
+    },
+  },
+  async run(event) {
+    // This function is polymorphic: it can be triggered as a cron job, to make
+    // sure we renew watch requests for specific files, or via HTTP request (the
+    // change payloads from Google)
+    const subscription = this._getSubscription();
+    const channelID = this._getChannelID();
+    const pageToken = this._getPageToken();
+
+    // Component was invoked by timer
+    if (event.timestamp) {
+      const {
+        newChannelID,
+        newPageToken,
+        expiration,
+        resourceId,
+      } = await this.googleDrive.renewSubscription(
+        this.drive,
+        subscription,
+        this.http.endpoint,
+        channelID,
+        pageToken,
+      );
+
+      this._setSubscription({
+        expiration,
+        resourceId,
+      });
+      this._setChannelID(newChannelID);
+      this._setPageToken(newPageToken);
+      return;
+    }
+
+    const { headers } = event;
+    if (!this.googleDrive.checkHeaders(headers, subscription, channelID)) {
+      return;
+    }
+
+    if (!includes(this.getUpdateTypes(), headers["x-goog-resource-state"])) {
+      console.log(
+        `Update type ${headers["x-goog-resource-state"]} not in list of updates to watch: `,
+        this.getUpdateTypes(),
+      );
+      return;
+    }
+
+    // We observed false positives where a single change to a document would trigger two changes:
+    // one to "properties" and another to "content,properties". But changes to properties
+    // alone are legitimate, most users just won't want this source to emit in those cases.
+    // If x-goog-changed is _only_ set to "properties", only move on if the user set the prop
+    if (
+      !this.watchForPropertiesChanges &&
+      headers["x-goog-changed"] === "properties"
+    ) {
+      console.log(
+        "Change to properties only, which this component is set to ignore. Exiting",
+      );
+      return;
+    }
+
+    const driveId = this.getDriveId();
+    const changedFilesStream = this.googleDrive.listChanges(pageToken, driveId);
+    for await (const changedFilesPage of changedFilesStream) {
+      const {
+        changedFiles,
+        nextPageToken,
+      } = changedFilesPage;
+
+      // Process all the changed files retrieved from the current page
+      await this.processChanges(changedFiles, headers);
+
+      // After successfully processing the changed files, we store the page
+      // token of the next page
+      this._setPageToken(nextPageToken);
+    }
+  },
+};

package/sources/new-files-instant/new-files-instant.mjs

@@ -0,0 +1,95 @@
+import common from "../common-webhook.mjs";
+import {
+  GOOGLE_DRIVE_NOTIFICATION_ADD,
+  GOOGLE_DRIVE_NOTIFICATION_CHANGE,
+} from "../../constants.mjs";
+
+export default {
+  ...common,
+  key: "google_drive-new-files-instant",
+  name: "New Files (Instant)",
+  description: "Emit new event any time a new file is added in your linked Google Drive",
+  version: "0.0.15",
+  type: "source",
+  dedupe: "unique",
+  props: {
+    ...common.props,
+    folders: {
+      type: "string[]",
+      label: "Folders",
+      description:
+        "(Optional) The folders you want to watch for changes. Leave blank to watch for any new file in the Drive.",
+      optional: true,
+      default: [],
+      options({ prevContext }) {
+        const { nextPageToken } = prevContext;
+        const baseOpts = {
+          q: "mimeType = 'application/vnd.google-apps.folder'",
+        };
+        const opts = this.isMyDrive()
+          ? baseOpts
+          : {
+            ...baseOpts,
+            corpora: "drive",
+            driveId: this.getDriveId(),
+            includeItemsFromAllDrives: true,
+            supportsAllDrives: true,
+          };
+        return this.googleDrive.listFilesOptions(nextPageToken, opts);
+      },
+    },
+  },
+  hooks: {
+    ...common.hooks,
+    async activate() {
+      await common.hooks.activate.bind(this)();
+      this._setLastFileCreatedTime(Date.now());
+    },
+  },
+  methods: {
+    ...common.methods,
+    _getLastFileCreatedTime() {
+      return this.db.get("lastFileCreatedTime");
+    },
+    _setLastFileCreatedTime(lastFileCreatedTime) {
+      this.db.set("lastFileCreatedTime", lastFileCreatedTime);
+    },
+    shouldProcess(file) {
+      const watchedFolders = new Set(this.folders);
+      return (
+        watchedFolders.size == 0 ||
+        (file.parents && file.parents.some((p) => watchedFolders.has(p)))
+      );
+    },
+    getUpdateTypes() {
+      return [
+        GOOGLE_DRIVE_NOTIFICATION_ADD,
+        GOOGLE_DRIVE_NOTIFICATION_CHANGE,
+      ];
+    },
+    async processChanges(changedFiles) {
+      const lastFileCreatedTime = this._getLastFileCreatedTime();
+      let maxCreatedTime = lastFileCreatedTime;
+
+      for (const file of changedFiles) {
+        const fileInfo = await this.googleDrive.getFile(file.id);
+        const createdTime = Date.parse(fileInfo.createdTime);
+        if (
+          !this.shouldProcess(fileInfo) ||
+          createdTime < lastFileCreatedTime
+        ) {
+          continue;
+        }
+
+        this.$emit(fileInfo, {
+          summary: `New File: ${fileInfo.name}`,
+          id: file.id,
+          ts: createdTime,
+        });
+
+        maxCreatedTime = Math.max(createdTime, maxCreatedTime);
+        this._setLastFileCreatedTime(maxCreatedTime);
+      }
+    },
+  },
+};

package/sources/new-or-modified-comments/new-or-modified-comments.mjs

@@ -0,0 +1,104 @@
+// This source processes changes to any files in a user's Google Drive,
+// implementing strategy enumerated in the Push Notifications API docs:
+// https://developers.google.com/drive/api/v3/push and here:
+// https://developers.google.com/drive/api/v3/manage-changes
+//
+// This source has two interfaces:
+//
+// 1) The HTTP requests tied to changes in the user's Google Drive
+// 2) A timer that runs on regular intervals, renewing the notification channel as needed
+
+import common from "../common-webhook.mjs";
+import { GOOGLE_DRIVE_NOTIFICATION_CHANGE } from "../../constants.mjs";
+
+export default {
+  ...common,
+  key: "google_drive-new-or-modified-comments",
+  name: "New or Modified Comments",
+  description:
+    "Emits a new event any time a file comment is added, modified, or deleted in your linked Google Drive",
+  version: "0.0.9",
+  type: "source",
+  // Dedupe events based on the "x-goog-message-number" header for the target channel:
+  // https://developers.google.com/drive/api/v3/push#making-watch-requests
+  dedupe: "unique",
+  hooks: {
+    ...common.hooks,
+    async activate() {
+      await common.hooks.activate.bind(this)();
+      this._setInitTime(Date.now());
+    },
+    async deactivate() {
+      await common.hooks.deactivate.bind(this)();
+      this._setInitTime(null);
+    },
+  },
+  methods: {
+    ...common.methods,
+    _getInitTime() {
+      return this.db.get("initTime");
+    },
+    _setInitTime(initTime) {
+      this.db.set("initTime", initTime);
+    },
+    _getLastCommentTimeForFile(fileId) {
+      return this.db.get(fileId) || this._getInitTime();
+    },
+    _updateLastCommentTimeForFile(fileId, commentTime) {
+      this.db.set(fileId, commentTime);
+    },
+    getUpdateTypes() {
+      return [
+        GOOGLE_DRIVE_NOTIFICATION_CHANGE,
+      ];
+    },
+    generateMeta(data, headers) {
+      const {
+        id: commentId,
+        content: summary,
+        modifiedTime: tsString,
+      } = data;
+      const { "x-goog-message-number": eventId } = headers;
+      return {
+        id: `${commentId}-${eventId}`,
+        summary,
+        ts: Date.parse(tsString),
+      };
+    },
+    async processChanges(changedFiles, headers) {
+      for (const file of changedFiles) {
+        const lastCommentTimeForFile = this._getLastCommentTimeForFile(file.id);
+        let maxModifiedTime = lastCommentTimeForFile;
+        const commentsStream = this.googleDrive.listComments(
+          file.id,
+          lastCommentTimeForFile,
+        );
+
+        for await (const comment of commentsStream) {
+          const commentTime = Date.parse(comment.modifiedTime);
+          if (commentTime <= lastCommentTimeForFile) {
+            continue;
+          }
+
+          const eventToEmit = {
+            comment,
+            file,
+            change: {
+              state: headers["x-goog-resource-state"],
+              resourceURI: headers["x-goog-resource-uri"],
+
+              // Additional details about the changes. Possible values: content,
+              // parents, children, permissions.
+              changed: headers["x-goog-changed"],
+            },
+          };
+          const meta = this.generateMeta(comment, headers);
+          this.$emit(eventToEmit, meta);
+
+          maxModifiedTime = Math.max(maxModifiedTime, commentTime);
+          this._updateLastCommentTimeForFile(file.id, maxModifiedTime);
+        }
+      }
+    },
+  },
+};

package/sources/new-or-modified-files/new-or-modified-files.mjs

@@ -0,0 +1,66 @@
+// This source processes changes to any files in a user's Google Drive,
+// implementing strategy enumerated in the Push Notifications API docs:
+// https://developers.google.com/drive/api/v3/push and here:
+// https://developers.google.com/drive/api/v3/manage-changes
+//
+// This source has two interfaces:
+//
+// 1) The HTTP requests tied to changes in the user's Google Drive
+// 2) A timer that runs on regular intervals, renewing the notification channel as needed
+
+import common from "../common-webhook.mjs";
+import {
+  GOOGLE_DRIVE_NOTIFICATION_ADD,
+  GOOGLE_DRIVE_NOTIFICATION_CHANGE,
+  GOOGLE_DRIVE_NOTIFICATION_UPDATE,
+} from "../../constants.mjs";
+
+export default {
+  ...common,
+  key: "google_drive-new-or-modified-files",
+  name: "New or Modified Files",
+  description:
+    "Emits a new event any time any file in your linked Google Drive is added, modified, or deleted",
+  version: "0.0.20",
+  type: "source",
+  // Dedupe events based on the "x-goog-message-number" header for the target channel:
+  // https://developers.google.com/drive/api/v3/push#making-watch-requests
+  dedupe: "unique",
+  methods: {
+    ...common.methods,
+    getUpdateTypes() {
+      return [
+        GOOGLE_DRIVE_NOTIFICATION_ADD,
+        GOOGLE_DRIVE_NOTIFICATION_CHANGE,
+        GOOGLE_DRIVE_NOTIFICATION_UPDATE,
+      ];
+    },
+    generateMeta(data, headers) {
+      const {
+        id: fileId,
+        name: summary,
+        modifiedTime: tsString,
+      } = data;
+      const { "x-goog-message-number": eventId } = headers;
+      return {
+        id: `${fileId}-${eventId}`,
+        summary,
+        ts: Date.parse(tsString),
+      };
+    },
+    async processChanges(changedFiles, headers) {
+      for (const file of changedFiles) {
+        const eventToEmit = {
+          file,
+          change: {
+            state: headers["x-goog-resource-state"],
+            resourceURI: headers["x-goog-resource-uri"],
+            changed: headers["x-goog-changed"], // "Additional details about the changes. Possible values: content, parents, children, permissions"
+          },
+        };
+        const meta = this.generateMeta(file, headers);
+        this.$emit(eventToEmit, meta);
+      }
+    },
+  },
+};

package/sources/new-or-modified-folders/new-or-modified-folders.mjs

@@ -0,0 +1,86 @@
+// This source processes changes to any files in a user's Google Drive,
+// implementing strategy enumerated in the Push Notifications API docs:
+// https://developers.google.com/drive/api/v3/push and here:
+// https://developers.google.com/drive/api/v3/manage-changes
+//
+// This source has two interfaces:
+//
+// 1) The HTTP requests tied to changes in the user's Google Drive
+// 2) A timer that runs on regular intervals, renewing the notification channel as needed
+
+import common from "../common-webhook.mjs";
+import {
+  GOOGLE_DRIVE_NOTIFICATION_ADD,
+  GOOGLE_DRIVE_NOTIFICATION_CHANGE,
+  GOOGLE_DRIVE_NOTIFICATION_UPDATE,
+} from "../../constants.mjs";
+
+export default {
+  ...common,
+  key: "google_drive-new-or-modified-folders",
+  name: "New or Modified Folders",
+  description:
+    "Emits a new event any time any folder in your linked Google Drive is added, modified, or deleted",
+  version: "0.0.9",
+  type: "source",
+  // Dedupe events based on the "x-goog-message-number" header for the target channel:
+  // https://developers.google.com/drive/api/v3/push#making-watch-requests
+  dedupe: "unique",
+  methods: {
+    ...common.methods,
+    _getLastModifiedTimeForFile(fileId) {
+      return this.db.get(fileId);
+    },
+    _setModifiedTimeForFile(fileId, modifiedTime) {
+      this.db.set(fileId, modifiedTime);
+    },
+    getUpdateTypes() {
+      return [
+        GOOGLE_DRIVE_NOTIFICATION_ADD,
+        GOOGLE_DRIVE_NOTIFICATION_CHANGE,
+        GOOGLE_DRIVE_NOTIFICATION_UPDATE,
+      ];
+    },
+    generateMeta(data, ts) {
+      const {
+        id: fileId,
+        name: summary,
+      } = data;
+      return {
+        id: `${fileId}-${ts}`,
+        summary,
+        ts,
+      };
+    },
+    async processChanges(changedFiles, headers) {
+      const files = changedFiles.filter(
+        // API docs that define Google Drive folders:
+        // https://developers.google.com/drive/api/v3/folder
+        (file) => file.mimeType === "application/vnd.google-apps.folder",
+      );
+
+      for (const file of files) {
+        // The changelog is updated each time a folder is opened. Check the
+        // folder's `modifiedTime` to see if the folder has been modified.
+        const fileInfo = await this.googleDrive.getFile(file.id);
+
+        const lastModifiedTimeForFile = this._getLastModifiedTimeForFile(file.id);
+        const modifiedTime = Date.parse(fileInfo.modifiedTime);
+        if (lastModifiedTimeForFile == modifiedTime) continue;
+
+        const eventToEmit = {
+          file,
+          change: {
+            state: headers["x-goog-resource-state"],
+            resourceURI: headers["x-goog-resource-uri"],
+            changed: headers["x-goog-changed"], // "Additional details about the changes. Possible values: content, parents, children, permissions"
+          },
+        };
+        const meta = this.generateMeta(file, modifiedTime);
+        this.$emit(eventToEmit, meta);
+
+        this._setModifiedTimeForFile(file.id, modifiedTime);
+      }
+    },
+  },
+};