@pipedream/google_drive 0.3.2 → 0.4.1

package/package.json

@@ -1,21 +1,24 @@
 {
-    "name": "@pipedream/google_drive",
-    "version": "0.3.2",
-    "description": "Pipedream Google_drive Components",
-    "main": "google_drive.app.js",
-    "keywords": [
-        "pipedream",
-        "google_drive"
-    ],
-    "homepage": "https://pipedream.com/apps/google_drive",
-    "author": "Pipedream <support@pipedream.com> (https://pipedream.com/)",
-    "license": "MIT",
-    "dependencies": {
-        "axios": "^0.21.1",
-        "googleapis": "^67.0.0"
-    },
-    "gitHead": "bf0e1cb3ad6da248c6125f9b512c59bfe35bf1fa",
-    "publishConfig": {
-        "access": "public"
-    }
-}
+  "name": "@pipedream/google_drive",
+  "version": "0.4.1",
+  "description": "Pipedream Google_drive Components",
+  "main": "google_drive.app.mjs",
+  "keywords": [
+    "pipedream",
+    "google_drive"
+  ],
+  "homepage": "https://pipedream.com/apps/google_drive",
+  "author": "Pipedream <support@pipedream.com> (https://pipedream.com/)",
+  "license": "MIT",
+  "dependencies": {
+    "@googleapis/drive": "^2.3.0",
+    "@pipedream/platform": "^0.9.0",
+    "axios": "^0.21.1",
+    "mime-db": "^1.51.0",
+    "uuid": "^8.3.2"
+  },
+  "gitHead": "e12480b94cc03bed4808ebc6b13e7fdb3a1ba535",
+  "publishConfig": {
+    "access": "public"
+  }
+}

package/sources/changes-to-specific-files/changes-to-specific-files.mjs

@@ -0,0 +1,226 @@
+import cronParser from "cron-parser";
+import includes from "lodash/includes.js";
+import { v4 as uuid } from "uuid";
+
+import { MY_DRIVE_VALUE } from "../../constants.mjs";
+
+import changesToSpecificFiles from "../changes-to-specific-files-shared-drive/changes-to-specific-files-shared-drive.mjs";
+
+/**
+ * This source uses the Google Drive API's
+ * {@link https://developers.google.com/drive/api/v3/reference/files/watch files: watch}
+ * endpoint to subscribe to changes to specific files in the user's drive.
+ */
+export default {
+  ...changesToSpecificFiles,
+  key: "google_drive-changes-to-specific-files",
+  name: "Changes to Specific Files",
+  description: "Watches for changes to specific files, emitting an event any time a change is made to one of those files. To watch for changes to [shared drive](https://support.google.com/a/users/answer/9310351) files, use the **Changes to Specific Files (Shared Drive)** source instead.",
+  version: "0.0.19",
+  type: "source",
+  // Dedupe events based on the "x-goog-message-number" header for the target channel:
+  // https://developers.google.com/drive/api/v3/push#making-watch-requests
+  dedupe: "unique",
+  props: {
+    ...changesToSpecificFiles.props,
+    drive: {
+      type: "string",
+      label: "Drive",
+      description: "Defaults to `My Drive`. To use a [Shared Drive](https://support.google.com/a/users/answer/9310351), use the **Changes to Specific Files (Shared Drive)** source instead.",
+      optional: true,
+      default: MY_DRIVE_VALUE,
+    },
+    updateTypes: {
+      propDefinition: [
+        changesToSpecificFiles.props.googleDrive,
+        "updateTypes",
+      ],
+    },
+  },
+  hooks: {
+    ...changesToSpecificFiles.hooks,
+    async activate() {
+      // Called when a component is created or updated. Handles all the logic
+      // for starting and stopping watch notifications tied to the desired
+      // files.
+
+      // You can pass the same channel ID in watch requests for multiple files, so
+      // our channel ID is fixed for this component to simplify the state we have to
+      // keep track of.
+      const channelID = this._getChannelID() || uuid();
+
+      // Subscriptions are keyed on Google's resourceID, "an opaque value that
+      // identifies the watched resource". This value is included in request
+      // headers, allowing us to look up the watched resource.
+      let subscriptions = this._getSubscriptions() || {};
+
+      const files = this.files;
+      for (const fileID of files) {
+        const {
+          expiration,
+          resourceId,
+        } = await this.googleDrive.activateFileHook(
+          channelID,
+          this.http.endpoint,
+          fileID,
+        );
+        // The fileID must be kept with the subscription metadata so we can
+        // renew the watch request for this specific file when it expires.
+        subscriptions[resourceId] = {
+          expiration,
+          fileID,
+        };
+      }
+
+      // Save metadata on the subscription so we can stop / renew later
+      this._setSubscriptions(subscriptions);
+      this._setChannelID(channelID);
+    },
+    async deactivate() {
+      const channelID = this._getChannelID();
+      if (!channelID) {
+        console.log(
+          "Channel not found, cannot stop notifications for non-existent channel",
+        );
+        return;
+      }
+
+      const subscriptions = this._getSubscriptions() || {};
+      for (const resourceId of Object.keys(subscriptions)) {
+        await this.googleDrive.stopNotifications(channelID, resourceId);
+      }
+
+      // Reset DB state
+      this._setSubscriptions({});
+      this._setChannelID(null);
+    },
+  },
+  methods: {
+    ...changesToSpecificFiles.methods,
+    _getSubscriptions() {
+      return this.db.get("subscriptions") || {};
+    },
+    _setSubscriptions(subscriptions) {
+      this.db.set("subscriptions", subscriptions);
+    },
+    _getNextTimerEventTimestamp(event) {
+      if (event.cron) {
+        return cronParser
+          .parseExpression(event.cron)
+          .next()
+          .getTime();
+      }
+      if (event.interval_seconds) {
+        return Date.now() + event.interval_seconds * 1000;
+      }
+    },
+    async renewFileSubscriptions(event) {
+      // Assume subscription & channelID may all be undefined at
+      // this point Handle their absence appropriately.
+      const subscriptions = this._getSubscriptions() || {};
+      const channelID = this._getChannelID() || uuid();
+
+      const nextRunTimestamp = this._getNextTimerEventTimestamp(event);
+
+      for (const [
+        currentResourceId,
+        metadata,
+      ] of Object.entries(subscriptions)) {
+        const { fileID } = metadata;
+
+        const subscription = {
+          ...metadata,
+          resourceId: currentResourceId,
+        };
+        const {
+          expiration,
+          resourceId,
+        } = await this.googleDrive.renewFileSubscription(
+          subscription,
+          this.http.endpoint,
+          channelID,
+          fileID,
+          nextRunTimestamp,
+        );
+        subscriptions[resourceId] = {
+          expiration,
+          fileID,
+        };
+      }
+      this._setSubscriptions(subscriptions);
+      this._setChannelID(channelID);
+    },
+  },
+  async run(event) {
+    // This function is polymorphic: it can be triggered as a cron job, to make sure we renew
+    // watch requests for specific files, or via HTTP request (the change payloads from Google)
+
+    // Component was invoked by timer
+    if (event.timestamp) {
+      return this.renewFileSubscriptions(event);
+    }
+
+    const channelID = this._getChannelID();
+    let subscriptions = this._getSubscriptions() || {};
+
+    const { headers } = event;
+
+    if (
+      !headers["x-goog-resource-state"] ||
+      !headers["x-goog-resource-id"] ||
+      !headers["x-goog-resource-uri"] ||
+      !headers["x-goog-message-number"]
+    ) {
+      console.log("Request missing necessary headers: ", headers);
+      return;
+    }
+
+    const incomingChannelID = headers["x-goog-channel-id"];
+    if (incomingChannelID !== channelID) {
+      console.log(
+        `Channel ID of ${incomingChannelID} not equal to deployed component channel of ${channelID}`,
+      );
+      return;
+    }
+
+    if (subscriptions[headers["x-goog-resource-id"]] === undefined) {
+      console.log(
+        `Resource ID of ${headers["x-goog-resource-id"]} not currently being tracked. Exiting`,
+      );
+      return;
+    }
+
+    if (!includes(this.updateTypes, headers["x-goog-resource-state"])) {
+      console.log(
+        `Update type ${headers["x-goog-resource-state"]} not in list of updates to watch: `,
+        this.updateTypes,
+      );
+      return;
+    }
+
+    // We observed false positives where a single change to a document would trigger two changes:
+    // one to "properties" and another to "content,properties". But changes to properties
+    // alone are legitimate, most users just won't want this source to emit in those cases.
+    // If x-goog-changed is _only_ set to "properties", only move on if the user set the prop
+    if (
+      !this.watchForPropertiesChanges &&
+      headers["x-goog-changed"] === "properties"
+    ) {
+      console.log(
+        "Change to properties only, which this component is set to ignore. Exiting",
+      );
+      return;
+    }
+
+    const file = await this.googleDrive.getFileMetadata(
+      headers["x-goog-resource-uri"],
+    );
+
+    if (!file || !Object.keys(file).length) {
+      console.log("No file metadata returned, nothing to emit");
+      return;
+    }
+
+    this.processChange(file, headers);
+  },
+};

package/sources/changes-to-specific-files-shared-drive/changes-to-specific-files-shared-drive.mjs

@@ -0,0 +1,110 @@
+// This source processes changes to specific files in a user's Google Drive,
+// implementing strategy enumerated in the Push Notifications API docs:
+// https://developers.google.com/drive/api/v3/push .
+//
+// This source has two interfaces:
+//
+// 1) The HTTP requests tied to changes in files in the user's Google Drive
+// 2) A timer that runs on regular intervals, renewing the notification channel as needed
+
+import common from "../common-webhook.mjs";
+
+import {
+  GOOGLE_DRIVE_NOTIFICATION_CHANGE,
+  GOOGLE_DRIVE_NOTIFICATION_ADD,
+  GOOGLE_DRIVE_NOTIFICATION_UPDATE,
+} from "../../constants.mjs";
+
+/**
+ * This source uses the Google Drive API's
+ * {@link https://developers.google.com/drive/api/v3/reference/changes/watch changes: watch}
+ * endpoint to subscribe to changes to the user's drive or a shard drive.
+ */
+export default {
+  ...common,
+  key: "google_drive-changes-to-specific-files-shared-drive",
+  name: "Changes to Specific Files (Shared Drive)",
+  description: "Watches for changes to specific files in a shared drive, emitting an event any time a change is made to one of those files",
+  version: "0.0.1",
+  type: "source",
+  // Dedupe events based on the "x-goog-message-number" header for the target channel:
+  // https://developers.google.com/drive/api/v3/push#making-watch-requests
+  dedupe: "unique",
+  props: {
+    ...common.props,
+    files: {
+      type: "string[]",
+      label: "Files",
+      description: "The files you want to watch for changes.",
+      optional: true,
+      default: [],
+      options({ prevContext }) {
+        const { nextPageToken } = prevContext;
+        const baseOpts = {};
+        const opts = this.isMyDrive()
+          ? baseOpts
+          : {
+            ...baseOpts,
+            corpora: "drive",
+            driveId: this.getDriveId(),
+            includeItemsFromAllDrives: true,
+            supportsAllDrives: true,
+          };
+        return this.googleDrive.listFilesOptions(nextPageToken, opts);
+      },
+    },
+  },
+  methods: {
+    ...common.methods,
+    getUpdateTypes() {
+      return [
+        GOOGLE_DRIVE_NOTIFICATION_ADD,
+        GOOGLE_DRIVE_NOTIFICATION_CHANGE,
+        GOOGLE_DRIVE_NOTIFICATION_UPDATE,
+      ];
+    },
+    generateMeta(data, headers) {
+      const {
+        id: fileId,
+        name: fileName,
+        modifiedTime: tsString,
+      } = data;
+      const {
+        "x-goog-message-number": eventId,
+        "x-goog-resource-state": resourceState,
+      } = headers;
+
+      return {
+        id: `${fileId}-${eventId}`,
+        summary: `${resourceState.toUpperCase()} - ${
+          fileName || "Untitled"
+        }`,
+        ts: Date.parse(tsString),
+      };
+    },
+    isFileRelevant(file) {
+      return this.files.includes(file.id);
+    },
+    async processChange(file, headers) {
+      const eventToEmit = {
+        file,
+        change: {
+          state: headers["x-goog-resource-state"],
+          resourceURI: headers["x-goog-resource-uri"],
+          changed: headers["x-goog-changed"], // "Additional details about the changes. Possible values: content, parents, children, permissions"
+        },
+      };
+      const meta = this.generateMeta(file, headers);
+      this.$emit(eventToEmit, meta);
+    },
+    async processChanges(changedFiles, headers) {
+      for (const file of changedFiles) {
+        if (!this.isFileRelevant(file)) {
+          console.log(`Skipping event for irrelevant file ${file.id}`);
+          continue;
+        }
+        this.processChange(file, headers);
+      }
+    },
+  },
+};

package/sources/common-webhook.mjs

@@ -0,0 +1,201 @@
+import includes from "lodash/includes.js";
+import { v4 as uuid } from "uuid";
+
+import googleDrive from "../google_drive.app.mjs";
+import { WEBHOOK_SUBSCRIPTION_RENEWAL_SECONDS } from "../constants.mjs";
+
+export default {
+  props: {
+    googleDrive,
+    db: "$.service.db",
+    http: "$.interface.http",
+    drive: {
+      propDefinition: [
+        googleDrive,
+        "watchedDrive",
+      ],
+      description: "Defaults to My Drive. To select a [Shared Drive](https://support.google.com/a/users/answer/9310351) instead, select it from this list.",
+      optional: false,
+    },
+    watchForPropertiesChanges: {
+      propDefinition: [
+        googleDrive,
+        "watchForPropertiesChanges",
+      ],
+    },
+    timer: {
+      label: "Push notification renewal schedule",
+      description:
+        "The Google Drive API requires occasional renewal of push notification subscriptions. **This runs in the background, so you should not need to modify this schedule**.",
+      type: "$.interface.timer",
+      static: {
+        intervalSeconds: WEBHOOK_SUBSCRIPTION_RENEWAL_SECONDS,
+      },
+    },
+  },
+  hooks: {
+    async activate() {
+      // Called when a component is created or updated. Handles all the logic
+      // for starting and stopping watch notifications tied to the desired
+      // files.
+      const channelID = uuid();
+      const {
+        startPageToken,
+        expiration,
+        resourceId,
+      } = await this.googleDrive.activateHook(
+        channelID,
+        this.http.endpoint,
+        this.getDriveId(),
+      );
+
+      // We use and increment the pageToken as new changes arrive, in run()
+      this._setPageToken(startPageToken);
+
+      // Save metadata on the subscription so we can stop / renew later
+      // Subscriptions are tied to Google's resourceID, "an opaque value that
+      // identifies the watched resource". This value is included in request headers
+      this._setSubscription({
+        resourceId,
+        expiration,
+      });
+      this._setChannelID(channelID);
+    },
+    async deactivate() {
+      const channelID = this._getChannelID();
+      const { resourceId } = this._getSubscription();
+      await this.googleDrive.deactivateHook(channelID, resourceId);
+
+      this._setSubscription(null);
+      this._setChannelID(null);
+      this._setPageToken(null);
+    },
+  },
+  methods: {
+    _getSubscription() {
+      return this.db.get("subscription");
+    },
+    _setSubscription(subscription) {
+      this.db.set("subscription", subscription);
+    },
+    _getChannelID() {
+      return this.db.get("channelID");
+    },
+    _setChannelID(channelID) {
+      this.db.set("channelID", channelID);
+    },
+    _getPageToken() {
+      return this.db.get("pageToken");
+    },
+    _setPageToken(pageToken) {
+      this.db.set("pageToken", pageToken);
+    },
+    isMyDrive(drive = this.drive) {
+      return googleDrive.methods.isMyDrive(drive);
+    },
+    getDriveId(drive = this.drive) {
+      return googleDrive.methods.getDriveId(drive);
+    },
+    /**
+     * This method returns the types of updates/events from Google Drive that
+     * the event source should listen to. This base implementation returns an
+     * empty list, which means that any event source that extends this module
+     * and that does not refine this implementation will essentially ignore
+     * every incoming event from Google Drive.
+     *
+     * @returns
+     * @type {UpdateType[]}
+     */
+    getUpdateTypes() {
+      return [];
+    },
+    /**
+     * This method is responsible for processing a list of changed files
+     * according to the event source's purpose. As an abstract method, it must
+     * be implemented by every event source that extends this module.
+     *
+     * @param {object[]} [changedFiles] - the list of file changes, as [defined
+     * by the API](https://bit.ly/3h7WeUa)
+     * @param {object} [headers] - an object containing the request headers of
+     * the webhook call made by Google Drive
+     */
+    processChanges() {
+      throw new Error("processChanges is not implemented");
+    },
+  },
+  async run(event) {
+    // This function is polymorphic: it can be triggered as a cron job, to make
+    // sure we renew watch requests for specific files, or via HTTP request (the
+    // change payloads from Google)
+    const subscription = this._getSubscription();
+    const channelID = this._getChannelID();
+    const pageToken = this._getPageToken();
+
+    // Component was invoked by timer
+    if (event.timestamp) {
+      const {
+        newChannelID,
+        newPageToken,
+        expiration,
+        resourceId,
+      } = await this.googleDrive.renewSubscription(
+        this.drive,
+        subscription,
+        this.http.endpoint,
+        channelID,
+        pageToken,
+      );
+
+      this._setSubscription({
+        expiration,
+        resourceId,
+      });
+      this._setChannelID(newChannelID);
+      this._setPageToken(newPageToken);
+      return;
+    }
+
+    const { headers } = event;
+    if (!this.googleDrive.checkHeaders(headers, subscription, channelID)) {
+      return;
+    }
+
+    if (!includes(this.getUpdateTypes(), headers["x-goog-resource-state"])) {
+      console.log(
+        `Update type ${headers["x-goog-resource-state"]} not in list of updates to watch: `,
+        this.getUpdateTypes(),
+      );
+      return;
+    }
+
+    // We observed false positives where a single change to a document would trigger two changes:
+    // one to "properties" and another to "content,properties". But changes to properties
+    // alone are legitimate, most users just won't want this source to emit in those cases.
+    // If x-goog-changed is _only_ set to "properties", only move on if the user set the prop
+    if (
+      !this.watchForPropertiesChanges &&
+      headers["x-goog-changed"] === "properties"
+    ) {
+      console.log(
+        "Change to properties only, which this component is set to ignore. Exiting",
+      );
+      return;
+    }
+
+    const driveId = this.getDriveId();
+    const changedFilesStream = this.googleDrive.listChanges(pageToken, driveId);
+    for await (const changedFilesPage of changedFilesStream) {
+      const {
+        changedFiles,
+        nextPageToken,
+      } = changedFilesPage;
+
+      // Process all the changed files retrieved from the current page
+      await this.processChanges(changedFiles, headers);
+
+      // After successfully processing the changed files, we store the page
+      // token of the next page
+      this._setPageToken(nextPageToken);
+    }
+  },
+};

package/sources/new-files-instant/new-files-instant.mjs

@@ -0,0 +1,95 @@
+import common from "../common-webhook.mjs";
+import {
+  GOOGLE_DRIVE_NOTIFICATION_ADD,
+  GOOGLE_DRIVE_NOTIFICATION_CHANGE,
+} from "../../constants.mjs";
+
+export default {
+  ...common,
+  key: "google_drive-new-files-instant",
+  name: "New Files (Instant)",
+  description: "Emit new event any time a new file is added in your linked Google Drive",
+  version: "0.0.15",
+  type: "source",
+  dedupe: "unique",
+  props: {
+    ...common.props,
+    folders: {
+      type: "string[]",
+      label: "Folders",
+      description:
+        "(Optional) The folders you want to watch for changes. Leave blank to watch for any new file in the Drive.",
+      optional: true,
+      default: [],
+      options({ prevContext }) {
+        const { nextPageToken } = prevContext;
+        const baseOpts = {
+          q: "mimeType = 'application/vnd.google-apps.folder'",
+        };
+        const opts = this.isMyDrive()
+          ? baseOpts
+          : {
+            ...baseOpts,
+            corpora: "drive",
+            driveId: this.getDriveId(),
+            includeItemsFromAllDrives: true,
+            supportsAllDrives: true,
+          };
+        return this.googleDrive.listFilesOptions(nextPageToken, opts);
+      },
+    },
+  },
+  hooks: {
+    ...common.hooks,
+    async activate() {
+      await common.hooks.activate.bind(this)();
+      this._setLastFileCreatedTime(Date.now());
+    },
+  },
+  methods: {
+    ...common.methods,
+    _getLastFileCreatedTime() {
+      return this.db.get("lastFileCreatedTime");
+    },
+    _setLastFileCreatedTime(lastFileCreatedTime) {
+      this.db.set("lastFileCreatedTime", lastFileCreatedTime);
+    },
+    shouldProcess(file) {
+      const watchedFolders = new Set(this.folders);
+      return (
+        watchedFolders.size == 0 ||
+        (file.parents && file.parents.some((p) => watchedFolders.has(p)))
+      );
+    },
+    getUpdateTypes() {
+      return [
+        GOOGLE_DRIVE_NOTIFICATION_ADD,
+        GOOGLE_DRIVE_NOTIFICATION_CHANGE,
+      ];
+    },
+    async processChanges(changedFiles) {
+      const lastFileCreatedTime = this._getLastFileCreatedTime();
+      let maxCreatedTime = lastFileCreatedTime;
+
+      for (const file of changedFiles) {
+        const fileInfo = await this.googleDrive.getFile(file.id);
+        const createdTime = Date.parse(fileInfo.createdTime);
+        if (
+          !this.shouldProcess(fileInfo) ||
+          createdTime < lastFileCreatedTime
+        ) {
+          continue;
+        }
+
+        this.$emit(fileInfo, {
+          summary: `New File: ${fileInfo.name}`,
+          id: file.id,
+          ts: createdTime,
+        });
+
+        maxCreatedTime = Math.max(createdTime, maxCreatedTime);
+        this._setLastFileCreatedTime(maxCreatedTime);
+      }
+    },
+  },
+};