@pipedream/dropbox 0.3.3 → 0.3.5

package/dropbox.app.mjs

@@ -0,0 +1,484 @@
+import dropbox from "dropbox";
+import fetch from "isomorphic-fetch";
+import get from "lodash/get.js";
+import config from "./config.mjs";
+import isString from "lodash/isString.js";
+import isEmpty from "lodash/isEmpty.js";
+
+const Dropbox = dropbox.Dropbox;
+
+export default {
+  type: "app",
+  app: "dropbox",
+  propDefinitions: {
+    pathFolder: {
+      type: "string",
+      label: "Path",
+      description: "The folder path. (Please use a valid path to filter the values)",
+      useQuery: true,
+      withLabel: true,
+      async options({
+        prevContext,
+        query,
+        returnSimpleString,
+      }) {
+        if (prevContext?.reachedLastPage) {
+          return [];
+        }
+        return this.getPathOptions(
+          query,
+          prevContext?.cursor,
+          {
+            omitFiles: true,
+            returnSimpleString,
+          },
+        );
+      },
+    },
+    pathFile: {
+      type: "string",
+      label: "Path",
+      description: "The file path. (Please use a valid path to filter the values)",
+      useQuery: true,
+      withLabel: true,
+      async options({
+        prevContext,
+        query,
+        returnSimpleString,
+      }) {
+        if (prevContext?.reachedLastPage) {
+          return [];
+        }
+        return this.getPathOptions(
+          query,
+          prevContext?.cursor,
+          {
+            omitFolders: true,
+            returnSimpleString,
+          },
+        );
+      },
+    },
+    pathFileFolder: {
+      type: "string",
+      label: "Path",
+      description: "The file or folder path. (Please use a valid path to filter the values)",
+      useQuery: true,
+      withLabel: true,
+      async options({
+        prevContext,
+        query,
+        returnSimpleString,
+      }) {
+        if (prevContext?.reachedLastPage) {
+          return [];
+        }
+        return this.getPathOptions(
+          query,
+          prevContext?.cursor,
+          {
+            returnSimpleString,
+          },
+        );
+      },
+    },
+    fileRevision: {
+      type: "string",
+      label: "Revision",
+      description: "The file revision",
+      withLabel: true,
+      async options({ path }) {
+        return this.getFileRevisionOptions(get(path, "value", path));
+      },
+    },
+    recursive: {
+      type: "boolean",
+      label: "Recursive",
+      description: "Also watch sub-directories and their contents.",
+      default: false,
+    },
+    query: {
+      type: "string",
+      label: "Query",
+      description: "The string to search for. May match across multiple fields based on the request arguments.",
+    },
+    limit: {
+      type: "integer",
+      label: "Limit",
+      description: "Specify a max amount of register to be fetched. Defaults to `100` if left blank.",
+      optional: true,
+      min: 1,
+    },
+  },
+  methods: {
+    async sdk() {
+      const baseClientOpts = {
+        accessToken: this.$auth.oauth_access_token,
+        fetch,
+      };
+
+      // In order to properly set the [root
+      // path](https://www.dropbox.com/developers/reference/path-root-header-modes)
+      // to use in every API request we first need to extract some information
+      // from the authenticated user's account, for which we need to create a
+      // client and issue an API request.
+      const dpx = new Dropbox(baseClientOpts);
+      const { result } = await dpx.usersGetCurrentAccount();
+
+      const pathRoot = JSON.stringify({
+        ".tag": "root",
+        "root": result.root_info.root_namespace_id,
+      });
+      return new Dropbox({
+        ...baseClientOpts,
+        pathRoot,
+      });
+    },
+    normalizeError(err) {
+      if (!err) {
+        throw new Error("Unknown error");
+      }
+
+      if (isString(err.error)) {
+        throw new Error(err.error);
+      }
+
+      if (err.error?.error_summary) {
+        throw new Error(err.error.error_summary);
+      }
+
+      throw new Error(JSON.stringify(err));
+    },
+    async getFileRevisionOptions(path) {
+      try {
+        const dpx = await this.sdk();
+        const revisions = await dpx.filesListRevisions({
+          path,
+          mode: {
+            ".tag": "id",
+          },
+        });
+        return revisions.result?.entries?.map((revision) => ({
+          label: `${revision.name} - ${revision.server_modified}`,
+          value: revision.rev,
+        }));
+      } catch (err) {
+        this.normalizeError(err);
+      }
+    },
+    async getPathOptions(path, prevCursor, opts = {}) {
+      try {
+        const {
+          omitFolders,
+          omitFiles,
+        } = opts;
+
+        const LIMIT = config.GET_PATH_OPTIONS.DEFAULT_MAX_RESULTS;
+
+        let data = [];
+        let res = null;
+        let nextCursor = null;
+        let lastPage = false;
+        path = path === "/" || path === null
+          ? ""
+          : path;
+        const dpx = await this.sdk();
+
+        if (prevCursor) {
+          res = await dpx.filesListFolderContinue({
+            cursor: prevCursor,
+          });
+        } else {
+          res = await dpx.filesListFolder({
+            path,
+            limit: LIMIT,
+            recursive: true,
+          });
+        }
+
+        data = res.result.entries.map((item) => ({
+          path: item.path_display,
+          type: item[".tag"],
+        }));
+
+        if (res.result.has_more) {
+          nextCursor = res.result.cursor;
+        } else {
+          lastPage = true;
+        }
+
+        if (omitFiles) {
+          data = data.filter((item) => item.type !== "file");
+        }
+
+        if (omitFolders) {
+          data = data.filter((item) => item.type !== "folder");
+        }
+
+        data = data.map((item) => (item.path));
+
+        data.sort((a, b) => {
+          return a > b ?
+            1 :
+            -1;
+        });
+
+        return {
+          options: data,
+          context: {
+            cursor: nextCursor,
+            reachedLastPage: lastPage,
+          },
+        };
+      } catch (err) {
+        console.log(err);
+        return [];
+      }
+    },
+    async initState(context) {
+      const {
+        path,
+        recursive,
+      } = context;
+      try {
+        let fixedPath = path == "/" ?
+          "" :
+          path;
+        fixedPath = typeof (fixedPath) === "object" ?
+          fixedPath.value :
+          fixedPath;
+
+        const dpx = await this.sdk();
+        let response = await dpx.filesListFolderGetLatestCursor({
+          path: fixedPath,
+          recursive,
+        });
+        if (response.result) {
+          response = response.result;
+        }
+        const { cursor } = response;
+        const state = {
+          path: fixedPath,
+          recursive,
+          cursor,
+        };
+        return state;
+      } catch (err) {
+        console.log(err);
+        throw `Error connecting to Dropbox API to get latest cursor for folder: ${path}${recursive
+          ? " (recursive)"
+          : ""
+        }`;
+      }
+    },
+    async getState(context, state) {
+      const {
+        path,
+        recursive,
+      } = context;
+      const fixedPath = typeof (path) === "object" ?
+        path.value :
+        path;
+      if (state == null || state.path != fixedPath || state.recursive != recursive) {
+        state = await this.initState(context);
+      }
+      return state;
+    },
+    async getUpdates(context, dbState) {
+      let ret = [];
+      const state = await this.getState(context, dbState);
+      if (state) {
+        try {
+          let [
+            cursor,
+            hasMore,
+            entries,
+          ] =
+            [
+              state.cursor,
+              true,
+              null,
+            ];
+          while (hasMore) {
+            const dpx = await this.sdk();
+            let response = await dpx.filesListFolderContinue({
+              cursor,
+            });
+            if (response.result) {
+              response = response.result;
+            }
+            ({
+              entries,
+              cursor,
+              has_more: hasMore,
+            } = response);
+            ret = ret.concat(entries);
+          }
+          state.cursor = cursor;
+        } catch (err) {
+          console.log(err);
+          throw `Error connecting to Dropbox API to get list of updated files/folders for cursor: ${state.cursor}`;
+        }
+      }
+      return {
+        ret,
+        state,
+      };
+    },
+    async createFolder(args) {
+      try {
+        const dpx = await this.sdk();
+        return await dpx.filesCreateFolderV2(args);
+      } catch (err) {
+        this.normalizeError(err);
+      }
+    },
+    async listFileRevisions(args) {
+      try {
+        const dpx = await this.sdk();
+        return await dpx.filesListRevisions(args);
+      } catch (err) {
+        this.normalizeError(err);
+      }
+    },
+    async listFilesFolders(params, limit) {
+      try {
+        const dpx = await this.sdk();
+        let data = [];
+        let cursor = null;
+
+        const args = {
+          ...params,
+          limit: limit <= config.LIST_FILES_IN_FOLDER.DEFAULT_MAX_RESULTS
+            ? limit
+            : config.LIST_FILES_IN_FOLDER.DEFAULT_MAX_RESULTS,
+        };
+
+        let res = await dpx.filesListFolder(args);
+
+        if (!res.result?.has_more || limit <= config.LIST_FILES_IN_FOLDER.DEFAULT_MAX_RESULTS) {
+          return res.result?.entries;
+        }
+
+        data = res.result?.entries;
+        cursor = res.result?.cursor;
+        do {
+          const res = await dpx.filesListFolderContinue({
+            cursor,
+          });
+          data = data.concat(res.result?.entries);
+          cursor = res.result?.cursor;
+          if (!res.result?.has_more || data.length >= limit) {
+            break;
+          }
+        } while (true);
+        return data;
+      } catch (err) {
+        this.normalizeError(err);
+      }
+    },
+    async filesMove(args) {
+      try {
+        const dpx = await this.sdk();
+        return await dpx.filesMoveV2(args);
+      } catch (err) {
+        this.normalizeError(err);
+      }
+    },
+    async restoreFile(args) {
+      try {
+        const dpx = await this.sdk();
+        return await dpx.filesRestore(args);
+      } catch (err) {
+        this.normalizeError(err);
+      }
+    },
+    async createSharedLink(args) {
+      try {
+        const dpx = await this.sdk();
+        const links = await dpx.sharingListSharedLinks({
+          path: args.path,
+        });
+        if (links.result?.links.length > 0) {
+          return await dpx.sharingModifySharedLinkSettings({
+            ...args,
+            path: undefined,
+            url: links.result?.links[0].url,
+            remove_expiration: isEmpty(args.remove_expiration)
+              ? false
+              : args.remove_expiration,
+          });
+        } else {
+          return await dpx.sharingCreateSharedLinkWithSettings({
+            ...args,
+            remove_expiration: undefined,
+          });
+        }
+      } catch (err) {
+        this.normalizeError(err);
+      }
+    },
+    async deleteFileFolder(args) {
+      try {
+        const dpx = await this.sdk();
+        return await dpx.filesDeleteV2(args);
+      } catch (err) {
+        this.normalizeError(err);
+      }
+    },
+    async uploadFile(args) {
+      try {
+        const dpx = await this.sdk();
+        return await dpx.filesUpload(args);
+      } catch (err) {
+        this.normalizeError(err);
+      }
+    },
+    async downloadFile(args) {
+      try {
+        const dpx = await this.sdk();
+        return await dpx.filesDownload(args);
+      } catch (err) {
+        this.normalizeError(err);
+      }
+    },
+    async searchFilesFolders(params, limit) {
+      try {
+        const dpx = await this.sdk();
+        let data = [];
+        let cursor = null;
+
+        const args = {
+          ...params,
+          options: {
+            ...params.options,
+            max_results: limit <= config.SEARCH_FILE_FOLDERS.DEFAULT_MAX_RESULTS
+              ? limit
+              : config.SEARCH_FILE_FOLDERS.DEFAULT_MAX_RESULTS,
+          },
+        };
+        let res = await dpx.filesSearchV2(args);
+
+        if (!res.result?.has_more || limit <= config.SEARCH_FILE_FOLDERS.DEFAULT_MAX_RESULTS) {
+          return res.result?.matches;
+        }
+
+        data = res.result?.matches;
+        cursor = res.result?.cursor;
+        do {
+          const res = await dpx.filesSearchContinueV2({
+            cursor,
+          });
+          data = data.concat(res.result?.matches);
+          cursor = res.result?.cursor;
+          if (!res.result?.has_more || data.length >= limit) {
+            break;
+          }
+        } while (true);
+        return data;
+      } catch (err) {
+        this.normalizeError(err);
+      }
+    },
+  },
+};

package/package.json

@@ -1,21 +1,21 @@
 {
-    "name": "@pipedream/dropbox",
-    "version": "0.3.3",
-    "description": "Pipedream Dropbox Components",
-    "main": "dropbox.app.js",
-    "keywords": [
-        "pipedream",
-        "dropbox"
-    ],
-    "homepage": "https://pipedream.com/apps/dropbox",
-    "author": "Pipedream <support@pipedream.com> (https://pipedream.com/)",
-    "license": "MIT",
-    "dependencies": {
-        "dropbox": "^8.2.0",
-        "isomorphic-fetch": "^3.0.0"
-    },
-    "gitHead": "12ad36100ee95629c1b22923c79923bfff156ca7",
-    "publishConfig": {
-        "access": "public"
-    }
-}
+  "name": "@pipedream/dropbox",
+  "version": "0.3.5",
+  "description": "Pipedream Dropbox Components",
+  "main": "dropbox.app.js",
+  "keywords": [
+    "pipedream",
+    "dropbox"
+  ],
+  "homepage": "https://pipedream.com/apps/dropbox",
+  "author": "Pipedream <support@pipedream.com> (https://pipedream.com/)",
+  "license": "MIT",
+  "dependencies": {
+    "dropbox": "^8.2.0",
+    "isomorphic-fetch": "^3.0.0"
+  },
+  "gitHead": "e12480b94cc03bed4808ebc6b13e7fdb3a1ba535",
+  "publishConfig": {
+    "access": "public"
+  }
+}

package/sources/all-updates/all-updates.mjs

@@ -0,0 +1,59 @@
+import common from "../common.mjs";
+
+export default {
+  ...common,
+  dedupe: "unique",
+  type: "source",
+  key: "dropbox-all-updates",
+  name: "New or Modified File or Folder",
+  version: "0.0.8",
+  description: "Emit new event when a file or folder is added or modified. Make sure the number of files/folders in the watched folder does not exceed 4000.",
+  props: {
+    ...common.props,
+    includeMediaInfo: {
+      label: "Include Media Info",
+      type: "boolean",
+      description: "Emit media info for photo and video files (incurs an additional API call)",
+      default: false,
+    },
+    includeLink: {
+      label: "Include Link",
+      type: "boolean",
+      description: "Emit temporary download link for files (incurs an additional API call)",
+      default: false,
+    },
+  },
+  hooks: {
+    async activate() {
+      await this.getHistoricalEvents([
+        "file",
+        "folder",
+      ]);
+      const state = await this.dropbox.initState(this);
+      this._setDropboxState(state);
+    },
+  },
+  async run() {
+    const state = this._getDropboxState();
+    const {
+      ret: updates, state: newState,
+    } = await this.dropbox.getUpdates(this, state);
+    this._setDropboxState(newState);
+    for (let update of updates) {
+      let file = {
+        ...update,
+      };
+      if (this.includeMediaInfo) {
+        file = await this.getMediaInfo(update);
+      }
+      if (this.includeLink) {
+        file.link = await this.getTemporaryLink(update);
+      }
+      // new unique identification from merging the file id and the last file revision
+      const id = update[".tag"] === "file"
+        ? `${file.id}-${file.rev}`
+        : `${file.id}-${newState.cursor}`;
+      this.$emit(file, this.getMeta(id, file.path_display || file.id));
+    }
+  },
+};

package/sources/common.mjs

@@ -0,0 +1,120 @@
+import dropbox from "../dropbox.app.mjs";
+
+export default {
+  props: {
+    dropbox,
+    db: "$.service.db",
+    path: {
+      propDefinition: [
+        dropbox,
+        "pathFolder",
+      ],
+    },
+    recursive: {
+      propDefinition: [
+        dropbox,
+        "recursive",
+      ],
+    },
+    dropboxApphook: {
+      type: "$.interface.apphook",
+      appProp: "dropbox",
+      static: [],
+    },
+  },
+  methods: {
+    _getDropboxState() {
+      return this.db.get("dropbox_state");
+    },
+    _setDropboxState(state) {
+      this.db.set("dropbox_state", state);
+    },
+    async getHistoricalEvents(fileTypes = []) {
+      const files = await this.dropbox.listFilesFolders({
+        path: this.path?.value || this.path,
+        recursive: this.recursive,
+        include_media_info: this.includeMediaInfo,
+      });
+      let count = 0;
+      for (const file of files) {
+        if (!fileTypes.includes(file[".tag"])) {
+          continue;
+        }
+        this.$emit(file, this.getMeta(file.id, file.path_display || file.id));
+        count++;
+        if (count >= 25) {
+          break;
+        }
+      }
+    },
+    getMeta(id, summary) {
+      return {
+        id,
+        summary,
+        ts: Date.now(),
+      };
+    },
+    async isNewFile(update, lastFileModTime) {
+      try {
+        const dpx = await this.dropbox.sdk();
+        let revisions = await dpx.filesListRevisions({
+          path: update.id,
+          mode: {
+            ".tag": "id",
+          },
+          limit: 10,
+        });
+        if (revisions.result) {
+          revisions = revisions.result;
+        }
+        if (revisions.entries.length > 1) {
+          const oldest = revisions.entries.pop();
+          if (lastFileModTime && lastFileModTime >= oldest.client_modified) {
+            return false;
+          }
+        }
+        return true;
+      } catch (err) {
+        console.log(err);
+        this.dropbox.normalizeError(err);
+      }
+    },
+    async getMediaInfo(update) {
+      try {
+        const dpx = await this.dropbox.sdk();
+        const info = await dpx.filesGetMetadata({
+          path: update.path_lower,
+          include_media_info: true,
+        });
+        if (info.result) {
+          return info.result;
+        }
+        return update;
+      } catch (err) {
+        console.log(err);
+        this.dropbox.normalizeError(err);
+      }
+    },
+    async getTemporaryLink(update) {
+      try {
+        const dpx = await this.dropbox.sdk();
+        let response = await dpx.filesGetTemporaryLink({
+          path: update.path_lower,
+        });
+        if (response.result) {
+          response = response.result;
+        }
+        const { link } = response;
+        return link;
+      } catch (err) {
+        // returning null because not all files supports download links,
+        // but the source should still emit update events to those files
+        if (err?.error?.error_summary?.startsWith("unsupported_file")) {
+          return null;
+        }
+        console.log(err);
+        this.dropbox.normalizeError(err);
+      }
+    },
+  },
+};