@pipedream/twitter 2.3.3 → 2.3.4

package/dist/sources/new-user-followed/new-user-followed.mjs

@@ -1,47 +1,743 @@
+// components/twitter/sources/new-user-followed/new-user-followed.ts
 import { defineSource } from "@pipedream/types";
-import common from "../common/base.mjs";
-import { getUserSummary as getItemSummary } from "../common/getItemSummary.mjs";
-import { getUserFields } from "../../common/methods.mjs";
-import cacheUserId from "../common/cacheUserId.mjs";
-import { getObjIncludes, getUserIncludeIds, } from "../../common/addObjIncludes.mjs";
-const DOCS_LINK = "https://developer.twitter.com/en/docs/twitter-api/users/follows/api-reference/get-users-id-following";
-const MAX_RESULTS_PER_PAGE = 1000;
-export default defineSource({
-    ...common,
-    key: "twitter-new-user-followed",
-    name: "New User Followed by User",
-    description: `Emit new event when the specified User follows another User [See the documentation](${DOCS_LINK})`,
-    version: "2.1.0",
-    type: "source",
-    props: {
-        ...common.props,
-        userNameOrId: {
-            propDefinition: [
-                common.props.app,
-                "userNameOrId",
-            ],
-        },
+
+// components/twitter/sources/common/base.ts
+import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform";
+
+// components/twitter/app/twitter.app.ts
+import OAuth from "oauth-1.0a";
+import crypto from "crypto";
+import { defineApp } from "@pipedream/types";
+import {
+  axios,
+  transformConfigForOauth,
+  ConfigurationError
+} from "@pipedream/platform";
+
+// components/twitter/common/errorMessage.ts
+var ERROR_URL = "https://developer.twitter.com/en/docs/twitter-api/getting-started/about-twitter-api#item0";
+var ERROR_MESSAGE = `Check if your app is properly configured on the [Twitter Developer Portal](${ERROR_URL}), and if your plan has access to the endpoint used by this component.`;
+var ACTION_ERROR_MESSAGE = `${ERROR_MESSAGE} Additional debugging information has been exported from the step.`;
+var PROBLEM_BASE_URL = "https://api.twitter.com/2/problems";
+var ERROR_BY_TYPE = {
+  [`${PROBLEM_BASE_URL}/client-forbidden`]: "This X (Twitter) account appears to be on the Free plan, which only has access to **Create Tweets**. [Refer to Twitter's developer docs](https://developer.twitter.com/en/docs/twitter-api/getting-started/about-twitter-api#v2-access-leve) for the latest info on API access.",
+  ["about:blank"]: "A generic problem with no additional information beyond that provided by the HTTP status code.",
+  [`${PROBLEM_BASE_URL}/invalid-request`]: "A problem that indicates this request is invalid. If your request takes a POST body, ensure the contents is valid JSON and matches the [OpenAPI spec](https://api.twitter.com/2/openapi.json).",
+  [`${PROBLEM_BASE_URL}/resource-not-found`]: "A problem that indicates that a given Post, User, etc. does not exist.",
+  [`${PROBLEM_BASE_URL}/not-authorized-for-resource`]: "A problem that indicates you are not allowed to see a particular Post, User, etc.",
+  [`${PROBLEM_BASE_URL}/disallowed-resource`]: "A problem that indicates that the resource requested violates the precepts of this API.",
+  [`${PROBLEM_BASE_URL}/unsupported-authentication`]: "A problem that indicates that the authentication used is not supported.",
+  [`${PROBLEM_BASE_URL}/usage-capped`]: "A problem that indicates that a usage cap has been exceeded.",
+  [`${PROBLEM_BASE_URL}/streaming-connection`]: "A problem that indicates something is wrong with the connection.",
+  [`${PROBLEM_BASE_URL}/client-disconnected`]: "Your client has gone away.",
+  [`${PROBLEM_BASE_URL}/operational-disconnect`]: "You have been disconnected for operational reasons.",
+  [`${PROBLEM_BASE_URL}/rule-cap`]: "You have exceeded the maximum number of rules.",
+  [`${PROBLEM_BASE_URL}/rule-length`]: "You have exceeded the maximum number of characters allowed on your query or rule based on your access level. See [access levels](https://developer.twitter.com/en/docs/twitter-api/getting-started/about-twitter-api.html#Access).",
+  [`${PROBLEM_BASE_URL}/invalid-rules`]: "The rule you have submitted is invalid.",
+  [`${PROBLEM_BASE_URL}/duplicate-rules`]: "The rule you have submitted is a duplicate.",
+  [`${PROBLEM_BASE_URL}/conflict`]: "The rule you have submitted is in conflict with another rule.",
+  [`${PROBLEM_BASE_URL}/noncompliant-rules`]: "The rule you have submitted is not compliant with Twitter's Automation Rules",
+  [`${PROBLEM_BASE_URL}/not-authorized-for-field`]: "A problem that indicates you are not allowed to see a particular field.",
+  [`${PROBLEM_BASE_URL}/oauth1-permissions`]: "A problem that indicates you are not allowed to see a particular field.",
+  [`${PROBLEM_BASE_URL}/resource-unavailable`]: "A problem that indicates that a given Post, User, etc. is temporarily unavailable."
+};
+
+// components/twitter/app/twitter.app.ts
+var twitter_app_default = defineApp({
+  type: "app",
+  app: "twitter",
+  propDefinitions: {
+    maxResults: {
+      type: "integer",
+      label: "Max Results",
+      description: "Maximum amount of items to return. The maximum amount per request for each endpoint is included in the API documentation.",
+      optional: true
+    },
+    listId: {
+      type: "string",
+      label: "List ID",
+      description: "Select a **List** owned by the authenticated user, or use a custom *List ID*.",
+      async options() {
+        const userId = await this.getAuthenticatedUserId();
+        const lists = await this.getUserOwnedLists({
+          userId
+        });
+        return lists.data?.map(({
+          id,
+          name
+        }) => ({
+          label: name,
+          value: id
+        })) ?? [];
+      }
+    },
+    userNameOrId: {
+      type: "string",
+      label: "User Name or ID",
+      description: "A Twitter username (handle) prefixed with `@` (e.g. `@pipedream`). You can also use the string `me` to use the authenticated user (default), or a numeric User ID.",
+      optional: true,
+      default: "me"
+    },
+    // See comment on "../common/additionalProps.ts"
+    /*
+    includeAllFields: {
+      type: "boolean",
+      label: "Include All Metadata",
+      description: "If set to `false`, you can choose which fields will be returned for each data type.",
+      optional: true,
+      default: true,
+      reloadProps: true,
+    },
+    */
+    tweetId: {
+      type: "string",
+      label: "Tweet ID",
+      description: "The numerical ID of the tweet."
+    },
+    query: {
+      type: "string",
+      label: "Query",
+      description: "One query for matching Tweets. See the [Twitter API guide on building queries](https://developer.twitter.com/en/docs/twitter-api/tweets/search/integrate/build-a-query)."
+    }
+  },
+  methods: {
+    throwError(data) {
+      const errorMessage = ERROR_BY_TYPE[data?.type];
+      if (!errorMessage) {
+        return;
+      }
+      throw new ConfigurationError(errorMessage);
+    },
+    _getAuthHeader(config) {
+      const {
+        developer_consumer_key: devKey,
+        developer_consumer_secret: devSecret,
+        oauth_access_token: key,
+        oauth_refresh_token: secret
+      } = this.$auth;
+      const consumer = {
+        key: devKey,
+        secret: devSecret
+      };
+      const oauth = new OAuth({
+        consumer,
+        signature_method: "HMAC-SHA1",
+        hash_function(base_string, key2) {
+          return crypto.createHmac("sha1", key2).update(base_string).digest("base64");
+        }
+      });
+      const token = {
+        key,
+        secret
+      };
+      const requestData = transformConfigForOauth(config);
+      return oauth.toHeader(oauth.authorize(requestData, token));
+    },
+    _getBaseUrl() {
+      return "https://api.twitter.com/2";
+    },
+    async _httpRequest({
+      $ = this,
+      headers,
+      specialAuth = false,
+      throwError = true,
+      fallbackError,
+      ...args
+    }) {
+      const maxRetries = 3;
+      let response;
+      let counter = 1;
+      const config = {
+        baseURL: this._getBaseUrl(),
+        ...args
+      };
+      const authConfig = specialAuth ? {
+        ...config,
+        params: {},
+        data: {}
+      } : config;
+      const axiosConfig = {
+        debug: true,
+        ...config,
+        headers: {
+          ...headers,
+          ...this._getAuthHeader(authConfig)
+        }
+      };
+      do {
+        try {
+          response = await axios($, axiosConfig);
+        } catch (err) {
+          console.log(`Request error on attempt #${counter}: `, err);
+          if (counter === maxRetries) {
+            if (throwError && err.response?.data) {
+              this.throwError(err.response.data);
+            }
+            if (fallbackError) {
+              throw new ConfigurationError(fallbackError);
+            }
+            throw err;
+          }
+        }
+      } while (!response && ++counter <= maxRetries);
+      return response;
+    },
+    async _paginatedRequest({
+      maxResults = 100,
+      maxPerPage = 100,
+      params,
+      ...args
+    }) {
+      const totalData = [], totalErrors = [], totalIncludes = {};
+      let paginationToken;
+      let resultCount = 0;
+      do {
+        const perPage = Math.min(maxResults - resultCount, maxPerPage);
+        const {
+          data,
+          errors,
+          meta: { next_token },
+          includes
+        } = await this._httpRequest({
+          params: {
+            ...params,
+            max_results: perPage,
+            pagination_token: paginationToken
+          },
+          ...args
+        });
+        if (data) {
+          totalData.push(...Array.isArray(data) ? data : [
+            data
+          ]);
+        }
+        if (errors) {
+          totalErrors.push(...Array.isArray(errors) ? errors : [
+            errors
+          ]);
+        }
+        if (includes) {
+          Object.entries(includes).forEach(
+            ([
+              key,
+              value
+            ]) => {
+              if (!totalIncludes[key]) totalIncludes[key] = [];
+              totalIncludes[key].push(...value);
+            }
+          );
+        }
+        paginationToken = next_token;
+        resultCount += perPage;
+      } while (paginationToken && resultCount < maxResults);
+      return {
+        data: totalData,
+        errors: totalErrors,
+        includes: totalIncludes
+      };
+    },
+    async addUserToList({
+      listId,
+      ...args
+    }) {
+      return this._httpRequest({
+        method: "POST",
+        url: `/lists/${listId}/members`,
+        ...args
+      });
+    },
+    async createTweet(args) {
+      return this._httpRequest({
+        throwError: false,
+        method: "POST",
+        url: "/tweets",
+        ...args
+      });
+    },
+    async deleteTweet({
+      tweetId,
+      ...args
+    }) {
+      return this._httpRequest({
+        method: "DELETE",
+        url: `/tweets/${tweetId}`,
+        ...args
+      });
+    },
+    async followUser(args) {
+      const id = await this.getAuthenticatedUserId();
+      return this._httpRequest({
+        method: "POST",
+        url: `/users/${id}/following`,
+        ...args
+      });
+    },
+    async getListTweets({
+      listId,
+      ...args
+    }) {
+      return this._paginatedRequest({
+        url: `/lists/${listId}/tweets`,
+        ...args
+      });
+    },
+    async getTweet({
+      tweetId,
+      ...args
+    }) {
+      return this._httpRequest({
+        url: `/tweets/${tweetId}`,
+        ...args
+      });
+    },
+    async getAuthenticatedUser(args) {
+      return this._httpRequest({
+        url: "/users/me",
+        ...args
+      });
+    },
+    async getAuthenticatedUserId() {
+      const response = await this.getAuthenticatedUser();
+      return response.data.id;
+    },
+    async getDirectMessages(args) {
+      return this._paginatedRequest({
+        url: "/dm_events",
+        ...args
+      });
+    },
+    async getUserLikedTweets({
+      userId,
+      ...args
+    }) {
+      return this._paginatedRequest({
+        url: `/users/${userId}/liked_tweets`,
+        ...args
+      });
+    },
+    async getUserOwnedLists({
+      userId,
+      ...args
+    }) {
+      return this._httpRequest({
+        url: `/users/${userId}/owned_lists`,
+        ...args
+      });
     },
-    methods: {
-        ...common.methods,
-        ...cacheUserId,
-        getUserFields,
-        getEntityName() {
-            return "User Followed";
+    async getUserFollowedLists({
+      userId,
+      ...args
+    }) {
+      return this._paginatedRequest({
+        url: `/users/${userId}/followed_lists`,
+        ...args
+      });
+    },
+    async getUserMentions({
+      userId,
+      ...args
+    }) {
+      return this._paginatedRequest({
+        url: `/users/${userId}/mentions`,
+        ...args
+      });
+    },
+    async getUserTweets({
+      userId,
+      ...args
+    }) {
+      return this._paginatedRequest({
+        url: `/users/${userId}/tweets`,
+        ...args
+      });
+    },
+    async getUserByUsername(username) {
+      return this._httpRequest({
+        url: `/users/by/username/${username}`
+      });
+    },
+    async getUser({
+      userId,
+      ...args
+    }) {
+      return this._httpRequest({
+        url: `/users/${userId}`,
+        ...args
+      });
+    },
+    async likeTweet(args) {
+      const id = await this.getAuthenticatedUserId();
+      return this._httpRequest({
+        method: "POST",
+        url: `/users/${id}/likes`,
+        ...args
+      });
+    },
+    async getUserFollowers({
+      userId,
+      ...args
+    }) {
+      return this._paginatedRequest({
+        url: `/users/${userId}/followers`,
+        ...args
+      });
+    },
+    async getUserFollowing({
+      userId,
+      ...args
+    }) {
+      return this._paginatedRequest({
+        url: `/users/${userId}/following`,
+        ...args
+      });
+    },
+    async retweet(args) {
+      const id = await this.getAuthenticatedUserId();
+      return this._httpRequest({
+        method: "POST",
+        url: `/users/${id}/retweets`,
+        ...args
+      });
+    },
+    async searchTweets(args) {
+      return this._paginatedRequest({
+        url: "/tweets/search/recent",
+        ...args
+      });
+    },
+    async sendMessage({
+      userId,
+      ...args
+    }) {
+      return this._httpRequest({
+        method: "POST",
+        url: `/dm_conversations/with/${userId}/messages`,
+        ...args
+      });
+    },
+    async unfollowUser({
+      userId,
+      ...args
+    }) {
+      const id = await this.getAuthenticatedUserId();
+      return this._httpRequest({
+        method: "DELETE",
+        url: `/users/${id}/following/${userId}`,
+        ...args
+      });
+    },
+    async unlikeTweet({
+      tweetId,
+      ...args
+    }) {
+      const id = await this.getAuthenticatedUserId();
+      return this._httpRequest({
+        method: "DELETE",
+        url: `/users/${id}/likes/${tweetId}`,
+        ...args
+      });
+    },
+    async uploadMedia(args) {
+      return this._httpRequest({
+        baseURL: "https://upload.twitter.com/1.1",
+        url: "/media/upload.json",
+        method: "POST",
+        headers: {
+          "Content-Type": `multipart/form-data; boundary=${args.data.getBoundary()}`
         },
-        getItemSummary,
-        async getResources(maxResults) {
-            const userId = await this.getCachedUserId();
-            const params = {
-                $: this,
-                maxPerPage: MAX_RESULTS_PER_PAGE,
-                maxResults: maxResults ?? MAX_RESULTS_PER_PAGE,
-                params: this.getUserFields(),
-                userId,
-            };
-            const { data, includes, } = await this.app.getUserFollowing(params);
-            data.forEach((user) => user.includes = getObjIncludes(user, includes, getUserIncludeIds));
-            return data;
+        specialAuth: true,
+        ...args
+      });
+    }
+  }
+});
+
+// components/twitter/common/appValidation.ts
+var appValidation_default = {
+  props: {
+    app: {
+      ...twitter_app_default,
+      reloadProps: true
+    }
+  },
+  async additionalProps() {
+    const q = "Pipedream";
+    const data = await this.getRecentTweets({
+      params: {
+        query: q
+      },
+      validateStatus: () => true
+    });
+    this.app.throwError(data);
+    return {};
+  },
+  methods: {
+    getRecentTweets(args = {}) {
+      return this.app._httpRequest({
+        url: "/tweets/search/recent",
+        ...args
+      });
+    }
+  }
+};
+
+// components/twitter/sources/common/base.ts
+var base_default = {
+  ...appValidation_default,
+  props: {
+    ...appValidation_default.props,
+    db: "$.service.db",
+    timer: {
+      type: "$.interface.timer",
+      default: {
+        intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL
+      }
+    }
+  },
+  hooks: {
+    async deploy() {
+      const q = "Pipedream";
+      const data = await this.getRecentTweets({
+        params: {
+          query: q
         },
+        validateStatus: () => true
+      });
+      this.app.throwError(data);
+      await this.getAndProcessData();
+    }
+  },
+  methods: {
+    ...appValidation_default.methods,
+    getEntityName() {
+      return "Entity";
+    },
+    getItemSummary() {
+      throw new Error("getItemSummary() not implemented in component");
+    },
+    async getResources() {
+      throw new Error("getResources() not implemented in component");
+    },
+    getSavedIds() {
+      return this.db.get("savedEntityIds");
+    },
+    setSavedIds(data) {
+      this.db.set("savedEntityIds", data);
+    },
+    async getAndProcessData(maxResults) {
+      const data = await this.getResources(maxResults);
+      if (data) {
+        const savedIds = this.getSavedIds() ?? [];
+        data.filter(({ id }) => !savedIds.includes(id)).reverse().forEach((obj) => {
+          this.emitEvent(obj);
+          savedIds.push(obj.id);
+        });
+        this.setSavedIds(savedIds);
+      }
+    },
+    emitEvent(data) {
+      const ts = Date.now();
+      const { id } = data;
+      this.$emit(data, {
+        id,
+        summary: `New ${this.getEntityName()}: "${this.getItemSummary(data)}"`,
+        ts
+      });
+    }
+  },
+  async run() {
+    try {
+      await this.getAndProcessData(50);
+    } catch (err) {
+      throw new Error(`${ERROR_MESSAGE} Error response: ${JSON.stringify(err)}`);
+    }
+  }
+};
+
+// components/twitter/sources/common/getItemSummary.ts
+function getUserSummary({ username }) {
+  return username;
+}
+
+// components/twitter/common/methods.ts
+import { ConfigurationError as ConfigurationError2 } from "@pipedream/platform";
+
+// components/twitter/common/dataFields.ts
+var TWEET_FIELD_OPTIONS = [
+  "attachments",
+  "author_id",
+  "context_annotations",
+  "conversation_id",
+  "created_at",
+  "edit_controls",
+  "entities",
+  "geo",
+  "id",
+  "in_reply_to_user_id",
+  "lang",
+  // "non_public_metrics",
+  "public_metrics",
+  // "organic_metrics",
+  // "promoted_metrics",
+  "possibly_sensitive",
+  "referenced_tweets",
+  "reply_settings",
+  "source",
+  "text",
+  "withheld"
+];
+var USER_FIELD_OPTIONS = [
+  "created_at",
+  "description",
+  "entities",
+  "id",
+  "location",
+  "name",
+  "pinned_tweet_id",
+  "profile_image_url",
+  "protected",
+  "public_metrics",
+  "url",
+  "username",
+  "verified",
+  "verified_type",
+  "withheld"
+];
+
+// components/twitter/common/expansions.ts
+var USER_EXPANSION_OPTIONS = [
+  "pinned_tweet_id"
+];
+
+// components/twitter/common/methods.ts
+async function getUserId() {
+  let { userNameOrId: id } = this;
+  if (id === "me" || id === void 0) {
+    id = await this.app.getAuthenticatedUserId();
+  } else if (id.startsWith("@")) {
+    const userData = await this.app.getUserByUsername(id.slice(1));
+    id = userData?.data?.id;
+    if (!id) throw new ConfigurationError2("**User not found!** Check the `User Name or ID` prop.");
+  }
+  return id;
+}
+function getUserFields() {
+  return {
+    "expansions": USER_EXPANSION_OPTIONS.join(),
+    "tweet.fields": TWEET_FIELD_OPTIONS.join(),
+    "user.fields": USER_FIELD_OPTIONS.join()
+  };
+}
+
+// components/twitter/sources/common/cacheUserId.ts
+async function getCachedUserId() {
+  const { userNameOrId: name } = this;
+  const cache = this.db.get("cachedUser");
+  if (cache?.name === name) {
+    return cache.id;
+  }
+  const id = await this.getUserId();
+  this.db.set("cachedUser", {
+    id,
+    name
+  });
+  return id;
+}
+var cacheUserId_default = {
+  getCachedUserId,
+  getUserId
+};
+
+// components/twitter/common/addObjIncludes.ts
+function flatClearIncludeIds(obj) {
+  return Object.fromEntries(
+    Object.entries(obj).map(([
+      key,
+      value
+    ]) => [
+      key,
+      value.filter((e) => e !== void 0).flat()
+    ])
+  );
+}
+function getUserIncludeIds(obj) {
+  return flatClearIncludeIds({
+    tweetIds: [
+      obj.pinned_tweet_id
+    ]
+  });
+}
+function getObjIncludes(obj, includes = {}, getIncludeIds) {
+  const {
+    userIds,
+    userNames,
+    tweetIds,
+    pollIds,
+    placeIds,
+    mediaKeys
+  } = getIncludeIds(obj);
+  const result = {
+    media: includes.media?.filter((item) => mediaKeys?.includes(item.media_key)),
+    users: includes.users?.filter(
+      (item) => userIds?.includes(item.id) || userNames?.includes(item.username)
+    ),
+    places: includes.places?.filter((item) => placeIds?.includes(item.id)),
+    polls: includes.polls?.filter((item) => pollIds?.includes(item.id)),
+    tweets: includes.tweets?.filter((item) => tweetIds?.includes(item.id))
+  };
+  return Object.fromEntries(
+    Object.entries(result).filter(([
+      ,
+      value
+    ]) => value?.length > 0)
+  );
+}
+
+// components/twitter/sources/new-user-followed/new-user-followed.ts
+var DOCS_LINK = "https://developer.twitter.com/en/docs/twitter-api/users/follows/api-reference/get-users-id-following";
+var MAX_RESULTS_PER_PAGE = 1e3;
+var new_user_followed_default = defineSource({
+  ...base_default,
+  key: "twitter-new-user-followed",
+  name: "New User Followed by User",
+  description: `Emit new event when the specified User follows another User [See the documentation](${DOCS_LINK})`,
+  version: "2.1.0",
+  type: "source",
+  props: {
+    ...base_default.props,
+    userNameOrId: {
+      propDefinition: [
+        base_default.props.app,
+        "userNameOrId"
+      ]
+    }
+  },
+  methods: {
+    ...base_default.methods,
+    ...cacheUserId_default,
+    getUserFields,
+    getEntityName() {
+      return "User Followed";
     },
+    getItemSummary: getUserSummary,
+    async getResources(maxResults) {
+      const userId = await this.getCachedUserId();
+      const params = {
+        $: this,
+        maxPerPage: MAX_RESULTS_PER_PAGE,
+        maxResults: maxResults ?? MAX_RESULTS_PER_PAGE,
+        params: this.getUserFields(),
+        userId
+      };
+      const {
+        data,
+        includes
+      } = await this.app.getUserFollowing(params);
+      data.forEach((user) => user.includes = getObjIncludes(user, includes, getUserIncludeIds));
+      return data;
+    }
+  }
 });
+export {
+  new_user_followed_default as default
+};