@pipedream/twitter 0.3.3 → 0.3.4

package/sources/common/followers.mjs

@@ -0,0 +1,158 @@
+import twitter from "../../twitter.app.mjs";
+
+export default {
+  props: {
+    db: "$.service.db",
+    twitter,
+    timer: {
+      label: "Polling interval",
+      description: "Pipedream will poll the Twitter API on this schedule",
+      type: "$.interface.timer",
+      default: {
+        intervalSeconds: 60 * 15,
+      },
+    },
+  },
+  hooks: {
+    async activate() {
+      await this._provisionFollowersCache();
+    },
+  },
+  deactivate() {
+    this._clearFollowersCache();
+  },
+  methods: {
+    /**
+     * This method returns the size of the internal cache used by the event
+     * source to keep track of the latest Twitter followers of the corresponding
+     * account.
+     *
+     * The size represents the maximum amount of entries/ID's that will be
+     * cached at any given time (or 0 for unlimited). Keep in mind that the
+     * Pipedream platform itself imposes size limits as well in terms of DB
+     * usage.
+     *
+     * @returns The maximum amount of Twitter followers entries to be cached at
+     * any given moment (or 0 for unlimited).
+     */
+    getFollowersCacheSize() {
+      return 0;
+    },
+    getScreenName() {
+      // When screen name is not explicitly provided, the Twitter API defaults
+      // it to the user making the API request
+      return undefined;
+    },
+    /**
+     * This function provides the list of relevant, follower-related, user ID's
+     * that are used by the event source to emit events. How each event source
+     * that implements it depends on the purpose of the event source itself. For
+     * example, an event source that emits an event for every new follower will
+     * implement this function in such a way that its result is a list of user
+     * ID's for each new follower detected.
+     *
+     * @returns the list of relevant user ID's for this event source to process
+     */
+    getRelevantIds() {
+      return [];
+    },
+    generateMeta(data) {
+      const {
+        timestamp,
+        user,
+      } = data;
+      const {
+        id_str: id,
+        screen_name: summary,
+      } = user;
+      // Event timestamp is expressed in seconds, whereas Javascript timestamps
+      // are expressed in milliseconds
+      const ts = timestamp * 1000;
+      return {
+        id,
+        summary,
+        ts,
+      };
+    },
+    _clearFollowersCache() {
+      this.setFollowersCache([]);
+    },
+    async _provisionFollowersCache() {
+      const screenName = this.getScreenName();
+      const followerIdsGen = this.twitter.scanFollowerIds(screenName);
+      const maxFollowerListSize = Math.max(this.getFollowersCacheSize(), 0);
+      const result = [];
+      for await (const id of followerIdsGen) {
+        if (maxFollowerListSize !== 0 && maxFollowerListSize === result.length) {
+          break;
+        }
+
+        result.push(id);
+      }
+
+      this.setFollowersCache(result);
+      return result;
+    },
+    getFollowersCache() {
+      return this.db.get("followers");
+    },
+    setFollowersCache(followers) {
+      const followersCacheSize = Math.max(this.getFollowersCacheSize(), 0);
+      const trimmedFollowers = followersCacheSize !== 0
+        ? followers.slice(0, followersCacheSize)
+        : followers;
+      this.db.set("followers", trimmedFollowers);
+      console.log(`
+        Updated followers cache: ${trimmedFollowers.length} records
+      `);
+    },
+    /**
+     * This generator method scans the list of Twitter followers until it finds
+     * the ID of a follower that was already processed.
+     *
+     * @param {string[]} processedFollowerIds a list of the ID's of the most
+     * recent Twitter followers processed by the event source
+     * @yields the ID of a new Twitter follower
+     */
+    async *scanNewFollowers(processedFollowerIds = []) {
+      const processeedFollowerIdsSet = new Set(processedFollowerIds);
+      const screenName = this.getScreenName();
+      const followerIdsGen = this.twitter.scanFollowerIds(screenName);
+      for await (const id of followerIdsGen) {
+        if (processeedFollowerIdsSet.has(id)) {
+          break;
+        }
+        yield id;
+      }
+    },
+    async getUnfollowers() {
+      const prevFollowers = this.getFollowersCache();
+      const currFollowers = await this._provisionFollowersCache();
+      const currFollowersSet = new Set(currFollowers);
+      return prevFollowers.filter((pf) => !currFollowersSet.has(pf));
+    },
+  },
+  async run(event) {
+    const { timestamp } = event;
+    const relevantIds = await this.getRelevantIds();
+    if (relevantIds.length <= 0) {
+      console.log(`
+        No changes in followers data for this event source to emit a new event
+      `);
+      return;
+    }
+
+    const users = await this.twitter.lookupUsers({
+      userIdArray: relevantIds,
+    });
+    users.forEach((user) => {
+      const data = {
+        timestamp,
+        user,
+      };
+      user.profile_url = `https://twitter.com/${user.screen_name}/`;
+      const meta = this.generateMeta(data);
+      this.$emit(user, meta);
+    });
+  },
+};

package/sources/common/tweets.mjs

@@ -0,0 +1,154 @@
+import twitter from "../../twitter.app.mjs";
+
+export default {
+  props: {
+    db: "$.service.db",
+    twitter,
+    count: {
+      propDefinition: [
+        twitter,
+        "count",
+      ],
+    },
+    maxRequests: {
+      propDefinition: [
+        twitter,
+        "maxRequests",
+      ],
+    },
+    timer: {
+      label: "Polling interval",
+      description: "Pipedream will poll the Twitter API on this schedule",
+      type: "$.interface.timer",
+      default: {
+        intervalSeconds: 60 * 15,
+      },
+    },
+  },
+  methods: {
+    _compareByIdAsc({ id_str: a }, { id_str: b }) {
+      const A = BigInt(a);
+      const B = BigInt(b);
+      if (A < B) return -1;
+      if (A > B) return 1;
+      return 0;
+    },
+    sortTweets(tweets) {
+      return tweets
+        .sort(this._compareByIdAsc);
+    },
+    /**
+     * This function returns the Twitter screen name of the subject account
+     *
+     * @returns a string containing the relevant Twitter screen name
+     */
+    getScreenName() {
+      throw new Error("getScreenName is not implemented");
+    },
+    /**
+     * This function provides the list of options for the `tweetId` user prop.
+     * It is meant to be called by Pipedream during the setup of the user prop
+     * and not during normal operations of the event source.
+     *
+     * @param {object}  context the context object for the pagination of the
+     * prop options
+     * @param {object}  context.prevContext the context object of a previous
+     * call to this method
+     * @returns an object containing the list of Tweet ID options for the
+     * `tweetId` user prop and the context for pagination
+     */
+    async tweetIdOptions(context) {
+      const { prevContext = {} } = context;
+      const {
+        screenName = await this.getScreenName(),
+        sinceId = "1",
+      } = prevContext;
+
+      const userTimelineOpts = {
+        screenName,
+        sinceId,
+        count: 10,
+        trim_user: true,
+        exclude_replies: false,
+        include_rts: false,
+      };
+      const tweets = await this.twitter.getUserTimeline(userTimelineOpts);
+      if (tweets.length === 0) {
+        // There are no more tweets to go through
+        return {
+          options: null,
+          context: prevContext,
+        };
+      }
+
+      const sortedTweets = this.sortTweets(tweets);
+      const { id_str: lastId } = sortedTweets[sortedTweets.length - 1];
+      const options = sortedTweets.map(({
+        full_text: fullText,
+        id_str: idStr,
+      }) => ({
+        label: fullText,
+        value: idStr,
+      }));
+
+      return {
+        options,
+        context: {
+          screenName,
+          sinceId: lastId,
+        },
+      };
+    },
+    getSinceId() {
+      return this.db.get("sinceId") || "1";
+    },
+    setSinceId(sinceId = "1") {
+      this.db.set("sinceId", sinceId);
+    },
+    generateMeta(tweet) {
+      const {
+        created_at: createdAt,
+        full_text: fullText,
+        id_str: id,
+        text,
+      } = tweet;
+      const summary = fullText || text;
+      const ts = this.twitter.parseDate(createdAt);
+      return {
+        id,
+        summary,
+        ts,
+      };
+    },
+    /**
+     * The purpose of this function is to retrieve the relevant Tweets for the
+     * event source that implements it. For example, if the event source emits
+     * an event for each new Tweet of a specific user, the implementation of
+     * this function will perform the retrieval of such Tweets and return it as
+     * a list of [Tweet
+     * objects](https://developer.twitter.com/en/docs/twitter-api/v1/data-dictionary/object-model/tweet)
+     *
+     * @returns a list of Tweet objects for which to emit new events
+     */
+    retrieveTweets() {
+      throw new Error("retrieveTweets is not implemented");
+    },
+  },
+  async run() {
+    const tweets = await this.retrieveTweets();
+    if (tweets.length === 0) {
+      console.log("No new tweets available. Skipping...");
+      return;
+    }
+
+    const sortedTweets = this.sortTweets(tweets);
+    const { id_str: lastId } = sortedTweets[sortedTweets.length - 1];
+    this.setSinceId(lastId);
+
+    // Emit array of tweet objects
+    sortedTweets.forEach((tweet) => {
+      const meta = this.generateMeta(tweet);
+      this.$emit(tweet, meta);
+    });
+  },
+};

package/sources/{language-codes.js → language-codes.mjs}

@@ -1,4 +1,8 @@
-module.exports = [
+export default [
+  {
+    English: "Undetected",
+    alpha2: "und",
+  },
   {
     English: "Abkhazian",
     alpha2: "ab",
@@ -734,5 +738,5 @@ module.exports = [
   {
     English: "Zulu",
     alpha2: "zu",
-  }
+  },
 ];

package/sources/my-liked-tweets/my-liked-tweets.mjs

@@ -0,0 +1,29 @@
+import twitter from "../../twitter.app.mjs";
+
+export default {
+  key: "twitter-my-liked-tweets",
+  name: "My Liked Tweets",
+  description: "Emit new Tweets you like on Twitter",
+  version: "0.0.7",
+  type: "source",
+  props: {
+    twitter,
+    timer: {
+      label: "Polling interval",
+      description: "Pipedream will poll the Twitter API on this schedule",
+      type: "$.interface.timer",
+      default: {
+        intervalSeconds: 60 * 15,
+      },
+    },
+  },
+  dedupe: "unique",
+  async run() {
+    (await this.twitter.getLikedTweets()).reverse().forEach((tweet) => {
+      this.$emit(this.twitter.enrichTweet(tweet), {
+        id: tweet.id_str,
+        summary: tweet.full_text,
+      });
+    });
+  },
+};

package/sources/my-tweets/my-tweets.mjs

@@ -0,0 +1,48 @@
+import base from "../common/tweets.mjs";
+
+export default {
+  ...base,
+  key: "twitter-my-tweets",
+  name: "My Tweets",
+  description: "Emit new Tweets you post to Twitter",
+  version: "0.0.9",
+  type: "source",
+  props: {
+    ...base.props,
+    includeRetweets: {
+      propDefinition: [
+        base.props.twitter,
+        "includeRetweets",
+      ],
+    },
+    includeReplies: {
+      propDefinition: [
+        base.props.twitter,
+        "includeReplies",
+      ],
+    },
+  },
+  methods: {
+    ...base.methods,
+    shouldExcludeReplies() {
+      return this.includeReplies === "exclude";
+    },
+    shouldIncludeRetweets() {
+      return this.includeRetweets !== "exclude";
+    },
+    async getScreenName() {
+      const { screen_name: screenName } = await this.twitter.verifyCredentials();
+      return screenName;
+    },
+    async retrieveTweets() {
+      const screenName = await this.getScreenName();
+      return this.twitter.getUserTimeline({
+        screenName,
+        count: this.count,
+        sinceId: this.getSinceId(),
+        excludeReplies: this.shouldExcludeReplies(),
+        includeRts: this.shouldIncludeRetweets(),
+      });
+    },
+  },
+};

package/sources/new-follower-of-me/new-follower-of-me.mjs

@@ -0,0 +1,69 @@
+import common from "../common/followers.mjs";
+
+export default {
+  ...common,
+  key: "twitter-new-follower-of-me",
+  name: "New Follower of Me",
+  description: "Emit new event when a user follows you on Twitter",
+  version: "0.0.8",
+  type: "source",
+  props: {
+    ...common.props,
+    followersCacheSize: {
+      type: "integer",
+      label: "Followers Cache Size",
+      description: "The maximum amount of follower ID's that will be cached at any given time",
+      min: 10,
+      max: 100,
+      default: 100,
+    },
+  },
+  methods: {
+    ...common.methods,
+    _getHasExecuted() {
+      return this.db.get("hasExecuted");
+    },
+    _setHasExecuted(hasExecuted) {
+      this.db.set("hasExecuted", hasExecuted);
+    },
+    _wasComponentExecuted() {
+      return !!this._getHasExecuted();
+    },
+    _markComponentAsExecuted() {
+      this._setHasExecuted(true);
+    },
+    getFollowersCacheSize() {
+      return this.followersCacheSize;
+    },
+    async getRelevantIds() {
+      const mostRecentFollowers = this.getFollowersCache();
+      if (!this._wasComponentExecuted()) {
+        this._markComponentAsExecuted();
+
+        // The first time this event source is executed, it will emit an event
+        // for the most recent followers.
+        // We reverse the list of new followers so that the event source emits the
+        // events for each follower in the same order that the followers followed
+        // the account, from least to most recent.
+        return mostRecentFollowers.reverse();
+      }
+
+      const newFollowersGen = this.scanNewFollowers(mostRecentFollowers);
+      const newFollowers = [];
+      for await (const id of newFollowersGen) {
+        newFollowers.push(id);
+      }
+
+      const followersCache = [
+        ...newFollowers,
+        ...mostRecentFollowers,
+      ];
+      this.setFollowersCache(followersCache);
+
+      // We reverse the list of new followers so that the event source emits the
+      // events for each follower in the same order that the followers followed
+      // the account, from least to most recent.
+      return newFollowers.reverse();
+    },
+  },
+};

package/sources/new-follower-of-user/new-follower-of-user.mjs

@@ -0,0 +1,25 @@
+import base from "../new-follower-of-me/new-follower-of-me.mjs";
+
+export default {
+  ...base,
+  key: "twitter-new-follower-of-user",
+  name: "New Follower of User",
+  description: "Emit new event when a specific user gains a follower",
+  version: "0.0.8",
+  type: "source",
+  props: {
+    ...base.props,
+    screenName: {
+      propDefinition: [
+        base.props.twitter,
+        "screenName",
+      ],
+    },
+  },
+  methods: {
+    ...base.methods,
+    getScreenName() {
+      return this.screenName;
+    },
+  },
+};

package/sources/new-list-followed/new-list-followed.mjs

@@ -0,0 +1,24 @@
+import common from "../common/followed.mjs";
+
+export default {
+  ...common,
+  key: "twitter-new-list-followed",
+  name: "New List Followed",
+  description: "Emit new event when the authenticated user follows a List",
+  version: "0.0.1",
+  type: "source",
+  dedupe: "unique",
+  methods: {
+    ...common.methods,
+    async getFollowed() {
+      return this.twitter.getLists({});
+    },
+    generateMeta(list) {
+      return {
+        id: list.id,
+        summary: list.name,
+        ts: Date.now(),
+      };
+    },
+  },
+};

package/sources/new-trends-by-geo/new-trends-by-geo.mjs

@@ -0,0 +1,39 @@
+import twitter from "../../twitter.app.mjs";
+
+export default {
+  key: "twitter-new-trends-by-geo",
+  name: "New Trends by Geo",
+  description: "Emit new event when a new topic is trending on Twitter in a specific geographic location",
+  version: "0.0.7",
+  type: "source",
+  props: {
+    twitter,
+    trendLocation: {
+      propDefinition: [
+        twitter,
+        "trendLocation",
+      ],
+    },
+    timer: {
+      label: "Polling interval",
+      description: "Pipedream will poll the Twitter API on this schedule",
+      type: "$.interface.timer",
+      default: {
+        intervalSeconds: 60 * 15,
+      },
+    },
+  },
+  dedupe: "unique",
+  async run() {
+    (await this.twitter.getTrends({
+      id: this.trendLocation,
+    })).forEach((geo) => {
+      geo.trends.reverse().forEach((trend) => {
+        this.$emit(trend, {
+          id: trend.query,
+          summary: trend.name,
+        });
+      });
+    });
+  },
+};

package/sources/new-tweet-in-list/new-tweet-in-list.mjs

@@ -0,0 +1,49 @@
+import base from "../common/tweets.mjs";
+
+export default {
+  ...base,
+  key: "twitter-new-tweet-in-list",
+  name: "New Tweet in List",
+  description: "Emit new Tweets posted by members of a list",
+  version: "0.0.5",
+  type: "source",
+  props: {
+    ...base.props,
+    includeRetweets: {
+      propDefinition: [
+        base.props.twitter,
+        "includeRetweets",
+      ],
+    },
+    list: {
+      propDefinition: [
+        base.props.twitter,
+        "list",
+      ],
+    },
+    includeEntities: {
+      type: "boolean",
+      label: "Entities",
+      description: `
+        Include the 'entities' node, which offers a variety of metadata about the
+        tweet in a discreet structure, including: user_mentions, urls, and hashtags
+      `,
+      default: false,
+    },
+  },
+  methods: {
+    ...base.methods,
+    shouldIncludeRetweets() {
+      return this.includeRetweets !== "exclude";
+    },
+    retrieveTweets() {
+      return this.twitter.getListTweets({
+        listId: this.list,
+        count: this.count,
+        sinceId: this.getSinceId(),
+        includeEntities: this.includeEntities,
+        includeRetweets: this.shouldIncludeRetweets(),
+      });
+    },
+  },
+};

package/sources/new-unfollower-of-me/{new-unfollower-of-me.js → new-unfollower-of-me.mjs}

@@ -1,11 +1,12 @@
-const common = require("../common-followers");
+import common from "../common/followers.mjs";
 
-module.exports = {
+export default {
   ...common,
   key: "twitter-new-unfollower-of-me",
   name: "New Unfollower of Me",
-  description: "Emit an event when a user unfollows you on Twitter",
-  version: "0.0.1",
+  description: "Emit new event when a user unfollows you on Twitter",
+  version: "0.0.9",
+  type: "source",
   methods: {
     ...common.methods,
     getRelevantIds() {

package/sources/new-unfollower-of-user/new-unfollower-of-user.mjs

@@ -0,0 +1,25 @@
+import base from "../new-unfollower-of-me/new-unfollower-of-me.mjs";
+
+export default {
+  ...base,
+  key: "twitter-new-unfollower-of-user",
+  name: "New Unfollower of User",
+  description: "Emit new event when a specific user loses a follower on Twitter",
+  version: "0.0.7",
+  type: "source",
+  props: {
+    ...base.props,
+    screenName: {
+      propDefinition: [
+        base.props.twitter,
+        "screenName",
+      ],
+    },
+  },
+  methods: {
+    ...base.methods,
+    getScreenName() {
+      return this.screenName;
+    },
+  },
+};

package/sources/new-user-followed/new-user-followed.mjs

@@ -0,0 +1,25 @@
+import common from "../common/followed.mjs";
+
+export default {
+  ...common,
+  key: "twitter-new-user-followed",
+  name: "New User Followed",
+  description: "Emit new event when the authenticated user follows a User",
+  version: "0.0.1",
+  type: "source",
+  dedupe: "unique",
+  methods: {
+    ...common.methods,
+    async getFollowed() {
+      const { users } = await this.twitter.getUserFollows({});
+      return users;
+    },
+    generateMeta(user) {
+      return {
+        id: user.id,
+        summary: user.name,
+        ts: Date.now(),
+      };
+    },
+  },
+};