@pipedream/twitter 0.3.11 → 1.1.0

package/sources/common/followed.mjs

@@ -1,38 +0,0 @@
-import twitter from "../../twitter.app.mjs";
-import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform";
-
-export default {
-  props: {
-    twitter,
-    db: "$.service.db",
-    timer: {
-      label: "Polling interval",
-      description: "Pipedream will poll the Twitter API on this schedule",
-      type: "$.interface.timer",
-      default: {
-        intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL,
-      },
-    },
-  },
-  methods: {
-    _getPreviousIds() {
-      return this.db.get("previousIds") || [];
-    },
-    _setPreviousIds(previousIds) {
-      this.db.set("previousIds", previousIds);
-    },
-  },
-  async run() {
-    const previousIds = this._getPreviousIds();
-    const followedItems = await this.getFollowed();
-    for (const item of followedItems.reverse()) {
-      if (previousIds.includes(item.id)) {
-        continue;
-      }
-      previousIds.push(item.id);
-      const meta = this.generateMeta(item);
-      this.$emit(item, meta);
-    }
-    this._setPreviousIds(previousIds);
-  },
-};

package/sources/common/followers.mjs

@@ -1,159 +0,0 @@
-import twitter from "../../twitter.app.mjs";
-import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform";
-
-export default {
-  props: {
-    db: "$.service.db",
-    twitter,
-    timer: {
-      label: "Polling interval",
-      description: "Pipedream will poll the Twitter API on this schedule",
-      type: "$.interface.timer",
-      default: {
-        intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL,
-      },
-    },
-  },
-  hooks: {
-    async activate() {
-      await this._provisionFollowersCache();
-    },
-  },
-  deactivate() {
-    this._clearFollowersCache();
-  },
-  methods: {
-    /**
-     * This method returns the size of the internal cache used by the event
-     * source to keep track of the latest Twitter followers of the corresponding
-     * account.
-     *
-     * The size represents the maximum amount of entries/ID's that will be
-     * cached at any given time (or 0 for unlimited). Keep in mind that the
-     * Pipedream platform itself imposes size limits as well in terms of DB
-     * usage.
-     *
-     * @returns The maximum amount of Twitter followers entries to be cached at
-     * any given moment (or 0 for unlimited).
-     */
-    getFollowersCacheSize() {
-      return 0;
-    },
-    getScreenName() {
-      // When screen name is not explicitly provided, the Twitter API defaults
-      // it to the user making the API request
-      return undefined;
-    },
-    /**
-     * This function provides the list of relevant, follower-related, user ID's
-     * that are used by the event source to emit events. How each event source
-     * that implements it depends on the purpose of the event source itself. For
-     * example, an event source that emits an event for every new follower will
-     * implement this function in such a way that its result is a list of user
-     * ID's for each new follower detected.
-     *
-     * @returns the list of relevant user ID's for this event source to process
-     */
-    getRelevantIds() {
-      return [];
-    },
-    generateMeta(data) {
-      const {
-        timestamp,
-        user,
-      } = data;
-      const {
-        id_str: id,
-        screen_name: summary,
-      } = user;
-      // Event timestamp is expressed in seconds, whereas Javascript timestamps
-      // are expressed in milliseconds
-      const ts = timestamp * 1000;
-      return {
-        id,
-        summary,
-        ts,
-      };
-    },
-    _clearFollowersCache() {
-      this.setFollowersCache([]);
-    },
-    async _provisionFollowersCache() {
-      const screenName = this.getScreenName();
-      const followerIdsGen = this.twitter.scanFollowerIds(screenName);
-      const maxFollowerListSize = Math.max(this.getFollowersCacheSize(), 0);
-      const result = [];
-      for await (const id of followerIdsGen) {
-        if (maxFollowerListSize !== 0 && maxFollowerListSize === result.length) {
-          break;
-        }
-
-        result.push(id);
-      }
-
-      this.setFollowersCache(result);
-      return result;
-    },
-    getFollowersCache() {
-      return this.db.get("followers");
-    },
-    setFollowersCache(followers) {
-      const followersCacheSize = Math.max(this.getFollowersCacheSize(), 0);
-      const trimmedFollowers = followersCacheSize !== 0
-        ? followers.slice(0, followersCacheSize)
-        : followers;
-      this.db.set("followers", trimmedFollowers);
-      console.log(`
-        Updated followers cache: ${trimmedFollowers.length} records
-      `);
-    },
-    /**
-     * This generator method scans the list of Twitter followers until it finds
-     * the ID of a follower that was already processed.
-     *
-     * @param {string[]} processedFollowerIds a list of the ID's of the most
-     * recent Twitter followers processed by the event source
-     * @yields the ID of a new Twitter follower
-     */
-    async *scanNewFollowers(processedFollowerIds = []) {
-      const processeedFollowerIdsSet = new Set(processedFollowerIds);
-      const screenName = this.getScreenName();
-      const followerIdsGen = this.twitter.scanFollowerIds(screenName);
-      for await (const id of followerIdsGen) {
-        if (processeedFollowerIdsSet.has(id)) {
-          break;
-        }
-        yield id;
-      }
-    },
-    async getUnfollowers() {
-      const prevFollowers = this.getFollowersCache();
-      const currFollowers = await this._provisionFollowersCache();
-      const currFollowersSet = new Set(currFollowers);
-      return prevFollowers.filter((pf) => !currFollowersSet.has(pf));
-    },
-  },
-  async run(event) {
-    const { timestamp } = event;
-    const relevantIds = await this.getRelevantIds();
-    if (relevantIds.length <= 0) {
-      console.log(`
-        No changes in followers data for this event source to emit a new event
-      `);
-      return;
-    }
-
-    const users = await this.twitter.lookupUsers({
-      userIdArray: relevantIds,
-    });
-    users.forEach((user) => {
-      const data = {
-        timestamp,
-        user,
-      };
-      user.profile_url = `https://twitter.com/${user.screen_name}/`;
-      const meta = this.generateMeta(data);
-      this.$emit(user, meta);
-    });
-  },
-};

package/sources/common/tweets.mjs

@@ -1,155 +0,0 @@
-import twitter from "../../twitter.app.mjs";
-import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform";
-
-export default {
-  props: {
-    db: "$.service.db",
-    twitter,
-    count: {
-      propDefinition: [
-        twitter,
-        "count",
-      ],
-    },
-    maxRequests: {
-      propDefinition: [
-        twitter,
-        "maxRequests",
-      ],
-    },
-    timer: {
-      label: "Polling interval",
-      description: "Pipedream will poll the Twitter API on this schedule",
-      type: "$.interface.timer",
-      default: {
-        intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL,
-      },
-    },
-  },
-  methods: {
-    _compareByIdAsc({ id_str: a }, { id_str: b }) {
-      const A = BigInt(a);
-      const B = BigInt(b);
-      if (A < B) return -1;
-      if (A > B) return 1;
-      return 0;
-    },
-    sortTweets(tweets) {
-      return tweets
-        .sort(this._compareByIdAsc);
-    },
-    /**
-     * This function returns the Twitter screen name of the subject account
-     *
-     * @returns a string containing the relevant Twitter screen name
-     */
-    getScreenName() {
-      throw new Error("getScreenName is not implemented");
-    },
-    /**
-     * This function provides the list of options for the `tweetId` user prop.
-     * It is meant to be called by Pipedream during the setup of the user prop
-     * and not during normal operations of the event source.
-     *
-     * @param {object}  context the context object for the pagination of the
-     * prop options
-     * @param {object}  context.prevContext the context object of a previous
-     * call to this method
-     * @returns an object containing the list of Tweet ID options for the
-     * `tweetId` user prop and the context for pagination
-     */
-    async tweetIdOptions(context) {
-      const { prevContext = {} } = context;
-      const {
-        screenName = await this.getScreenName(),
-        sinceId = "1",
-      } = prevContext;
-
-      const userTimelineOpts = {
-        screenName,
-        sinceId,
-        count: 10,
-        trim_user: true,
-        exclude_replies: false,
-        include_rts: false,
-      };
-      const tweets = await this.twitter.getUserTimeline(userTimelineOpts);
-      if (tweets.length === 0) {
-        // There are no more tweets to go through
-        return {
-          options: null,
-          context: prevContext,
-        };
-      }
-
-      const sortedTweets = this.sortTweets(tweets);
-      const { id_str: lastId } = sortedTweets[sortedTweets.length - 1];
-      const options = sortedTweets.map(({
-        full_text: fullText,
-        id_str: idStr,
-      }) => ({
-        label: fullText,
-        value: idStr,
-      }));
-
-      return {
-        options,
-        context: {
-          screenName,
-          sinceId: lastId,
-        },
-      };
-    },
-    getSinceId() {
-      return this.db.get("sinceId") || "1";
-    },
-    setSinceId(sinceId = "1") {
-      this.db.set("sinceId", sinceId);
-    },
-    generateMeta(tweet) {
-      const {
-        created_at: createdAt,
-        full_text: fullText,
-        id_str: id,
-        text,
-      } = tweet;
-      const summary = fullText || text;
-      const ts = this.twitter.parseDate(createdAt);
-      return {
-        id,
-        summary,
-        ts,
-      };
-    },
-    /**
-     * The purpose of this function is to retrieve the relevant Tweets for the
-     * event source that implements it. For example, if the event source emits
-     * an event for each new Tweet of a specific user, the implementation of
-     * this function will perform the retrieval of such Tweets and return it as
-     * a list of [Tweet
-     * objects](https://developer.twitter.com/en/docs/twitter-api/v1/data-dictionary/object-model/tweet)
-     *
-     * @returns a list of Tweet objects for which to emit new events
-     */
-    retrieveTweets() {
-      throw new Error("retrieveTweets is not implemented");
-    },
-  },
-  async run() {
-    const tweets = await this.retrieveTweets();
-    if (tweets.length === 0) {
-      console.log("No new tweets available. Skipping...");
-      return;
-    }
-
-    const sortedTweets = this.sortTweets(tweets);
-    const { id_str: lastId } = sortedTweets[sortedTweets.length - 1];
-    this.setSinceId(lastId);
-
-    // Emit array of tweet objects
-    sortedTweets.forEach((tweet) => {
-      const meta = this.generateMeta(tweet);
-      this.$emit(tweet, meta);
-    });
-  },
-};