dev_orbit 0.2.0 → 0.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 051a39f2fff550c076f0a7bec373c7634f0b6516a9173121ba4b9facebfa157c
-  data.tar.gz: acad969ccbc5ac6dd29b176f069e4a94eb00cec1ffa69c301a9c5da5284e3d6a
+  metadata.gz: 49d580cf75ef85e8e082e763b2c48232431ce5f945edb6f28050a16d98f717b8
+  data.tar.gz: 65f47c79b4f40649c9e36a7a4186c854340d335e503d503927a81cb06027c1d2
 SHA512:
-  metadata.gz: 6953c67373ff7f8ae654a87916045b54c2254736809fd074ff26f6a46b400a9e6144226eaf29d2247e3e532877735b378adc77e28b29532e3d4325cf5b22ebce
-  data.tar.gz: e1a4069e95e2b5ff86b14625132bef5e5342c55e1e2d4cd767345fbd53e114cbb5ed5515826e554ffbeb04d929f1bdab1322b93868504a651a0a2ac0afef5cac
+  metadata.gz: bda03fdecb838c3b11f217a7a5da316cb05a64942aaadad0b6bf17d82aae00792af8ceed8250c259e7b3de2933836e338c9a6730eb584a897544ff7061cae5b5
+  data.tar.gz: 4caf442b3598ec3727604049825d6cfc90f8a47bbdc5b0942a2693b82e8a8d6eb1550f6dd4c48a5ddfd805dc4849fbead1213923229e5356d3421430c0847e7a

data/CHANGELOG.md

@@ -41,4 +41,14 @@
 ## [0.1.0] - 2021-05-06
 
 - Correct CLI `CheckFollowers` action
-- Update gem specifications
+- Update gem specifications
+## [0.4.1] - 2021-07-01
+
+- Remove debugging output from processing DEV comments
+## [0.5.0] - 2021-07-08
+
+- Add pagination for all DEV interactions
+- Dynamically filter what to send to Orbit based on last item of that type in the Orbit workspace
+## [0.5.1] - 2021-07-11
+
+- Fix URL query params path for organization comments

data/Gemfile.lock

@@ -1,36 +1,36 @@
 PATH
   remote: .
   specs:
-    dev_orbit (0.1.3)
+    dev_orbit (0.5.1)
       actionview (~> 6.1)
       activesupport (~> 6.1)
       http (~> 4.4)
       json (~> 2.5)
-      orbit_activities (~> 0.1)
+      orbit_activities (~> 0.2)
       thor (~> 1.1)
       zeitwerk (~> 2.4)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    actionview (6.1.3.2)
-      activesupport (= 6.1.3.2)
+    actionview (6.1.4)
+      activesupport (= 6.1.4)
       builder (~> 3.1)
       erubi (~> 1.4)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.1, >= 1.2.0)
-    activesupport (6.1.3.2)
+    activesupport (6.1.4)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
       tzinfo (~> 2.0)
       zeitwerk (~> 2.3)
-    addressable (2.7.0)
+    addressable (2.8.0)
       public_suffix (>= 2.0.2, < 5.0)
     ast (2.4.2)
     builder (3.2.4)
     byebug (11.1.3)
-    concurrent-ruby (1.1.8)
+    concurrent-ruby (1.1.9)
     crack (0.4.5)
       rexml
     crass (1.0.6)
@@ -38,7 +38,7 @@ GEM
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
     erubi (1.10.0)
-    ffi (1.15.0)
+    ffi (1.15.3)
     ffi-compiler (1.0.1)
       ffi (>= 1.0.0)
       rake
@@ -48,7 +48,7 @@ GEM
       http-cookie (~> 1.0)
       http-form_data (~> 2.2)
       http-parser (~> 1.2.0)
-    http-cookie (1.0.3)
+    http-cookie (1.0.4)
       domain_name (~> 0.5)
     http-form_data (2.3.0)
     http-parser (1.2.3)
@@ -56,27 +56,19 @@ GEM
     i18n (1.8.10)
       concurrent-ruby (~> 1.0)
     json (2.5.1)
-    loofah (2.9.1)
+    loofah (2.10.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
     minitest (5.14.4)
-    nokogiri (1.11.5-arm64-darwin)
+    nokogiri (1.11.7-arm64-darwin)
       racc (~> 1.4)
-    nokogiri (1.11.5-x86_64-darwin)
-      racc (~> 1.4)
-    nokogiri (1.11.5-x86_64-linux)
-      racc (~> 1.4)
-    nokogiri (1.11.3-x86_64-darwin)
-      racc (~> 1.4)
-    nokogiri (1.11.3-x86_64-linux)
-      racc (~> 1.4)
-    orbit_activities (0.1.0)
+    orbit_activities (0.2.2)
       http (~> 4.4)
       json (~> 2.5)
       rake (~> 13.0)
       zeitwerk (~> 2.4)
     parallel (1.20.1)
-    parser (3.0.0.0)
+    parser (3.0.2.0)
       ast (~> 2.4.1)
     public_suffix (4.0.6)
     racc (1.5.2)
@@ -86,7 +78,7 @@ GEM
     rails-html-sanitizer (1.3.0)
       loofah (~> 2.3)
     rainbow (3.0.0)
-    rake (13.0.3)
+    rake (13.0.6)
     regexp_parser (2.1.1)
     rexml (3.2.5)
     rspec (3.10.0)
@@ -102,17 +94,17 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.10.0)
     rspec-support (3.10.2)
-    rubocop (1.11.0)
+    rubocop (1.18.3)
       parallel (~> 1.10)
       parser (>= 3.0.0.0)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml
-      rubocop-ast (>= 1.2.0, < 2.0)
+      rubocop-ast (>= 1.7.0, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 3.0)
-    rubocop-ast (1.4.1)
-      parser (>= 2.7.1.5)
+    rubocop-ast (1.7.0)
+      parser (>= 3.0.1.1)
     ruby-progressbar (1.11.0)
     thor (1.1.0)
     tzinfo (2.0.4)
@@ -121,7 +113,7 @@ GEM
       unf_ext
     unf_ext (0.0.7.7)
     unicode-display_width (2.0.0)
-    webmock (3.12.1)
+    webmock (3.13.0)
       addressable (>= 2.3.6)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
@@ -129,8 +121,6 @@ GEM
 
 PLATFORMS
   arm64-darwin-20
-  x86_64-darwin-19
-  x86_64-linux
 
 DEPENDENCIES
   byebug

data/README.md

@@ -54,6 +54,15 @@ client = DevOrbit::Client.new(
 )
 ```
 
+### Performing a Historical Import
+
+By default, the integration will only import comments that are newer than the newest DEV comment in your Orbit workspace. You may want to perform a one-time historical import to fetch all your previous DEV comments and bring them into your Orbit workspace. To do so, instantiate your `client` with the `historical_import` flag:
+
+```ruby
+client = DevOrbit::Client.new(
+  historical_import: true
+)
+```
 ### Post New DEV Comments from a DEV User to Orbit Workspace
 
 You can use the gem to get new DEV comments on your DEV user by invoking the `#comments` method on your `client` instance:
@@ -74,7 +83,6 @@ client.organization_comments
 
 This method will fetch all your organization articles from DEV, gather their respective comments, filter them for anything within the past day, and then send them to your Orbit workspace via the Orbit API.
 
-
 ### Post New DEV Followers to Orbit Workspace
 
 You can use the gem to get new DEV followers by invoking the `#followers` method on your `client` instance:
@@ -106,6 +114,7 @@ $ ORBIT_API_KEY='...' ORBIT_WORKSPACE='...' DEV_API_KEY='...' DEV_USERNAME='...'
 ```bash
 $ ORBIT_API_KEY='...' ORBIT_WORKSPACE='...' DEV_API_KEY='...' DEV_ORGANIZATION='...' bundle exec dev_orbit --check-organization-comments
 ```
+**Add the `--historical-import` flag to your CLI command to perform a historical import of all your DEV comments using the CLI.**
 
 ## Contributing
 

data/bin/dev_orbit

@@ -4,6 +4,7 @@ require 'optparse'
 check_comments = false
 check_followers = false
 check_org_comments = false
+historical_import = false
 
 options = {}
 choices = OptionParser.new do |opts|
@@ -21,6 +22,9 @@ choices = OptionParser.new do |opts|
   opts.on("--check-organization-comments", "Check for new DEV comments for an organization") do
     check_org_comments = true
   end
+  opts.on("--historical-import", "Performa historical import of all new DEV comments") do
+    historical_import = true
+  end
 end.parse!
 
 $LOAD_PATH.unshift(File.expand_path('../lib/dev_orbit', __dir__))
@@ -33,6 +37,7 @@ require_relative '../scripts/check_org_comments'
 if check_comments
   puts "Checking for new DEV comments within the past day and posting them to your Orbit workspace..."
   ARGV[0] = 'render'
+  ARGV[1] = historical_import
   DevOrbit::Scripts::CheckComments.start(ARGV)
 end
 
@@ -45,5 +50,6 @@ end
 if check_org_comments
   puts "Checking for new DEV organization comments and posting them to your Orbit workspace..."
   ARGV[0] = 'render'
+  ARGV[1] = historical_import
   DevOrbit::Scripts::CheckOrgComments.start(ARGV)
 end

data/dev_orbit.gemspec

@@ -35,7 +35,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "json", "~> 2.5"
   spec.add_dependency "zeitwerk", "~> 2.4"
   spec.add_dependency "thor", "~> 1.1"
-  spec.add_dependency "orbit_activities", "~> 0.1"
+  spec.add_dependency "orbit_activities", "~> 0.2"
   spec.add_development_dependency "rspec", "~> 3.4"
   spec.add_development_dependency "webmock", "~> 3.12"
 

data/lib/dev_orbit/client.rb

@@ -22,13 +22,17 @@
 # @option params [String] :orbit_api_key
 #   The API key for the Orbit API
 #
+# @option params [Boolean] :historical_import
+#   Perform a historical import on all DEV comments
+#   Boolean, default to false
+#
 # @param [Hash] params
 #
 # @return [DevOrbit::Client]
 #
 module DevOrbit
   class Client
-    attr_accessor :dev_username, :dev_api_key, :dev_organization, :orbit_workspace, :orbit_api_key
+    attr_accessor :dev_username, :dev_api_key, :dev_organization, :orbit_workspace, :orbit_api_key, :historical_import
 
     def initialize(params = {})
       @orbit_api_key = params.fetch(:orbit_api_key, ENV["ORBIT_API_KEY"])
@@ -36,6 +40,7 @@ module DevOrbit
       @dev_api_key = params.fetch(:dev_api_key, ENV["DEV_API_KEY"])
       @dev_username = params.fetch(:dev_username, ENV["DEV_USERNAME"])
       @dev_organization = params.fetch(:dev_organization, ENV["DEV_ORGANIZATION"])
+      @historical_import = params.fetch(:historical_import, false)
     end
 
     # Fetch new comments from DEV and post them to the Orbit workspace
@@ -44,7 +49,8 @@ module DevOrbit
         api_key: @dev_api_key,
         username: @dev_username,
         workspace_id: @orbit_workspace,
-        orbit_api_key: @orbit_api_key
+        orbit_api_key: @orbit_api_key,
+        historical_import: @historical_import
       ).process_comments(type: type)
     end
 

data/lib/dev_orbit/dev.rb

@@ -13,17 +13,20 @@ module DevOrbit
       @api_key = params.fetch(:api_key, ENV["DEV_API_KEY"])
       @workspace_id = params.fetch(:workspace_id, ENV["ORBIT_WORKSPACE_ID"])
       @orbit_api_key = params.fetch(:orbit_api_key, ENV["ORBIT_API_KEY"])
+      @historical_import = params.fetch(:historical_import, false)
     end
 
     def process_comments(type:)
       articles = get_articles(type: type)
 
+      return if articles.empty? || articles.nil?
+
       articles.each do |article|
         comments = get_article_comments(article["id"])
 
         next if comments.nil? || comments.empty?
 
-        DevOrbit::Orbit.call(
+        return DevOrbit::Orbit.new(
           type: "comments",
           data: {
             comments: comments,
@@ -31,60 +34,91 @@ module DevOrbit
             url: article["url"]
           },
           workspace_id: @workspace_id,
-          api_key: @orbit_api_key
-        )
+          api_key: @orbit_api_key,
+          historical_import: @historical_import
+        ).call
       end
     end
 
     def process_followers
       followers = get_followers
 
-      followers.each do |follower|
-        next if follower.nil? || follower.empty?
-
-        DevOrbit::Orbit.call(
-          type: "followers",
-          data: {
-            follower: follower
-          },
-          workspace_id: @workspace_id,
-          api_key: @orbit_api_key
-        )
-      end
+      return if followers.empty? || followers.nil?
+
+      DevOrbit::Orbit.new(
+        type: "followers",
+        data: {
+          followers: followers
+        },
+        workspace_id: @workspace_id,
+        api_key: @orbit_api_key,
+        historical_import: @historical_import,
+        last_orbit_member_timestamp: last_orbit_member_timestamp
+      ).call
     end
 
     private
 
     def get_articles(type:)
-      if type == "user"
-        url = URI("https://dev.to/api/articles?username=#{@username}&top=1")
-      end
+      page = 1
+      articles = []
+      looped_at_least_once = false
 
-      if type == "organization"
-        url = URI("https://dev.to/api/organizations/#{@organization}/articles")
-      end
+      while page >= 1
+        page += 1 if looped_at_least_once
+        url = URI("https://dev.to/api/articles?username=#{@username}&page=#{page}&per_page=1000") if type == "user"
 
-      https = Net::HTTP.new(url.host, url.port)
-      https.use_ssl = true
+        if type == "organization"
+          url = URI("https://dev.to/api/organizations/#{@organization}/articles?page=#{page}&per_page=1000")
+        end
 
-      request = Net::HTTP::Get.new(url)
+        https = Net::HTTP.new(url.host, url.port)
+        https.use_ssl = true
 
-      response = https.request(request)
+        request = Net::HTTP::Get.new(url)
+
+        response = https.request(request)
 
-      JSON.parse(response.body) if DevOrbit::Utils.valid_json?(response.body)
+        break if response.code == "404"
+
+        response = JSON.parse(response.body) if DevOrbit::Utils.valid_json?(response.body)
+
+        articles << response unless response.empty? || response.nil?
+        looped_at_least_once = true
+
+        break if response.empty? || response.nil?
+      end
+
+      articles.flatten!
     end
 
     def get_followers
-      url = URI("https://dev.to/api/followers/users")
-      https = Net::HTTP.new(url.host, url.port)
-      https.use_ssl = true
+      page = 1
+      followers = []
+      looped_at_least_once = false
 
-      request = Net::HTTP::Get.new(url)
-      request["api_key"] = @api_key
+      while page >= 1
+        page += 1 if looped_at_least_once
+        url = URI("https://dev.to/api/followers/users?page=#{page}&per_page=1000")
+        https = Net::HTTP.new(url.host, url.port)
+        https.use_ssl = true
 
-      response = https.request(request)
+        request = Net::HTTP::Get.new(url)
+        request["api_key"] = @api_key
+
+        response = https.request(request)
+
+        break if response.code == "404"
+
+        response = JSON.parse(response.body) if DevOrbit::Utils.valid_json?(response.body)
 
-      JSON.parse(response.body) if DevOrbit::Utils.valid_json?(response.body)
+        followers << response unless response.empty? || response.nil?
+        looped_at_least_once = true
+
+        break if response.empty? || response.nil?
+      end
+
+      followers.flatten!
     end
 
     def get_article_comments(id)
@@ -100,12 +134,27 @@ module DevOrbit
 
       return if comments.nil? || comments.empty?
 
-      filter_comments(comments)
+      comments
     end
 
-    def filter_comments(comments)
-      comments.select do |comment|
-        comment["created_at"] <= 1.day.ago
+    def last_orbit_member_timestamp
+      @last_orbit_member_timestamp ||= begin
+        url = URI("https://app.orbit.love/api/v1/#{@workspace_id}/members?direction=DESC&items=10&identity=devto&sort=created_at")
+
+        http = Net::HTTP.new(url.host, url.port)
+        http.use_ssl = true
+
+        request = Net::HTTP::Get.new(url)
+        request["Accept"] = "application/json"
+        request["Authorization"] = "Bearer #{@orbit_api_key}"
+        request["User-Agent"] = "community-ruby-dev-orbit/#{DevOrbit::VERSION}"
+
+        response = http.request(request)
+        response = JSON.parse(response.body)
+
+        return nil if response["data"].nil? || response["data"].empty?
+
+        response["data"][0]["attributes"]["created_at"]
       end
     end
   end

data/lib/dev_orbit/interactions/follower.rb

@@ -43,8 +43,7 @@ module DevOrbit
         {
           member: {
             name: @name.include?("_") ? @name.split("_").map(&:capitalize).join(" ") : @name,
-            devto: @username,
-            url: "https://dev.to#{@url}"
+            devto: @username
           }
         }
       end

data/lib/dev_orbit/orbit.rb

@@ -2,31 +2,101 @@
 
 require "net/http"
 require "json"
+require "active_support/time"
 
 module DevOrbit
   class Orbit
-    def self.call(type:, data:, workspace_id:, api_key:)
-      if type == "comments"
-        data[:comments].each do |comment|
+    def initialize(type:, data:, workspace_id:, api_key:, historical_import: false, last_orbit_member_timestamp: nil)
+      @type = type
+      @data = data
+      @workspace_id = workspace_id
+      @api_key = api_key
+      @historical_import = historical_import
+      @last_orbit_activity_timestamp = last_orbit_activity_timestamp(type)
+      @last_orbit_member_timestamp = last_orbit_member_timestamp
+    end
+    
+    def call
+      if @type == "comments"
+        times = 0
+        
+        @data[:comments].each do |comment|
+          unless @historical_import && @last_orbit_activity_timestamp
+            next if (comment["created_at"] || comment[:created_at]) < @last_orbit_activity_timestamp unless @last_orbit_activity_timestamp.nil? 
+          end
+
+          if @last_orbit_activity_timestamp && @historical_import == false
+            next if (comment["created_at"] || comment[:created_at]) < @last_orbit_activity_timestamp
+          end
+
+          times += 1
+
           DevOrbit::Interactions::Comment.new(
-            article_title: data.transform_keys(&:to_sym)[:title],
+            article_title: @data.transform_keys(&:to_sym)[:title],
             comment: comment.transform_keys(&:to_sym),
-            url: data[:url],
-            workspace_id: workspace_id,
-            api_key: api_key
+            url: @data[:url],
+            workspace_id: @workspace_id,
+            api_key: @api_key
           )
         end
+
+        output = "Sent #{times} new DEV comments to your Orbit workspace"
+        
+        puts output
+        return output
       end
 
-      if type == "followers"
-        DevOrbit::Interactions::Follower.new(
-          id: data[:follower]["id"],
-          name: data[:follower]["name"],
-          username: data[:follower]["username"],
-          url: data[:follower]["path"],
-          workspace_id: workspace_id,
-          api_key: api_key
-        )
+      if @type == "followers"
+        counter = 0
+        @data[:followers].each do |follower|
+          next if follower.nil? || follower.empty?
+
+          unless @historical_import && @last_orbit_member_timestamp
+            next if follower["created_at"] < @last_orbit_member_timestamp unless @last_orbit_member_timestamp.nil?
+          end
+  
+          if @last_orbit_member_timestamp && @historical_import == false
+            next if follower["created_at"] < @last_orbit_member_timestamp
+          end
+
+          sleep(20) if counter % 100 == 0 && counter != 0 # API rate limiting
+
+          DevOrbit::Interactions::Follower.new(
+            id: follower["id"],
+            name: follower["name"],
+            username: follower["username"],
+            url: follower["path"],
+            workspace_id: @workspace_id,
+            api_key: @api_key
+          )
+          counter += 1
+        end
+        output = "Sent #{counter} new followers to Orbit"
+
+        puts output
+        return output
+      end
+    end
+
+    private
+
+    def last_orbit_activity_timestamp(type)
+      @last_orbit_activity_timestamp ||= begin
+        if type == "comments"
+          activity_type = "custom:dev:comment"
+        end
+
+        if type == "followers"
+          return nil
+        end
+
+        OrbitActivities::Request.new(
+          api_key: @api_key,
+          workspace_id: @workspace_id,
+          user_agent: "community-ruby-dev-orbit/#{DevOrbit::VERSION}",
+          action: "latest_activity_timestamp",
+          filters: { activity_type: activity_type }
+        ).response
       end
     end
   end

data/lib/dev_orbit/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DevOrbit
-  VERSION = "0.2.0"
+  VERSION = "0.5.1"
 end

data/scripts/check_comments.rb

@@ -8,8 +8,8 @@ module DevOrbit
   module Scripts
     class CheckComments < Thor
       desc "render", "check for new DEV comments and push them to Orbit"
-      def render
-        client = DevOrbit::Client.new
+      def render(*params)
+        client = DevOrbit::Client.new(historical_import: params[0])
         client.comments
       end
     end

data/scripts/check_org_comments.rb

@@ -8,8 +8,8 @@ module DevOrbit
   module Scripts
     class CheckOrgComments < Thor
       desc "render", "check for new DEV comments on an organization and push them to Orbit"
-      def render
-        client = DevOrbit::Client.new
+      def render(*params)
+        client = DevOrbit::Client.new(historical_import: params[0])
         client.organization_comments
       end
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dev_orbit
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.5.1
 platform: ruby
 authors:
 - Orbit DevRel
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-06-02 00:00:00.000000000 Z
+date: 2021-07-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -101,14 +101,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: '0.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: '0.2'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement