trellodon 0.2.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9931b6c34e6a8fc7c74261de329037838ac5fc098b5436e146f3c6a38068ec09
-  data.tar.gz: a81d60ff2a430f2dd23a7ae7d4ca0ddb958f47e1539ffa16fb9f2ca298ad3301
+  metadata.gz: a61911e92509a77f13aa90f9f56c14d038dbe443b4e1063d28f66b6745b04432
+  data.tar.gz: '079609a773107e786a7d795fe7d23564ded2aa66e8d587a6fd22b4e288e15478'
 SHA512:
-  metadata.gz: 3831a6d818d121e245b6a6f87ab11eb80f58ac7814e4fd10423b59d131a2e3a52234ed724b25aeb5f98de5473a1350d67f330d95faeff1f9e05025ca2e379d0e
-  data.tar.gz: 0adca9ec2bef56ae020c840e912a3a4276c9da1b5eefb66f294215232b994c52e8bb148c896ff292eac683c824a9b9cfbaf6bee3725948523be1375e2dde5f8c
+  metadata.gz: 617119b48c1f2385c87d57ffd2683933097ed1aae11a51577f65e16012ab1f58b65187f57884123ed04ba522797925826e3cf202e7d4dad9d1fd63a3d736bcef
+  data.tar.gz: 30c1f1bcb2df2c69a3c8d0cebc93a597f16d2207d33bef979670134d306bd0173cc97948f38f739cf1d6b5eef0a7a602112fb0de40021d078d0188280ca4d039

data/CHANGELOG.md

@@ -2,6 +2,14 @@
 
 ## [master]
 
+## [0.3.0] - 2022-04-12 🚀🚀🚀
+
+- Handle missing members. ([@palkan][])
+
+- Use inline scheduler by default. ([@palkan][])
+
+- Replace `--sync` with `--concurrency=0`. ([@palkan][])
+
 ## [0.2.1] - 2022-04-12 🚀
 
 - Misc changes.

data/README.md

@@ -31,10 +31,12 @@ To generate them, go to the [trello.com/app-key](https://trello.com/app-key) pag
 ## Usage
 
 ```sh
-Usage: trellodon dump
+Usage: trellodon dump [options]
 Options:
         --board VALUE                Board URL or ID
         --out VALUE                  Destination folder path
+        --concurrency VALUE          Amount of processing threads (default: 4). Set to 0 to execute API requests in-line
+        --clear-auth                 Remove saved api credentials
     -v, --version                    Print current version
     -h, --help                       Print help
 ```

data/lib/{trellodon/api_executor.rb → .rbnext/2.7/trellodon/executor.rb}

@@ -1,10 +1,7 @@
 # frozen_string_literal: true
 
-require "trellodon/api_client"
-require "trellodon/config"
-
 module Trellodon
-  class APIExecutor
+  class Executor
     BOARD_ID_REGEX = /\/b\/([^\/]+)\//
     private_constant :BOARD_ID_REGEX
 
@@ -16,7 +13,7 @@ module Trellodon
       @scheduler = scheduler
       check_credentials!
 
-      @api_client = APIClient.new(api_key: @api_key, api_token: @api_token)
+      @client = Client.new(api_key: @api_key, api_token: @api_token)
     end
 
     def download(board_pointer)
@@ -25,13 +22,13 @@ module Trellodon
       startup_time = Time.now
       logger.debug "Fetching board 🚀️️"
       board = scheduler.post do
-        api_client.fetch_board(board_id).tap { formatter.board_added(_1) }
+        client.fetch_board(board_id).tap { |_1| formatter.board_added(_1) }
       end.value
 
-      logger.debug "Fetching cards in board with comments and attachments 🐢"
+      logger.debug "Fetching #{board.card_ids.size} cards from the board with comments and attachments 🐢"
       board.card_ids.map do |card_id|
         scheduler.post do
-          api_client.fetch_card(card_id).tap { formatter.card_added(_1) }
+          client.fetch_card(card_id).tap { |_1| formatter.card_added(_1) }
         end
       end.map(&:value)
 
@@ -41,7 +38,7 @@ module Trellodon
 
     private
 
-    attr_reader :api_client, :board_id, :logger, :scheduler, :formatter
+    attr_reader :client, :board_id, :logger, :scheduler, :formatter
 
     def check_credentials!
       return if @api_key.to_s.size.positive? && @api_token.to_s.size.positive?

data/lib/.rbnext/3.0/trellodon/client.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+require "trello"
+require "trellodon/entities"
+
+module Trellodon
+  class Client
+    class DeletedMember < Struct.new(:id)
+      def full_name ;  "[DELETED]"; end
+
+      def username ;  "__deleted__"; end
+    end
+
+    def initialize(api_key:, api_token:, logger: Config.logger)
+      @logger = logger
+      @client = Trello::Client.new(
+        developer_public_key: api_key,
+        member_token: api_token
+      )
+      @members = {}
+    end
+
+    def fetch_board(board_id)
+      retrying do
+        board = @client.find(:boards, board_id)
+
+        Board.new(
+          id: board.id,
+          short_id: board.short_url.chars.last(8).join,
+          name: board.name,
+          lists: lists_from(board),
+          card_ids: @client.find_many(Trello::Card, "/boards/#{board_id}/cards", fields: "id").map(&:id),
+          last_activity_date: board.last_activity_date
+        )
+      end
+    end
+
+    def fetch_card(card_id)
+      retrying do
+        card = @client.find(:cards, card_id)
+
+        Card.new(
+          id: card.id,
+          short_id: card.short_url.chars.last(8).join,
+          name: card.name,
+          desc: card.desc,
+          labels: card.labels.map(&:name),
+          list_id: card.list_id,
+          comments: comments_from(card),
+          attachments: attachments_from(card),
+          checklists: checklists_from(card),
+          last_activity_date: card.last_activity_date
+        )
+      end
+    end
+
+    private
+
+    attr_reader :logger
+
+    def retrying(&block)
+      attempt = 0
+      begin
+        block.call
+      rescue Trello::Error => err
+        raise unless err.status_code == 429
+        attempt += 1
+        cooldown = 2**attempt + rand(2**attempt) - 2**(attempt - 1)
+        logger.warn "API limit exceeded, cool down for #{cooldown}s"
+        sleep cooldown
+        retry
+      end
+    end
+
+    def comments_from(card)
+      card.comments.map do |comment|
+        Comment.new(
+          text: comment.data["text"],
+          date: comment.date,
+          creator: member_from(comment)
+        )
+      end
+    end
+
+    def attachments_from(card)
+      card.attachments.map do |attach|
+        Attachment.new(
+          file_name: attach.file_name,
+          mime_type: attach.mime_type,
+          bytes: attach.bytes,
+          date: attach.date,
+          name: attach.name,
+          is_upload: attach.is_upload,
+          headers: attach.is_upload ? {"Authorization" => "OAuth oauth_consumer_key=\"#{@client.configuration.developer_public_key}\", oauth_token=\"#{@client.configuration.member_token}\""} : {},
+          url: attach.url
+        )
+      end
+    end
+
+    def lists_from(board)
+      board.lists.map do |list|
+        List.new(
+          id: list.id,
+          name: list.name,
+          short_id: board.short_url.chars.last(8).join
+        )
+      end
+    end
+
+    def checklists_from(card)
+      card.checklists.map do |checklist|
+        Checklist.new(
+          id: checklist.id,
+          name: checklist.name,
+          items: checklist_items_from(checklist)
+        )
+      end
+    end
+
+    def checklist_items_from(checklist)
+      checklist.items.map do |item|
+        ChecklistItem.new(
+          id: item.id,
+          name: item.name,
+          state: item.state
+        )
+      end
+    end
+
+    def member_from(comment)
+      return @members[comment.creator_id] if @members.has_key? comment.creator_id
+
+      member =
+        begin
+          @client.find(:members, comment.creator_id)
+        rescue Trello::Error => err
+          raise unless err.status_code == 404
+
+          DeletedMember.new(comment.creator_id)
+        end
+
+      @members[comment.creator_id] = Member.new(
+        id: member.id,
+        full_name: member.full_name,
+        username: member.username
+      )
+    end
+  end
+end

data/lib/.rbnext/3.1/trellodon/schedulers/concurrent.rb

@@ -8,7 +8,7 @@ module Trellodon
     class Concurrent < Base
       using RubyNext
 
-      DEFAULT_MAX_THREADS = 50
+      DEFAULT_MAX_THREADS = 4
 
       attr_reader :max_threads
 

data/lib/trellodon/cli.rb

@@ -46,8 +46,7 @@ module Trellodon
         opts.banner = "Usage: trellodon dump \e[34m[options]\e[0m\n"\
                       "Options:\n"
 
-        opts.on("--sync", "Process API requests synchronously")
-        opts.on("--concurrency VALUE", "Amount of processing threads (default: 50)")
+        opts.on("--concurrency VALUE", "Amount of processing threads (default: 4)")
         opts.on("--clear-auth", "Remove saved api credentials") do
           config.reload(api_token: nil, api_key: nil)
         end
@@ -103,7 +102,7 @@ module Trellodon
 
     def initialize
       @config = Config.new
-      @scheduler = Schedulers::Concurrent.new
+      @scheduler = Schedulers::Inline.new
     end
 
     def run
@@ -111,15 +110,16 @@ module Trellodon
 
       need_to_fill_config = config.to_h[:api_key].nil?
 
-      executor = Trellodon::APIExecutor.new(
+      executor = Trellodon::Executor.new(
         api_key: config.api_key,
         api_token: config.api_token,
         formatter: Trellodon::Formatters::Markdown.new(output_dir: config.out),
         scheduler: scheduler
       )
-      executor.download(config.board)
 
       ask_save_credentials if need_to_fill_config
+
+      executor.download(config.board)
     end
 
     private
@@ -152,14 +152,12 @@ module Trellodon
     end
 
     def change_scheduler(params)
-      if params.key?(:sync)
-        @scheduler = Schedulers::Inline.new
-        return
-      end
-
       return unless params[:concurrency]
 
-      threads = params[:concurrency].to_i
+      threads = Integer(params[:concurrency])
+
+      return @scheduler = Schedulers::Inline.new if threads.zero?
+
       raise "Value of concurrency option must be integer and greater than zero" unless threads.positive?
 
       @scheduler = Schedulers::Concurrent.new(threads)

data/lib/trellodon/{api_client.rb → client.rb}

@@ -4,7 +4,13 @@ require "trello"
 require "trellodon/entities"
 
 module Trellodon
-  class APIClient
+  class Client
+    class DeletedMember < Struct.new(:id)
+      def full_name = "[DELETED]"
+
+      def username = "__deleted__"
+    end
+
     def initialize(api_key:, api_token:, logger: Config.logger)
       @logger = logger
       @client = Trello::Client.new(
@@ -124,7 +130,15 @@ module Trellodon
     def member_from(comment)
       return @members[comment.creator_id] if @members.has_key? comment.creator_id
 
-      member = @client.find(:members, comment.creator_id)
+      member =
+        begin
+          @client.find(:members, comment.creator_id)
+        rescue Trello::Error => err
+          raise unless err.status_code == 404
+
+          DeletedMember.new(comment.creator_id)
+        end
+
       @members[comment.creator_id] = Member.new(
         id: member.id,
         full_name: member.full_name,

data/lib/{.rbnext/2.7/trellodon/api_executor.rb → trellodon/executor.rb}

@@ -1,10 +1,7 @@
 # frozen_string_literal: true
 
-require "trellodon/api_client"
-require "trellodon/config"
-
 module Trellodon
-  class APIExecutor
+  class Executor
     BOARD_ID_REGEX = /\/b\/([^\/]+)\//
     private_constant :BOARD_ID_REGEX
 
@@ -16,7 +13,7 @@ module Trellodon
       @scheduler = scheduler
       check_credentials!
 
-      @api_client = APIClient.new(api_key: @api_key, api_token: @api_token)
+      @client = Client.new(api_key: @api_key, api_token: @api_token)
     end
 
     def download(board_pointer)
@@ -25,13 +22,13 @@ module Trellodon
       startup_time = Time.now
       logger.debug "Fetching board 🚀️️"
       board = scheduler.post do
-        api_client.fetch_board(board_id).tap { |_1| formatter.board_added(_1) }
+        client.fetch_board(board_id).tap { formatter.board_added(_1) }
       end.value
 
-      logger.debug "Fetching cards in board with comments and attachments 🐢"
+      logger.debug "Fetching #{board.card_ids.size} cards from the board with comments and attachments 🐢"
       board.card_ids.map do |card_id|
         scheduler.post do
-          api_client.fetch_card(card_id).tap { |_1| formatter.card_added(_1) }
+          client.fetch_card(card_id).tap { formatter.card_added(_1) }
         end
       end.map(&:value)
 
@@ -41,7 +38,7 @@ module Trellodon
 
     private
 
-    attr_reader :api_client, :board_id, :logger, :scheduler, :formatter
+    attr_reader :client, :board_id, :logger, :scheduler, :formatter
 
     def check_credentials!
       return if @api_key.to_s.size.positive? && @api_token.to_s.size.positive?

data/lib/trellodon/schedulers/concurrent.rb

@@ -8,7 +8,7 @@ module Trellodon
     class Concurrent < Base
       using RubyNext
 
-      DEFAULT_MAX_THREADS = 50
+      DEFAULT_MAX_THREADS = 4
 
       attr_reader :max_threads
 

data/lib/trellodon/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Trellodon
-  VERSION = "0.2.1"
+  VERSION = "0.3.0"
 end

data/lib/trellodon.rb

@@ -7,7 +7,8 @@ RubyNext::Language.setup_gem_load_path(transpile: true)
 
 require "trellodon/version"
 require "trellodon/config"
-require "trellodon/api_executor"
+require "trellodon/client"
+require "trellodon/executor"
 
 require "trellodon/entities"
 require "trellodon/schedulers"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: trellodon
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.0
 platform: ruby
 authors:
 - fargelus
@@ -100,12 +100,12 @@ files:
 - bin/console
 - bin/setup
 - bin/trellodon
-- lib/.rbnext/2.7/trellodon/api_executor.rb
+- lib/.rbnext/2.7/trellodon/executor.rb
+- lib/.rbnext/3.0/trellodon/client.rb
 - lib/.rbnext/3.1/trellodon/schedulers/concurrent.rb
 - lib/trellodon.rb
-- lib/trellodon/api_client.rb
-- lib/trellodon/api_executor.rb
 - lib/trellodon/cli.rb
+- lib/trellodon/client.rb
 - lib/trellodon/config.rb
 - lib/trellodon/entities.rb
 - lib/trellodon/entities/attachment.rb
@@ -116,6 +116,7 @@ files:
 - lib/trellodon/entities/comment.rb
 - lib/trellodon/entities/list.rb
 - lib/trellodon/entities/member.rb
+- lib/trellodon/executor.rb
 - lib/trellodon/formatters.rb
 - lib/trellodon/formatters/base.rb
 - lib/trellodon/formatters/markdown.rb