trellodon 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a61911e92509a77f13aa90f9f56c14d038dbe443b4e1063d28f66b6745b04432
-  data.tar.gz: '079609a773107e786a7d795fe7d23564ded2aa66e8d587a6fd22b4e288e15478'
+  metadata.gz: 7210dc1a4a18474871607069363ab1d1acdfeb283588ef52403a38699e24bebb
+  data.tar.gz: 465aa0681c0c9e2a3744a94ad06b17e8eaf35bfdc90204e36d6aa35359765134
 SHA512:
-  metadata.gz: 617119b48c1f2385c87d57ffd2683933097ed1aae11a51577f65e16012ab1f58b65187f57884123ed04ba522797925826e3cf202e7d4dad9d1fd63a3d736bcef
-  data.tar.gz: 30c1f1bcb2df2c69a3c8d0cebc93a597f16d2207d33bef979670134d306bd0173cc97948f38f739cf1d6b5eef0a7a602112fb0de40021d078d0188280ca4d039
+  metadata.gz: 3895577372463c4d917c0c91f557789d298d5183263b72475bf213a44bb4d17deefac8e1ce5eb13b65b93540ae8442dfb43a4123d255abbe9a403700307353dc
+  data.tar.gz: d7e099a0bdb52d541152522f317e3b66ed89aa0128639be8731bb0c99c677e26b815bf1d6d1b61754bfa72998df7b5649967b8a28c23c05d5d561abd775b841c

data/CHANGELOG.md

@@ -2,6 +2,12 @@
 
 ## [master]
 
+## [0.4.0] - 2022-04-27 🗂️
+
+- Resolve folder name collisions in markdown formatter ([@rinasergeeva][])
+- Add user info to attachments and checklists ([@rinasergeeva][])
+- Verbose option in cli. ([@fargelus][])
+
 ## [0.3.0] - 2022-04-12 🚀🚀🚀
 
 - Handle missing members. ([@palkan][])
@@ -29,3 +35,4 @@
 
 [@fargelus]: https://github.com/fargelus
 [@rinasergeeva]: https://github.com/rinasergeeva
+[@palkan]: https://github.com/palkan

data/README.md

@@ -9,6 +9,7 @@ A Ruby tool to export a Trello board and convert it into a set of folders and ma
 * Description
 * Labels
 * Comments
+* Checklists
 * Attachments
 
 <img src="https://evilmartians.com/badges/sponsored-by-evil-martians.svg" alt="Sponsored by Evil Martians" width="236" height="54"></a>
@@ -30,6 +31,14 @@ To generate them, go to the [trello.com/app-key](https://trello.com/app-key) pag
 
 ## Usage
 
+Copy the board URL and run the following command:
+
+```sh
+trellodon dump --board=<url>
+```
+
+Other supported options:
+
 ```sh
 Usage: trellodon dump [options]
 Options:
@@ -37,7 +46,8 @@ Options:
         --out VALUE                  Destination folder path
         --concurrency VALUE          Amount of processing threads (default: 4). Set to 0 to execute API requests in-line
         --clear-auth                 Remove saved api credentials
-    -v, --version                    Print current version
+    -v, --verbose                    Print all processes output
+    -V, --version                    Print current version
     -h, --help                       Print help
 ```
 
@@ -90,3 +100,9 @@ Some card description
 **date**: 2022-03-16
 **url**: https://trello.com/1/cards/#{card_id}/attachments/#{attachment.id}/download/image.png
 ```
+
+### Trello API Limits 🚧
+
+Trello API limits the total amount of requests for each API Key & token.
+By default, 300 requests are available per 10 seconds for each API key and no more than 100 requests per 10 second interval for each token.
+If a request exceeds the limit, Trello will return a 429 error. Additional information see [here](https://help.trello.com/article/838-api-rate-limits).

data/lib/.rbnext/2.7/trellodon/executor.rb

@@ -5,12 +5,12 @@ module Trellodon
     BOARD_ID_REGEX = /\/b\/([^\/]+)\//
     private_constant :BOARD_ID_REGEX
 
-    def initialize(api_key:, api_token:, scheduler:, formatter:, logger: Config.logger)
-      @api_key = api_key
-      @api_token = api_token
-      @formatter = formatter
-      @logger = logger
-      @scheduler = scheduler
+    def initialize(opts)
+      @api_key = opts.fetch(:api_key)
+      @api_token = opts.fetch(:api_token)
+      @formatter = opts.fetch(:formatter)
+      @logger = opts.fetch(:logger) { Config.logger }
+      @scheduler = opts.fetch(:scheduler)
       check_credentials!
 
       @client = Client.new(api_key: @api_key, api_token: @api_token)

data/lib/.rbnext/3.0/trellodon/client.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "uri"
 require "trello"
 require "trellodon/entities"
 
@@ -17,7 +18,7 @@ module Trellodon
         developer_public_key: api_key,
         member_token: api_token
       )
-      @members = {}
+      @members = Concurrent::Map.new
     end
 
     def fetch_board(board_id)
@@ -26,7 +27,6 @@ module Trellodon
 
         Board.new(
           id: board.id,
-          short_id: board.short_url.chars.last(8).join,
           name: board.name,
           lists: lists_from(board),
           card_ids: @client.find_many(Trello::Card, "/boards/#{board_id}/cards", fields: "id").map(&:id),
@@ -41,7 +41,6 @@ module Trellodon
 
         Card.new(
           id: card.id,
-          short_id: card.short_url.chars.last(8).join,
           name: card.name,
           desc: card.desc,
           labels: card.labels.map(&:name),
@@ -77,7 +76,7 @@ module Trellodon
         Comment.new(
           text: comment.data["text"],
           date: comment.date,
-          creator: member_from(comment)
+          creator: member_from(comment.creator_id)
         )
       end
     end
@@ -85,12 +84,13 @@ module Trellodon
     def attachments_from(card)
       card.attachments.map do |attach|
         Attachment.new(
-          file_name: attach.file_name,
+          file_name: attach.is_upload ? URI.decode_www_form_component(attach.file_name) : attach.file_name,
           mime_type: attach.mime_type,
           bytes: attach.bytes,
           date: attach.date,
           name: attach.name,
           is_upload: attach.is_upload,
+          added_by: member_from(attach.member_id),
           headers: attach.is_upload ? {"Authorization" => "OAuth oauth_consumer_key=\"#{@client.configuration.developer_public_key}\", oauth_token=\"#{@client.configuration.member_token}\""} : {},
           url: attach.url
         )
@@ -102,7 +102,7 @@ module Trellodon
         List.new(
           id: list.id,
           name: list.name,
-          short_id: board.short_url.chars.last(8).join
+          board_id: list.board_id
         )
       end
     end
@@ -122,28 +122,31 @@ module Trellodon
         ChecklistItem.new(
           id: item.id,
           name: item.name,
-          state: item.state
+          state: item.state,
+          member: member_from(item.member_id),
+          due_date: item.due_date
         )
       end
     end
 
-    def member_from(comment)
-      return @members[comment.creator_id] if @members.has_key? comment.creator_id
-
-      member =
-        begin
-          @client.find(:members, comment.creator_id)
-        rescue Trello::Error => err
-          raise unless err.status_code == 404
-
-          DeletedMember.new(comment.creator_id)
-        end
-
-      @members[comment.creator_id] = Member.new(
-        id: member.id,
-        full_name: member.full_name,
-        username: member.username
-      )
+    def member_from(id)
+      return nil unless id
+
+      @members.compute_if_absent(id) do
+        member =
+          begin
+            @client.find(:members, id)
+          rescue Trello::Error => err
+            raise unless err.status_code == 404
+
+            DeletedMember.new(id)
+          end
+        Member.new(
+          id: member.id,
+          full_name: member.full_name,
+          username: member.username
+        )
+      end
     end
   end
 end

data/lib/trellodon/cli.rb

@@ -50,8 +50,9 @@ module Trellodon
         opts.on("--clear-auth", "Remove saved api credentials") do
           config.reload(api_token: nil, api_key: nil)
         end
+        opts.on("-v", "--verbose", "Print all processes output")
 
-        opts.on_tail("-v", "--version", "Print current version") do
+        opts.on_tail("-V", "--version", "Print current version") do
           puts Trellodon::VERSION
           exit
         end
@@ -103,28 +104,25 @@ module Trellodon
     def initialize
       @config = Config.new
       @scheduler = Schedulers::Inline.new
+      @logger = Config.logger
     end
 
     def run
       check_command!
 
       need_to_fill_config = config.to_h[:api_key].nil?
-
-      executor = Trellodon::Executor.new(
-        api_key: config.api_key,
-        api_token: config.api_token,
-        formatter: Trellodon::Formatters::Markdown.new(output_dir: config.out),
-        scheduler: scheduler
-      )
+      executor = Trellodon::Executor.new(executor_options)
 
       ask_save_credentials if need_to_fill_config
 
       executor.download(config.board)
     end
 
+    attr_reader :scheduler, :logger
+
     private
 
-    attr_reader :config, :scheduler
+    attr_reader :config
 
     def prompt
       Prompt.instance
@@ -139,13 +137,24 @@ module Trellodon
       end
 
       if command == "dump"
-        change_scheduler(params) if params.key?(:sync) || params.key?(:concurrency)
+        change_scheduler(params)
+        @logger.level = 0 if params.key?(:verbose)
         return
       end
 
       raise "Unknown command: #{command}. Available commands: dump"
     end
 
+    def executor_options
+      {
+        api_key: config.api_key,
+        api_token: config.api_token,
+        formatter: Trellodon::Formatters::Markdown.new(output_dir: config.out, logger: logger),
+        scheduler: scheduler,
+        logger: logger
+      }
+    end
+
     def ask_save_credentials
       answer = prompt.yes?("Would you like to save api creds so next time trellodon won't asking it?", default: true)
       config.update_config_file! if answer

data/lib/trellodon/client.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "uri"
 require "trello"
 require "trellodon/entities"
 
@@ -17,7 +18,7 @@ module Trellodon
         developer_public_key: api_key,
         member_token: api_token
       )
-      @members = {}
+      @members = Concurrent::Map.new
     end
 
     def fetch_board(board_id)
@@ -26,7 +27,6 @@ module Trellodon
 
         Board.new(
           id: board.id,
-          short_id: board.short_url.chars.last(8).join,
           name: board.name,
           lists: lists_from(board),
           card_ids: @client.find_many(Trello::Card, "/boards/#{board_id}/cards", fields: "id").map(&:id),
@@ -41,7 +41,6 @@ module Trellodon
 
         Card.new(
           id: card.id,
-          short_id: card.short_url.chars.last(8).join,
           name: card.name,
           desc: card.desc,
           labels: card.labels.map(&:name),
@@ -77,7 +76,7 @@ module Trellodon
         Comment.new(
           text: comment.data["text"],
           date: comment.date,
-          creator: member_from(comment)
+          creator: member_from(comment.creator_id)
         )
       end
     end
@@ -85,12 +84,13 @@ module Trellodon
     def attachments_from(card)
       card.attachments.map do |attach|
         Attachment.new(
-          file_name: attach.file_name,
+          file_name: attach.is_upload ? URI.decode_www_form_component(attach.file_name) : attach.file_name,
           mime_type: attach.mime_type,
           bytes: attach.bytes,
           date: attach.date,
           name: attach.name,
           is_upload: attach.is_upload,
+          added_by: member_from(attach.member_id),
           headers: attach.is_upload ? {"Authorization" => "OAuth oauth_consumer_key=\"#{@client.configuration.developer_public_key}\", oauth_token=\"#{@client.configuration.member_token}\""} : {},
           url: attach.url
         )
@@ -102,7 +102,7 @@ module Trellodon
         List.new(
           id: list.id,
           name: list.name,
-          short_id: board.short_url.chars.last(8).join
+          board_id: list.board_id
         )
       end
     end
@@ -122,28 +122,31 @@ module Trellodon
         ChecklistItem.new(
           id: item.id,
           name: item.name,
-          state: item.state
+          state: item.state,
+          member: member_from(item.member_id),
+          due_date: item.due_date
         )
       end
     end
 
-    def member_from(comment)
-      return @members[comment.creator_id] if @members.has_key? comment.creator_id
-
-      member =
-        begin
-          @client.find(:members, comment.creator_id)
-        rescue Trello::Error => err
-          raise unless err.status_code == 404
-
-          DeletedMember.new(comment.creator_id)
-        end
-
-      @members[comment.creator_id] = Member.new(
-        id: member.id,
-        full_name: member.full_name,
-        username: member.username
-      )
+    def member_from(id)
+      return nil unless id
+
+      @members.compute_if_absent(id) do
+        member =
+          begin
+            @client.find(:members, id)
+          rescue Trello::Error => err
+            raise unless err.status_code == 404
+
+            DeletedMember.new(id)
+          end
+        Member.new(
+          id: member.id,
+          full_name: member.full_name,
+          username: member.username
+        )
+      end
     end
   end
 end

data/lib/trellodon/config.rb

@@ -10,7 +10,7 @@ module Trellodon
     attr_config :api_token, :api_key
 
     def self.logger
-      Logger.new($stdout)
+      Logger.new($stdout, level: Logger::INFO)
     end
   end
 end

data/lib/trellodon/entities/attachment.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Trellodon
-  Attachment = Struct.new("Attachment", :file_name, :mime_type, :bytes, :date, :name, :is_upload, :headers, :url, keyword_init: true)
+  Attachment = Struct.new("Attachment", :file_name, :mime_type, :bytes, :date, :name, :is_upload, :headers, :url, :added_by, keyword_init: true)
 end

data/lib/trellodon/entities/board.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Trellodon
-  Board = Struct.new("Board", :id, :short_id, :name, :card_ids, :lists, :last_activity_date, keyword_init: true) do
+  Board = Struct.new("Board", :id, :name, :card_ids, :lists, :last_activity_date, keyword_init: true) do
     def get_list(list_id)
       return nil if lists.nil?
       lists.find { |list| list.id == list_id }

data/lib/trellodon/entities/card.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Trellodon
-  Card = Struct.new("Card", :id, :short_id, :name, :desc, :list_id, :labels, :comments, :attachments, :checklists, :last_activity_date, keyword_init: true)
+  Card = Struct.new("Card", :id, :name, :desc, :list_id, :labels, :comments, :attachments, :checklists, :last_activity_date, keyword_init: true)
 end

data/lib/trellodon/entities/card_event.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module Trellodon
+  CREATE_CARD_TYPE = "createCard"
+  CardEvent = Struct.new(:type, :data, :card, :list, :board, :date, keyword_init: true)
+end

data/lib/trellodon/entities/checklist_item.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Trellodon
-  ChecklistItem = Struct.new("ChecklistItem", :id, :name, :state, keyword_init: true) do
+  ChecklistItem = Struct.new("ChecklistItem", :id, :name, :state, :member, :due_date, keyword_init: true) do
     def checked?
       state == "complete"
     end

data/lib/trellodon/entities/list.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Trellodon
-  List = Struct.new("List", :id, :short_id, :name, keyword_init: true)
+  List = Struct.new("List", :id, :board_id, :name, keyword_init: true)
 end

data/lib/trellodon/executor.rb

@@ -5,12 +5,12 @@ module Trellodon
     BOARD_ID_REGEX = /\/b\/([^\/]+)\//
     private_constant :BOARD_ID_REGEX
 
-    def initialize(api_key:, api_token:, scheduler:, formatter:, logger: Config.logger)
-      @api_key = api_key
-      @api_token = api_token
-      @formatter = formatter
-      @logger = logger
-      @scheduler = scheduler
+    def initialize(opts)
+      @api_key = opts.fetch(:api_key)
+      @api_token = opts.fetch(:api_token)
+      @formatter = opts.fetch(:formatter)
+      @logger = opts.fetch(:logger) { Config.logger }
+      @scheduler = opts.fetch(:scheduler)
       check_credentials!
 
       @client = Client.new(api_key: @api_key, api_token: @api_token)

data/lib/trellodon/formatters/markdown.rb

@@ -11,10 +11,16 @@ module Trellodon
 
       MD_FILENAME = "README.md"
       ATTACHMENTS_DIRNAME = "attachments"
+      DIRNAME_MAXLENGTH = 50
 
       def initialize(output_dir:, **opts)
         super(**opts)
         @output_dir = output_dir
+        @board_dirname = Concurrent::Map.new
+        @board_subdirs = Concurrent::Map.new
+        @list_dirname = Concurrent::Map.new
+        @list_subdirs = Concurrent::Map.new
+        @card_dirname = Concurrent::Map.new
       end
 
       def card_added(card)
@@ -37,10 +43,7 @@ module Trellodon
         list = @board.get_list(card.list_id)
         raise "List #{card.list_id} is not found" if list.nil?
 
-        card_dir = File.join(@output_dir,
-          "#{board.name} [#{board.short_id}]",
-          "#{list.name} [#{list.short_id}]",
-          "#{card.name} [#{card.short_id}]")
+        card_dir = File.join(@output_dir, board_dirname(board), list_dirname(list), card_dirname(card))
         FileUtils.mkdir_p(card_dir) unless File.directory?(card_dir)
         card_dir
       end
@@ -51,11 +54,57 @@ module Trellodon
         attachments_dir
       end
 
+      def board_dirname(board)
+        @board_dirname.compute_if_absent(board.id) { sanitize(board.name) }
+      end
+
+      def list_dirname(list)
+        @list_dirname.compute_if_absent(list.id) do
+          uniq_dirname(
+            dir: sanitize(list.name),
+            entity: list,
+            dirlist: @board_subdirs,
+            parent_id: list.board_id
+          )
+        end
+      end
+
+      def card_dirname(card)
+        @card_dirname.compute_if_absent(card.id) do
+          uniq_dirname(
+            dir: sanitize(card.name),
+            entity: card,
+            dirlist: @list_subdirs,
+            parent_id: card.list_id
+          )
+        end
+      end
+
+      def uniq_dirname(dir:, entity:, dirlist:, parent_id:)
+        return dir unless [List, Card].include?(entity.class)
+
+        dirlist.put_if_absent(parent_id, Concurrent::Map.new)
+        dirlist[parent_id].put_if_absent(dir, Concurrent::Array.new([entity.id]))
+        dirlist[parent_id][dir].push(entity.id) unless dirlist[parent_id][dir].include?(entity.id)
+        idx = dirlist[parent_id][dir].index(entity.id)
+        idx > 0 ? dir + "(#{idx})" : dir
+      end
+
+      def sanitize(name, separator: "")
+        name.gsub(/[\/\\"*?<>|:]+/, separator).slice(0, DIRNAME_MAXLENGTH)
+      end
+
       def download_attachments(card)
         return if card.attachments.nil? || card.attachments.empty?
+
         attachments_path = card_attachments_path(card)
         card.attachments.select(&:is_upload).each do |att|
-          IO.copy_stream(URI.parse(att.url).open(att.headers), File.join(attachments_path, att.file_name))
+          att_file_path = File.join(attachments_path, att.file_name)
+          if File.file?(att_file_path)
+            logger.warn "Attachment file already exists: #{att_file_path}"
+          else
+            IO.copy_stream(URI.parse(att.url).open(att.headers), att_file_path)
+          end
         end
       end
 
@@ -73,6 +122,7 @@ module Trellodon
           ---
           title: #{card.name}
           last_updated_at: #{card.last_activity_date}
+          dumped_at: #{Time.now}
           labels: #{create_card_labels(card)}
           ---
         EOS
@@ -96,11 +146,13 @@ module Trellodon
 
       def create_card_labels(card)
         return "" if card.labels.nil? || card.labels.empty?
+
         card.labels.map { |label| "\n - " + label }.join
       end
 
       def create_card_checklists(card)
         return "" if card.checklists.nil? || card.checklists.empty?
+
         <<~EOS
 
           ## Checklists
@@ -117,11 +169,15 @@ module Trellodon
       end
 
       def create_card_checklist_item(item)
-        "\n- [#{item.checked? ? "x" : " "}] #{item.name}"
+        formatted_item = "\n- [#{item.checked? ? "x" : " "}] #{item.name}"
+        formatted_item += " | *assigned_to: #{create_member(item.member)}*" if item.member
+        formatted_item += " | *due: #{item.due_date}*" if item.due_date
+        formatted_item
       end
 
       def create_card_comments(card)
         return "" if card.comments.nil? || card.comments.empty?
+
         <<~EOS
 
           ## Comments
@@ -132,13 +188,15 @@ module Trellodon
       def create_card_comment(comment)
         <<~EOS
 
-          **#{comment.creator.full_name} (@#{comment.creator.username}) at #{comment.date}**  
+          ### #{create_member(comment.creator)} at #{comment.date}
+
           #{comment.text}
         EOS
       end
 
       def create_card_attachments(card)
         return "" if card.attachments.nil? || card.attachments.empty?
+
         <<~EOS
 
           ## Attachments
@@ -152,6 +210,7 @@ module Trellodon
           ### #{attachment.name}
 
           **date**: #{attachment.date}  
+          **added_by**: #{create_member(attachment.added_by)}  
           **url**: #{attachment.url}  
           **local copy**: #{create_attachment_local_link(attachment)}
         EOS
@@ -159,8 +218,14 @@ module Trellodon
 
       def create_attachment_local_link(attachment)
         return "-" unless attachment.is_upload
+
         "[#{attachment.file_name}](#{File.join("./", ATTACHMENTS_DIRNAME, attachment.file_name)})"
       end
+
+      def create_member(member)
+        return "" unless member
+        "#{member.full_name} (@#{member.username})"
+      end
     end
   end
 end

data/lib/trellodon/journal.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require "yaml"
+
+module Trellodon
+  class Journal
+    FILENAME = "journal.yml"
+
+    def initialize(dir)
+      @location = File.join(dir, FILENAME)
+      File.write(location, "") unless File.exist?(location)
+    end
+
+    def <<(record)
+      File.write(location, record.to_yaml, mode: "a+")
+    end
+
+    private attr_reader :location
+  end
+end

data/lib/trellodon/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Trellodon
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: trellodon
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - fargelus
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-04-12 00:00:00.000000000 Z
+date: 2022-04-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: anyway_config
@@ -69,7 +69,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: ruby-next-core
+  name: ruby-next
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -111,6 +111,7 @@ files:
 - lib/trellodon/entities/attachment.rb
 - lib/trellodon/entities/board.rb
 - lib/trellodon/entities/card.rb
+- lib/trellodon/entities/card_event.rb
 - lib/trellodon/entities/checklist.rb
 - lib/trellodon/entities/checklist_item.rb
 - lib/trellodon/entities/comment.rb
@@ -120,6 +121,7 @@ files:
 - lib/trellodon/formatters.rb
 - lib/trellodon/formatters/base.rb
 - lib/trellodon/formatters/markdown.rb
+- lib/trellodon/journal.rb
 - lib/trellodon/schedulers.rb
 - lib/trellodon/schedulers/base.rb
 - lib/trellodon/schedulers/concurrent.rb