fyodor 0.1.1

data/README.md

@@ -0,0 +1,62 @@
+# Fyodor
+
+Convert your Amazon Kindle highlights, notes and bookmarks into markdown files.
+
+
+## What is it about
+This application parses `My Clippings.txt` from your Kindle and generates a markdown file for each book/document, in the format `#{Author} - #{Title}.md`. This way, your annotations on the books you read are conveniently stored and easily managed.
+
+To read more about the motivation and what problem it tries to solve, [check this blog post](http://rafaelc.org/blog/export-all-your-kindle-highlights-and-notes/).
+
+[For samples of the output, click here.](samples/)
+
+
+## Features
+
+* Supports all the type of entries in your clippings file: highlights, notes, clips and bookmarks.
+* Automatic removal of empty or duplicate entries (the clippings file can get a lot of those).
+* Orders your entries by location/page on each book (the clippings file is ordered by time).
+* Easily configurable for your language, allowing you to get all features and beautiful output.
+* This software goes some length to be locale agnostic: basic parsing should work without configuration for any language. It should also work even if your clippings file has multiple locales.
+* Bookmarks are printed together and notes are formatted differently, for better visualization.
+* Output in a format that is clean and easy to edit/fiddle around: markdown.
+
+This program is based on the clippings file generated by Kindle 2019, but should work with other models.
+
+
+## Installation
+
+Install Ruby and run:
+
+```
+$ gem install fyodor
+```
+
+
+## Configuration
+
+If your Kindle is not in English, you should configure Fyodor so it knows how your `My Clippings.txt` calls some things (e.g. highlights, pages, etc). This is easily done in `parser` section of the config file, replacing the default English values.
+
+Note that basic parsing should still work without configuration, but you won't take advantage of many features, resulting in a dirtier output.
+
+To configure the application, [copy the sample config](https://github.com/rccavalcanti/fyodor/blob/master/fyodor.toml.sample) and place it at `~/.config/fyodor.toml`. Edit it as you like.
+
+The configuration file also allows you to set whether to print the time of each entry. On `[output]`, set `time` to `true` or `false`.
+
+
+## Running
+
+```
+$ fyodor CLIPPINGS_FILE [OUTPUT_DIR]
+```
+
+Where:
+* `CLIPPINGS_FILE` is the path for `My Clippings.txt`.
+* `OUTPUT_DIR` is the directory to write the markdown files. If none supplied, it will be `fyodor_output` in the current directory.
+
+
+## LICENSE
+
+Released under [GNU GPL v3](LICENSE).
+
+Copyright 2019 Rafael Cavalcanti <hi@rafaelc.org>

data/bin/fyodor

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+
+require "fyodor/cli"
+
+# Supress stack trace
+begin
+  Fyodor::CLI.new.main
+rescue StandardError => e
+  abort(e.message)
+end

data/lib/fyodor/book.rb

@@ -0,0 +1,42 @@
+require "forwardable"
+require "set"
+
+module Fyodor
+  class Book
+    extend Forwardable
+    include Enumerable
+
+    attr_reader :title, :author, :rej_dup
+
+    def_delegators :@entries, :each, :size
+
+    def initialize(title, author=nil)
+      raise "Book title can't be empty" if title.to_s.empty?
+
+      @title = title
+      @author = author
+      @entries = SortedSet.new
+      @rej_dup = 0
+    end
+
+    def <<(entry)
+      return if entry.empty?
+      # #add? returns nil if the entry was duplicated
+      @rej_dup += 1 if @entries.add?(entry).nil?
+    end
+
+    def basename
+      base = @author.to_s.empty? ? @title : "#{@author} - #{@title}"
+      base.strip.gsub(/[?*:|\/"<>]/,"_")
+    end
+
+    def count_types
+      list = group_by(&:type).map { |k, v| [k, v.size] }
+      Hash[list]
+    end
+
+    def count_desc_unparsed
+      count { |entry| ! entry.desc_parsed? }
+    end
+  end
+end

data/lib/fyodor/cli.rb

@@ -0,0 +1,43 @@
+require_relative "config_getter"
+require_relative "stats_printer"
+require_relative "library"
+require_relative "clippings_parser"
+require_relative "output_writer"
+require "pathname"
+
+module Fyodor
+  class CLI
+    def initialize
+      get_args
+      @config = ConfigGetter.new.config
+    end
+
+    def main
+      library = Library.new
+      ClippingsParser.new(@clippings_path, @config["parser"]).parse(library)
+      StatsPrinter.new(library).print
+      OutputWriter.new(library, @output_dir, @config["output"]).write_all
+    end
+
+
+    private
+
+    def get_args
+      if ARGV.count != 1 && ARGV.count != 2
+        puts "Usage: #{File.basename($0)} my_clippings_path [output_dir]"
+        exit 1
+      end
+
+      @clippings_path = get_path(ARGV[0])
+      @output_dir = ARGV[1].nil? ? default_output_dir : get_path(ARGV[1])
+    end
+
+    def get_path(path)
+      Pathname.new(path).expand_path
+    end
+
+    def default_output_dir
+      Pathname.new(Dir.pwd) + "fyodor_output"
+    end
+  end
+end

data/lib/fyodor/clippings_parser.rb

@@ -0,0 +1,37 @@
+require_relative "entry_parser"
+
+module Fyodor
+  class ClippingsParser
+    SEPARATOR = /^==========\r?\n$/
+    ENTRY_SIZE = 5
+
+    def initialize(clippings_path, parser_config)
+      @path = clippings_path
+      @config = parser_config
+    end
+
+    def parse(library)
+      lines = []
+      File.open(@path).each do |line|
+        lines << line
+        if end_entry?(lines)
+          library << parse_entry(lines)
+          lines.clear
+        end
+      end
+      raise "MyClippings is badly formatted" if lines.size > 0
+    end
+
+    private
+
+    def end_entry?(lines)
+      return false if lines.size < ENTRY_SIZE
+      return true if lines.size == ENTRY_SIZE && lines.last =~ SEPARATOR
+      raise "MyClippings is badly formatted"
+    end
+
+    def parse_entry(lines)
+      EntryParser.new(lines, @config).entry
+    end
+  end
+end

data/lib/fyodor/config_getter.rb

@@ -0,0 +1,51 @@
+require_relative "core_extensions/hash/merging"
+require "pathname"
+require "toml"
+
+module Fyodor
+  class ConfigGetter
+    PATHS = [Pathname.new(__FILE__).dirname + "../fyodor.toml",
+             Pathname.new("~/.config/fyodor.toml").expand_path]
+    DEFAULT = {
+      "parser" => {
+        "highlight" => "Your Highlight",
+        "note" => "Your Note",
+        "bookmark" => "Your Bookmark",
+        "clip" => "Clip This Article",
+        "loc" => "Location",
+        "page" => "page",
+        "time" => "Added on"
+      },
+      "output" => {
+        "time" => false
+      }
+    }
+
+    def config
+      @config ||= get_config
+    end
+
+
+    private
+
+    def get_config
+      Hash.include CoreExtensions::Hash::Merging
+
+      print_path
+      user_config = path.nil? ? {} : TOML.load_file(path)
+      DEFAULT.deep_merge(user_config)
+    end
+
+    def path
+      @path ||= PATHS.find { |path| path.exist? }
+    end
+
+    def print_path
+      if path.nil?
+        puts "No config found: using defaults.\n\n"
+      else
+        puts "Using config at #{path}\n\n"
+      end
+    end
+  end
+end

data/lib/fyodor/core_extensions/hash/merging.rb

@@ -0,0 +1,12 @@
+module Fyodor
+  module CoreExtensions
+    module Hash
+      module Merging
+        def deep_merge(second)
+          merger = proc { |key, v1, v2| Hash === v1 && Hash === v2 ? v1.merge(v2, &merger) : Array === v1 && Array === v2 ? v1 | v2 : [:undefined, nil, :nil].include?(v2) ? v1 : v2 }
+          self.merge(second.to_h, &merger)
+        end
+      end
+    end
+  end
+end

data/lib/fyodor/entry.rb

@@ -0,0 +1,63 @@
+module Fyodor
+  class Entry
+    TYPE = { note: "note",
+            highlight: "highlight",
+            bookmark: "bookmark",
+            clip: "clip" }
+
+    attr_reader :book_title, :book_author, :text, :desc, :type, :loc, :loc_start, :page, :time
+
+    def initialize(attrs)
+      @book_title = attrs[:book_title]
+      @book_author = attrs[:book_author]
+      @text = attrs[:text]
+      @desc = attrs[:desc]
+      @type = attrs[:type]
+      @loc = attrs[:loc]
+      # This is our comparable, we need it as a number.
+      @loc_start = attrs[:loc_start].to_i
+      @page = attrs[:page]
+      @time = attrs[:time]
+
+      raise ArgumentError, "Invalid Entry type" unless TYPE.value?(@type) || @type.nil?
+    end
+
+    def empty?
+      if @type == TYPE[:bookmark] || @type.nil?
+        @desc.strip == ""
+      else
+        @text.strip == ""
+      end
+    end
+
+    def desc_parsed?
+      @loc_start != 0 && ! @type.nil?
+    end
+
+    # Override this method to use a SortedSet.
+    def <=>(other)
+      @loc_start <=> other.loc_start
+    end
+
+    # Override the following methods for deduplication.
+    def ==(other)
+      return false if @type != other.type || @text != other.text
+
+      if desc_parsed? && other.desc_parsed?
+        @loc == other.loc
+      else
+        @desc == other.desc
+      end
+    end
+
+    alias eql? ==
+
+    def hash
+      if desc_parsed?
+        @text.hash ^ @type.hash ^ @loc.hash
+      else
+        @text.hash ^ @desc.hash
+      end
+    end
+  end
+end

data/lib/fyodor/entry_parser.rb

@@ -0,0 +1,93 @@
+require_relative "entry"
+
+module Fyodor
+  class EntryParser
+    def initialize(entry_lines, parser_config)
+      @lines = entry_lines
+      @config = parser_config
+      format_check
+    end
+
+    def entry
+      Entry.new({book_title: book[:title],
+                book_author: book[:author],
+                text: text,
+                desc: desc,
+                type: type,
+                loc: loc,
+                loc_start: loc_start,
+                page: page,
+                time: time})
+    end
+
+
+    private
+
+    def book
+      title, author = @lines[0].scan(regex_cap(:title_author)).first
+      # If book has no author, regex fails.
+      title = @lines[0] if title.nil?
+
+      {title: title.strip, author: author.to_s.strip}
+    end
+
+    def desc
+      @lines[1].delete_prefix("- ").strip
+    end
+
+    def type
+      Entry::TYPE.values.find { |type| @lines[1] =~ regex_type(type) }
+    end
+
+    def loc
+      @lines[1][regex_cap(:loc), 1]
+    end
+
+    def loc_start
+      @lines[1][regex_cap(:loc_start), 1].to_i
+    end
+
+    def page
+      @lines[1][regex_cap(:page), 1]
+    end
+
+    def time
+      @lines[1][regex_cap(:time), 1]
+    end
+
+    def text
+      @lines[3].strip
+    end
+
+    def regex_type(type)
+      s = Regexp.quote(@config[type])
+      /^- #{s}/
+    end
+
+    def regex_cap(item)
+      case item
+      when :title_author
+        /^(.*) \((.*)\)\r?\n$/
+      when :loc
+        s = Regexp.quote(@config["loc"])
+        /#{s} (\S+)/
+      when :loc_start
+        s = Regexp.quote(@config["loc"])
+        /#{s} (\d+)(-\d+)?/
+      when :page
+        s = Regexp.quote(@config["page"])
+        /#{s} (\S+)/
+      when :time
+        s = Regexp.quote(@config["time"])
+        /#{s} (.*)\r?\n$/
+      end
+    end
+
+    def format_check
+      raise "Entry must have five lines" unless @lines.size == 5
+      raise "Entry is badly formatted" if @lines[0].strip.empty?
+      raise "Entry is badly formatted" if @lines[1].strip.empty?
+      raise "Entry is badly formatted" unless @lines[2].strip.empty?
+    end
+  end
+end

data/lib/fyodor/library.rb

@@ -0,0 +1,57 @@
+require_relative "book"
+require "forwardable"
+
+module Fyodor
+  class Library
+    extend Forwardable
+    include Enumerable
+
+    def_delegators :@books, :each, :empty?, :size
+
+    def initialize
+      @books = []
+      @rej_empty = 0
+    end
+
+    def <<(entry)
+      if entry.empty?
+        @rej_empty += 1
+        return
+      end
+
+      book(entry.book_title, entry.book_author) << entry
+    end
+
+    def count_types
+      reduce({}) { |acc, book| acc.merge(book.count_types) { |key, val1, val2| val1 + val2 } }
+    end
+
+    def count_desc_unparsed
+      reduce(0) { |acc, book| acc + book.count_desc_unparsed }
+    end
+
+    def count_entries
+      reduce(0) { |acc, book| acc + book.size }
+    end
+
+    def rejected
+      {empty: @rej_empty, dup: count_rej_dup}
+    end
+
+
+    private
+
+    def book(title, author)
+      book = find { |book| book.title == title && book.author == author }
+      if book.nil?
+        book = Book.new(title, author)
+        @books << book
+      end
+      book
+    end
+
+    def count_rej_dup
+      reduce(0) { |acc, book| acc + book.rej_dup }
+    end
+  end
+end

data/lib/fyodor/md_generator.rb

@@ -0,0 +1,102 @@
+require_relative "strings"
+
+module Fyodor
+  class MdGenerator
+    include Strings
+
+    def initialize(book, config)
+      @book = book
+      @config = config
+    end
+
+    def content
+      header + body + bookmarks
+    end
+
+
+    private
+
+    def header
+      return <<~EOF
+      # #{@book.title}
+      #{"by #{@book.author}" unless @book.author.to_s.empty?}
+
+      #{header_counts}
+
+      EOF
+    end
+
+    def header_counts
+      output = ""
+      @book.count_types.each do |type, n|
+        output += "#{n} #{pluralize(type, n)}, " if n > 0
+      end
+      output.delete_suffix(", ")
+    end
+
+    def pluralize(type, n)
+      n == 1 ? SINGULAR[type] : PLURAL[type]
+    end
+
+    def body
+      entries = @book.reject { |entry| entry.type == Entry::TYPE[:bookmark] }
+      entries.size == 0 ? "" : entries_render(entries)
+    end
+
+    def bookmarks
+      bookmarks = @book.select { |entry| entry.type == Entry::TYPE[:bookmark] }
+      bookmarks.size == 0 ? "" : entries_render(bookmarks, "Bookmarks")
+    end
+
+    def entries_render(entries, title=nil)
+      output = "---\n\n"
+      output += "## #{title}\n\n" unless title.nil?
+      entries.each do |entry|
+        output += "#{entry_text(entry)}\n\n"
+        output += "<p style=\"text-align: right;\"><sup>#{entry_desc(entry)}</sup></p>\n\n"
+      end
+      output
+    end
+
+    def entry_text(entry)
+      case entry.type
+      when Entry::TYPE[:bookmark]
+        "* #{page(entry)}"
+      when Entry::TYPE[:note]
+
+        "> _#{text(entry)}_"
+      else
+        "> #{text(entry)}"
+      end
+    end
+
+    def entry_desc(entry)
+      return entry.desc unless entry.desc_parsed?
+
+      case entry.type
+      when Entry::TYPE[:bookmark]
+        time(entry)
+      else
+        (type(entry) + " @ " + page(entry) + " " + time(entry)).strip
+      end
+    end
+
+    def page(entry)
+      ((entry.page.nil? ? "" : "page #{entry.page}, ") +
+        (entry.loc.nil? ? "" : "loc. #{entry.loc}")).delete_suffix(", ")
+    end
+
+    def time(entry)
+      @config["time"] ? "[#{entry.time}]" : ""
+    end
+
+    def type(entry)
+      SINGULAR[entry.type]
+    end
+
+    def text(entry)
+      # Markdown needs no white space between text and formatters
+      entry.text.strip
+    end
+  end
+end

data/lib/fyodor/output_writer.rb

@@ -0,0 +1,35 @@
+require_relative "md_generator"
+
+module Fyodor
+  class OutputWriter
+    def initialize(library, output_dir, config)
+      @library = library
+      @output_dir = output_dir
+      @output_dir.mkdir unless @output_dir.exist?
+      @config = config
+    end
+
+    def write_all
+      puts "\nWriting to #{@output_dir}..." unless @library.empty?
+      @library.each do |book|
+        content = MdGenerator.new(book, @config).content
+        File.open(path(book), "w") { |f| f.puts(content) }
+      end
+    end
+
+
+    private
+
+    def path(book)
+      path = @output_dir + "#{book.basename}.md"
+
+      i = 2
+      while(path.exist?)
+        path = @output_dir + "#{book.basename} - #{i}.md"
+        i += 1
+      end
+
+      path
+    end
+  end
+end

data/lib/fyodor/stats_printer.rb

@@ -0,0 +1,53 @@
+require_relative "strings"
+
+module Fyodor
+  class StatsPrinter
+    include Strings
+
+    def initialize(library)
+      @library = library
+    end
+
+    def print
+      num_books
+      rejected
+      types
+      pretty_output
+    end
+
+    private
+
+    def num_books
+      puts "=> #{@library.size} books found"
+    end
+
+    def types
+      ct = @library.count_types
+      PLURAL.each { |type, label| puts "#{label.capitalize.rjust(12)}: #{ct[type] || 0}" }
+      puts "-------------------"
+      puts "#{"TOTAL".rjust(12)}: #{ct.sum {|k, v| v}}\n\n"
+    end
+
+    def rejected
+      rejected = @library.rejected
+      puts "=> Ignored #{rejected[:empty]} empty and #{rejected[:dup]} duplicated entries."
+    end
+
+    def pretty_output
+      bad = @library.count_desc_unparsed
+      percent = (bad.to_f / @library.count_entries.to_f) * 100
+
+      if bad > 0
+        warn <<~EOF
+        We couldn't *improve* the output of #{bad} (#{percent.round(1)}%) entries.
+        Possible causes:
+        - Wrong strings in the config file (most probable).
+        - Your locale has some specificity the app is not aware. Please open an issue.
+        #{"- Your clippings file has more than one locale." if percent != 100}
+        EOF
+      else
+        puts "ALL entries were correctly parsed."
+      end
+    end
+  end
+end

data/lib/fyodor/strings.rb

@@ -0,0 +1,17 @@
+require_relative "entry"
+
+module Fyodor
+  module Strings
+    PLURAL = { Entry::TYPE[:highlight] => "highlights",
+               Entry::TYPE[:note] => "notes",
+               Entry::TYPE[:bookmark] => "bookmarks",
+               Entry::TYPE[:clip] => "clips",
+               nil => "unrecognized" }
+
+    SINGULAR = { Entry::TYPE[:highlight] => "highlight",
+                 Entry::TYPE[:note] => "note",
+                 Entry::TYPE[:bookmark] => "bookmark",
+                 Entry::TYPE[:clip] => "clip",
+                 nil=> "unrecognized" }
+  end
+end

data/lib/fyodor.rb

@@ -0,0 +1,3 @@
+require "fyodor/clippings_parser"
+require "fyodor/library"
+require "fyodor/output_writer"