nbk2pdf 1.0.0

data/lib/kindle_notebook/cli.rb

@@ -0,0 +1,274 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+# SPDX-FileCopyrightText: 2026 Sergey Avseyev
+# frozen_string_literal: true
+
+require "fileutils"
+require "optparse"
+require "pathname"
+
+module KindleNotebook
+  # The `nbk2pdf` command line.
+  class CLI
+    SUCCESS = 0
+    FAILURE = 1
+
+    VERSION_NOTICE = <<~NOTICE.freeze
+      #{PROGRAM} #{VERSION}
+      Copyright (C) 2026 Sergey Avseyev
+      License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>
+      This is free software: you are free to change and redistribute it.
+      There is NO WARRANTY, to the extent permitted by law.
+    NOTICE
+
+    SYNC = "sync"
+
+    # Where sync puts things inside its destination.
+    NOTEBOOKS_DIR = "notebooks"
+    DOCUMENTS_DIR = "documents"
+
+    BANNER = <<~BANNER.freeze
+      Convert Kindle Scribe notebooks to PDF.
+
+      Usage: #{PROGRAM} [options] PATH...
+             #{PROGRAM} #{SYNC} [options] [DIR]
+
+        PATH may be an `nbk` file, a notebook directory, or any directory
+        containing notebooks (they are found recursively). It may also be a
+        PDF on the Kindle, to get a copy with what was written on it.
+
+        #{SYNC} converts every notebook on a Kindle attached over USB into
+        DIR/#{NOTEBOOKS_DIR}, and every PDF on it that has been written on into
+        DIR/#{DOCUMENTS_DIR}, in the folders it has under documents/ on the
+        Kindle. DIR is the current directory when left out. The device is
+        mounted over MTP first if it is not mounted already, and is left
+        mounted afterwards.
+
+      Options:
+    BANNER
+
+    def self.start(argv, out: $stdout, err: $stderr) = new(out: out, err: err).run(argv)
+
+    def initialize(out: $stdout, err: $stderr, device: nil)
+      @out = out
+      @err = err
+      @device = device
+      @options = { template: true, quiet: false, annotations: false }
+    end
+
+    # Runs the command and returns the process exit status.
+    def run(argv)
+      catch(:halt) do
+        argv = argv.dup
+        sync = argv.first == SYNC && argv.shift
+        arguments = parser.parse(argv)
+        next sync_device(arguments) if sync
+
+        arguments.empty? ? fail_with(parser.help) : convert(arguments)
+      end
+    rescue OptionParser::ParseError, Error => e
+      @err.puts(e.message)
+      FAILURE
+    end
+
+    private
+
+    attr_reader :options
+
+    def device = @device ||= Device.new
+
+    def parser
+      @parser ||= OptionParser.new do |opts|
+        opts.banner = BANNER
+        define_output_options(opts)
+        define_rendering_options(opts)
+        opts.on("-h", "--help", "Show this message") { halt(opts.help) }
+        opts.on("-v", "--version", "Show the version") { halt(VERSION_NOTICE.chomp) }
+      end
+    end
+
+    def define_output_options(opts)
+      opts.on("-o", "--output PATH", "Output file, or directory for several notebooks") do |path|
+        options[:output] = path
+      end
+      opts.on("-n", "--name NAME", "Override the output file name (single notebook only)") do |name|
+        options[:name] = name
+      end
+      opts.on("-t", "--titles FILE", "JSON/YAML map of notebook uuid to the name you gave it") do |file|
+        options[:titles] = file
+      end
+    end
+
+    def define_rendering_options(opts)
+      opts.on("--[no-]template", "Draw the notebook's ruled background (default: yes)") do |draw|
+        options[:template] = draw
+      end
+      opts.on("-p", "--paper SIZE", "Page size: letter (default), a4, native, ...") do |size|
+        options[:paper] = size
+      end
+      opts.on("-a", "--annotations", "Add ink to a PDF as annotations, not drawn into the page") do
+        options[:annotations] = true
+      end
+      opts.on("-q", "--quiet", "Only report errors") { options[:quiet] = true }
+    end
+
+    def convert(paths)
+      sources = paths.flat_map { |path| Converter.document?(path) ? [File.expand_path(path)] : Notebook.find(path) }
+      sources = with_documents(sources)
+      return fail_with("no notebooks found in #{paths.join(", ")}") if sources.empty?
+
+      if options[:name] && sources.size > 1
+        return fail_with("--name needs a single notebook, but #{sources.size} were found")
+      end
+
+      convert_each(sources, options[:output])
+    end
+
+    # `sync` takes its destination as a positional argument, since the notebooks
+    # come from the device rather than from a path.
+    def sync_device(arguments)
+      return fail_with("#{SYNC} takes one directory at most") if arguments.size > 1
+      return fail_with("--name cannot be used with #{SYNC}") if options[:name]
+
+      destination = arguments.first || options[:output] || "."
+      targets = sync_targets(device_notebooks, destination)
+      return fail_with("the Kindle holds no notebooks") if targets.empty?
+
+      convert_targets(targets) { device_vanished? }
+    end
+
+    # [source, output directory] for everything on the device: notebooks side
+    # by side, and each PDF under the same folders it sits in on the Kindle.
+    # The directories are created here, so that each is written into rather
+    # than taken for the name of an output file.
+    def sync_targets(notebooks, destination)
+      targets = notebooks.map do |notebook|
+        pdf = Document.pdf_for(notebook)
+        next [notebook, File.join(destination, NOTEBOOKS_DIR)] unless pdf
+
+        library = Pathname(File.join(File.dirname(notebook, 3), DOCUMENTS_DIR))
+        [pdf, File.join(destination, DOCUMENTS_DIR, Pathname(File.dirname(pdf)).relative_path_from(library).to_s)]
+      end.uniq
+      targets.map(&:last).uniq.each { |directory| FileUtils.mkdir_p(directory) }
+      targets
+    end
+
+    # A Kindle can drop off the bus part way through, which otherwise shows up
+    # as every remaining notebook having gone missing.
+    def device_vanished?
+      return false if device.mounted?
+
+      @err.puts("the Kindle disconnected part way through; #{Device::RECONNECT}")
+      true
+    end
+
+    # The ink written on a PDF is kept in a notebook of its own, which is swapped
+    # for the PDF it belongs to so that the ink ends up on a copy of it.
+    def with_documents(sources) = sources.map { |source| Document.pdf_for(source) || source }.uniq
+
+    def device_notebooks
+      @out.puts("mounting the Kindle over MTP...") unless options[:quiet] || device.mounted?
+      device.notebooks
+    end
+
+    def convert_each(sources, output)
+      output = destination_for(sources, output)
+      convert_targets(sources.map { |source| [source, output] })
+    end
+
+    def convert_targets(targets, &)
+      failures = count_failures(targets, &)
+      report_missing_names
+      failures.zero? ? SUCCESS : FAILURE
+    end
+
+    # Notebooks the device could not name are offered for filling in, since the
+    # Kindle keeps its names where MTP cannot reach them.
+    def report_missing_names
+      return if options[:quiet] || titles.missing.zero?
+
+      pending = titles.missing
+      added = titles.record_missing
+      @out.puts(
+        if added.positive?
+          "#{pluralize(added, "notebook")} had no name; added to #{titles.path} to fill in"
+        else
+          "#{pluralize(pending, "notebook")} had no name; pass --titles FILE to record them"
+        end
+      )
+    end
+
+    # Several notebooks cannot share one file name, so a destination that does
+    # not exist yet is created as a directory rather than written to as a file.
+    def destination_for(sources, output)
+      return output unless output && sources.size > 1
+
+      FileUtils.mkdir_p(output)
+      output
+    end
+
+    def count_failures(targets, &stop)
+      failures = 0
+      targets.each do |source, output|
+        next if convert_one(source, output)
+
+        failures += 1
+        break if stop&.call
+      end
+      failures
+    end
+
+    def convert_one(source, output)
+      result = converter.convert(source, output: output, name: options[:name])
+      result.warnings.each { |warning| @err.puts("#{source}: #{warning}") }
+      note_missing_name(result)
+      report(result) unless options[:quiet]
+      true
+    rescue Error => e
+      @err.puts("#{source}: #{e.message}")
+      false
+    end
+
+    def converter
+      @converter ||= Converter.new(titles: titles, render_template: options[:template],
+                                   paper: PdfRenderer.paper(options[:paper]),
+                                   annotations: options[:annotations])
+    end
+
+    def titles = @titles ||= Titles.load(options[:titles])
+
+    def report(result)
+      pages = pluralize(result.page_count, "page")
+      summary = [result.document ? "ink on #{pages}" : pages, pluralize(result.stroke_count, "stroke")]
+      summary << problems(result) if result.warnings.any?
+      @out.puts("#{result.source} -> #{result.output} (#{summary.join(", ")})")
+    end
+
+    # A document's warnings are about placing notes as well as decoding strokes.
+    def problems(result)
+      return "#{pluralize(result.warnings.size, "warning")}, see above" if result.document
+
+      "#{pluralize(result.warnings.size, "stroke")} could not be decoded"
+    end
+
+    def note_missing_name(result)
+      return if result.name
+
+      titles.note_missing(result.uuid,
+                          "#{pluralize(result.page_count, "page")}, " \
+                          "last written #{result.written_at.strftime("%Y-%m-%d")}")
+    end
+
+    def pluralize(count, noun) = "#{count} #{noun}#{"s" unless count == 1}"
+
+    def fail_with(message)
+      @err.puts(message)
+      FAILURE
+    end
+
+    # OptionParser has no way to stop parsing, so --help and --version unwind.
+    def halt(message)
+      @out.puts(message)
+      throw(:halt, SUCCESS)
+    end
+  end
+end

data/lib/kindle_notebook/container.rb

@@ -0,0 +1,82 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+# SPDX-FileCopyrightText: 2026 Sergey Avseyev
+# frozen_string_literal: true
+
+module KindleNotebook
+  # A Kindle Scribe `nbk` file is an ordinary SQLite database that has been
+  # sliced up and interleaved with 1024-byte signature blocks: one block sits
+  # after the first kibibyte of database bytes, and another after every
+  # mebibyte that follows.
+  #
+  #   [ 1 KiB db ][ 1 KiB sig ][ 1 MiB db ][ 1 KiB sig ][ 1 MiB db ] ...
+  #
+  # Stripping those blocks yields a database that SQLite opens cleanly.
+  module Container
+    BLOCK_SIZE = 1024
+    CHUNK_SIZE = 1024 * 1024
+    STRIDE = BLOCK_SIZE + CHUNK_SIZE
+    FIRST_BLOCK_OFFSET = BLOCK_SIZE
+
+    MAGIC = "SQLite format 3\0"
+    HEADER_SIZE = 100
+
+    class FormatError < Error; end
+
+    module_function
+
+    # Offsets of every signature block for a container of the given size.
+    def block_offsets(size)
+      offsets = []
+      offset = FIRST_BLOCK_OFFSET
+      while offset + BLOCK_SIZE <= size
+        offsets << offset
+        offset += STRIDE
+      end
+      offsets
+    end
+
+    # Database size in bytes as declared by the SQLite header.
+    def database_size(data)
+      raise FormatError, "not a SQLite database" unless data.byteslice(0, MAGIC.bytesize) == MAGIC
+      raise FormatError, "truncated SQLite header" if data.bytesize < HEADER_SIZE
+
+      page_size = data.byteslice(16, 2).unpack1("n")
+      page_size = 65_536 if page_size == 1
+      page_count = data.byteslice(28, 4).unpack1("N")
+      raise FormatError, "unusable SQLite header" if page_size.zero? || page_count.zero?
+
+      page_size * page_count
+    end
+
+    # Returns the plain SQLite bytes carried by +data+.
+    def unwrap(data)
+      data = data.b
+      expected = database_size(data)
+      extra = data.bytesize - expected
+      return data if extra.zero?
+
+      raise FormatError, "database is shorter than its header claims" if extra.negative?
+
+      offsets = block_offsets(data.bytesize)
+      unless offsets.size * BLOCK_SIZE == extra
+        raise FormatError,
+              "unexpected container layout: #{extra} stray bytes, " \
+              "#{offsets.size} signature blocks"
+      end
+
+      out = +""
+      cursor = 0
+      offsets.each do |offset|
+        out << data.byteslice(cursor, offset - cursor)
+        cursor = offset + BLOCK_SIZE
+      end
+      out << data.byteslice(cursor, data.bytesize - cursor)
+      out
+    end
+
+    def unwrap_file(source, destination)
+      File.binwrite(destination, unwrap(File.binread(source)))
+      destination
+    end
+  end
+end

data/lib/kindle_notebook/converter.rb

@@ -0,0 +1,112 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+# SPDX-FileCopyrightText: 2026 Sergey Avseyev
+# frozen_string_literal: true
+
+require "fileutils"
+require "json"
+
+module KindleNotebook
+  # Turns notebooks into PDF files on disk, working out a sensible file name for
+  # each one, and adds the ink written on a PDF to a copy of it.
+  class Converter
+    # For a document, +page_count+ counts the pages that carry ink.
+    Result = Struct.new(:source, :output, :uuid, :name, :page_count, :stroke_count, :warnings,
+                        :written_at, :document, keyword_init: true)
+
+    # +titles+ maps a notebook uuid (or book id) to the name you know it by on
+    # the device, since standalone notebooks do not store their own title.
+    # +annotations+ adds ink to a document as annotations rather than drawing
+    # it into the page.
+    def initialize(titles: {}, render_template: true, paper: PdfRenderer.paper, annotations: false)
+      @titles = titles
+      @renderer = PdfRenderer.new(render_template: render_template, paper: paper)
+      @documents = DocumentRenderer.new(annotations: annotations)
+      @written = {}
+    end
+
+    def self.document?(path) = File.extname(path).casecmp?(".pdf")
+
+    # Converts one notebook, or annotates one PDF. +output+ may be a file path
+    # or a directory.
+    def convert(source, output: nil, name: nil)
+      return convert_document(source, output, name) if self.class.document?(source)
+
+      notebook = Notebook.open(source)
+      pages = notebook.pages
+      raise Error, "#{source}: notebook has no pages" if pages.empty?
+
+      stem = name ? notebook.sanitize(name) : notebook.basename(titles: @titles)
+      destination = resolve_output(output, stem, File.dirname(notebook.path))
+      FileUtils.mkdir_p(File.dirname(destination))
+      @renderer.render(pages, destination, title: notebook.known_name(@titles) || File.basename(destination, ".pdf"))
+      notebook_result(notebook, destination, pages)
+    end
+
+    private
+
+    def notebook_result(notebook, destination, pages)
+      Result.new(
+        source: notebook.path,
+        output: destination,
+        uuid: notebook.uuid,
+        name: notebook.known_name(@titles),
+        page_count: pages.size,
+        stroke_count: pages.sum { |page| page.strokes.size },
+        warnings: notebook.warnings,
+        written_at: notebook.modified_at,
+        document: false
+      )
+    end
+
+    # The copy goes into the current directory unless told otherwise: next to
+    # the original it would have the original's name.
+    def convert_document(source, output, name)
+      document = Document.open(source)
+      stem = (name && Notebook.safe_name(name)) || File.basename(document.path, ".*")
+      destination = resolve_output(output, stem, Dir.pwd)
+      if File.expand_path(destination) == document.path ||
+         (File.exist?(destination) && File.identical?(destination, document.path))
+        raise Error, "refusing to write over the original; choose another place with -o"
+      end
+
+      claim(destination, document.path)
+
+      FileUtils.mkdir_p(File.dirname(destination))
+      notes = @documents.render(document, destination)
+      document_result(document, destination, stem, notes)
+    end
+
+    def document_result(document, destination, stem, notes)
+      Result.new(
+        source: document.path,
+        output: destination,
+        uuid: document.id,
+        name: stem,
+        page_count: notes.map(&:page_index).uniq.size,
+        stroke_count: notes.sum { |note| note.page.strokes.size },
+        warnings: document.warnings,
+        written_at: File.mtime(document.notebook_path),
+        document: true
+      )
+    end
+
+    # Two PDFs of the same name in different folders would otherwise end up
+    # in one file, the second silently replacing the first.
+    def claim(destination, source)
+      earlier = @written[File.expand_path(destination)]
+      raise Error, "#{destination} was already written from #{earlier}; convert this one with -o" if earlier
+
+      @written[File.expand_path(destination)] = source
+    end
+
+    def resolve_output(output, stem, default_directory)
+      if output.nil?
+        File.join(default_directory, "#{stem}.pdf")
+      elsif File.directory?(output) || output.end_with?(File::SEPARATOR)
+        File.join(output, "#{stem}.pdf")
+      else
+        output
+      end
+    end
+  end
+end

data/lib/kindle_notebook/device.rb

@@ -0,0 +1,145 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+# SPDX-FileCopyrightText: 2026 Sergey Avseyev
+# frozen_string_literal: true
+
+require "open3"
+require "rbconfig"
+
+module KindleNotebook
+  # Locates the notebook folder on a Kindle attached over USB.
+  #
+  # A Kindle speaks MTP rather than presenting a block device, so it cannot be
+  # mounted with mount(8). On Linux, gvfs exposes MTP devices as FUSE paths
+  # under the session runtime directory once `gio mount` has attached them,
+  # which is what this drives.
+  class Device
+    NOTEBOOKS = ".notebooks"
+    ACTIVATION_ROOT = %r{activation_root=(mtp://\S+)}
+
+    # Amazon's USB vendor id, so a missing device can be told apart from one
+    # gvfs has not offered yet.
+    KINDLE_VENDOR = "1949"
+    USB_DEVICES = "/sys/bus/usb/devices"
+
+    # Detaching and reattaching makes a Kindle re-enumerate, during which gvfs
+    # briefly stops listing it, so both steps wait rather than fail outright.
+    # The waits are against the clock, not a count of sleeps, because probing
+    # gvfs costs about half a second each time.
+    MOUNT_TIMEOUT = 20
+    VOLUME_TIMEOUT = 10
+    POLL_INTERVAL = 0.25
+
+    # A Kindle that has dropped off the bus does not come back on its own.
+    RECONNECT = "unplug the USB cable and plug it back in, then try again"
+
+    class Unavailable < Error; end
+
+    # The external commands this needs, isolated so tests can stand in for them.
+    class Shell
+      def capture(*argv)
+        output, status = Open3.capture2e(*argv)
+        [output, status.success?]
+      rescue Errno::ENOENT
+        ["", false]
+      end
+
+      # capture returns false for a command that is not installed at all.
+      def available?(command) = capture(command, "--version").last
+
+      def sleep(seconds) = Kernel.sleep(seconds)
+
+      def now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+
+    def initialize(shell: Shell.new, runtime_dir: nil, host_os: RbConfig::CONFIG["host_os"],
+                   usb_devices: USB_DEVICES)
+      @shell = shell
+      @runtime_dir = runtime_dir || ENV["XDG_RUNTIME_DIR"] || "/run/user/#{Process.uid}"
+      @host_os = host_os
+      @usb_devices = usb_devices
+    end
+
+    # The device's `.notebooks` folder, mounting the Kindle first when it is
+    # not mounted already.
+    def notebooks_path
+      require_linux
+      existing = mounted_notebooks
+      return existing if existing
+
+      mount
+      mounted_notebooks || raise(Unavailable, "mounted the Kindle but found no #{NOTEBOOKS} folder on it")
+    end
+
+    # Every notebook on the device. Only the top level counts: the clipboard's
+    # scratch buffer sits a level deeper and is not a notebook of its own.
+    def notebooks(root = notebooks_path)
+      Dir.glob(File.join(root, "*", "nbk")).select { |path| File.file?(path) }.sort
+    end
+
+    # True when a Kindle is already mounted and holds notebooks.
+    def mounted? = !mounted_notebooks.nil?
+
+    # True when a device with Amazon's USB vendor id is plugged in.
+    def connected?
+      Dir.glob(File.join(@usb_devices, "*", "idVendor")).any? do |path|
+        File.read(path).strip == KINDLE_VENDOR
+      rescue SystemCallError
+        false
+      end
+    end
+
+    private
+
+    def require_linux
+      return if @host_os.match?(/linux/i)
+
+      raise Unavailable, "connecting to a Kindle over MTP is only implemented for Linux, not #{@host_os}"
+    end
+
+    # gvfs names each MTP mount after the device, and the storage sits one
+    # level in ("Internal Storage" on a Scribe).
+    def mounted_notebooks
+      Dir.glob(File.join(@runtime_dir, "gvfs", "mtp:host=*", "*", NOTEBOOKS))
+         .find { |path| File.directory?(path) }
+    end
+
+    def mount
+      raise Unavailable, "gio is not installed; it comes with gvfs" unless @shell.available?("gio")
+
+      uri = activation_root
+      output, ok = @shell.capture("gio", "mount", uri)
+      raise Unavailable, "could not mount #{uri}: #{output.strip}" unless ok
+
+      wait_for_mount
+    end
+
+    # A Kindle that is not on the bus at all will not turn up by waiting, so
+    # that case is reported straight away; a Kindle that is plugged in may still
+    # be settling, and gets the benefit of the wait.
+    def activation_root
+      raise Unavailable, "no Kindle found on USB; connect it with a cable and try again" unless connected?
+
+      wait_for(VOLUME_TIMEOUT) { listed_activation_root } ||
+        raise(Unavailable, "a Kindle is connected but gvfs is not offering it over MTP. " \
+                           "Unlock the screen, and if it stays hidden, #{RECONNECT}")
+    end
+
+    def listed_activation_root
+      listing, ok = @shell.capture("gio", "mount", "-li")
+      ok ? listing[ACTIVATION_ROOT, 1] : nil
+    end
+
+    def wait_for_mount = wait_for(MOUNT_TIMEOUT) { mounted_notebooks }
+
+    def wait_for(timeout)
+      deadline = @shell.now + timeout
+      loop do
+        found = yield
+        return found if found
+        return nil if @shell.now >= deadline
+
+        @shell.sleep(POLL_INTERVAL)
+      end
+    end
+  end
+end