nbk2pdf 1.0.0

data/lib/kindle_notebook/document.rb

@@ -0,0 +1,133 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+# SPDX-FileCopyrightText: 2026 Sergey Avseyev
+# frozen_string_literal: true
+
+module KindleNotebook
+  # A PDF on the Kindle and the handwriting written on it.
+  #
+  # The device never changes the PDF. Opening it writes a sidecar next to it,
+  # and writing on it keeps the ink in a notebook of its own:
+  #
+  #   documents/.../<name>.pdf
+  #   documents/.../<name>.sdr/<id>/book.sdr/*.yjr    one note per page written on
+  #   .notebooks/<id>!!PDOC!!notebook/nbk             the ink, one page per note
+  #
+  # Each note in the `.yjr` gives a position on the PDF and the notebook page
+  # holding the ink.
+  class Document
+    # Ink for one PDF page: +page_index+ counts from zero, +page+ is the
+    # notebook page carrying the strokes and the frame they were written in.
+    Note = Struct.new(:page_index, :page, keyword_init: true)
+
+    NOTEBOOKS = ".notebooks"
+    ANNOTATIONS = "*/book.sdr/*.yjr"
+    HANDWRITTEN_NOTE = "annotation.personal.handwritten_note"
+
+    # A position reads like `AckEAAAAAAAA:65538`. The number after the colon
+    # holds the page index in its upper 16 bits; the lower 16 were 2 on every
+    # page seen. What the base64 part encodes is not known, and it is not used.
+    PAGE_SHIFT = 16
+    POSITION_NUMBER = /:(\d+)\z/
+
+    attr_reader :path, :id, :notebook_path
+
+    # Ink that could not be placed or decoded.
+    attr_reader :warnings
+
+    def self.open(path) = new(path)
+
+    # The PDF a device notebook holds ink for, or nil for an ordinary notebook
+    # or one whose document is not a PDF on the device.
+    def self.pdf_for(notebook_path)
+      folder = File.basename(File.dirname(notebook_path))
+      id, kind = folder.split("!!", 2)
+      return nil unless kind
+
+      storage = File.dirname(notebook_path, 3)
+      Dir.glob("documents/**/*.sdr/#{id}", base: storage).each do |sidecar|
+        stem = File.join(storage, File.dirname(sidecar).delete_suffix(".sdr"))
+        pdf = ["#{stem}.pdf", "#{stem}.PDF"].find { |candidate| File.file?(candidate) }
+        return pdf if pdf
+      end
+      nil
+    end
+
+    def initialize(path)
+      @path = File.expand_path(path)
+      raise Error, "no such document: #{path}" unless File.file?(@path)
+
+      @sidecar = sidecar
+      @id = File.basename(@sidecar)
+      @notebook_path = find_notebook
+      @warnings = []
+    end
+
+    def notes
+      @notes ||= place(Notebook.open(@notebook_path))
+    end
+
+    private
+
+    # The folder holding the notes. The device also writes a `kpp_` copy of it
+    # without any, which the glob leaves out.
+    def sidecar
+      root = @path.sub(/\.pdf\z/i, ".sdr")
+      raise Error, "#{@path} is not a PDF" if root == @path
+
+      notes = Dir.glob(ANNOTATIONS, base: root)
+      raise Error, "#{@path} has not been opened on the Kindle: no #{File.basename(root)} beside it" if notes.empty?
+
+      File.join(root, notes.max_by { |file| File.mtime(File.join(root, file)) }.split("/").first)
+    end
+
+    # The ink sits in the storage root's notebook folder, however deep the
+    # document is filed.
+    def find_notebook
+      storage = File.dirname(@sidecar)
+      storage = File.dirname(storage) until File.directory?(File.join(storage, NOTEBOOKS)) || root?(storage)
+      found = Dir.glob("#{NOTEBOOKS}/#{@id}!!*/nbk", base: storage).first
+      raise Error, "no handwriting for #{@path}: #{NOTEBOOKS}/#{@id}!!...!!notebook not found" unless found
+
+      File.join(storage, found)
+    end
+
+    def root?(dir) = File.dirname(dir) == dir
+
+    def place(notebook)
+      @warnings.concat(notebook.warnings)
+      pages = notebook.pages.to_h { |page| [page.id, page] }
+      placed = []
+
+      positions.each do |position, note_id|
+        page = pages.delete(note_id)
+        index = page_index(position)
+        problem = placement_problem(page, index, position)
+        problem ? @warnings << "note #{note_id}: #{problem}" : placed << Note.new(page_index: index, page: page)
+      end
+      pages.each_key { |note_id| @warnings << "note #{note_id}: not attached to any page of the document" }
+      placed.sort_by(&:page_index)
+    end
+
+    def placement_problem(page, index, position)
+      if page.nil? then "its ink is missing from #{@notebook_path}"
+      elsif index.nil? then "position #{position} names no page"
+      elsif page.frame.nil? then "page #{index + 1} has no frame to place it in"
+      end
+    end
+
+    # [position, notebook page id] for every handwritten note in the sidecar.
+    def positions
+      Dir.glob("book.sdr/*.yjr", base: @sidecar).flat_map do |file|
+        Krds.parse_file(File.join(@sidecar, file)).find_all(HANDWRITTEN_NOTE).map do |note|
+          strings = note.strings
+          [strings.first, strings.last]
+        end
+      end
+    end
+
+    def page_index(position)
+      number = position.to_s[POSITION_NUMBER, 1]
+      number && (number.to_i >> PAGE_SHIFT)
+    end
+  end
+end

data/lib/kindle_notebook/document_renderer.rb

@@ -0,0 +1,247 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+# SPDX-FileCopyrightText: 2026 Sergey Avseyev
+# frozen_string_literal: true
+
+require "combine_pdf"
+require "prawn"
+
+module KindleNotebook
+  # Writes a copy of a PDF with the handwriting the Kindle holds for it.
+  #
+  # By default the ink is drawn into each page over what is already there, so
+  # every viewer and printer shows it. With +annotations+ each stroke becomes
+  # a PDF ink annotation instead, which a viewer can hide, move or delete. Its
+  # appearance is drawn by the same code as the page, so it keeps the taper.
+  class DocumentRenderer
+    # Annotation flag: print the annotation with the page.
+    PRINT = 4
+
+    def initialize(annotations: false)
+      @annotations = annotations
+      @renderer = PdfRenderer.new(render_template: false, paper: nil)
+    end
+
+    def render(document, destination)
+      pdf = load(document.path)
+      notes = on_pages(document, pdf.pages.size).reject { |note| note.page.strokes.empty? }
+      if notes.empty? && document.warnings.any?
+        raise Error, "none of the ink could be placed on the PDF: #{document.warnings.join("; ")}"
+      end
+
+      if @annotations
+        annotate(pdf, notes)
+      elsif notes.any?
+        stamp(pdf, notes)
+      end
+      File.binwrite(destination, pdf.to_pdf)
+      notes
+    end
+
+    private
+
+    def load(path)
+      CombinePDF.load(path, allow_optional_content: true)
+    rescue CombinePDF::ParsingError, CombinePDF::EncryptionError => e
+      raise Error, "cannot read #{path}: #{e.message}"
+    end
+
+    def on_pages(document, count)
+      beyond, within = document.notes.partition { |note| note.page_index >= count }
+      beyond.each do |note|
+        document.warnings << "note #{note.page.id}: ink for page #{note.page_index + 1}, " \
+                             "but the PDF has #{count}; is it the same file?"
+      end
+      within
+    end
+
+    # The ink is drawn into a document of its own, one page per note, and each
+    # of its pages is then laid over the page the note belongs to.
+    def stamp(pdf, notes)
+      overlay = Prawn::Document.new(skip_page_creation: true, margin: 0)
+      notes.each { |note| draw_overlay(overlay, pdf.pages[note.page_index], note.page) }
+      ink = CombinePDF.parse(overlay.render)
+      notes.each_with_index { |note, index| pdf.pages[note.page_index] << ink.pages[index] }
+    end
+
+    def draw_overlay(overlay, target, page)
+      overlay.start_new_page(size: box(target).last(2), margin: 0)
+      overlay.cap_style(:round)
+      overlay.join_style(:round)
+      overlay.transformation_matrix(*placement(target, page)) do
+        page.strokes.each { |stroke| @renderer.draw_stroke(overlay, page, stroke) }
+      end
+    end
+
+    def annotate(pdf, notes)
+      notes.each do |note|
+        target = pdf.pages[note.page_index]
+        matrix = placement(target, note.page)
+        added = note.page.strokes.map { |stroke| indirect(ink_annotation(note.page, stroke, matrix)) }
+        existing = target[:Annots]
+        existing = existing[:referenced_object] if existing.is_a?(Hash)
+        target[:Annots] = (existing.is_a?(Array) ? existing : []) + added
+      end
+    end
+
+    def ink_annotation(page, stroke, matrix)
+      appearance = Appearance.new(matrix)
+      @renderer.draw_stroke(appearance, page, stroke)
+      rect = appearance.bounds
+      {
+        Type: :Annot,
+        Subtype: :Ink,
+        F: PRINT,
+        Rect: rect,
+        InkList: [appearance.ink_list],
+        C: appearance.color,
+        BS: { W: appearance.width },
+        AP: { N: indirect(appearance.form(rect)) }
+      }
+    end
+
+    def indirect(object) = { is_reference_only: true, referenced_object: object }
+
+    # [x, y, width, height] of the visible page, in its own user space.
+    def box(pdf_page)
+      left, bottom, right, top = pdf_page.page_size.map(&:to_f)
+      [left, bottom, right - left, top - bottom]
+    end
+
+    # The matrix taking the notebook page's own points onto the frame it was
+    # written in, which is given as fractions of the PDF page from its top left
+    # corner. On a Letter page, for example, the Scribe's 3:4 canvas is fitted
+    # to the width and runs past the top and bottom edges by 12 points each.
+    def placement(pdf_page, page)
+      left, bottom, width, height = box(pdf_page)
+      frame = page.frame
+      frame_width = frame.width * width
+      frame_height = frame.height * height
+      frame_left = left + (frame.left * width)
+      frame_bottom = bottom + height - (frame.top * height) - frame_height
+      [frame_width / page.width_in_points, 0, 0, frame_height / page.height_in_points, frame_left, frame_bottom]
+    end
+
+    # Takes the drawing calls PdfRenderer#draw_stroke makes and writes them as
+    # PDF content operators, to become an annotation's appearance stream.
+    # Points arrive in the notebook page's own space; +matrix+ maps them onto
+    # the PDF page.
+    class Appearance
+      # Control point offset for a quarter circle drawn as a Bezier curve.
+      KAPPA = 0.5523
+
+      def initialize(matrix)
+        @matrix = matrix
+        @operators = []
+        @points = []
+        @alpha = 1.0
+        @width = 0.0
+        @color = "000000"
+      end
+
+      def stroke_color(hex)
+        @color = hex
+        @operators << "#{rgb(hex).join(" ")} RG"
+      end
+
+      def fill_color(hex) = @operators << "#{rgb(hex).join(" ")} rg"
+
+      def line_width(width)
+        @width = [@width, width].max
+        @operators << "#{number(width)} w"
+      end
+
+      def move_to(point) = @operators << "#{coordinates(point)} m"
+
+      def line_to(point) = @operators << "#{coordinates(point)} l"
+
+      def stroke = @operators << "S"
+
+      # Four Bezier quarters, counter-clockwise from the rightmost point.
+      def fill_circle(center, radius)
+        x, y = center
+        @operators << "#{format_point([x + radius, y])} m"
+        quarters(center, radius).each { |curve| @operators << "#{curve.map { |p| format_point(p) }.join(" ")} c" }
+        @operators << "f"
+        @points << center
+        @corners = [[x - radius, y - radius], [x + radius, y + radius]]
+      end
+
+      def transparent(opacity, _stroke_opacity = opacity)
+        @alpha = opacity
+        yield
+      end
+
+      def color = rgb(@color)
+
+      # The widest line drawn, in PDF page units.
+      def width = @width * scale
+
+      # The stroke's points on the PDF page, as an ink list wants them.
+      def ink_list = @points.flat_map { |point| transform(point) }
+
+      # [left, bottom, right, top] on the PDF page, wide enough for the line.
+      def bounds
+        placed = (@points + @corners.to_a).map { |point| transform(point) }
+        pad = width / 2
+        xs = placed.map(&:first)
+        ys = placed.map(&:last)
+        [xs.min - pad, ys.min - pad, xs.max + pad, ys.max + pad]
+      end
+
+      # A form whose bounding box equals the annotation's rectangle, so the
+      # viewer draws it without scaling it.
+      def form(rect)
+        {
+          Type: :XObject,
+          Subtype: :Form,
+          BBox: rect,
+          Resources: resources,
+          raw_stream_content: content
+        }
+      end
+
+      private
+
+      def quarters(center, radius)
+        x, y = center
+        k = radius * KAPPA
+        [
+          [[x + radius, y + k], [x + k, y + radius], [x, y + radius]],
+          [[x - k, y + radius], [x - radius, y + k], [x - radius, y]],
+          [[x - radius, y - k], [x - k, y - radius], [x, y - radius]],
+          [[x + k, y - radius], [x + radius, y - k], [x + radius, y]]
+        ]
+      end
+
+      def content
+        header = ["q", "#{@matrix.map { |value| number(value) }.join(" ")} cm", "1 J", "1 j"]
+        header << "/Ink gs" if @alpha < 1.0
+        (header + @operators + ["Q"]).join("\n")
+      end
+
+      def resources
+        return {} unless @alpha < 1.0
+
+        { ExtGState: { Ink: { Type: :ExtGState, CA: @alpha, ca: @alpha } } }
+      end
+
+      def coordinates(point)
+        @points << point unless @points.last == point
+        format_point(point)
+      end
+
+      def format_point(point) = point.map { |value| number(value) }.join(" ")
+
+      def transform(point)
+        a, _, _, d, e, f = @matrix
+        [(a * point[0]) + e, (d * point[1]) + f]
+      end
+
+      def scale = @matrix.first
+
+      def rgb(hex) = hex.scan(/../).map { |pair| (pair.to_i(16) / 255.0).round(4) }
+
+      def number(value) = format("%.4f", value).sub(/\.?0+\z/, "")
+    end
+  end
+end

data/lib/kindle_notebook/ion.rb

@@ -0,0 +1,243 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+# SPDX-FileCopyrightText: 2026 Sergey Avseyev
+# frozen_string_literal: true
+
+module KindleNotebook
+  # A reader for the subset of binary Amazon Ion 1.0 that Kindle Scribe
+  # notebooks are written in.
+  #
+  # Notebook fragments share one symbol table, stored in its own database row,
+  # which imports the (textless) `YJ_symbols` catalogue. Symbols that resolve
+  # into an imported range have no text, so they are rendered as `$<sid>` --
+  # the same convention Amazon's own tooling uses, and the reason field names
+  # such as `$258` show up in the decoded structures.
+  module Ion
+    BVM = "\xE0\x01\x00\xEA".b
+
+    SYSTEM_SYMBOLS = [
+      nil, "$ion", "$ion_1_0", "$ion_symbol_table", "name", "version",
+      "imports", "symbols", "max_id", "$ion_shared_symbol_table"
+    ].freeze
+
+    class ParseError < Error; end
+
+    # A value carrying Ion annotations, e.g. `$490::{...}`.
+    Annotated = Struct.new(:annotations, :value) do
+      def type = annotations.first
+    end
+
+    # Resolves symbol IDs to text using the system table plus whatever a
+    # notebook's shared symbol table declares.
+    class SymbolTable
+      def initialize(imported_max_id: 0, locals: [])
+        @imported_max_id = imported_max_id
+        @locals = locals
+      end
+
+      def self.system = new
+
+      def local_base = SYSTEM_SYMBOLS.size + @imported_max_id
+
+      def [](sid)
+        return "$#{sid}" if sid.nil? || sid <= 0
+        return SYSTEM_SYMBOLS[sid] || "$#{sid}" if sid < SYSTEM_SYMBOLS.size
+
+        text = @locals[sid - local_base] if sid >= local_base
+        text || "$#{sid}"
+      end
+
+      # Builds a table from a `$ion_symbol_table`-annotated struct.
+      def self.from_fragment(bytes)
+        value = Reader.new(bytes, symbols: system).read_all.last
+        value = value.value if value.is_a?(Annotated)
+        raise ParseError, "symbol table is not a struct" unless value.is_a?(Hash)
+
+        imports = Array(value["imports"])
+        imported = imports.sum { |i| i.is_a?(Hash) ? i["max_id"].to_i : 0 }
+        new(imported_max_id: imported, locals: Array(value["symbols"]))
+      end
+    end
+
+    # Streams top-level Ion values out of a byte string.
+    class Reader
+      def initialize(bytes, symbols: SymbolTable.system)
+        @data = bytes.b
+        @pos = 0
+        @symbols = symbols
+      end
+
+      def read_all
+        values = []
+        skip_bvm
+        while @pos < @data.bytesize
+          value = read_value
+          values << value unless value.equal?(NOP)
+          skip_bvm
+        end
+        values
+      end
+
+      NOP = Object.new.freeze
+      private_constant :NOP
+
+      private
+
+      def skip_bvm
+        @pos += 4 while @data.byteslice(@pos, 4) == BVM
+      end
+
+      def byte
+        b = @data.getbyte(@pos) or raise ParseError, "unexpected end of Ion stream"
+        @pos += 1
+        b
+      end
+
+      def take(count)
+        raise ParseError, "unexpected end of Ion stream" if @pos + count > @data.bytesize
+
+        slice = @data.byteslice(@pos, count)
+        @pos += count
+        slice
+      end
+
+      def var_uint
+        value = 0
+        loop do
+          b = byte
+          value = (value << 7) | (b & 0x7F)
+          return value if b.anybits?(0x80)
+        end
+      end
+
+      def var_int
+        b = byte
+        negative = b.anybits?(0x40)
+        value = b & 0x3F
+        until b.anybits?(0x80)
+          b = byte
+          value = (value << 7) | (b & 0x7F)
+        end
+        negative ? -value : value
+      end
+
+      def bytes_to_uint(length)
+        take(length).each_byte.reduce(0) { |acc, b| (acc << 8) | b }
+      end
+
+      def read_value
+        descriptor = byte
+        type = descriptor >> 4
+        arg = descriptor & 0x0F
+        return nil if arg == 0x0F
+
+        length = case arg
+                 when 0x0E then var_uint
+                 else arg
+                 end
+        dispatch(type, arg, length)
+      end
+
+      def dispatch(type, arg, length)
+        case type
+        when 0x0 then skip_pad(length)
+        when 0x1 then arg == 1
+        when 0x2 then bytes_to_uint(length)
+        when 0x3 then -bytes_to_uint(length)
+        when 0x4 then read_float(length)
+        when 0x5 then read_decimal(length)
+        when 0x6 then read_timestamp(length)
+        when 0x7 then @symbols[bytes_to_uint(length)]
+        when 0x8 then take(length).force_encoding(Encoding::UTF_8)
+        when 0x9, 0xA then take(length)
+        when 0xB, 0xC then read_sequence(length)
+        when 0xD then read_struct(arg, length)
+        when 0xE then read_annotated(length)
+        else raise ParseError, format("reserved Ion type 0x%X", type)
+        end
+      end
+
+      def skip_pad(length)
+        take(length)
+        NOP
+      end
+
+      def read_float(length)
+        case length
+        when 0 then 0.0
+        when 4 then take(4).unpack1("g")
+        when 8 then take(8).unpack1("G")
+        else raise ParseError, "invalid float width #{length}"
+        end
+      end
+
+      def read_decimal(length)
+        return 0r if length.zero?
+
+        stop = @pos + length
+        exponent = var_int
+        coefficient = read_signed_magnitude(stop - @pos)
+        coefficient * (10r**exponent)
+      end
+
+      def read_signed_magnitude(length)
+        return 0 if length.zero?
+
+        raw = take(length)
+        first = raw.getbyte(0)
+        negative = first.anybits?(0x80)
+        magnitude = raw.each_byte.with_index.reduce(0) do |acc, (b, i)|
+          (acc << 8) | (i.zero? ? b & 0x7F : b)
+        end
+        negative ? -magnitude : magnitude
+      end
+
+      # Timestamps are not used by stroke data; the raw bytes are preserved.
+      def read_timestamp(length) = take(length)
+
+      def read_sequence(length)
+        stop = @pos + length
+        values = []
+        while @pos < stop
+          value = read_value
+          values << value unless value.equal?(NOP)
+        end
+        values
+      end
+
+      def read_struct(arg, length)
+        length = var_uint if arg == 1
+        stop = @pos + length
+        fields = {}
+        while @pos < stop
+          name = @symbols[var_uint]
+          value = read_value
+          fields[name] = value unless value.equal?(NOP)
+        end
+        fields
+      end
+
+      def read_annotated(length)
+        stop = @pos + length
+        annot_length = var_uint
+        annot_stop = @pos + annot_length
+        annotations = []
+        annotations << @symbols[var_uint] while @pos < annot_stop
+        value = read_value
+        @pos = stop
+        Annotated.new(annotations.freeze, value)
+      end
+    end
+
+    module_function
+
+    # Strips any annotation wrappers from +value+.
+    def unwrap(value)
+      value = value.value while value.is_a?(Annotated)
+      value
+    end
+
+    def parse(bytes, symbols: SymbolTable.system)
+      Reader.new(bytes, symbols: symbols).read_all
+    end
+  end
+end