nbk2pdf 1.0.0

data/lib/kindle_notebook/krds.rb

@@ -0,0 +1,109 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+# SPDX-FileCopyrightText: 2026 Sergey Avseyev
+# frozen_string_literal: true
+
+module KindleNotebook
+  # Reads the Kindle's reader data store, the format of the `.yjr` and `.yjf`
+  # files it keeps beside a document for reading state and annotations.
+  #
+  #   signature  8 bytes
+  #   version    long
+  #   count      int, the number of top-level fields
+  #   fields     0xFE, name, then typed values and nested fields, then 0xFF
+  #
+  # Every value carries its own type byte, so the file reads without knowing
+  # what any field means. Names are strings without the type byte.
+  module Krds
+    SIGNATURE = "\x00\x00\x00\x00\x00\x1A\xB1\x26".b
+
+    BOOLEAN = 0
+    INT = 1
+    LONG = 2
+    UTF = 3
+    BYTE = 7
+    FIELD_BEGIN = 0xFE
+    FIELD_END = 0xFF
+
+    # A string whose flag byte is set has no length and no content.
+    NULL_STRING = 1
+
+    Field = Struct.new(:name, :items) do
+      # Every field called +name+, at any depth below this one.
+      def find_all(name)
+        items.grep(Field).flat_map { |field| (field.name == name ? [field] : []) + field.find_all(name) }
+      end
+
+      def strings = items.grep(String)
+    end
+
+    class FormatError < Error; end
+
+    module_function
+
+    # Returns the top-level fields of +data+ as one root Field.
+    def parse(data)
+      reader = Reader.new(data.b)
+      raise FormatError, "not a Kindle reader data file" unless reader.take(SIGNATURE.bytesize) == SIGNATURE
+
+      reader.value
+      count = reader.value
+      raise FormatError, "field count missing" unless count.is_a?(Integer)
+
+      Field.new("", Array.new(count) { reader.value })
+    end
+
+    def parse_file(path) = parse(File.binread(path))
+
+    # A cursor over the bytes; every call consumes what it returns.
+    class Reader
+      def initialize(data)
+        @data = data
+        @offset = 0
+      end
+
+      def take(size)
+        raise FormatError, "unexpected end of data at #{@offset}" if @offset + size > @data.bytesize
+
+        chunk = @data.byteslice(@offset, size)
+        @offset += size
+        chunk
+      end
+
+      def byte = take(1).getbyte(0)
+
+      # One value of any type, or a whole field with everything inside it.
+      def value
+        type = byte
+        case type
+        when BOOLEAN then byte != 0
+        when INT then take(4).unpack1("l>")
+        when LONG then take(8).unpack1("q>")
+        when UTF then string
+        when BYTE then byte
+        when FIELD_BEGIN then field
+        else raise FormatError, format("unknown value type 0x%<type>02X at %<at>d", type: type, at: @offset - 1)
+        end
+      end
+
+      def string
+        return nil if byte == NULL_STRING
+
+        take(take(2).unpack1("n")).force_encoding(Encoding::UTF_8)
+      end
+
+      def field
+        name = string
+        items = []
+        items << value until peek == FIELD_END
+        @offset += 1
+        Field.new(name, items)
+      end
+
+      def peek
+        raise FormatError, "unterminated field at #{@offset}" if @offset >= @data.bytesize
+
+        @data.getbyte(@offset)
+      end
+    end
+  end
+end

data/lib/kindle_notebook/notebook.rb

@@ -0,0 +1,442 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+# SPDX-FileCopyrightText: 2026 Sergey Avseyev
+# frozen_string_literal: true
+
+require "set"
+require "sqlite3"
+require "tmpdir"
+
+module KindleNotebook
+  # PDF user space is measured in points.
+  POINTS_PER_INCH = 72.0
+
+  # The brush the firmware records for the highlighter pen.
+  HIGHLIGHTER_BRUSH = 1
+
+  # A single pen stroke in absolute canvas units. +pressures+ is one reading
+  # per point, or nil when the stroke did not record any.
+  Stroke = Struct.new(:xs, :ys, :pressures, :thickness, :color, :brush_type, keyword_init: true) do
+    def dot? = xs.size < 2
+    def highlighter? = brush_type == HIGHLIGHTER_BRUSH
+  end
+
+  # One drawing primitive of a page template, in template units.
+  TemplateShape = Struct.new(:segments, :color, :width, keyword_init: true)
+
+  # The ruled/grid background a page was written on.
+  Template = Struct.new(:id, :kind, :width, :height, :shapes, keyword_init: true) do
+    def empty? = shapes.nil? || shapes.empty?
+  end
+
+  # Where a page's canvas sits on a document page, as fractions of that page's
+  # width and height measured from its top left corner.
+  Frame = Struct.new(:left, :top, :width, :height, keyword_init: true)
+
+  # One notebook page: a canvas, its background template and its ink. Ink
+  # written on a document also carries the +frame+ it was written in.
+  Page = Struct.new(:id, :canvas_width, :canvas_height, :ppi, :template, :strokes, :frame,
+                    keyword_init: true) do
+    def width_in_points = canvas_width / ppi * POINTS_PER_INCH
+    def height_in_points = canvas_height / ppi * POINTS_PER_INCH
+  end
+
+  # Reads a Kindle Scribe `nbk` file: unwraps the signature container, pulls the
+  # Ion fragments out of the embedded SQLite database and resolves them into
+  # pages, strokes and templates.
+  class Notebook
+    # Fragment types, named the way Amazon's own tooling renders them.
+    READING_ORDER = "$258"
+    DOCUMENT_DATA = "$538"
+    BOOK_METADATA = "$490"
+    PAGE_LIST = "$351"
+
+    # Field names inside those fragments.
+    SECTIONS = "$169"
+    SECTION_NAME = "$178"
+    SECTION_MEMBERS = "$170"
+    CONTENT = "$141"
+    CHILDREN = "$146"
+    REFERENCE = "$176"
+    SHAPES = "$250"
+    PATH_OPS = "$249"
+    SHAPE_COLOR = "$75"
+    SHAPE_WIDTH = "$76"
+    DIM_WIDTH = "$56"
+    DIM_HEIGHT = "$57"
+    DIM_VALUE = "$307"
+    DIM_UNIT = "$306"
+    PERCENT = "$314"
+    FRAME_LEFT = "$59"
+    FRAME_TOP = "$58"
+    METADATA_CATEGORIES = "$491"
+    CATEGORY_NAME = "$495"
+    CATEGORY_ENTRIES = "$258"
+    ENTRY_KEY = "$492"
+    ENTRY_VALUE = "$307"
+
+    DEFAULT_CANVAS_WIDTH = 15_624
+    DEFAULT_CANVAS_HEIGHT = 20_832
+    DEFAULT_PPI = 2520.0
+    DEFAULT_THICKNESS = 23.625
+    DEFAULT_BRUSH = 7
+
+    SYMBOL_TABLE_ID = "$ion_symbol_table"
+    TITLE_CATEGORY = "kindle_title_metadata"
+
+    # What each tree walk is looking for.
+    STROKE_LEAF = ->(node) { node["nmdl.type"] == "nmdl.stroke" || node.key?("nmdl.stroke_points") }
+    TEMPLATE_LEAF = ->(node) { node.key?(SHAPES) }
+    private_constant :STROKE_LEAF, :TEMPLATE_LEAF
+
+    class MalformedError < Error; end
+
+    attr_reader :path, :uuid
+
+    # Ink that could not be decoded, reported rather than silently dropped.
+    attr_reader :warnings
+
+    def self.open(path) = new(path)
+
+    # Finds every notebook under +root+ (or returns the notebook at +root+).
+    def self.find(root)
+      root = File.expand_path(root)
+      return [root] if File.file?(root)
+
+      direct = File.join(root, "nbk")
+      return [direct] if File.file?(direct)
+
+      Dir.glob(File.join(root, "**", "nbk")).select { |p| File.file?(p) }.sort
+    end
+
+    def initialize(path)
+      @path = File.expand_path(path)
+      @path = File.join(@path, "nbk") if File.directory?(@path)
+      raise Error, "no such notebook: #{path}" unless File.file?(@path)
+
+      @uuid = File.basename(File.dirname(@path))
+      @warnings = []
+      @fragments = load_fragments
+      @templates = {}
+    end
+
+    def modified_at = File.mtime(@path)
+
+    # The device-supplied title, when the notebook carries one. Standalone
+    # notebooks generally do not -- the Scribe keeps their names elsewhere.
+    def title = title_metadata["title"]
+
+    def book_id = title_metadata["book_id"]
+
+    def pages
+      @pages ||= page_ids.filter_map { |id| build_page(id) }
+    end
+
+    # The file name this notebook should be exported under, without extension.
+    #
+    #   1. an explicit override from +titles+ (keyed by uuid or book id)
+    #   2. the title stored in the notebook itself
+    #   3. the modification date followed by the notebook's uuid
+    def basename(titles: {})
+      name = known_name(titles)
+      return sanitize(name) if name
+
+      sanitize("#{modified_at.strftime("%Y-%m-%d")}_#{uuid}")
+    end
+
+    # The name this notebook is known by, or nil when it falls back to its uuid.
+    def known_name(titles = {})
+      [titles[uuid], titles[book_id], title].find { |name| present?(name) }
+    end
+
+    ILLEGAL_NAME_CHARS = %r{[\\/:*?"<>|\x00-\x1f]}
+
+    # Makes +name+ usable as a file name, falling back to the notebook's uuid
+    # when nothing recognisable survives.
+    def sanitize(name, limit: 120) = self.class.safe_name(name, limit: limit) || uuid
+
+    # +name+ made usable as a file name, or nil when nothing recognisable is left.
+    def self.safe_name(name, limit: 120)
+      safe = name.to_s.gsub(ILLEGAL_NAME_CHARS, "_").strip.sub(/[. ]+\z/, "")
+      safe = safe[0, limit].to_s.strip
+      safe.match?(/[[:alnum:]]/) ? safe : nil
+    end
+
+    private
+
+    def present?(value) = !value.nil? && !value.to_s.strip.empty?
+
+    def load_fragments
+      raw = Dir.mktmpdir("nbk") do |dir|
+        database = File.join(dir, "nbk.sqlite")
+        Container.unwrap_file(@path, database)
+        read_rows(database)
+      end
+
+      symbols = raw[SYMBOL_TABLE_ID]
+      symbols = symbols ? Ion::SymbolTable.from_fragment(symbols) : Ion::SymbolTable.system
+      raw.delete(SYMBOL_TABLE_ID)
+      raw.transform_values { |blob| Ion.parse(blob, symbols: symbols).last }
+    end
+
+    def read_rows(database)
+      rows = {}
+      SQLite3::Database.new(database, readonly: true) do |db|
+        db.execute("SELECT id, payload_value FROM fragments") do |id, blob|
+          rows[id] = blob.to_s.b
+        end
+        db.execute("SELECT id, payload_value, deleted FROM local_delta_fragments") do |id, blob, deleted|
+          deleted.to_i.zero? ? rows[id] = blob.to_s.b : rows.delete(id)
+        end
+      end
+      rows
+    rescue SQLite3::Exception => e
+      raise MalformedError, "cannot read notebook database: #{e.message}"
+    end
+
+    def body(id) = id && Ion.unwrap(@fragments[Ion.unwrap(id)])
+
+    def fragments_of_type(type)
+      @fragments.each_value.select { |value| value.is_a?(Ion::Annotated) && value.type == type }
+    end
+
+    def title_metadata
+      @title_metadata ||= begin
+        metadata = Ion.unwrap(fragments_of_type(BOOK_METADATA).first) || {}
+        categories = Array(metadata[METADATA_CATEGORIES]).map { |category| Ion.unwrap(category) }
+        category = categories.find { |c| Ion.unwrap(c[CATEGORY_NAME]) == TITLE_CATEGORY }
+        Array(category&.[](CATEGORY_ENTRIES)).each_with_object({}) do |entry, acc|
+          entry = Ion.unwrap(entry)
+          acc[Ion.unwrap(entry[ENTRY_KEY]).to_s] = Ion.unwrap(entry[ENTRY_VALUE])
+        end
+      end
+    end
+
+    def reading_order
+      @reading_order ||= Ion.unwrap(fragments_of_type(READING_ORDER).first) ||
+                         raise(MalformedError, "notebook has no reading order")
+    end
+
+    def document_data
+      @document_data ||= Ion.unwrap(fragments_of_type(DOCUMENT_DATA).first) || {}
+    end
+
+    def sections_of(source) = Array(source[SECTIONS]).map { |section| Ion.unwrap(section) }
+
+    def section_of(source, name) = sections_of(source).find { |s| Ion.unwrap(s[SECTION_NAME]) == name }
+
+    def members_of(section) = Array(section&.[](SECTION_MEMBERS)).map { |ref| Ion.unwrap(ref) }
+
+    def page_ids
+      members_of(section_of(reading_order, PAGE_LIST) || sections_of(reading_order).first)
+    end
+
+    def build_page(id)
+      page = body(id)
+      return nil unless page.is_a?(Hash)
+
+      Page.new(
+        id: id,
+        canvas_width: (page["nmdl.canvas_width"] || DEFAULT_CANVAS_WIDTH).to_i,
+        canvas_height: (page["nmdl.canvas_height"] || DEFAULT_CANVAS_HEIGHT).to_i,
+        ppi: (page["nmdl.normalized_ppi"] || DEFAULT_PPI).to_f,
+        template: template_for(page),
+        strokes: collect_strokes(page),
+        frame: frame_of(page)
+      )
+    end
+
+    # A page written on a document records the box its canvas occupies there,
+    # in percent of the document page; a notebook page records none.
+    def frame_of(page)
+      box = [FRAME_LEFT, FRAME_TOP, DIM_WIDTH, DIM_HEIGHT].map { |key| percentage(page[key]) }
+      return nil if box.any?(&:nil?)
+
+      left, top, width, height = box
+      Frame.new(left: left, top: top, width: width, height: height)
+    end
+
+    def percentage(value)
+      value = Ion.unwrap(value)
+      return nil unless value.is_a?(Hash) && Ion.unwrap(value[DIM_UNIT]) == PERCENT
+
+      Ion.unwrap(value[DIM_VALUE]).to_f / 100
+    end
+
+    def collect_strokes(page)
+      each_leaf(page[CONTENT], STROKE_LEAF).filter_map { |node| build_stroke(node) }
+    end
+
+    # Walks a content tree depth first -- so ink keeps its drawing order --
+    # following both fragment references and inline children, and returns every
+    # node +leaf+ recognises. A fragment is visited once, however often it is
+    # referenced.
+    def each_leaf(roots, leaf)
+      found = []
+      visited = Set.new
+      stack = Array(roots).dup
+
+      until stack.empty?
+        node = Ion.unwrap(stack.shift)
+        next unless node.is_a?(Hash)
+        next found << node if leaf.call(node)
+
+        stack.unshift(*children_of(node, visited))
+      end
+      found
+    end
+
+    def children_of(node, visited)
+      reference = Ion.unwrap(node[REFERENCE])
+      referenced = reference && visited.add?(reference) ? Array(body(reference)&.[](CHILDREN)) : []
+      referenced + Array(node[CONTENT]) + Array(node[CHILDREN])
+    end
+
+    def build_stroke(node)
+      points = Ion.unwrap(node["nmdl.stroke_points"])
+      count = points&.[]("nmdl.num_points").to_i
+      return nil if count < 1
+
+      origin = Array(Ion.unwrap(node["nmdl.stroke_bounds"]))
+      Stroke.new(
+        xs: coordinates(points["nmdl.position_x"], count, origin[0]),
+        ys: coordinates(points["nmdl.position_y"], count, origin[1]),
+        pressures: pressures(points["nmdl.pressure"], count),
+        **pen(node)
+      )
+    rescue StrokeDecoder::DecodeError => e
+      @warnings << "stroke #{Ion.unwrap(node["$598"]) || "?"}: #{e.message}"
+      nil
+    end
+
+    def pen(node)
+      {
+        thickness: (node["nmdl.thickness"] || DEFAULT_THICKNESS).to_f,
+        color: (node["nmdl.color"] || 0).to_i,
+        brush_type: (node["nmdl.brush_type"] || DEFAULT_BRUSH).to_i
+      }
+    end
+
+    # Unreadable pressure costs the stroke its taper, not the stroke itself.
+    def pressures(blob, count)
+      return nil unless blob
+
+      StrokeDecoder.decode(blob, count)
+    rescue StrokeDecoder::DecodeError
+      nil
+    end
+
+    # Decoded coordinates are relative to the stroke's bounding box origin.
+    def coordinates(blob, count, origin)
+      offset = origin.to_i
+      StrokeDecoder.decode(blob, count).map { |value| value + offset }
+    end
+
+    # A page names its template, or stores a placeholder symbol to say it has
+    # none; a page that names nothing at all takes the document's default.
+    def template_for(page)
+      id =
+        if page.key?("nmdl.template_id")
+          Ion.unwrap(page["nmdl.template_id"])
+        else
+          default_template_id
+        end
+      return nil unless @fragments.key?(id)
+
+      @templates.fetch(id) { @templates[id] = build_template(id) }
+    end
+
+    # The template a page without its own falls back to.
+    def default_template_id
+      @default_template_id ||= Ion.unwrap(document_data["nmdl.template_id"])
+    end
+
+    def build_template(id)
+      root = body(id)
+      return nil unless root.is_a?(Hash)
+
+      canvas = each_leaf([root], TEMPLATE_LEAF).first
+      return nil unless canvas
+
+      Template.new(
+        id: id,
+        kind: Ion.unwrap(root["nmdl.template_type"]),
+        width: dimension(canvas[DIM_WIDTH]),
+        height: dimension(canvas[DIM_HEIGHT]),
+        shapes: Array(canvas[SHAPES]).filter_map { |shape| build_shape(shape) }
+      )
+    end
+
+    def dimension(value)
+      value = Ion.unwrap(value)
+      return value.to_f if value.is_a?(Numeric)
+
+      value.is_a?(Hash) ? Ion.unwrap(value[DIM_VALUE]).to_f : 0.0
+    end
+
+    def build_shape(shape)
+      shape = Ion.unwrap(shape)
+      segments = PathOps.segments(Array(Ion.unwrap(shape[PATH_OPS])))
+      return nil if segments.empty?
+
+      TemplateShape.new(
+        segments: segments,
+        color: (shape[SHAPE_COLOR] || 0xFF00_0000).to_i,
+        width: (shape[SHAPE_WIDTH] || 1).to_f
+      )
+    end
+
+    # Template outlines are flat op streams: an opcode followed by its operands.
+    # These are the only opcodes the device writes. An unknown one ends the
+    # shape, because guessing how many operands it takes would desynchronise
+    # everything after it.
+    module PathOps
+      MOVE_TO = 0
+      LINE_TO = 1
+      CLOSE = 4
+
+      ARITY = { MOVE_TO => 2, LINE_TO => 2, CLOSE => 0 }.freeze
+
+      module_function
+
+      # Returns a list of polylines, each a list of [x, y] pairs. A run with no
+      # starting point, or with only one, draws nothing and is dropped, as does
+      # the whole shape when the op stream cannot be read to the end.
+      def segments(ops)
+        parsed = parse(ops)
+        return [] if parsed.nil?
+
+        parsed.chunk_while { |_, (opcode, _)| opcode != MOVE_TO }
+              .map { |run| outline(run) }
+              .reject { |line| line.size < 2 }
+      end
+
+      # One run of ops, starting at a move, becomes one polyline.
+      def outline(run)
+        points = []
+        run.each do |opcode, args|
+          case opcode
+          when MOVE_TO, LINE_TO then points << args
+          when CLOSE then points << points.first if points.size > 1
+          end
+        end
+        points
+      end
+
+      # Splits the stream into [opcode, operands] pairs, or nil on an opcode it
+      # does not know or operands that run off the end.
+      def parse(ops)
+        parsed = []
+        index = 0
+        while index < ops.size
+          opcode = ops[index].to_i
+          arity = ARITY[opcode]
+          return nil if arity.nil? || index + arity >= ops.size
+
+          parsed << [opcode, ops[index + 1, arity].map(&:to_f)]
+          index += 1 + arity
+        end
+        parsed
+      end
+    end
+  end
+end