mddir 0.1.0

data/lib/mddir/fetcher.rb

@@ -0,0 +1,213 @@
+# frozen_string_literal: true
+
+require "http-cookie"
+require "httpx"
+require "nokogiri"
+require "readability"
+require "reverse_markdown"
+require "yaml"
+
+module Mddir
+  class Fetcher # rubocop:disable Metrics/ClassLength
+    CONNECT_TIMEOUT = 15
+    READ_TIMEOUT = 30
+
+    READABILITY_TAGS = %w[
+      div p span
+      h1 h2 h3 h4 h5 h6
+      pre code
+      ul ol li
+      table thead tbody tfoot tr th td
+      blockquote
+      a img br hr
+      strong em b i u s del sub sup
+      dl dt dd
+      figure figcaption
+      details summary
+    ].freeze
+
+    READABILITY_ATTRIBUTES = %w[href src alt title lang class id style].freeze
+
+    def initialize(config, cookies_path: nil)
+      @config = config
+      @cookie_jar = load_cookies(cookies_path)
+      @client = build_client
+    end
+
+    def fetch(url)
+      response = request(url)
+      content_type = response.headers["content-type"].to_s
+
+      if content_type.include?("text/markdown")
+        process_markdown_response(url, response)
+      else
+        process_html_response(url, response)
+      end
+    end
+
+    private
+
+    def load_cookies(path)
+      return nil unless path && File.exist?(path)
+
+      jar = HTTP::CookieJar.new
+      jar.load(path, format: :cookiestxt, session: true)
+      jar
+    end
+
+    def build_client
+      HTTPX.plugin(:follow_redirects)
+           .with(
+             headers: {
+               "accept" => "text/markdown, text/html",
+               "user-agent" => @config.user_agent
+             },
+             timeout: { connect_timeout: CONNECT_TIMEOUT, read_timeout: READ_TIMEOUT }
+           )
+    end
+
+    def request(url)
+      headers = cookie_headers(url)
+      response = @client.get(url, headers: headers)
+      raise FetchError, response.error.message if response.is_a?(HTTPX::ErrorResponse)
+
+      response
+    end
+
+    def cookie_headers(url)
+      return {} unless @cookie_jar
+
+      uri = URI.parse(url)
+      cookie_value = HTTP::Cookie.cookie_value(@cookie_jar.cookies(uri))
+      cookie_value.empty? ? {} : { "cookie" => cookie_value }
+    end
+
+    def normalize_encoding(body, content_type)
+      body = body.dup
+      charset = content_type&.match(/charset=([^\s;]+)/i)&.captures&.first # rubocop:disable Style/SafeNavigationChainLength
+      body.force_encoding(charset || "UTF-8")
+      body.encode("UTF-8", invalid: :replace, undef: :replace)
+    end
+
+    def process_markdown_response(url, response)
+      body = normalize_encoding(response.body.to_s, response.headers["content-type"])
+      frontmatter, content = parse_frontmatter(body)
+      token_count, token_estimated = resolve_token_count(content, response.headers["x-markdown-tokens"])
+
+      Entry.new(
+        url:,
+        title: frontmatter["title"].to_s,
+        description: frontmatter["description"].to_s,
+        markdown: body,
+        conversion: "cloudflare",
+        token_count:,
+        token_estimated:
+      )
+    end
+
+    def parse_frontmatter(body)
+      if body.start_with?("---")
+        parts = body.split("---", 3)
+        if parts.length >= 3
+          frontmatter = YAML.safe_load(parts[1], permitted_classes: [Time]) || {}
+          return [frontmatter, parts[2].lstrip]
+        end
+      end
+
+      [{}, body]
+    end
+
+    def resolve_token_count(content, header)
+      if header
+        [header.to_i, false]
+      else
+        [(content.length / 4.0).ceil, true]
+      end
+    end
+
+    def process_html_response(url, response)
+      html = normalize_encoding(response.body.to_s, response.headers["content-type"])
+      document = Nokogiri::HTML(html)
+      title, article_html = extract_readable_content(html, document)
+      markdown = html_to_markdown(article_html)
+
+      Entry.new(
+        url:,
+        title:,
+        description: extract_description(document),
+        markdown:,
+        conversion: "local",
+        token_count: (markdown.length / 4.0).ceil,
+        token_estimated: true
+      )
+    end
+
+    def extract_readable_content(html, document)
+      title, article_html = run_readability(html)
+
+      if article_html.nil? || article_html.strip.empty?
+        warn "Warning: readability extracted no content, falling back to full body"
+        article_html = document.at("body")&.inner_html.to_s
+      end
+
+      title = extract_title(document) if title.empty?
+
+      [clean_title(title), article_html]
+    end
+
+    def run_readability(html)
+      readable = Readability::Document.new(html, tags: READABILITY_TAGS, attributes: READABILITY_ATTRIBUTES)
+      [readable.title.to_s, readable.content]
+    rescue StandardError
+      ["", nil]
+    end
+
+    def html_to_markdown(article_html)
+      article_html = article_html.encode("UTF-8", invalid: :replace, undef: :replace)
+      code_languages = extract_code_languages(article_html)
+      markdown = ReverseMarkdown.convert(article_html, github_flavored: true).force_encoding("UTF-8")
+      inject_code_languages(markdown, code_languages)
+    end
+
+    def extract_code_languages(html) # rubocop:disable Metrics/CyclomaticComplexity
+      fragment = Nokogiri::HTML.fragment(html)
+
+      fragment.css("pre").map do |pre|
+        pre["lang"] ||
+          pre["data-lang"] ||
+          pre.css("code").first&.[]("class")&.match(/language-(\w+)/)&.captures&.first # rubocop:disable Style/SafeNavigationChainLength
+      end
+    end
+
+    def inject_code_languages(markdown, languages) # rubocop:disable Metrics/MethodLength
+      index = 0
+
+      markdown.gsub(/^```\s*$/) do |match|
+        if index.even? && (index / 2) < languages.length
+          lang = languages[index / 2]
+          index += 1
+          lang ? "```#{lang}" : match
+        else
+          index += 1
+          match
+        end
+      end
+    end
+
+    def extract_description(document)
+      meta = document.at('meta[name="description"]')
+      meta ? meta["content"].to_s : ""
+    end
+
+    def extract_title(document)
+      title_tag = document.at("title")
+      title_tag ? title_tag.text.to_s.strip : ""
+    end
+
+    def clean_title(title)
+      title.sub(/\s*[|–—-]\s*[^|–—-]+\z/, "").strip
+    end
+  end
+
+  class FetchError < StandardError; end
+end

data/lib/mddir/global_index.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "yaml"
+
+module Mddir
+  module GlobalIndex
+    def self.path(config)
+      File.join(config.base_dir, "index.yml")
+    end
+
+    def self.load(config)
+      file = path(config)
+      return update!(config) unless File.exist?(file)
+
+      data = YAML.safe_load_file(file, permitted_classes: [Time])
+      return update!(config) unless data.is_a?(Hash)
+
+      data
+    rescue Psych::SyntaxError
+      update!(config)
+    end
+
+    def self.update!(config)
+      FileUtils.mkdir_p(config.base_dir)
+
+      collections = build_collections(config)
+
+      data = {
+        "collections" => collections,
+        "total_entries" => collections.sum { |_, info| info["entry_count"] },
+        "last_updated" => Time.now.utc.iso8601
+      }
+
+      File.write(path(config), YAML.dump(data))
+
+      data
+    end
+
+    def self.build_collections(config)
+      Collection.all(config).to_h do |collection|
+        [collection.name, {
+          "entry_count" => collection.entry_count,
+          "last_added" => collection.last_added&.to_s
+        }]
+      end
+    end
+
+    private_class_method :build_collections
+  end
+end

data/lib/mddir/search.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module Mddir
+  class Search
+    Result = Struct.new(:collection_name, :entry, :matches)
+    Match = Struct.new(:line_number, :snippet)
+
+    def initialize(config)
+      @config = config
+    end
+
+    def search(query, collection_name: nil)
+      collections = resolve_collections(collection_name)
+      return [] if collections.empty?
+
+      SearchIndex.open(@config) do |index|
+        collections.each { |collection| index.ensure_current!(collection) }
+
+        rows = index.query(query, collection_names: collections.map(&:name))
+        build_results(collections, rows, query)
+      end
+    end
+
+    private
+
+    def resolve_collections(collection_name)
+      if collection_name
+        collection = Collection.new(collection_name, @config)
+        collection.exist? ? [collection] : []
+      else
+        Collection.all(@config)
+      end
+    end
+
+    def build_results(collections, rows, query)
+      entries_lookup = build_entries_lookup(collections)
+      grouped = rows.group_by { |row| [row["collection"], row["filename"]] }
+
+      grouped.filter_map do |(collection_name, filename), file_rows|
+        entry = entries_lookup.dig(collection_name, filename)
+        next unless entry
+
+        Result.new(collection_name:, entry:, matches: build_matches(file_rows, query))
+      end
+    end
+
+    def build_entries_lookup(collections)
+      collections.to_h do |collection|
+        [collection.name, collection.entries.to_h { |entry| [entry["filename"], entry] }]
+      end
+    end
+
+    def build_matches(file_rows, query)
+      file_rows.map do |row|
+        snippet = extract_snippet(row["content"], query)
+        Match.new(line_number: row["line_number"].to_i, snippet: snippet)
+      end
+    end
+
+    def extract_snippet(line, query) # rubocop:disable Metrics/AbcSize
+      line = line.strip
+      index = line.downcase.index(query.downcase)
+      return line[0, 120] unless index
+
+      start = [index - 40, 0].max
+      finish = [index + query.length + 80, line.length].min
+      snippet = line[start...finish]
+      snippet = "...#{snippet}" if start.positive?
+      snippet = "#{snippet}..." if finish < line.length
+      snippet
+    end
+  end
+end

data/lib/mddir/search_index.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+require "sqlite3"
+
+module Mddir
+  class SearchIndex
+    def self.open(config)
+      index = new(config)
+      yield index
+    ensure
+      index&.close
+    end
+
+    def initialize(config)
+      @db = SQLite3::Database.new(File.join(config.base_dir, "search.db"))
+      @db.results_as_hash = true
+      setup_schema
+    end
+
+    def ensure_current!(collection)
+      row = @db.get_first_row("SELECT indexed_at FROM meta WHERE collection = ?", collection.name)
+      return if row && row["indexed_at"] >= index_mtime(collection)
+
+      reindex(collection)
+    end
+
+    def query(text, collection_names:)
+      escaped = text.gsub('"', '""')
+      placeholders = (["?"] * collection_names.size).join(", ")
+
+      @db.execute(
+        "SELECT collection, filename, line_number, content FROM search_lines " \
+        "WHERE search_lines MATCH ? AND collection IN (#{placeholders}) ORDER BY rank",
+        ["\"#{escaped}\"", *collection_names]
+      )
+    end
+
+    def remove_collection!(collection_name)
+      @db.execute("DELETE FROM search_lines WHERE collection = ?", collection_name)
+      @db.execute("DELETE FROM meta WHERE collection = ?", collection_name)
+    end
+
+    def close
+      @db.close
+    end
+
+    private
+
+    def setup_schema
+      @db.execute_batch(<<~SQL)
+        CREATE VIRTUAL TABLE IF NOT EXISTS search_lines USING fts5(
+          collection UNINDEXED,
+          filename UNINDEXED,
+          line_number UNINDEXED,
+          content,
+          tokenize='trigram case_sensitive 0'
+        );
+
+        CREATE TABLE IF NOT EXISTS meta (
+          collection TEXT PRIMARY KEY,
+          indexed_at REAL NOT NULL
+        );
+      SQL
+    end
+
+    def reindex(collection)
+      @db.transaction do
+        @db.execute("DELETE FROM search_lines WHERE collection = ?", collection.name)
+        index_collection_files(collection)
+        update_meta(collection)
+      end
+    end
+
+    def index_collection_files(collection)
+      collection.entries.each do |entry|
+        file_path = File.join(collection.path, entry["filename"])
+        next unless File.exist?(file_path)
+
+        index_file(collection.name, entry["filename"], file_path)
+      end
+    end
+
+    def update_meta(collection)
+      @db.execute(
+        "INSERT OR REPLACE INTO meta (collection, indexed_at) VALUES (?, ?)",
+        [collection.name, index_mtime(collection)]
+      )
+    end
+
+    def index_file(collection_name, filename, file_path)
+      lines = File.readlines(file_path, encoding: "UTF-8")
+
+      Utils.skip_frontmatter(lines).each do |line_number, line|
+        next if line.strip.empty?
+
+        @db.execute(
+          "INSERT INTO search_lines (collection, filename, line_number, content) VALUES (?, ?, ?, ?)",
+          [collection_name, filename, line_number, line]
+        )
+      end
+    end
+
+    def index_mtime(collection)
+      File.exist?(collection.index_path) ? File.mtime(collection.index_path).to_f : 0.0
+    end
+  end
+end

data/lib/mddir/server.rb

@@ -0,0 +1,152 @@
+# frozen_string_literal: true
+
+require "sinatra/base"
+require "kramdown"
+require "kramdown-parser-gfm"
+require "rouge"
+require "uri"
+
+module Mddir
+  class Server < Sinatra::Base # rubocop:disable Metrics/ClassLength
+    set :views, File.expand_path("../../views", __dir__)
+    set :public_folder, File.expand_path("../../public", __dir__)
+
+    enable :method_override
+
+    before do
+      @collection_names = Collection.all(config).map(&:name)
+    end
+
+    def self.start(config)
+      set :mddir_config, config
+      set :port, config.port
+      set :bind, "localhost"
+
+      puts "mddir server running at http://localhost:#{config.port}"
+      puts "Press Ctrl+C to stop"
+      run!
+    end
+
+    helpers do # rubocop:disable Metrics/BlockLength
+      def config
+        settings.mddir_config
+      end
+
+      def format_date(date_str)
+        return "" unless date_str
+
+        Time.parse(date_str.to_s).strftime("%b %d, %Y")
+      rescue ArgumentError
+        date_str.to_s
+      end
+
+      def domain_from_url(url)
+        URI.parse(url).host
+      rescue URI::InvalidURIError
+        url
+      end
+
+      def truncate(text, length = 200)
+        return "" unless text
+
+        text.length > length ? "#{text[0, length]}..." : text
+      end
+
+      def h(text)
+        Rack::Utils.escape_html(text.to_s)
+      end
+
+      def format_tokens(count)
+        return "" unless count
+
+        count >= 1000 ? "~#{(count / 1000.0).round(1)}k tokens" : "~#{count} tokens"
+      end
+
+      def highlight(text, query)
+        return h(text) unless query && !query.empty?
+
+        escaped_query = Regexp.escape(query)
+        h(text).gsub(/#{escaped_query}/i) { |m| "<mark>#{m}</mark>" }
+      end
+    end
+
+    get "/" do
+      @global = GlobalIndex.load(config)
+      @collections = (@global["collections"] || {}).sort_by { |name, _| name }
+
+      erb :home
+    end
+
+    get "/search" do
+      @query = params["q"].to_s.strip
+      @collection_filter = params["collection"]
+
+      if @query.empty?
+        @results = []
+      else
+        searcher = Search.new(config)
+        @results = searcher.search(@query, collection_name: @collection_filter)
+      end
+
+      erb :search
+    end
+
+    get "/:collection" do
+      collection = Collection.new(params[:collection], config)
+      halt 404, "Collection not found" unless collection.exist?
+
+      @collection = collection
+      @current_collection = collection.name
+      @entries = collection.entries.reverse
+
+      erb :collection
+    end
+
+    get "/:collection/:slug" do
+      collection = Collection.new(params[:collection], config)
+      halt 404, "Collection not found" unless collection.exist?
+
+      @collection = collection
+      @current_collection = collection.name
+      @entry = collection.entries.find { |entry| entry["slug"] == params[:slug] }
+      halt 404, "Entry not found" unless @entry
+
+      file_path = File.join(collection.path, @entry["filename"])
+      halt 404, "File not found" unless File.exist?(file_path)
+
+      raw = File.read(file_path, encoding: "UTF-8")
+      content = Utils.strip_frontmatter(raw)
+      @html_content = Kramdown::Document.new(
+        content,
+        input: "GFM",
+        syntax_highlighter: :rouge,
+        syntax_highlighter_opts: {
+          default_lang: "plaintext"
+        }
+      ).to_html
+
+      erb :reader
+    end
+
+    delete "/:collection" do
+      collection = Collection.new(params[:collection], config)
+      halt 404, "Collection not found" unless collection.exist?
+
+      collection.remove!
+
+      redirect "/"
+    end
+
+    delete "/:collection/:slug" do
+      collection = Collection.new(params[:collection], config)
+      halt 404, "Collection not found" unless collection.exist?
+
+      entry = collection.entries.find { |e| e["slug"] == params[:slug] }
+      halt 404, "Entry not found" unless entry
+
+      collection.remove_entry(entry["slug"])
+
+      redirect "/#{collection.name}"
+    end
+  end
+end

data/lib/mddir/utils.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Mddir
+  module Utils
+    def self.slugify(text)
+      text.downcase
+          .gsub(/[^a-z0-9]+/, "-")
+          .gsub(/-{2,}/, "-")
+          .gsub(/\A-|-\z/, "")
+    end
+
+    def self.strip_frontmatter(text)
+      if text.start_with?("---")
+        parts = text.split("---", 3)
+        parts.length >= 3 ? parts[2].lstrip : text
+      else
+        text
+      end
+    end
+
+    def self.skip_frontmatter(lines) # rubocop:disable Metrics/MethodLength
+      result = []
+      in_frontmatter = false
+
+      lines.each_with_index do |line, index|
+        line_number = index + 1
+
+        if line_number == 1 && line.strip == "---"
+          in_frontmatter = true
+          next
+        end
+
+        if in_frontmatter && line.strip == "---"
+          in_frontmatter = false
+          next
+        end
+
+        next if in_frontmatter
+
+        result << [line_number, line.chomp]
+      end
+
+      result
+    end
+  end
+end

data/lib/mddir/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Mddir
+  VERSION = "0.1.0"
+end

data/lib/mddir.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require_relative "mddir/version"
+require_relative "mddir/utils"
+require_relative "mddir/config"
+require_relative "mddir/global_index"
+require_relative "mddir/collection"
+require_relative "mddir/entry"
+require_relative "mddir/fetcher"
+require_relative "mddir/search_index"
+require_relative "mddir/search"
+require_relative "mddir/cli"
+
+module Mddir
+  class Error < StandardError; end
+end