markdownr 0.4.5 → 0.4.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 53062f10b38520d041fb4fab0e1155e1b19e15242409996dd82122716bc727d8
-  data.tar.gz: 17ee73d28a5ddc64a915d190f31c157c6d8c9319451c1fe754c3c41417bab9a9
+  metadata.gz: ae4a95fd815bc7acdaf267cf6ac5b056debfa7da5812b66c254e0ef002c01da6
+  data.tar.gz: cb4e91f2e5aab746588702770f331e20fea2d71e586c84835417cf5d6d27ffe7
 SHA512:
-  metadata.gz: 8c762e19d73d839cdcf1ff5f4e16c009f22b575a5672936eabef2d5131641d639ae26771bce19f25adbb287e6f7c5d71bb0561b76f302ea32db4b3ad41dc2937
-  data.tar.gz: e28d99ad58d6ee904aa3814f665fa31d0313605629be193a482b6f0ef30cce74868800a18cec6db0ecb0101f3182dc48d6b8b561a75d9e39f04db8b7daa9e443
+  metadata.gz: 20729eb32420c42496a531229d72baf7692cd0de81eed5578f48e1a20fdf844fe4235367174c24b27e6cabbccf1223eb17dfe73c40a35e2058475334dd4fff50
+  data.tar.gz: b2902ec6c5bc7ae048da0c4f5f6d54cadd09003f8fdf524b59e2b96f519d4ca8b9a6e0cd30635266dfc64fa678eca09103aad5366bd96a840c995f6eaa000111

data/lib/markdown_server/app.rb

@@ -8,6 +8,7 @@ require "uri"
 require "cgi"
 require "pathname"
 require "set"
+require "net/http"
 
 module MarkdownServer
   class App < Sinatra::Base
@@ -431,6 +432,102 @@ module MarkdownServer
         html
       end
 
+      FETCH_MAX_BYTES = 512_000
+      FETCH_TIMEOUT   = 5
+
+      # Tags kept as-is (attributes stripped)
+      ALLOWED_HTML = %w[p h1 h2 h3 h4 h5 h6 blockquote ul ol li
+                        pre br hr strong b em i sup sub code
+                        table tr td th].to_set
+      # Block containers — replaced with a newline (content kept)
+      BLOCK_HTML   = %w[div section aside figure figcaption
+                        thead tbody tfoot].to_set
+      # Elements removed completely, including their content
+      STRIP_FULL   = %w[script style nav header footer form input
+                        button select textarea svg iframe noscript].to_set
+
+      def fetch_external_page(url_str)
+        uri = URI.parse(url_str)
+        return nil unless uri.is_a?(URI::HTTP) || uri.is_a?(URI::HTTPS)
+        fetch_follow_redirects(uri, 5)
+      rescue
+        nil
+      end
+
+      def fetch_follow_redirects(uri, limit)
+        return nil if limit <= 0
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = (uri.scheme == "https")
+        http.open_timeout = FETCH_TIMEOUT
+        http.read_timeout = FETCH_TIMEOUT
+        req = Net::HTTP::Get.new(uri.request_uri)
+        req["User-Agent"] = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
+        req["Accept"] = "text/html,application/xhtml+xml;q=0.9,*/*;q=0.8"
+        req["Accept-Language"] = "en-US,en;q=0.5"
+        resp = http.request(req)
+        case resp
+        when Net::HTTPSuccess
+          ct = resp["content-type"].to_s
+          return nil unless ct.match?(/html|text/i)
+          body = resp.body.to_s
+          body = body.b[0, FETCH_MAX_BYTES].force_encoding("utf-8")
+          body.encode("utf-8", invalid: :replace, undef: :replace, replace: "?")
+        when Net::HTTPRedirection
+          loc = resp["Location"].to_s
+          new_uri = (URI.parse(loc) rescue nil)
+          return nil unless new_uri
+          new_uri = uri + new_uri unless new_uri.absolute?
+          return nil unless new_uri.is_a?(URI::HTTP) || new_uri.is_a?(URI::HTTPS)
+          fetch_follow_redirects(new_uri, limit - 1)
+        end
+      rescue
+        nil
+      end
+
+      def page_title(html)
+        html.match(/<title[^>]*>(.*?)<\/title>/im)&.then { |m|
+          m[1].gsub(/<[^>]+>/, "").gsub(/&amp;/i, "&").gsub(/&lt;/i, "<")
+              .gsub(/&gt;/i, ">").gsub(/&quot;/i, '"').gsub(/&#?\w+;/, "").strip
+        } || ""
+      end
+
+      def page_html(raw)
+        w = raw.dup
+        # Remove inert elements and their entire contents
+        STRIP_FULL.each { |t| w.gsub!(/<#{t}[^>]*>.*?<\/#{t}>/im, " ") }
+        w.gsub!(/<!--.*?-->/m, " ")
+
+        # Prefer a focused content block
+        content = w.match(/<article[^>]*>(.*?)<\/article>/im)&.[](1) ||
+                  w.match(/<main[^>]*>(.*?)<\/main>/im)&.[](1) ||
+                  w.match(/<body[^>]*>(.*?)<\/body>/im)&.[](1) ||
+                  w
+
+        # Rewrite tags: keep allowed (strip attrs), block→newline, rest→empty
+        out = content.gsub(/<(\/?)(\w+)[^>]*>/) do
+          slash, tag = $1, $2.downcase
+          if ALLOWED_HTML.include?(tag)  then "<#{slash}#{tag}>"
+          elsif BLOCK_HTML.include?(tag) then "\n"
+          else ""
+          end
+        end
+
+        # Decode HTML entities
+        out = out
+          .gsub(/&nbsp;/i, " ").gsub(/&amp;/i, "&").gsub(/&lt;/i, "<").gsub(/&gt;/i, ">")
+          .gsub(/&quot;/i, '"').gsub(/&apos;/i, "'")
+          .gsub(/&mdash;/i, "—").gsub(/&ndash;/i, "–").gsub(/&hellip;/i, "…")
+          .gsub(/&#(\d+);/)         { [$1.to_i].pack("U")      rescue " " }
+          .gsub(/&#x([\da-f]+);/i)  { [$1.to_i(16)].pack("U") rescue " " }
+          .gsub(/&\w+;/, " ")
+          .gsub(/[ \t]+/, " ")
+          .gsub(/\n{3,}/, "\n\n")
+          .gsub(/<(\w+)>\s*<\/\1>/, "")  # drop empty tags
+          .strip
+
+        out.length > 10_000 ? out[0, 10_000] : out
+      end
+
       def compile_regexes(query)
         words = query.split(/\s+/).reject(&:empty?)
         return nil if words.empty?
@@ -514,6 +611,17 @@ module MarkdownServer
       JSON.dump({ title: title.to_s, html: html })
     end
 
+    get "/fetch" do
+      content_type :json
+      url = params[:url].to_s.strip
+      halt 400, '{"error":"invalid url"}' unless url.match?(/\Ahttps?:\/\//i)
+
+      html = fetch_external_page(url)
+      halt 502, '{"error":"fetch failed"}' unless html
+
+      JSON.dump({ title: page_title(html), html: page_html(html) })
+    end
+
     get "/search/?*" do
       requested = params["splat"].first.to_s.chomp("/")
       @query = params[:q].to_s.strip

data/lib/markdown_server/version.rb

@@ -1,3 +1,3 @@
 module MarkdownServer
-  VERSION = "0.4.5"
+  VERSION = "0.4.7"
 end

data/views/layout.erb

@@ -1535,14 +1535,47 @@
                 });
             }
           }
-        } else {
-          var urlBody = '<div class="link-ctx-popup-url">' + escHtml(href) + '</div>';
-          if (isExternal(href)) {
-            var host = '';
-            try { host = new URL(href).hostname; } catch(e) {}
-            if (host) urlBody += '<p style="margin:0;color:#888;font-family:sans-serif;font-size:0.82rem">External: <strong style="color:#555">' + escHtml(host) + '</strong></p>';
+        } else if (isExternal(href)) {
+          var extKey = 'ext:' + href;
+          var extCached = cache[extKey];
+          if (extCached && typeof extCached === 'object') {
+            showPopup(x, y, extCached.title || label,
+              '<div class="link-ctx-popup-url">' + escHtml(href) + '</div>' +
+              extCached.html);
+          } else if (extCached === false) {
+            showPopup(x, y, label,
+              '<div class="link-ctx-popup-url">' + escHtml(href) + '</div>' +
+              '<p style="margin:0.5rem 0 0;color:#888;font-family:sans-serif;font-size:0.82rem">Could not fetch page content.</p>');
+          } else {
+            showPopup(x, y, label, '<p style="opacity:0.5;margin:0;font-family:sans-serif">Loading\u2026</p>');
+            if (extCached === undefined) {
+              cache[extKey] = null;
+              fetch('/fetch?url=' + encodeURIComponent(href))
+                .then(function(r) { return r.ok ? r.json() : null; })
+                .then(function(data) {
+                  if (!data || data.error) {
+                    cache[extKey] = false;
+                    updatePopup(
+                      '<div class="link-ctx-popup-url">' + escHtml(href) + '</div>' +
+                      '<p style="margin:0.5rem 0 0;color:#888;font-family:sans-serif;font-size:0.82rem">Could not fetch page content.</p>');
+                    return;
+                  }
+                  cache[extKey] = { title: data.title, html: data.html };
+                  updatePopup(
+                    '<div class="link-ctx-popup-url">' + escHtml(href) + '</div>' +
+                    data.html,
+                    data.title || label);
+                })
+                .catch(function() {
+                  cache[extKey] = false;
+                  updatePopup(
+                    '<div class="link-ctx-popup-url">' + escHtml(href) + '</div>' +
+                    '<p style="margin:0.5rem 0 0;color:#c44;font-family:sans-serif;font-size:0.82rem">Could not fetch page content.</p>');
+                });
+            }
           }
-          showPopup(x, y, label, urlBody);
+        } else {
+          showPopup(x, y, label, '<div class="link-ctx-popup-url">' + escHtml(href) + '</div>');
         }
       }
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: markdownr
 version: !ruby/object:Gem::Version
-  version: 0.4.5
+  version: 0.4.7
 platform: ruby
 authors:
 - Brian Dunn