ogthing 0.0.0-alpha.1

package/server/metadata.ts

@@ -0,0 +1,365 @@
+import type { MetaData, UaMode } from "./types";
+
+export const userAgents: Record<UaMode, string> = {
+  browser:
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
+  facebook: "facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)",
+  twitter: "Twitterbot/1.0",
+  slack: "Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)",
+  discord: "Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)",
+  telegram: "TelegramBot (like TwitterBot)",
+  whatsapp: "WhatsApp/2.23.20.0",
+  linkedin: "LinkedInBot/1.0 (compatible; Mozilla/5.0; Apache-HttpClient +http://www.linkedin.com)",
+};
+
+const entities: Record<string, string> = {
+  amp: "&",
+  lt: "<",
+  gt: ">",
+  quot: '"',
+  apos: "'",
+  nbsp: " ",
+  copy: "\u00a9",
+  reg: "\u00ae",
+  trade: "\u2122",
+  hellip: "\u2026",
+  mdash: "\u2014",
+  ndash: "\u2013",
+  lsquo: "\u2018",
+  rsquo: "\u2019",
+  ldquo: "\u201c",
+  rdquo: "\u201d",
+  deg: "\u00b0",
+  plusmn: "\u00b1",
+  middot: "\u00b7",
+  bull: "\u2022",
+  euro: "\u20ac",
+  pound: "\u00a3",
+  yen: "\u00a5",
+  cent: "\u00a2",
+  sect: "\u00a7",
+  para: "\u00b6",
+  laquo: "\u00ab",
+  raquo: "\u00bb",
+  times: "\u00d7",
+  divide: "\u00f7",
+  szlig: "\u00df",
+  Agrave: "\u00c0",
+  Aacute: "\u00c1",
+  Acirc: "\u00c2",
+  Atilde: "\u00c3",
+  Auml: "\u00c4",
+  Aring: "\u00c5",
+  Ccedil: "\u00c7",
+  Egrave: "\u00c8",
+  Eacute: "\u00c9",
+  Ecirc: "\u00ca",
+  Euml: "\u00cb",
+  Igrave: "\u00cc",
+  Iacute: "\u00cd",
+  Icirc: "\u00ce",
+  Iuml: "\u00cf",
+  Ntilde: "\u00d1",
+  Ograve: "\u00d2",
+  Oacute: "\u00d3",
+  Ocirc: "\u00d4",
+  Otilde: "\u00d5",
+  Ouml: "\u00d6",
+  Oslash: "\u00d8",
+  Ugrave: "\u00d9",
+  Uacute: "\u00da",
+  Ucirc: "\u00db",
+  Uuml: "\u00dc",
+  agrave: "\u00e0",
+  aacute: "\u00e1",
+  acirc: "\u00e2",
+  atilde: "\u00e3",
+  auml: "\u00e4",
+  aring: "\u00e5",
+  ccedil: "\u00e7",
+  egrave: "\u00e8",
+  eacute: "\u00e9",
+  ecirc: "\u00ea",
+  euml: "\u00eb",
+  igrave: "\u00ec",
+  iacute: "\u00ed",
+  icirc: "\u00ee",
+  iuml: "\u00ef",
+  ntilde: "\u00f1",
+  ograve: "\u00f2",
+  oacute: "\u00f3",
+  ocirc: "\u00f4",
+  otilde: "\u00f5",
+  ouml: "\u00f6",
+  oslash: "\u00f8",
+  ugrave: "\u00f9",
+  uacute: "\u00fa",
+  ucirc: "\u00fb",
+  uuml: "\u00fc",
+  yacute: "\u00fd",
+};
+
+export const decodeEntities = (value: string): string =>
+  value.replace(/&(#x?[0-9a-fA-F]+|[a-zA-Z]+);/g, (match, body: string) => {
+    if (body.startsWith("#x") || body.startsWith("#X")) {
+      const code = Number.parseInt(body.slice(2), 16);
+      return Number.isNaN(code) ? match : String.fromCodePoint(code);
+    }
+    if (body.startsWith("#")) {
+      const code = Number.parseInt(body.slice(1), 10);
+      return Number.isNaN(code) ? match : String.fromCodePoint(code);
+    }
+    return entities[body] ?? match;
+  });
+
+const collapseSpace = (value: string): string => value.replace(/\s+/g, " ").trim();
+
+interface TagInfo {
+  name: string;
+  attrs: Record<string, string>;
+}
+
+/** Parse the attributes of a single `<tag ...>` string. */
+export const parseTagAttributes = (raw: string): TagInfo | null => {
+  const match = /^<\s*([a-zA-Z][a-zA-Z0-9:-]*)/.exec(raw);
+  if (!match) return null;
+  const name = match[1].toLowerCase();
+  const attrs: Record<string, string> = {};
+  const body = raw.slice(match[0].length).replace(/>[^>]*$/, "");
+  const attrPattern =
+    /([:@a-zA-Z_][-.:a-zA-Z0-9_]*)\s*(?:=\s*("([^"]*)"|'([^']*)'|[^\s"'=<>`]+))?/g;
+  for (const attr of body.matchAll(attrPattern)) {
+    const key = attr[1].toLowerCase();
+    const value = attr[3] ?? attr[4] ?? attr[2] ?? "";
+    if (!(key in attrs)) attrs[key] = decodeEntities(value);
+  }
+  return { name, attrs };
+};
+
+/** Extract title/meta/link tags from an HTML document without a DOM dependency. */
+export const extractHeadTags = (html: string) => {
+  let head = html;
+  const headMatch = /<head[\s>]([\s\S]*?)<\/head\s*>/i.exec(html);
+  if (headMatch) {
+    head = headMatch[1];
+  } else {
+    // Some pages ship malformed HTML; fall back to everything before <body>.
+    const bodyIndex = html.search(/<body[\s>]/i);
+    if (bodyIndex >= 0) head = html.slice(0, bodyIndex);
+  }
+
+  const cleaned = head
+    .replace(/<!--[\s\S]*?-->/g, "")
+    .replace(/<script[\s>][\s\S]*?<\/script\s*>/gi, "")
+    .replace(/<style[\s>][\s\S]*?<\/style\s*>/gi, "")
+    .replace(/<!\[CDATA\[[\s\S]*?\]\]>/g, "");
+
+  const titleMatch = /<title[\s>]*>([\s\S]*?)<\/title\s*>/i.exec(cleaned);
+  const title = titleMatch ? collapseSpace(decodeEntities(titleMatch[1])) : undefined;
+
+  const htmlTag = /<html[^>]*>/i.exec(html);
+  const lang = htmlTag ? parseTagAttributes(htmlTag[0])?.attrs.lang : undefined;
+
+  const metas: TagInfo[] = [];
+  const links: TagInfo[] = [];
+  for (const tag of cleaned.matchAll(/<[a-zA-Z][^>]*>/g)) {
+    const info = parseTagAttributes(tag[0]);
+    if (!info) continue;
+    if (info.name === "meta") metas.push(info);
+    else if (info.name === "link" && info.attrs.rel && info.attrs.href !== undefined)
+      links.push(info);
+  }
+  return { title, lang, metas, links };
+};
+
+const firstMeta = (metas: TagInfo[], match: (attrs: Record<string, string>) => boolean) => {
+  for (const meta of metas) if (match(meta.attrs) && meta.attrs.content) return meta.attrs.content;
+  return undefined;
+};
+
+const allMetas = (metas: TagInfo[], match: (attrs: Record<string, string>) => boolean) =>
+  metas.filter((meta) => match(meta.attrs) && meta.attrs.content).map((meta) => meta.attrs.content);
+
+const firstLink = (links: TagInfo[], rels: string[]) => {
+  const wanted = new Set(rels);
+  for (const link of links) {
+    const rel = link.attrs.rel?.toLowerCase() ?? "";
+    if (rel.split(/\s+/).some((part) => wanted.has(part))) return link.attrs.href;
+  }
+  return undefined;
+};
+
+const makeAbsolute = (value: string | undefined, base: string): string | undefined => {
+  if (!value) return undefined;
+  try {
+    return new URL(value, base).toString();
+  } catch {
+    return value;
+  }
+};
+
+const linkKeys = ["icon", "href", "image", "url", "canonical"];
+
+/**
+ * Build the metadata object golbat-style from raw HTML and the final URL
+ * (after redirects). `probes` reports whether robots.txt/sitemap.xml exist.
+ */
+export const buildMetadata = (
+  html: string,
+  baseUrl: string,
+  probes?: { robotsFile?: boolean; sitemap?: boolean },
+): MetaData => {
+  const { title, lang, metas, links } = extractHeadTags(html);
+
+  const charset =
+    firstMeta(metas, (attrs) => "charset" in attrs) ??
+    firstMeta(metas, (attrs) => attrs["http-equiv"]?.toLowerCase() === "content-type");
+
+  const faviconHref =
+    firstLink(links, ["icon"]) ??
+    firstLink(links, ["shortcut icon"]) ??
+    firstLink(links, ["apple-touch-icon"]) ??
+    firstLink(links, ["apple-touch-icon-precomposed"]) ??
+    firstLink(links, ["mask-icon", "fluid-icon"]);
+
+  const metadata: MetaData = {
+    url: baseUrl,
+    title: title || firstMeta(metas, (attrs) => attrs.property === "og:title"),
+    description: firstMeta(metas, (attrs) => attrs.name === "description"),
+    ogTitle: firstMeta(metas, (attrs) => attrs.property === "og:title"),
+    ogDescription: firstMeta(metas, (attrs) => attrs.property === "og:description"),
+    ogImage: firstMeta(
+      metas,
+      (attrs) => attrs.property === "og:image" || attrs.property === "og:image:url",
+    ),
+    ogImageAll: allMetas(
+      metas,
+      (attrs) => attrs.property === "og:image" || attrs.property === "og:image:url",
+    ),
+    ogType: firstMeta(metas, (attrs) => attrs.property === "og:type"),
+    ogUrl: firstMeta(metas, (attrs) => attrs.property === "og:url"),
+    ogSiteName: firstMeta(metas, (attrs) => attrs.property === "og:site_name"),
+    ogLocale: firstMeta(metas, (attrs) => attrs.property === "og:locale"),
+    twitterCard: firstMeta(
+      metas,
+      (attrs) => attrs.name === "twitter:card" || attrs.property === "twitter:card",
+    ),
+    twitterTitle: firstMeta(
+      metas,
+      (attrs) => attrs.name === "twitter:title" || attrs.property === "twitter:title",
+    ),
+    twitterDescription: firstMeta(
+      metas,
+      (attrs) => attrs.name === "twitter:description" || attrs.property === "twitter:description",
+    ),
+    twitterImage: firstMeta(
+      metas,
+      (attrs) =>
+        attrs.name === "twitter:image" ||
+        attrs.property === "twitter:image" ||
+        attrs.name === "twitter:image:src",
+    ),
+    twitterImageAll: allMetas(
+      metas,
+      (attrs) =>
+        attrs.name === "twitter:image" ||
+        attrs.property === "twitter:image" ||
+        attrs.name === "twitter:image:src",
+    ),
+    twitterSite: firstMeta(
+      metas,
+      (attrs) => attrs.name === "twitter:site" || attrs.property === "twitter:site",
+    ),
+    twitterCreator: firstMeta(
+      metas,
+      (attrs) => attrs.name === "twitter:creator" || attrs.property === "twitter:creator",
+    ),
+    language: lang,
+    charset,
+    viewport: firstMeta(metas, (attrs) => attrs.name === "viewport"),
+    robots: firstMeta(metas, (attrs) => attrs.name === "robots"),
+    generator: firstMeta(metas, (attrs) => attrs.name === "generator"),
+    themeColor: firstMeta(metas, (attrs) => attrs.name === "theme-color"),
+    canonical: firstLink(links, ["canonical"]),
+    alternate: firstLink(links, ["alternate"]),
+    author: firstLink(links, ["author"]),
+    prev: firstLink(links, ["prev"]),
+    next: firstLink(links, ["next"]),
+    search: firstLink(links, ["search"]),
+    icon: firstLink(links, ["icon"]),
+    appleItunesApp: firstMeta(metas, (attrs) => attrs.name === "apple-itunes-app"),
+    appleMobileWebAppCapable: firstMeta(
+      metas,
+      (attrs) => attrs.name === "apple-mobile-web-app-capable",
+    ),
+    appleMobileWebAppTitle: firstMeta(
+      metas,
+      (attrs) => attrs.name === "apple-mobile-web-app-title",
+    ),
+    formatDetection: firstMeta(metas, (attrs) => attrs.name === "format-detection"),
+    favicon: faviconHref,
+  };
+
+  // Resolve every URL-shaped field against the final response URL.
+  for (const [key, value] of Object.entries(metadata)) {
+    const isUrlShaped =
+      linkKeys.some((fragment) => key.toLowerCase().includes(fragment)) || key === "favicon";
+    if (!isUrlShaped) continue;
+    if (typeof value === "string") metadata[key] = makeAbsolute(value, baseUrl);
+    else if (Array.isArray(value))
+      metadata[key] = value.map((entry) => makeAbsolute(entry, baseUrl) ?? entry);
+  }
+
+  metadata.favicon = metadata.favicon ?? makeAbsolute("/favicon.ico", baseUrl);
+
+  if (probes?.robotsFile) metadata.robotsFile = makeAbsolute("/robots.txt", baseUrl);
+  if (probes?.sitemap) metadata.sitemap = makeAbsolute("/sitemap.xml", baseUrl);
+
+  return metadata;
+};
+
+/** Collect every meta/link on the page as flat `meta_x` / `link_x` keys. */
+export const appendFullTags = (html: string, metadata: MetaData): void => {
+  const { metas, links } = extractHeadTags(html);
+  const sanitize = (value: string) => value.replace(/[^a-zA-Z0-9]/g, "_");
+  for (const meta of metas) {
+    const name = meta.attrs.name || meta.attrs.property || meta.attrs["http-equiv"];
+    if (name && meta.attrs.content) metadata[`meta_${sanitize(name)}`] = meta.attrs.content;
+  }
+  for (const link of links) {
+    if (link.attrs.href && link.attrs.rel)
+      metadata[`link_${sanitize(link.attrs.rel)}`] = link.attrs.href;
+  }
+};
+
+const escapeHtml = (value: string): string =>
+  value
+    .replaceAll("&", "&amp;")
+    .replaceAll("<", "&lt;")
+    .replaceAll(">", "&gt;")
+    .replaceAll('"', "&quot;");
+
+/** Rebuild the meta tags that would produce this metadata. */
+export const generateHtmlTags = (metadata: MetaData): string => {
+  const tags: string[] = [];
+  const pushMeta = (kind: "name" | "property", key: string, value?: string) => {
+    if (value) tags.push(`<meta ${kind}="${key}" content="${escapeHtml(value)}" />`);
+  };
+
+  if (metadata.title) tags.push(`<title>${escapeHtml(metadata.title)}</title>`);
+  pushMeta("name", "description", metadata.description);
+  if (metadata.canonical)
+    tags.push(`<link rel="canonical" href="${escapeHtml(metadata.canonical)}" />`);
+  pushMeta("property", "og:title", metadata.ogTitle);
+  pushMeta("property", "og:description", metadata.ogDescription);
+  pushMeta("property", "og:image", metadata.ogImage);
+  pushMeta("property", "og:type", metadata.ogType);
+  pushMeta("property", "og:url", metadata.ogUrl);
+  pushMeta("property", "og:site_name", metadata.ogSiteName);
+  pushMeta("name", "twitter:card", metadata.twitterCard);
+  pushMeta("name", "twitter:title", metadata.twitterTitle);
+  pushMeta("name", "twitter:description", metadata.twitterDescription);
+  pushMeta("name", "twitter:image", metadata.twitterImage);
+  pushMeta("name", "twitter:site", metadata.twitterSite);
+  return tags.join("\n");
+};

package/server/ogthing-api.ts

@@ -0,0 +1,130 @@
+import type { Connect, Plugin } from "vite";
+import { appendFullTags, buildMetadata, generateHtmlTags, userAgents } from "./metadata";
+import type { MetaData, UaMode } from "./types";
+
+const jsonHeaders = {
+  "content-type": "application/json; charset=utf-8",
+  "cache-control": "no-store",
+};
+
+const fetchTimeoutMs = Number(process.env.OGTHING_FETCH_TIMEOUT_MS || 15_000);
+const maxBodyBytes = 5_000_000;
+
+const sendJson = (response: import("node:http").ServerResponse, status: number, value: unknown) => {
+  response.statusCode = status;
+  for (const [name, headerValue] of Object.entries(jsonHeaders))
+    response.setHeader(name, headerValue);
+  response.end(JSON.stringify(value));
+};
+
+const headExists = async (url: string): Promise<boolean> => {
+  try {
+    const response = await fetch(url, {
+      method: "HEAD",
+      redirect: "follow",
+      signal: AbortSignal.timeout(Math.min(fetchTimeoutMs, 5_000)),
+    });
+    return response.ok;
+  } catch {
+    return false;
+  }
+};
+
+export const inspectUrl = async (
+  rawUrl: string,
+  uaMode: UaMode,
+  full: boolean,
+): Promise<MetaData> => {
+  const target = new URL(rawUrl);
+  if (!/^https?:$/.test(target.protocol))
+    throw Object.assign(new Error("only http and https urls are supported"), { status: 400 });
+
+  const userAgent = userAgents[uaMode] ?? userAgents.browser;
+  const response = await fetch(target, {
+    redirect: "follow",
+    signal: AbortSignal.timeout(fetchTimeoutMs),
+    headers: {
+      "user-agent": userAgent,
+      accept: "text/html,application/xhtml+xml;q=0.9,*/*;q=0.8",
+      "accept-language": "en-US,en;q=0.9",
+    },
+  });
+
+  if (!response.ok) {
+    throw Object.assign(
+      new Error(`the site responded ${response.status} ${response.statusText}`.trim()),
+      { status: response.status },
+    );
+  }
+
+  const html = (await response.text()).slice(0, maxBodyBytes);
+  const finalUrl = response.url || target.toString();
+  const [robotsFile, sitemap] = await Promise.all([
+    headExists(new URL("/robots.txt", finalUrl).toString()),
+    headExists(new URL("/sitemap.xml", finalUrl).toString()),
+  ]);
+
+  const metadata = buildMetadata(html, finalUrl, { robotsFile, sitemap });
+  metadata.statusCode = response.status;
+  if (full) appendFullTags(html, metadata);
+  return metadata;
+};
+
+export function ogthingApi(): Plugin {
+  return {
+    name: "ogthing-api",
+    configureServer(server) {
+      server.middlewares.use((request, response, next) => {
+        void handleApiRequest(request, response, next);
+      });
+    },
+  };
+}
+
+export const handleApiRequest = async (
+  request: import("node:http").IncomingMessage,
+  response: import("node:http").ServerResponse,
+  next: Connect.NextFunction,
+) => {
+  if (!request.url?.startsWith("/api/")) {
+    next();
+    return;
+  }
+  const url = new URL(request.url, "http://127.0.0.1");
+
+  if (url.pathname === "/api/health" && request.method === "GET") {
+    sendJson(response, 200, { ok: true, service: "ogthing", platform: process.platform });
+    return;
+  }
+
+  if (url.pathname === "/api/metadata" && request.method === "GET") {
+    const rawUrl = url.searchParams.get("url");
+    if (!rawUrl) {
+      sendJson(response, 400, { error: "a url parameter is required" });
+      return;
+    }
+    let formatted = rawUrl.trim();
+    if (!/^https?:\/\//i.test(formatted)) formatted = `https://${formatted}`;
+    try {
+      const uaMode = (url.searchParams.get("ua") as UaMode) || "browser";
+      const full = url.searchParams.get("full") !== "false";
+      const metadata = await inspectUrl(formatted, uaMode, full);
+      metadata.htmlTags = generateHtmlTags(metadata);
+      sendJson(response, 200, metadata);
+    } catch (error) {
+      const err = error as Error & { status?: number; cause?: Error };
+      let message = err.message || "could not fetch the url";
+      const causeCode = (err.cause as { code?: string } | undefined)?.code;
+      const causeMessage = (err.cause as Error | undefined)?.message ?? "";
+      if (/refused/i.test(`${causeCode} ${causeMessage}`) || /unable to connect/i.test(message))
+        message = "connection refused - is the server running?";
+      else if (causeCode === "ENOTFOUND") message = "host not found - check the address";
+      else if (err.name === "TimeoutError" || /timeout|timed out/i.test(message))
+        message = "the site took too long to respond";
+      sendJson(response, err.status && err.status >= 400 ? err.status : 502, { error: message });
+    }
+    return;
+  }
+
+  sendJson(response, 404, { error: "not found" });
+};

package/server/types.ts

@@ -0,0 +1,53 @@
+export type UaMode =
+  | "browser"
+  | "facebook"
+  | "twitter"
+  | "slack"
+  | "discord"
+  | "telegram"
+  | "whatsapp"
+  | "linkedin";
+
+export interface MetaData {
+  url?: string;
+  statusCode?: number;
+  htmlTags?: string;
+  title?: string;
+  description?: string;
+  ogTitle?: string;
+  ogDescription?: string;
+  ogImage?: string;
+  ogImageAll?: string[];
+  ogType?: string;
+  ogUrl?: string;
+  ogSiteName?: string;
+  ogLocale?: string;
+  twitterCard?: string;
+  twitterTitle?: string;
+  twitterDescription?: string;
+  twitterImage?: string;
+  twitterImageAll?: string[];
+  twitterSite?: string;
+  twitterCreator?: string;
+  canonical?: string;
+  favicon?: string;
+  language?: string;
+  charset?: string;
+  viewport?: string;
+  robots?: string;
+  generator?: string;
+  themeColor?: string;
+  alternate?: string;
+  author?: string;
+  prev?: string;
+  next?: string;
+  search?: string;
+  icon?: string;
+  appleItunesApp?: string;
+  appleMobileWebAppCapable?: string;
+  appleMobileWebAppTitle?: string;
+  formatDetection?: string;
+  robotsFile?: string;
+  sitemap?: string;
+  [key: string]: string | string[] | number | undefined;
+}