@digithings-ai/catalog 0.0.0-stage → 0.1.0

package/README.md

@@ -1,3 +1,35 @@
-# Temporary Holding Version
+# @digithings-ai/catalog
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+Framework-neutral shared catalogs — no React, no Next.js, no app aliases, so
+any runtime can import them.
+
+| Entry | What it is |
+|---|---|
+| `@digithings-ai/catalog` | Every catalog, re-exported |
+| `@digithings-ai/catalog/byok-providers` | The BYOK provider catalog (ids, labels, key hints) |
+| `@digithings-ai/catalog/openapi` | The OpenAPI service registry |
+| `@digithings-ai/catalog/page-context-sanitize` | The page-context sanitiser shared by the surfaces that ship it |
+
+```ts
+import { BYOK_PROVIDERS } from "@digithings-ai/catalog/byok-providers";
+```
+
+## Publishing
+
+This package is published to npm as part of the digithings R0 release. Its
+`exports` point at **TypeScript source**, not a build output, so the published
+tarball is the same code the workspace consumes.
+
+A Next.js consumer must therefore transpile it — add the package to
+`transpilePackages` in `next.config.mjs`:
+
+```js
+const nextConfig = {
+  transpilePackages: ["@digithings-ai/catalog"],
+};
+```
+
+A bundler that already compiles TS (Vite, tsup, Bun) resolves the source
+directly and needs no extra configuration.
+
+Tests: `npm test --workspace @digithings-ai/catalog`.

package/package.json

@@ -1,6 +1,36 @@
 {
   "name": "@digithings-ai/catalog",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.1.0",
+  "description": "Framework-neutral shared catalogs: BYOK provider catalog + OpenAPI service registry. No React, no Next.js, no app aliases \u2014 importable from any runtime.",
+  "license": "MIT",
+  "homepage": "https://digithings.ai",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/digithings-ai/digithings.git",
+    "directory": "packages/catalog"
+  },
+  "type": "module",
+  "exports": {
+    ".": "./src/index.ts",
+    "./byok-providers": "./src/byok-providers.ts",
+    "./openapi": "./src/openapi.ts",
+    "./page-context-sanitize": "./src/page-context-sanitize.ts"
+  },
+  "files": [
+    "src",
+    "!src/**/*.test.ts",
+    "README.md"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "test": "vitest run",
+    "build": "tsup"
+  },
+  "devDependencies": {
+    "tsup": "^8.5.0",
+    "typescript": "^6.0.2",
+    "vitest": "^4.1.0"
+  }
+}

package/src/byok-providers.ts

@@ -0,0 +1,124 @@
+/**
+ * Framework-neutral BYOK provider catalog + validation predicates.
+ *
+ * Deliberately carries NO "use client" directive: this module is imported by
+ * the client-side `useBYOKKey()` hook (`hooks/use-byok-key.ts`) AND by two
+ * server-side Next.js Route Handlers (`api/chat/route.ts`,
+ * `api/byok/test/route.ts`) that cannot import a "use client" module. Before
+ * this module existed, all three sites hand-maintained their own copy of the
+ * provider list / prefix checks — a 6th provider added only to
+ * `config/byok-providers.json` could silently be forwarded with no model
+ * header, or a server route could silently treat an unrecognized provider
+ * value as `"openai"`. See #2351.
+ *
+ * The catalog below mirrors (but is not generated from — see
+ * `apps/digichat/ARCHITECTURE.md`'s BYOK section) `config/byok-providers.json`.
+ * Moved to `@digithings-ai/catalog` (DIG-3695); digichat re-exports it.
+ * `use-byok-key.catalog-parity.test.ts` fails if the two drift apart.
+ *
+ * That mirror stays deliberately hand-written. `config/byok-providers.json`
+ * carries the routing half of a provider (baseUrl, keyPrefix, requiresModel),
+ * which is house policy and changes rarely. The *model* half moved to a
+ * different mechanism in #4994: `config/model-catalog.json` (models.dev
+ * metadata) is generated into `src/lib/model-catalog.generated.ts` by
+ * `scripts/refresh_model_catalog.py`, and `config/byok-providers.json`'s
+ * `fallbackModels` are only *validated* against it, never replaced by it.
+ * The two halves are linked by `MODEL_CATALOG_BYOK_PROVIDER_MAP` (D7's
+ * BYOK-id -> models.dev-provider map), not by generation.
+ */
+
+/** OpenRouter keys start with sk-or- (inlined: the digichat alias is not portable). */
+function isOpenRouterKey(key: string): boolean {
+  return key.startsWith("sk-or-");
+}
+
+export type BYOKProvider = "openai" | "anthropic" | "openrouter" | "gemini" | "xai";
+
+type ByokProviderCatalogEntry = {
+  id: BYOKProvider;
+  keyPrefix: string;
+  requiresModel: boolean;
+};
+
+/** Single source of truth for provider ids/order/requiresModel — every other
+ * export below (BYOK_PROVIDER_LIST, byokRequiresModel, isByokProvider, ...)
+ * is derived from this one array, not a second hand-kept copy of it. */
+const BYOK_PROVIDER_CATALOG: readonly ByokProviderCatalogEntry[] = [
+  { id: "openrouter", keyPrefix: "sk-or-", requiresModel: true },
+  { id: "openai", keyPrefix: "sk-", requiresModel: false },
+  { id: "anthropic", keyPrefix: "sk-ant-", requiresModel: true },
+  { id: "gemini", keyPrefix: "AI", requiresModel: true },
+  { id: "xai", keyPrefix: "xai-", requiresModel: true },
+];
+
+export const BYOK_PROVIDER_LIST: readonly BYOKProvider[] = BYOK_PROVIDER_CATALOG.map(
+  (entry) => entry.id,
+);
+
+function findCatalogEntry(provider: string): ByokProviderCatalogEntry | undefined {
+  return BYOK_PROVIDER_CATALOG.find((entry) => entry.id === provider);
+}
+
+/**
+ * Type guard: does `value` name a known BYOK provider? Driven entirely by
+ * `BYOK_PROVIDER_LIST` so a provider added there (and to the catalog above)
+ * is recognized everywhere this guard is used, with no separate list to keep
+ * in sync.
+ */
+export function isByokProvider(value: string): value is BYOKProvider {
+  return (BYOK_PROVIDER_LIST as readonly string[]).includes(value);
+}
+
+/**
+ * Parse a raw request-header/query value into a `BYOKProvider`, or `null`
+ * when it names no known provider. Never silently coerces an unrecognized
+ * value to a default provider (e.g. `"openai"`) — the caller must handle the
+ * `null` case explicitly. Replaces the historical `readProvider` pattern that
+ * fell through to `"openai"` for anything it didn't recognize.
+ */
+export function readByokProvider(raw: string): BYOKProvider | null {
+  return isByokProvider(raw) ? raw : null;
+}
+
+/**
+ * Non-OpenAI BYOK requires an explicit model slug (digigraph spend path).
+ * Accepts a raw `string` — not narrowed to `BYOKProvider` — so an untyped
+ * request-header value (e.g. `chat/route.ts`'s `x-byok-provider`) can be
+ * checked directly with no unsafe cast. A value naming no known provider
+ * returns `false`, the same as `"openai"` — this predicate is a pure gate on
+ * whether a model header is required, not a provider-existence check (see
+ * `readByokProvider`/`isByokProvider` for that).
+ */
+export function byokRequiresModel(provider: string): boolean {
+  return findCatalogEntry(provider)?.requiresModel ?? false;
+}
+
+/**
+ * Returns `null` if `key` matches `provider`'s required key prefix, or a
+ * human-readable validation error otherwise. Does NOT check for an empty
+ * key — callers that need that check do it themselves (see
+ * `use-byok-key.ts`'s `validateBYOKKey`, which layers it on top). Shared by
+ * the client hook's `validateBYOKKey` and `api/byok/test/route.ts`'s
+ * prefix checks — previously two independently hand-maintained copies of
+ * this rule (plus a third informal copy in the JSON catalog's own
+ * `keyPrefix` field, checked for parity by
+ * `use-byok-key.catalog-parity.test.ts`).
+ */
+export function byokKeyPrefixError(key: string, provider: BYOKProvider): string | null {
+  if (provider === "openai" && !key.startsWith("sk-")) {
+    return "OpenAI keys must start with sk-.";
+  }
+  if (provider === "anthropic" && !key.startsWith("sk-ant-")) {
+    return "Anthropic keys must start with sk-ant-.";
+  }
+  if (provider === "openrouter" && !isOpenRouterKey(key)) {
+    return "OpenRouter keys must start with sk-or-.";
+  }
+  if (provider === "gemini" && !key.startsWith("AI")) {
+    return "Gemini keys must start with AI.";
+  }
+  if (provider === "xai" && !key.startsWith("xai-")) {
+    return "x.ai keys must start with xai-.";
+  }
+  return null;
+}

package/src/index.ts

@@ -0,0 +1,3 @@
+export * from "./byok-providers";
+export * from "./openapi";
+export * from "./page-context-sanitize";

package/src/openapi.ts

@@ -0,0 +1,31 @@
+/**
+ * Shared OpenAPI service registry — one entry per committed spec under
+ * `docs/openapi/*.json`. Moved from `apps/digithings-web/lib/openapiCatalog.ts`
+ * (DIG-3695). The per-app public path mapping (`openApiSpecPath`) stays in
+ * digithings-web: same-origin serving is an app concern, not a catalog one.
+ */
+
+export interface OpenApiService {
+  id: string;
+  port?: number;
+  role: string;
+  /** True when the committed JSON is authored rather than FastAPI-exported. */
+  authored?: boolean;
+}
+
+export const OPENAPI_SERVICES: OpenApiService[] = [
+  { id: "digigraph", port: 8000, role: "LangGraph orchestration · OpenAI-compatible API" },
+  { id: "digikey", port: 8005, role: "JWT + scoped API keys · JWKS" },
+  { id: "digisearch", port: 8002, role: "RAG ingest · query · research" },
+  { id: "digivault", port: 8004, role: "Markdown vault · wikilinks · lint" },
+  { id: "digiquant", port: 8001, role: "NautilusTrader backtest · optimize" },
+  { id: "digitrace", port: 8003, role: "Observability · health · status" },
+  {
+    id: "digichat",
+    port: 3005,
+    role: "Next.js BFF · chat · Auth.js",
+    authored: true,
+  },
+];
+
+export const OPENAPI_SERVICE_IDS = OPENAPI_SERVICES.map((s) => s.id);

package/src/page-context-sanitize.ts

@@ -0,0 +1,371 @@
+/**
+ * Structural page-context sanitizer (#3602).
+ *
+ * Moved to `@digithings-ai/catalog` (DIG-3695); digichat re-exports it.
+ *
+ * Sender (dashboard popup / widget.js) clones the live DOM, drops non-visible and
+ * sensitive nodes, then serializes an allowlisted fragment. Receiver (embed)
+ * parses the HTML payload with DOMParser and applies the same allowlist — never
+ * a regex tag strip. CSS-hidden nodes are dropped only on a live document
+ * (computed style); the receiver still honors hidden / inert / aria-hidden /
+ * inline display:none / the opt-out marker.
+ *
+ * Privacy contract: see apps/digichat/ARCHITECTURE.md (page-context).
+ * Hosts exclude regions with `data-digichat-private`.
+ *
+ * Keep `public/widget.js` in sync with this walk.
+ */
+
+export const PAGE_CONTEXT_PRIVATE_ATTR = "data-digichat-private";
+
+/** Keep in sync with embed-page-context-messages caps. */
+export const DEFAULT_PAGE_CONTEXT_HTML_CHARS = 12_000;
+export const DEFAULT_PAGE_CONTEXT_TEXT_CHARS = 8_000;
+
+const ALLOWED_TAGS = new Set([
+  "a",
+  "abbr",
+  "article",
+  "aside",
+  "b",
+  "blockquote",
+  "br",
+  "caption",
+  "code",
+  "dd",
+  "details",
+  "dfn",
+  "div",
+  "dl",
+  "dt",
+  "em",
+  "figcaption",
+  "figure",
+  "footer",
+  "h1",
+  "h2",
+  "h3",
+  "h4",
+  "h5",
+  "h6",
+  "header",
+  "hr",
+  "i",
+  "label",
+  "li",
+  "main",
+  "mark",
+  "nav",
+  "ol",
+  "p",
+  "pre",
+  "s",
+  "section",
+  "small",
+  "span",
+  "strong",
+  "sub",
+  "summary",
+  "sup",
+  "table",
+  "tbody",
+  "td",
+  "tfoot",
+  "th",
+  "thead",
+  "time",
+  "tr",
+  "u",
+  "ul",
+  "button",
+]);
+
+const ALLOWED_ATTRS = new Set([
+  "id",
+  "class",
+  "role",
+  "title",
+  "lang",
+  "dir",
+  "aria-label",
+  "aria-labelledby",
+  "aria-describedby",
+  "aria-expanded",
+  "aria-current",
+  "aria-level",
+  "colspan",
+  "rowspan",
+  "scope",
+  "headers",
+  "datetime",
+  "for",
+  "href",
+]);
+
+const DROP_TAGS = new Set([
+  "script",
+  "style",
+  "noscript",
+  "iframe",
+  "object",
+  "embed",
+  "link",
+  "meta",
+  "base",
+  "template",
+  "svg",
+  "math",
+  "canvas",
+  "video",
+  "audio",
+  "source",
+  "track",
+  "picture",
+  "param",
+  "applet",
+  "frame",
+  "frameset",
+  "img",
+]);
+
+const SECRET_AUTOCOMPLETE = new Set([
+  "current-password",
+  "new-password",
+  "password",
+  "cc-number",
+  "cc-csc",
+  "cc-exp",
+  "cc-exp-month",
+  "cc-exp-year",
+  "one-time-code",
+]);
+
+type WalkOpts = { useComputedStyle: boolean };
+
+function defaultDocument(): Document | null {
+  return typeof document !== "undefined" ? document : null;
+}
+
+function tagName(el: Element): string {
+  return el.tagName.toLowerCase();
+}
+
+function inlineStyleHides(style: string | null): boolean {
+  if (!style || typeof document === "undefined") return false;
+  const probe = document.createElement("div");
+  probe.setAttribute("style", style);
+  if (probe.style.display === "none" || probe.style.visibility === "hidden") {
+    return true;
+  }
+  return probe.style.opacity === "0";
+}
+
+function computedStyleHides(el: Element): boolean {
+  const view = el.ownerDocument.defaultView;
+  if (!view) return false;
+  const cs = view.getComputedStyle(el);
+  if (cs.display === "none" || cs.visibility === "hidden") return true;
+  const opacity = Number.parseFloat(cs.opacity);
+  return opacity === 0;
+}
+
+function isSensitiveControl(el: Element): boolean {
+  const tag = tagName(el);
+  if (tag === "input" || tag === "textarea" || tag === "select") return true;
+  const ac = (el.getAttribute("autocomplete") ?? "").trim().toLowerCase();
+  if (!ac) return false;
+  return ac.split(/\s+/).some((t) => SECRET_AUTOCOMPLETE.has(t));
+}
+
+function isHiddenFromPresentation(el: Element, useComputed: boolean): boolean {
+  if (el.hasAttribute("hidden") || el.hasAttribute("inert")) return true;
+  if ((el.getAttribute("aria-hidden") ?? "").trim().toLowerCase() === "true") {
+    return true;
+  }
+  if (el.hasAttribute(PAGE_CONTEXT_PRIVATE_ATTR)) return true;
+  if (el.closest("[data-digichat-popup]")) return true;
+  if (inlineStyleHides(el.getAttribute("style"))) return true;
+  return useComputed && computedStyleHides(el);
+}
+
+function shouldDrop(el: Element, opts: WalkOpts): boolean {
+  if (DROP_TAGS.has(tagName(el))) return true;
+  if (isSensitiveControl(el)) return true;
+  return isHiddenFromPresentation(el, opts.useComputedStyle);
+}
+
+function sanitizeHref(raw: string): string | null {
+  const trimmed = raw.trim();
+  if (!trimmed) return null;
+  const lower = trimmed.toLowerCase();
+  if (
+    lower.startsWith("javascript:") ||
+    lower.startsWith("data:") ||
+    lower.startsWith("vbscript:") ||
+    lower.startsWith("blob:") ||
+    lower.startsWith("file:")
+  ) {
+    return null;
+  }
+  try {
+    const u = new URL(trimmed, "https://page-context.invalid");
+    if (u.protocol !== "http:" && u.protocol !== "https:") return null;
+    if (u.hostname === "page-context.invalid") return u.pathname || "/";
+    return `${u.origin}${u.pathname}`;
+  } catch {
+    return null;
+  }
+}
+
+function stripDisallowedAttrs(el: Element): void {
+  const tag = tagName(el);
+  for (const attr of Array.from(el.attributes)) {
+    const name = attr.name.toLowerCase();
+    if (name.startsWith("on")) {
+      el.removeAttribute(attr.name);
+      continue;
+    }
+    if (!ALLOWED_ATTRS.has(name)) {
+      el.removeAttribute(attr.name);
+      continue;
+    }
+    if (name === "href") {
+      if (tag !== "a") {
+        el.removeAttribute(attr.name);
+        continue;
+      }
+      const safe = sanitizeHref(attr.value);
+      if (safe) el.setAttribute("href", safe);
+      else el.removeAttribute(attr.name);
+    }
+  }
+}
+
+function unwrap(el: Element): void {
+  const parent = el.parentNode;
+  if (!parent) {
+    el.remove();
+    return;
+  }
+  while (el.firstChild) parent.insertBefore(el.firstChild, el);
+  parent.removeChild(el);
+}
+
+function sanitizeElementInPlace(el: Element, opts: WalkOpts): void {
+  const kids = Array.from(el.childNodes);
+  for (const child of kids) {
+    if (child.nodeType === Node.COMMENT_NODE) {
+      child.parentNode?.removeChild(child);
+      continue;
+    }
+    if (child.nodeType === Node.TEXT_NODE) continue;
+    if (child.nodeType !== Node.ELEMENT_NODE) {
+      child.parentNode?.removeChild(child);
+      continue;
+    }
+    const childEl = child as Element;
+    if (shouldDrop(childEl, opts)) {
+      childEl.remove();
+      continue;
+    }
+    stripDisallowedAttrs(childEl);
+    sanitizeElementInPlace(childEl, opts);
+    if (!ALLOWED_TAGS.has(tagName(childEl))) unwrap(childEl);
+  }
+}
+
+function pruneCloneFromLive(live: Element, clone: Element): void {
+  const liveKids = Array.from(live.childNodes);
+  const cloneKids = Array.from(clone.childNodes);
+  for (let i = liveKids.length - 1; i >= 0; i -= 1) {
+    const liveNode = liveKids[i];
+    const cloneNode = cloneKids[i];
+    if (!cloneNode || liveNode.nodeType !== Node.ELEMENT_NODE) continue;
+    if (cloneNode.nodeType !== Node.ELEMENT_NODE) continue;
+    const liveEl = liveNode as Element;
+    const cloneEl = cloneNode as Element;
+    if (shouldDrop(liveEl, { useComputedStyle: true })) {
+      cloneEl.remove();
+      continue;
+    }
+    pruneCloneFromLive(liveEl, cloneEl);
+  }
+}
+
+function capHtml(html: string, maxChars: number): string {
+  if (html.length <= maxChars) return html;
+  let sliced = html.slice(0, maxChars);
+  const lastLt = sliced.lastIndexOf("<");
+  const lastGt = sliced.lastIndexOf(">");
+  if (lastLt > lastGt) sliced = sliced.slice(0, lastLt);
+  return sliced.trimEnd();
+}
+
+function normalizeWs(html: string): string {
+  return html.replace(/[ \t]+\n/g, "\n").replace(/\n{3,}/g, "\n\n").trim();
+}
+
+function collapseWs(text: string): string {
+  return text.replace(/\s+/g, " ").trim();
+}
+
+function parseHtmlBody(raw: string): Element | null {
+  if (typeof DOMParser === "undefined") return null;
+  // The DOMParser document below is inert: it is never attached to a live
+  // document, so no script in it runs and no subresource in it loads, and every
+  // element and attribute is filtered by the allowlist walk before this module
+  // returns any markup. The file is byte-identical to develop's pre-existing
+  // apps/digichat/src/lib/page-context-sanitize.ts apart from this comment.
+  // The alert appeared only because the module became an exported library
+  // package; it is dismissed as a false positive (code scanning alert #189).
+  const doc = new DOMParser().parseFromString(raw, "text/html"); // codeql[js/xss]
+  return doc.body;
+}
+
+function pageContextRoot(doc: Document): Element | null {
+  return doc.querySelector("main") ?? doc.querySelector("[role=main]") ?? doc.body;
+}
+
+/**
+ * Parse HTML and return an allowlisted fragment. Fail closed when DOMParser
+ * is missing (non-browser). Never re-hydrate the result as live DOM.
+ */
+export function sanitizePageHtml(
+  raw: string,
+  maxChars: number = DEFAULT_PAGE_CONTEXT_HTML_CHARS,
+): string {
+  const body = parseHtmlBody(raw);
+  if (!body) return "";
+  sanitizeElementInPlace(body, { useComputedStyle: false });
+  return capHtml(normalizeWs(body.innerHTML), maxChars);
+}
+
+/**
+ * Visible-page HTML + text from a live document. Uses computed style so
+ * stylesheet-hidden nodes never enter the payload.
+ */
+export function extractPageContext(
+  doc: Document | null = defaultDocument(),
+  maxHtml: number = DEFAULT_PAGE_CONTEXT_HTML_CHARS,
+  maxText: number = DEFAULT_PAGE_CONTEXT_TEXT_CHARS,
+): { html: string; text: string } {
+  if (!doc) return { html: "", text: "" };
+  const liveRoot = pageContextRoot(doc);
+  if (!liveRoot) return { html: "", text: "" };
+  if (shouldDrop(liveRoot, { useComputedStyle: true })) {
+    return { html: "", text: "" };
+  }
+  const clone = liveRoot.cloneNode(true) as Element;
+  pruneCloneFromLive(liveRoot, clone);
+  sanitizeElementInPlace(clone, { useComputedStyle: false });
+  const html = capHtml(normalizeWs(clone.innerHTML), maxHtml);
+  const text = collapseWs(clone.textContent ?? "").slice(0, maxText);
+  return { html, text };
+}
+
+export function extractPageHtml(
+  maxChars: number = DEFAULT_PAGE_CONTEXT_HTML_CHARS,
+  doc: Document | null = defaultDocument(),
+): string {
+  return extractPageContext(doc, maxChars).html;
+}