@hulistmi/hulistmi 1.0.0

package/src/lib/fetch.ts

@@ -0,0 +1,165 @@
+import { UPSTREAM_CONTRACT } from "./upstream-contract";
+
+export class NotFoundError extends Error {}
+export class UpstreamPolicyError extends Error {}
+export class UpstreamSizeError extends Error {}
+export class UpstreamTimeoutError extends Error {}
+
+export const HULISTMI_USER_AGENT =
+  "hulistmi-ai/1.0 (+https://hulistmi.ai/#bot)";
+export const UPSTREAM_TIMEOUT_MS = 10_000;
+export const MAX_UPSTREAM_RESPONSE_BYTES = 1_048_576;
+export const MAX_RENDERED_MARKDOWN_BYTES = 524_288;
+export const MAX_MCP_REQUEST_BYTES = 131_072;
+
+export interface VerifiedHuaweiRequest {
+  url: string;
+  headers?: Record<string, string>;
+  body: unknown;
+}
+
+function assertAllowedHuaweiUrl(input: string): URL {
+  const url = new URL(input);
+  if (
+    url.protocol !== "https:" ||
+    !UPSTREAM_CONTRACT.allowedHosts.includes(url.hostname) ||
+    !UPSTREAM_CONTRACT.allowedPaths.includes(url.pathname)
+  ) {
+    throw new Error(`Unverified Huawei upstream URL: ${input}`);
+  }
+  return url;
+}
+
+function collectVerifiedRequests(): VerifiedHuaweiRequest[] {
+  return [
+    ...Object.values(UPSTREAM_CONTRACT.catalogs).map((entry) => entry.request),
+    ...Object.values(UPSTREAM_CONTRACT.documents).flatMap((doc) =>
+      [
+        doc.checkCenterGrayUser,
+        doc.getDocumentById,
+        "getCenterRootNodeTree" in doc ? doc.getCenterRootNodeTree : undefined,
+        "getCenterDocument" in doc ? doc.getCenterDocument : undefined,
+      ].filter((request): request is VerifiedHuaweiRequest => Boolean(request)),
+    ),
+    {
+      url: UPSTREAM_CONTRACT.search.url,
+      headers: UPSTREAM_CONTRACT.search.headers,
+      body: UPSTREAM_CONTRACT.search.bodyForUIAbility,
+    },
+  ];
+}
+
+function isVerifiedHeaderValue(name: string, value: string): boolean {
+  return collectVerifiedRequests().some((request) =>
+    Object.entries(request.headers ?? {}).some(
+      ([key, verified]) => key.toLowerCase() === name && verified === value,
+    ),
+  );
+}
+
+function verifiedHeaders(request: VerifiedHuaweiRequest): HeadersInit {
+  const headers = new Headers({
+    "Content-Type": "application/json",
+    Accept: "application/json",
+    "User-Agent": HULISTMI_USER_AGENT,
+  });
+  for (const [key, value] of Object.entries(request.headers ?? {})) {
+    const lower = key.toLowerCase();
+    if (
+      (lower === "origin" || lower === "referer") &&
+      isVerifiedHeaderValue(lower, value)
+    )
+      headers.set(key, value);
+  }
+  return headers;
+}
+
+async function readCappedText(
+  response: Response,
+  maxBytes: number,
+): Promise<string> {
+  const contentLength = Number(response.headers.get("Content-Length") ?? "0");
+  if (contentLength > maxBytes)
+    throw new UpstreamSizeError(
+      "Huawei upstream response exceeded maximum size",
+    );
+  if (!response.body) return response.text();
+
+  const reader = response.body.getReader();
+  const chunks: Uint8Array[] = [];
+  let total = 0;
+  while (true) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    total += value.byteLength;
+    if (total > maxBytes) {
+      await reader.cancel();
+      throw new UpstreamSizeError(
+        "Huawei upstream response exceeded maximum size",
+      );
+    }
+    chunks.push(value);
+  }
+
+  const bytes = new Uint8Array(total);
+  let offset = 0;
+  for (const chunk of chunks) {
+    bytes.set(chunk, offset);
+    offset += chunk.byteLength;
+  }
+  return new TextDecoder().decode(bytes);
+}
+
+export async function fetchHuaweiJson<T>(
+  request: VerifiedHuaweiRequest,
+  maxBytes = MAX_UPSTREAM_RESPONSE_BYTES,
+): Promise<T> {
+  const verifiedUrl = assertAllowedHuaweiUrl(request.url);
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), UPSTREAM_TIMEOUT_MS);
+
+  let response: Response;
+  try {
+    response = await fetch(verifiedUrl, {
+      method: "POST",
+      headers: verifiedHeaders(request),
+      body: JSON.stringify(request.body),
+      signal: controller.signal,
+    });
+  } catch (error) {
+    if (error instanceof DOMException && error.name === "AbortError")
+      throw new UpstreamTimeoutError("Huawei upstream request timed out");
+    throw error;
+  } finally {
+    clearTimeout(timeout);
+  }
+
+  const robots = response.headers.get("X-Robots-Tag")?.toLowerCase() ?? "";
+  if (
+    robots.includes("noai") ||
+    robots.includes("none") ||
+    robots.includes("noindex")
+  ) {
+    throw new UpstreamPolicyError(
+      "Upstream policy disallows rendering this content",
+    );
+  }
+
+  if (!response.ok)
+    throw new Error(`Huawei upstream request failed: ${response.status}`);
+  const data = JSON.parse(await readCappedText(response, maxBytes)) as T & {
+    code?: number | string;
+  };
+  if (data.code === 404 || data.code === "404")
+    throw new NotFoundError("Huawei document not found");
+  return data;
+}
+
+export function assertRenderedMarkdownWithinLimit(content: string): string {
+  if (
+    new TextEncoder().encode(content).byteLength > MAX_RENDERED_MARKDOWN_BYTES
+  ) {
+    throw new UpstreamSizeError("Rendered markdown exceeded maximum size");
+  }
+  return content;
+}

package/src/lib/guides/fetch.ts

@@ -0,0 +1,8 @@
+import { fetchHarmonyDocumentPageData } from "../documents";
+import type { HarmonyDocumentValue } from "../types";
+
+export async function fetchGuidePageData(
+  path: string,
+): Promise<HarmonyDocumentValue> {
+  return fetchHarmonyDocumentPageData("harmonyos-guides", path);
+}

package/src/lib/guides/index.ts

@@ -0,0 +1,3 @@
+export { fetchGuidePageData } from "./fetch";
+export { renderGuideMarkdown } from "./render";
+export type { HarmonyDocumentResponse, HarmonyDocumentValue } from "./types";

package/src/lib/guides/render.ts

@@ -0,0 +1,13 @@
+import { renderDocumentMarkdown } from "../render";
+import type { HarmonyDocumentValue } from "../types";
+
+export function renderGuideMarkdown(
+  value: HarmonyDocumentValue,
+  path: string,
+): string {
+  return renderDocumentMarkdown(
+    value,
+    `harmonyos-guides/${path}`,
+    "HarmonyOS Guides",
+  );
+}

package/src/lib/guides/types.ts

@@ -0,0 +1,5 @@
+export type {
+  HarmonyContentBlock,
+  HarmonyDocumentResponse,
+  HarmonyDocumentValue,
+} from "../types";

package/src/lib/mcp.ts

@@ -0,0 +1,119 @@
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { z } from "zod";
+import { fetchHarmonyOSCatalog, renderCatalogMarkdown } from "./catalog";
+import { assertRenderedMarkdownWithinLimit } from "./fetch";
+import { fetchGuidePageData, renderGuideMarkdown } from "./guides";
+import { fetchReferencePageData, renderReferenceMarkdown } from "./reference";
+import { renderSearchMarkdown, searchHarmonyOSDocs } from "./search";
+
+export const MCP_SERVER_INFO = {
+  name: "hulistmi.ai",
+  version: "1.0.0",
+} as const;
+
+const readOnlyAnnotations = {
+  readOnlyHint: true,
+  destructiveHint: false,
+  idempotentHint: true,
+  openWorldHint: true,
+} as const;
+
+export const TOOL_DEFINITIONS = {
+  searchHarmonyOSDocumentation: {
+    description: "Search HarmonyOS developer documentation.",
+    http: { path: "/search", query: { q: "query" } },
+  },
+  fetchHarmonyOSDocumentation: {
+    description: "Fetch a HarmonyOS documentation page as Markdown.",
+    http: { path: "/{path}", query: {} },
+  },
+  fetchHarmonyOSCatalog: {
+    description: "Fetch a HarmonyOS documentation catalog.",
+    http: {
+      path: "/catalog",
+      query: { catalogName: "catalogName", language: "language", depth: "depth" },
+    },
+  },
+} as const;
+
+export function createMcpServer(): McpServer {
+  const server = new McpServer(MCP_SERVER_INFO);
+  server.registerTool(
+    "searchHarmonyOSDocumentation",
+    {
+      title: "Search HarmonyOS Documentation",
+      description: TOOL_DEFINITIONS.searchHarmonyOSDocumentation.description,
+      inputSchema: { query: z.string().min(1).max(120) },
+      annotations: readOnlyAnnotations,
+    },
+    async ({ query }) => ({
+      content: [
+        {
+          type: "text",
+          text: assertRenderedMarkdownWithinLimit(
+            renderSearchMarkdown(await searchHarmonyOSDocs(query)),
+          ),
+        },
+      ],
+    }),
+  );
+  server.registerTool(
+    "fetchHarmonyOSDocumentation",
+    {
+      title: "Fetch HarmonyOS Documentation",
+      description: TOOL_DEFINITIONS.fetchHarmonyOSDocumentation.description,
+      inputSchema: { path: z.string().min(1) },
+      annotations: readOnlyAnnotations,
+    },
+    async ({ path }) => {
+      const normalized = path.replace(/^\/+/, "");
+      const content = normalized.startsWith("harmonyos-references/")
+        ? renderReferenceMarkdown(
+            await fetchReferencePageData(
+              normalized.replace("harmonyos-references/", ""),
+            ),
+            normalized.replace("harmonyos-references/", ""),
+          )
+        : renderGuideMarkdown(
+            await fetchGuidePageData(
+              normalized.replace("harmonyos-guides/", ""),
+            ),
+            normalized.replace("harmonyos-guides/", ""),
+          );
+      return {
+        content: [
+          { type: "text", text: assertRenderedMarkdownWithinLimit(content) },
+        ],
+      };
+    },
+  );
+  server.registerTool(
+    "fetchHarmonyOSCatalog",
+    {
+      title: "Fetch HarmonyOS Catalog",
+      description: TOOL_DEFINITIONS.fetchHarmonyOSCatalog.description,
+      inputSchema: {
+        catalogName: z
+          .enum(["harmonyos-guides", "harmonyos-references"])
+          .default("harmonyos-guides"),
+        language: z.literal("en").default("en"),
+        depth: z.number().int().min(1).optional(),
+      },
+      annotations: readOnlyAnnotations,
+    },
+    async ({ catalogName, language, depth }) => ({
+      content: [
+        {
+          type: "text",
+          text: assertRenderedMarkdownWithinLimit(
+            renderCatalogMarkdown(
+              await fetchHarmonyOSCatalog(catalogName, language),
+              depth,
+            ),
+          ),
+        },
+      ],
+    }),
+  );
+  return server;
+}

package/src/lib/rate-limit.ts

@@ -0,0 +1,27 @@
+export interface RateLimitBinding {
+  limit(options: { key: string }): Promise<{ success: boolean }>;
+}
+
+export interface RateLimitEnv {
+  RATE_LIMITER?: RateLimitBinding;
+}
+
+export async function enforceRateLimit(
+  request: Request,
+  env: RateLimitEnv,
+): Promise<Response | null> {
+  if (!env.RATE_LIMITER) return null;
+  const url = new URL(request.url);
+  const ip = request.headers.get("CF-Connecting-IP") ?? "anonymous";
+  const route = url.pathname.split("/")[1] || "root";
+  const result = await env.RATE_LIMITER.limit({ key: `${ip}:${route}` });
+  if (result.success) return null;
+  return new Response(JSON.stringify({ error: "Rate limit exceeded" }), {
+    status: 429,
+    headers: {
+      "Content-Type": "application/json",
+      "Retry-After": "60",
+      "Cache-Control": "no-store",
+    },
+  });
+}

package/src/lib/reference/fetch.ts

@@ -0,0 +1,8 @@
+import { fetchHarmonyDocumentPageData } from "../documents";
+import type { HarmonyDocumentValue } from "../types";
+
+export async function fetchReferencePageData(
+  path: string,
+): Promise<HarmonyDocumentValue> {
+  return fetchHarmonyDocumentPageData("harmonyos-references", path);
+}

package/src/lib/reference/index.ts

@@ -0,0 +1,3 @@
+export { fetchReferencePageData } from "./fetch";
+export { renderReferenceMarkdown } from "./render";
+export type { HarmonyDocumentResponse, HarmonyDocumentValue } from "./types";

package/src/lib/reference/render.ts

@@ -0,0 +1,13 @@
+import { renderDocumentMarkdown } from "../render";
+import type { HarmonyDocumentValue } from "../types";
+
+export function renderReferenceMarkdown(
+  value: HarmonyDocumentValue,
+  path: string,
+): string {
+  return renderDocumentMarkdown(
+    value,
+    `harmonyos-references/${path}`,
+    "HarmonyOS References",
+  );
+}

package/src/lib/reference/types.ts

@@ -0,0 +1,5 @@
+export type {
+  HarmonyContentBlock,
+  HarmonyDocumentResponse,
+  HarmonyDocumentValue,
+} from "../types";

package/src/lib/render.ts

@@ -0,0 +1,216 @@
+import * as cheerio from "cheerio";
+import type { HarmonyDocumentValue } from "./types";
+import { generateHuaweiDocUrl } from "./url";
+
+interface NodeLike {
+  type?: string;
+  tagName?: string;
+  name?: string;
+  data?: string;
+  children?: NodeLike[];
+}
+
+export function htmlToMarkdown(html: string): string {
+  const $ = cheerio.load(html);
+  $("script,style,noscript").remove();
+
+  const root = $("body").length ? $("body") : $.root();
+  return renderChildren(root.get(0) as NodeLike, $)
+    .replace(/\n{3,}/g, "\n\n")
+    .trim();
+}
+
+function renderChildren(node: NodeLike, $: cheerio.CheerioAPI): string {
+  return (node.children ?? [])
+    .map((child) => renderBlock(child, $))
+    .filter(Boolean)
+    .join("\n\n");
+}
+
+function renderBlock(node: NodeLike, $: cheerio.CheerioAPI): string {
+  if (node.type === "text") return normalizeText(node.data ?? "");
+  if (node.type !== "tag") return "";
+
+  const tag = (node.tagName ?? node.name ?? "").toLowerCase();
+  const element = $(node);
+  switch (tag) {
+    case "h1":
+    case "h2":
+    case "h3":
+    case "h4":
+    case "h5":
+    case "h6": {
+      const depth = Number(tag.slice(1));
+      return `${"#".repeat(depth)} ${inlineText(node, $)}`;
+    }
+    case "p":
+      return inlineText(node, $);
+    case "pre":
+      return `\`\`\`\n${element.text().trim()}\n\`\`\``;
+    case "ul":
+      return renderList(node, $, false);
+    case "ol":
+      return renderList(node, $, true);
+    case "table":
+      return renderTable(node, $);
+    case "br":
+      return "\n";
+    case "hr":
+      return "---";
+    case "blockquote":
+      return renderChildren(node, $)
+        .split("\n")
+        .map((line) => (line ? `> ${line}` : ">"))
+        .join("\n");
+    case "div":
+    case "section":
+    case "article":
+    case "main":
+    case "body":
+    case "thead":
+    case "tbody":
+    case "tfoot":
+      return renderChildren(node, $);
+    default:
+      return hasBlockChildren(node)
+        ? renderChildren(node, $)
+        : inlineText(node, $);
+  }
+}
+
+function renderList(
+  node: NodeLike,
+  $: cheerio.CheerioAPI,
+  ordered: boolean,
+): string {
+  return (node.children ?? [])
+    .filter((child) => child.type === "tag" && tagName(child) === "li")
+    .map((item, index) => {
+      const marker = ordered ? `${index + 1}.` : "-";
+      const text = inlineTextWithoutNestedBlocks(item, $);
+      const nested = (item.children ?? [])
+        .filter((child) => ["ul", "ol"].includes(tagName(child)))
+        .map((child) =>
+          renderList(child, $, tagName(child) === "ol")
+            .split("\n")
+            .map((line) => `  ${line}`)
+            .join("\n"),
+        )
+        .filter(Boolean)
+        .join("\n");
+      return nested ? `${marker} ${text}\n${nested}` : `${marker} ${text}`;
+    })
+    .join("\n");
+}
+
+function renderTable(node: NodeLike, $: cheerio.CheerioAPI): string {
+  const rows = $(node)
+    .find("tr")
+    .toArray()
+    .map((row) =>
+      $(row)
+        .children("th,td")
+        .toArray()
+        .map((cell) => escapeTableCell(inlineText(cell as NodeLike, $))),
+    )
+    .filter((cells) => cells.length > 0);
+  if (rows.length === 0) return "";
+
+  const columnCount = Math.max(...rows.map((row) => row.length));
+  const normalized = rows.map((row) =>
+    Array.from({ length: columnCount }, (_, index) => row[index] ?? ""),
+  );
+  const [header, ...body] = normalized;
+  return [
+    tableRow(header),
+    tableRow(header.map(() => "---")),
+    ...body.map(tableRow),
+  ].join("\n");
+}
+
+function tableRow(cells: string[]): string {
+  return `| ${cells.join(" | ")} |`;
+}
+
+function escapeTableCell(text: string): string {
+  return text.replace(/\|/g, "\\|").replace(/\n+/g, "<br>");
+}
+
+function inlineTextWithoutNestedBlocks(
+  node: NodeLike,
+  $: cheerio.CheerioAPI,
+): string {
+  return (node.children ?? [])
+    .filter((child) => !["ul", "ol"].includes(tagName(child)))
+    .map((child) => inlineNode(child, $))
+    .join("")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+function inlineText(node: NodeLike, $: cheerio.CheerioAPI): string {
+  return (node.children ?? [])
+    .map((child) => inlineNode(child, $))
+    .join("")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+function inlineNode(node: NodeLike, $: cheerio.CheerioAPI): string {
+  if (node.type === "text") return node.data ?? "";
+  if (node.type !== "tag") return "";
+
+  const tag = tagName(node);
+  if (tag === "br") return "\n";
+  if (tag === "code") return `\`${normalizeText($(node).text())}\``;
+  if (tag === "a") {
+    const text = inlineText(node, $);
+    const href = $(node).attr("href");
+    return href && text ? `[${text}](${href})` : text;
+  }
+  if (tag === "img") {
+    const alt = $(node).attr("alt") ?? $(node).attr("title") ?? "";
+    return alt ? `![${alt}](${$(node).attr("src") ?? ""})` : "";
+  }
+  return inlineText(node, $);
+}
+
+function hasBlockChildren(node: NodeLike): boolean {
+  return (node.children ?? []).some((child) =>
+    [
+      "h1",
+      "h2",
+      "h3",
+      "h4",
+      "h5",
+      "h6",
+      "p",
+      "ul",
+      "ol",
+      "table",
+      "pre",
+      "div",
+      "section",
+      "article",
+    ].includes(tagName(child)),
+  );
+}
+
+function tagName(node: NodeLike): string {
+  return (node.tagName ?? node.name ?? "").toLowerCase();
+}
+
+function normalizeText(text: string): string {
+  return text.replace(/\s+/g, " ").trim();
+}
+
+export function renderDocumentMarkdown(
+  value: HarmonyDocumentValue,
+  path: string,
+  category: string,
+): string {
+  const title = value.title || "Untitled";
+  const sourceUrl = generateHuaweiDocUrl(path);
+  const body = htmlToMarkdown(value.content?.content ?? "");
+  return `---\ntitle: ${title}\nsource: ${sourceUrl}\ntimestamp: ${new Date().toISOString()}\ncategory: ${category}\nlanguage: en\n---\n\n# ${title}\n\n${body}\n\n---\n\n*Extracted by [hulistmi.ai](https://hulistmi-ai.y6vd2dkjgb.workers.dev) - Making HarmonyOS docs AI-readable.*\n*This is unofficial content. Source documentation belongs to Huawei.*\n`;
+}

package/src/lib/search.ts

@@ -0,0 +1,81 @@
+import { fetchHuaweiJson } from "./fetch";
+import { UPSTREAM_CONTRACT } from "./upstream-contract";
+
+export interface SearchResult {
+  title: string;
+  url: string;
+  description: string;
+  breadcrumbs: string[];
+  tags: string[];
+  type: string;
+}
+
+interface HuaweiSearchResponse {
+  searchResult?: Array<{ developerInfos?: HuaweiDeveloperInfo[] }>;
+}
+
+interface HuaweiDeveloperInfo {
+  name?: string;
+  url?: string;
+  description?: string;
+  highlightInfos?: string[];
+  ext?: string;
+}
+
+export async function searchHarmonyOSDocs(
+  query: string,
+): Promise<{ query: string; results: SearchResult[] }> {
+  const trimmed = query.trim();
+  if (!trimmed) throw new Error("Search query is required");
+  if (trimmed.length > UPSTREAM_CONTRACT.search.maxQueryLength)
+    throw new Error("Search query is too long");
+  const data = await fetchHuaweiJson<HuaweiSearchResponse>({
+    url: UPSTREAM_CONTRACT.search.url,
+    headers: UPSTREAM_CONTRACT.search.headers,
+    body: { ...UPSTREAM_CONTRACT.search.bodyForUIAbility, keyWord: trimmed },
+  });
+  return {
+    query: trimmed,
+    results: normalizeSearchResults(data).slice(
+      0,
+      UPSTREAM_CONTRACT.search.maxLength,
+    ),
+  };
+}
+
+function normalizeSearchResults(data: HuaweiSearchResponse): SearchResult[] {
+  return (data.searchResult ?? [])
+    .flatMap((group) => group.developerInfos ?? [])
+    .map((item) => ({
+      title: item.name ?? "Untitled",
+      url: item.url?.startsWith("http")
+        ? item.url
+        : `https://developer.huawei.com${item.url ?? ""}`,
+      description: item.description ?? item.highlightInfos?.join(" ") ?? "",
+      breadcrumbs: [],
+      tags: parseTags(item.ext),
+      type: "documentation",
+    }));
+}
+
+function parseTags(ext: string | undefined): string[] {
+  if (!ext) return [];
+  try {
+    const parsed = JSON.parse(ext) as { domain?: string; nextSubType?: string };
+    return [parsed.domain, parsed.nextSubType].filter(Boolean) as string[];
+  } catch {
+    return [];
+  }
+}
+
+export function renderSearchMarkdown(result: {
+  query: string;
+  results: SearchResult[];
+}): string {
+  const lines = [`# HarmonyOS search: ${result.query}`, ""];
+  for (const item of result.results) {
+    lines.push(`- [${item.title}](${item.url})`);
+    if (item.description) lines.push(`  ${item.description}`);
+  }
+  return lines.join("\n");
+}