@hulistmi/hulistmi 1.0.0

package/src/index.ts

@@ -0,0 +1,260 @@
+import { StreamableHTTPTransport } from "@hono/mcp";
+import type { Context } from "hono";
+import { Hono } from "hono";
+import { fetchHarmonyOSCatalog, renderCatalogMarkdown } from "./lib/catalog";
+import {
+  assertRenderedMarkdownWithinLimit,
+  MAX_MCP_REQUEST_BYTES,
+  NotFoundError,
+  UpstreamPolicyError,
+  UpstreamSizeError,
+} from "./lib/fetch";
+import { fetchGuidePageData, renderGuideMarkdown } from "./lib/guides";
+import { createMcpServer, MCP_SERVER_INFO } from "./lib/mcp";
+import { enforceRateLimit } from "./lib/rate-limit";
+import {
+  fetchReferencePageData,
+  renderReferenceMarkdown,
+} from "./lib/reference";
+import { renderSearchMarkdown, searchHarmonyOSDocs } from "./lib/search";
+import {
+  createSkillIndex,
+  loadSkill,
+  SKILL_NAME,
+  skillHeaders,
+  skillIndexHeaders,
+} from "./lib/skill";
+import { buildWebMcpManifest } from "./lib/webmcp";
+
+export interface Env {
+  ASSETS: Fetcher;
+  RATE_LIMITER?: {
+    limit(options: { key: string }): Promise<{ success: boolean }>;
+  };
+}
+
+const app = new Hono<{ Bindings: Env }>();
+const ROBOTS_HEADER = "noindex, nofollow, noarchive";
+const DOC_CACHE = "public, max-age=3600, s-maxage=86400";
+const SHORT_CACHE = "public, max-age=300, s-maxage=600";
+
+function origin(c: Context): string {
+  return new URL(c.req.url).origin;
+}
+
+function wantsJson(c: Context): boolean {
+  return c.req.header("Accept")?.includes("application/json") ?? false;
+}
+
+function setNoIndex(c: Context, cacheControl: string): void {
+  c.header("X-Robots-Tag", ROBOTS_HEADER);
+  c.header("Cache-Control", cacheControl);
+  c.header("Vary", "Accept");
+}
+
+async function sha256(text: string): Promise<string> {
+  const data = new TextEncoder().encode(text);
+  const digest = await crypto.subtle.digest("SHA-256", data);
+  return `"${Array.from(new Uint8Array(digest))
+    .map((byte) => byte.toString(16).padStart(2, "0"))
+    .join("")}"`;
+}
+
+async function assertMcpBodyWithinLimit(
+  request: Request,
+): Promise<Response | null> {
+  if (request.method === "GET" || request.method === "HEAD") return null;
+  const declared = Number(request.headers.get("Content-Length") ?? "NaN");
+  if (!Number.isFinite(declared) || declared > MAX_MCP_REQUEST_BYTES)
+    return bodyTooLarge();
+  const reader = request.clone().body?.getReader();
+  if (!reader) return null;
+  let total = 0;
+  while (true) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    total += value.byteLength;
+    if (total > MAX_MCP_REQUEST_BYTES) {
+      await reader.cancel();
+      return bodyTooLarge();
+    }
+  }
+  return null;
+}
+
+function bodyTooLarge(): Response {
+  return new Response(JSON.stringify({ error: "Request body too large" }), {
+    status: 413,
+    headers: {
+      "Content-Type": "application/json",
+      "Cache-Control": "no-store",
+    },
+  });
+}
+
+async function renderDocument(
+  c: Context,
+  catalogName: "harmonyos-guides" | "harmonyos-references",
+  path: string,
+): Promise<Response> {
+  const data =
+    catalogName === "harmonyos-guides"
+      ? await fetchGuidePageData(path)
+      : await fetchReferencePageData(path);
+  const content =
+    catalogName === "harmonyos-guides"
+      ? renderGuideMarkdown(data, path)
+      : renderReferenceMarkdown(data, path);
+  const bounded = assertRenderedMarkdownWithinLimit(content);
+  const sourceUrl = `https://developer.huawei.com/consumer/en/doc/${catalogName}/${path}`;
+  setNoIndex(c, DOC_CACHE);
+  c.header("Content-Location", sourceUrl);
+  c.header("ETag", await sha256(bounded));
+  if (wantsJson(c)) return c.json({ url: sourceUrl, content: bounded });
+  return c.text(bounded, 200, {
+    "Content-Type": "text/markdown; charset=utf-8",
+  });
+}
+
+async function publicLimit(c: Context, next: () => Promise<void>) {
+  const blocked = await enforceRateLimit(c.req.raw, c.env);
+  if (blocked) return blocked;
+  await next();
+}
+
+app.use("/search", publicLimit);
+app.use("/catalog", publicLimit);
+app.use("/mcp", publicLimit);
+app.use("/consumer/en/doc/harmonyos-guides/*", publicLimit);
+app.use("/consumer/en/doc/harmonyos-references/*", publicLimit);
+
+app.get("/", async (c) =>
+  c.env.ASSETS.fetch(new Request(new URL("/index.html", c.req.url))),
+);
+
+app.get("/bot", (c) =>
+  c.text(
+    "hulistmi.ai uses transparent, on-demand requests for HarmonyOS documentation and identifies itself with hulistmi-ai/1.0 (+https://hulistmi-ai.y6vd2dkjgb.workers.dev/#bot).",
+    200,
+    {
+      "Content-Type": "text/plain; charset=utf-8",
+      "Cache-Control": SHORT_CACHE,
+    },
+  ),
+);
+
+app.get("/consumer/en/doc/harmonyos-guides/:path{.+}", async (c) =>
+  renderDocument(c, "harmonyos-guides", c.req.param("path")),
+);
+app.get("/consumer/en/doc/harmonyos-references/:path{.+}", async (c) =>
+  renderDocument(c, "harmonyos-references", c.req.param("path")),
+);
+
+app.get("/catalog", async (c) => {
+  const catalogName = c.req.query("catalogName") ?? "harmonyos-guides";
+  const language = c.req.query("language") ?? "en";
+  const depthRaw = c.req.query("depth");
+  const depth = depthRaw ? Number(depthRaw) : undefined;
+  if (
+    !["harmonyos-guides", "harmonyos-references"].includes(catalogName) ||
+    language !== "en" ||
+    (depth !== undefined && (!Number.isFinite(depth) || depth < 1))
+  )
+    return c.json({ error: "Unsupported catalog" }, 400);
+  const catalog = await fetchHarmonyOSCatalog(catalogName, language);
+  setNoIndex(c, SHORT_CACHE);
+  if (wantsJson(c)) return c.json(catalog);
+  return c.text(
+    assertRenderedMarkdownWithinLimit(renderCatalogMarkdown(catalog, depth)),
+    200,
+    { "Content-Type": "text/markdown; charset=utf-8" },
+  );
+});
+
+app.get("/search", async (c) => {
+  const query = c.req.query("q") ?? "";
+  if (!query.trim() || query.length > 120)
+    return c.json({ error: "Invalid search query" }, 400);
+  const result = await searchHarmonyOSDocs(query);
+  setNoIndex(c, SHORT_CACHE);
+  if (wantsJson(c)) return c.json(result);
+  return c.text(
+    assertRenderedMarkdownWithinLimit(renderSearchMarkdown(result)),
+    200,
+    { "Content-Type": "text/markdown; charset=utf-8" },
+  );
+});
+
+app.get("/.well-known/mcp/server-card.json", (c) =>
+  c.json({
+    serverInfo: MCP_SERVER_INFO,
+    endpoint: `${origin(c)}/mcp`,
+    transports: ["streamable-http"],
+  }),
+);
+
+app.get("/webmcp/manifest.json", (c) => c.json(buildWebMcpManifest(origin(c))));
+
+app.get("/.well-known/api-catalog", (c) =>
+  c.json(
+    {
+      linkset: [
+        {
+          anchor: `${origin(c)}/mcp`,
+          item: [
+            {
+              href: `${origin(c)}/.well-known/mcp/server-card.json`,
+              rel: "service-desc",
+            },
+            { href: `${origin(c)}/webmcp/manifest.json`, rel: "manifest" },
+          ],
+        },
+      ],
+    },
+    200,
+    {
+      "Content-Type": "application/linkset+json; charset=utf-8",
+      "Cache-Control": SHORT_CACHE,
+    },
+  ),
+);
+
+app.get("/.well-known/agent-skills/index.json", async (c) => {
+  const skill = await loadSkill(c.env.ASSETS, origin(c));
+  return c.json(await createSkillIndex(skill), 200, skillIndexHeaders);
+});
+
+app.get(`/.well-known/agent-skills/${SKILL_NAME}/SKILL.md`, async (c) => {
+  const skill = await loadSkill(c.env.ASSETS, origin(c));
+  return new Response(skill.bytes, { headers: skillHeaders });
+});
+
+app.all("/mcp", async (c) => {
+  const tooLarge = await assertMcpBodyWithinLimit(c.req.raw);
+  if (tooLarge) return tooLarge;
+  const mcpServer = createMcpServer();
+  const transport = new StreamableHTTPTransport();
+  await mcpServer.connect(transport);
+  return transport.handleRequest(c);
+});
+
+app.onError((err, c) => {
+  c.header("Cache-Control", "no-store");
+  c.header("X-Robots-Tag", ROBOTS_HEADER);
+  if (err instanceof NotFoundError) return c.json({ error: "Not found" }, 404);
+  if (err instanceof UpstreamPolicyError)
+    return c.json(
+      { error: "Upstream policy prevents rendering this content" },
+      502,
+    );
+  if (err instanceof UpstreamSizeError)
+    return c.json({ error: "Upstream content is too large" }, 502);
+  if (
+    err instanceof Error &&
+    /invalid|unsupported|required|too long/i.test(err.message)
+  )
+    return c.json({ error: err.message }, 400);
+  return c.json({ error: "Unable to render HarmonyOS documentation" }, 502);
+});
+
+export default app;

package/src/lib/catalog.ts

@@ -0,0 +1,113 @@
+import { fetchHuaweiJson, NotFoundError, UpstreamSizeError } from "./fetch";
+import { UPSTREAM_CONTRACT } from "./upstream-contract";
+
+const MAX_CATALOG_ITEMS = 20_000;
+const MAX_CATALOG_UPSTREAM_BYTES = 5_000_000;
+const SUPPORTED_CATALOGS = new Set([
+  "harmonyos-guides",
+  "harmonyos-references",
+]);
+
+export interface HarmonyCatalogItem {
+  title: string;
+  path?: string;
+  children: HarmonyCatalogItem[];
+}
+
+export interface HarmonyCatalog {
+  catalogName: string;
+  language: "en";
+  items: HarmonyCatalogItem[];
+}
+
+export async function fetchHarmonyOSCatalog(
+  catalogName: string,
+  language = "en",
+): Promise<HarmonyCatalog> {
+  if (!SUPPORTED_CATALOGS.has(catalogName))
+    throw new NotFoundError("Unsupported HarmonyOS catalog");
+  if (language !== "en")
+    throw new Error("Unsupported HarmonyOS catalog language");
+  const request =
+    UPSTREAM_CONTRACT.catalogs[
+      catalogName as keyof typeof UPSTREAM_CONTRACT.catalogs
+    ].request;
+  const data = await fetchHuaweiJson<{
+    code: number | string;
+    value?: unknown;
+  }>(request, MAX_CATALOG_UPSTREAM_BYTES);
+  if (data.code !== 0 && data.code !== "0")
+    throw new Error("Huawei catalog response changed shape");
+  const items = normalizeCatalogItems(data.value);
+  if (countItems(items) > MAX_CATALOG_ITEMS)
+    throw new UpstreamSizeError("Huawei catalog exceeded maximum item count");
+  return { catalogName, language: "en", items };
+}
+
+function normalizeCatalogItems(value: unknown): HarmonyCatalogItem[] {
+  const source = Array.isArray(value)
+    ? value
+    : isRecord(value) && Array.isArray(value.catalogTreeList)
+      ? value.catalogTreeList
+      : [];
+  if (!Array.isArray(source)) return [];
+  return source.map((item) => {
+    const record = item as Record<string, unknown>;
+    return {
+      title: String(
+        record.nodeName ??
+          record.title ??
+          record.name ??
+          record.label ??
+          "Untitled",
+      ),
+      path:
+        typeof record.relateDocument === "string"
+          ? record.relateDocument
+          : typeof record.fileName === "string"
+            ? record.fileName
+            : undefined,
+      children: normalizeCatalogItems(record.children ?? record.childList),
+    };
+  });
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return Boolean(value) && typeof value === "object" && !Array.isArray(value);
+}
+
+function countItems(items: HarmonyCatalogItem[]): number {
+  return items.reduce(
+    (count, item) => count + 1 + countItems(item.children),
+    0,
+  );
+}
+
+function renderTreeItems(
+  items: HarmonyCatalogItem[],
+  depth: number,
+  maxDepth: number | undefined,
+): string[] {
+  if (!items.length || (maxDepth !== undefined && depth > maxDepth)) return [];
+  const lines: string[] = [];
+  for (const item of items) {
+    const indent = "  ".repeat(depth - 1);
+    lines.push(`${indent}- ${item.title}${item.path ? `: ${item.path}` : ""}`);
+    if (item.children.length)
+      lines.push(...renderTreeItems(item.children, depth + 1, maxDepth));
+  }
+  return lines;
+}
+
+export function renderCatalogMarkdown(
+  catalog: HarmonyCatalog,
+  maxDepth?: number,
+): string {
+  const title =
+    catalog.catalogName === "harmonyos-guides"
+      ? "HarmonyOS Guides Catalog"
+      : "HarmonyOS References Catalog";
+  const lines = [`# ${title}`, ""];
+  lines.push(...renderTreeItems(catalog.items, 1, maxDepth));
+  return lines.join("\n");
+}

package/src/lib/cli-endpoints.ts

@@ -0,0 +1,62 @@
+import { huaweiUrlToPath, normalizeDocsPath } from "./url";
+
+export type CliCommand =
+  | { command: "fetch"; input: string; json: boolean }
+  | { command: "search"; query: string; json: boolean }
+  | { command: "serve"; port?: number };
+
+const CLI_DOC_PREFIX = "consumer/en/doc/";
+
+export function resolveFetchEndpoint(input: string): string {
+  const trimmed = input.trim();
+  if (!trimmed) throw new Error("Fetch input cannot be empty");
+  let path: string;
+  if (/^https?:\/\//i.test(trimmed)) {
+    path = huaweiUrlToPath(trimmed);
+  } else {
+    path = normalizeDocsPath(trimmed);
+  }
+  // Strip the full prefix if already present
+  if (path.startsWith(CLI_DOC_PREFIX)) {
+    path = path.slice(CLI_DOC_PREFIX.length);
+  }
+  // Only add prefix for known HarmonyOS doc paths
+  if (
+    path.startsWith("harmonyos-guides/") ||
+    path.startsWith("harmonyos-references/")
+  ) {
+    return `/${CLI_DOC_PREFIX}${path}`;
+  }
+  return `/${path}`;
+}
+
+export function resolveSearchEndpoint(query: string): string {
+  const trimmed = query.trim();
+  if (!trimmed) throw new Error("Search query cannot be empty");
+  return `/search?q=${encodeURIComponent(trimmed)}`;
+}
+
+export function parseCliArgs(argv: string[]): CliCommand {
+  const [command, ...rest] = argv;
+  const json = rest.includes("--json");
+  const values = rest.filter((value) => value !== "--json");
+
+  if (command === "fetch") {
+    const input = values[0];
+    if (!input) throw new Error("Usage: hulistmi fetch <url-or-path> [--json]");
+    return { command, input, json };
+  }
+  if (command === "search") {
+    const query = values.join(" ");
+    if (!query) throw new Error("Usage: hulistmi search <query> [--json]");
+    return { command, query, json };
+  }
+  if (command === "serve") {
+    const portFlag = values.indexOf("--port");
+    const port = portFlag >= 0 ? Number(values[portFlag + 1]) : undefined;
+    return { command, port };
+  }
+  throw new Error(
+    "Usage: hulistmi fetch <url-or-path> [--json] | hulistmi search <query> [--json] | hulistmi serve [--port 8787]",
+  );
+}

package/src/lib/documents.ts

@@ -0,0 +1,164 @@
+import {
+  fetchHuaweiJson,
+  NotFoundError,
+  type VerifiedHuaweiRequest,
+} from "./fetch";
+import type { HarmonyDocumentResponse, HarmonyDocumentValue } from "./types";
+import { UPSTREAM_CONTRACT } from "./upstream-contract";
+
+interface DocumentContractEntry {
+  checkCenterGrayUser: VerifiedHuaweiRequest;
+  getDocumentById: VerifiedHuaweiRequest;
+  getCenterRootNodeTree?: VerifiedHuaweiRequest;
+  getCenterDocument?: VerifiedHuaweiRequest;
+}
+
+interface GrayUserResponse {
+  code: number | string;
+  value?: {
+    isGrayUser?: number | boolean;
+    centerPrefix?: string;
+    level2NodeAlias?: string;
+    isApi?: number | boolean;
+    filename?: string;
+  };
+}
+
+const DOCUMENTS = UPSTREAM_CONTRACT.documents as Record<
+  string,
+  DocumentContractEntry
+>;
+const DOCUMENT_SLUG_PATTERN = /^[a-z0-9][a-z0-9-]*$/;
+const GRAY_ID = "11111111111111111111111111111111";
+const CHECK_CENTER_GRAY_USER_URL =
+  "https://svc-drcn.developer.huawei.com/community/servlet/consumer/cn/documentPortal/checkCenterGrayUser";
+const GET_DOCUMENT_BY_ID_URL =
+  "https://svc-drcn.developer.huawei.com/community/servlet/consumer/cn/documentPortal/getDocumentById";
+const GET_CENTER_ROOT_NODE_TREE_URL =
+  "https://svc-drcn.developer.huawei.com/community/servlet/consumer/cn/documentPortal/getCenterRootNodeTree";
+const GET_CENTER_DOCUMENT_URL =
+  "https://svc-drcn.developer.huawei.com/community/servlet/consumer/cn/documentPortal/getCenterDocument";
+
+export async function fetchHarmonyDocumentPageData(
+  catalogName: "harmonyos-guides" | "harmonyos-references",
+  path: string,
+): Promise<HarmonyDocumentValue> {
+  const entry =
+    DOCUMENTS[documentKey(catalogName, path)] ?? buildEntry(catalogName, path);
+
+  const grayResponse = await fetchHuaweiJson<GrayUserResponse>(
+    entry.checkCenterGrayUser,
+  );
+  if (isCenterDocument(grayResponse)) {
+    const centerRequests =
+      entry.getCenterRootNodeTree && entry.getCenterDocument
+        ? {
+            getCenterRootNodeTree: entry.getCenterRootNodeTree,
+            getCenterDocument: entry.getCenterDocument,
+          }
+        : buildCenterRequests(grayResponse, path);
+    await fetchHuaweiJson(centerRequests.getCenterRootNodeTree);
+    return fetchAndValidateDocument(centerRequests.getCenterDocument);
+  }
+
+  return fetchAndValidateDocument(entry.getDocumentById);
+}
+
+function documentKey(
+  catalogName: "harmonyos-guides" | "harmonyos-references",
+  path: string,
+): string {
+  return `${catalogName}/${normalizeDocumentSlug(path)}`;
+}
+
+function buildEntry(
+  catalogName: "harmonyos-guides" | "harmonyos-references",
+  path: string,
+): DocumentContractEntry {
+  const slug = normalizeDocumentSlug(path);
+  if (!DOCUMENT_SLUG_PATTERN.test(slug))
+    throw new NotFoundError(`Unknown HarmonyOS document: ${path}`);
+
+  return {
+    checkCenterGrayUser: {
+      url: CHECK_CENTER_GRAY_USER_URL,
+      headers: {},
+      body: {
+        catalogName,
+        language: "en",
+        fileName: slug,
+        grayId: GRAY_ID,
+      },
+    },
+    getDocumentById: {
+      url: GET_DOCUMENT_BY_ID_URL,
+      headers: {},
+      body: {
+        objectId: slug,
+        nodeAlias: null,
+        catalogName,
+        language: "en",
+      },
+    },
+  };
+}
+
+function normalizeDocumentSlug(path: string): string {
+  return path.replace(/^\/+/, "").replace(/\/+$/, "").toLowerCase();
+}
+
+function isCenterDocument(response: GrayUserResponse): boolean {
+  return (
+    (response.code === 0 || response.code === "0") &&
+    Boolean(response.value?.isGrayUser)
+  );
+}
+
+function buildCenterRequests(
+  response: GrayUserResponse,
+  path: string,
+): Pick<DocumentContractEntry, "getCenterRootNodeTree" | "getCenterDocument"> {
+  const value = response.value;
+  const fileName = value?.filename ?? normalizeDocumentSlug(path);
+  if (
+    !value?.centerPrefix ||
+    !value.level2NodeAlias ||
+    !DOCUMENT_SLUG_PATTERN.test(fileName)
+  )
+    throw new Error("HarmonyOS center document response changed shape");
+
+  const body = {
+    centerPrefix: value.centerPrefix,
+    language: "en",
+    level2NodeAlias: value.level2NodeAlias,
+    isApi: value.isApi ? 1 : 0,
+    fileName,
+  };
+  return {
+    getCenterRootNodeTree: {
+      url: GET_CENTER_ROOT_NODE_TREE_URL,
+      headers: {},
+      body,
+    },
+    getCenterDocument: {
+      url: GET_CENTER_DOCUMENT_URL,
+      headers: {},
+      body,
+    },
+  };
+}
+
+async function fetchAndValidateDocument(
+  request: VerifiedHuaweiRequest,
+): Promise<HarmonyDocumentValue> {
+  const response = await fetchHuaweiJson<HarmonyDocumentResponse>(request);
+  if (response.code !== 0 && response.code !== "0")
+    throw new Error("HarmonyOS document response changed shape");
+  if (
+    !response.value ||
+    response.value.status !== "4" ||
+    !response.value.content?.content
+  )
+    throw new Error("HarmonyOS document content is unavailable");
+  return response.value;
+}