@nshipster/sosumi 1.0.0 → 1.0.4

package/src/lib/search.ts

@@ -14,208 +14,225 @@ export interface SearchResponse {
   results: SearchResult[]
 }
 
-class SearchResultParser {
-  private results: SearchResult[] = []
-  private currentResult: Partial<SearchResult> = {}
-  private currentBreadcrumbs: string[] = []
-  private currentTags: string[] = []
-  private isInResultTitle = false
-  private isInResultDescription = false
-  private isInBreadcrumb = false
-  private isInTag = false
-
-  getResults(): SearchResult[] {
-    return this.results
+// Apple's current search backend, discovered from
+// https://developer.apple.com/search/scripts/search.js (May 2026)
+//
+// Historical context:
+//   - The legacy /search/ HTML scraper broke when Apple switched to a JS-rendered SPA
+//   - PR #54 upstream (NSHipster/sosumi.ai) targeted /search/services/search.php with
+//     NDJSON-style streamed events; that endpoint is now also gone (404)
+//   - The current backend is a plain JSON POST API on Apple's MSC infrastructure
+const APPLE_SEARCH_SERVICE_URL = "https://devintserv.msc.sbz.apple.com/api/v1/search"
+const DEFAULT_TARGET_RESULT_LOCALE = "en"
+const TARGET_RESULT_LOCALE_BY_BASE_NAME = new Map([
+  ["en", "en"],
+  ["zh-CN", "zh-CN"],
+  ["ja-JP", "ja-JP"],
+  ["ko-KR", "ko-KR"],
+  ["fr-FR", "fr-FR"],
+  ["de-DE", "de-DE"],
+  ["pt-BR", "pt-BR"],
+  ["es-LA", "es-lamr"],
+  ["es-419", "es-lamr"],
+  ["it-IT", "it-IT"],
+])
+
+type JsonRecord = Record<string, unknown>
+
+export async function searchAppleDeveloperDocs(query: string): Promise<SearchResponse> {
+  const results = await searchAppleDeveloperDocsViaService(query)
+  return { query, results }
+}
+
+async function searchAppleDeveloperDocsViaService(query: string): Promise<SearchResult[]> {
+  const response = await fetch(APPLE_SEARCH_SERVICE_URL, {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+      Accept: "application/json",
+      // The MSC backend requires a browser-style Origin/Referer pair to accept the request.
+      Origin: "https://developer.apple.com",
+      Referer: "https://developer.apple.com/search/",
+      "User-Agent": getRandomUserAgent(),
+    },
+    body: JSON.stringify({
+      text: query,
+      targetResultLocale: resolveTargetResultLocale(),
+    }),
+  })
+
+  if (!response.ok) {
+    throw new Error(`Search request failed: ${response.status}`)
   }
 
-  private resetCurrentResult() {
-    this.currentResult = {}
-    this.currentBreadcrumbs = []
-    this.currentTags = []
-    this.isInResultTitle = false
-    this.isInResultDescription = false
-    this.isInBreadcrumb = false
-    this.isInTag = false
+  const data = await readSearchResponseJson(response)
+  const rawResults = Array.isArray(data.results) ? (data.results as unknown[]) : []
+  return extractSearchResults(rawResults)
+}
+
+async function readSearchResponseJson(response: Response): Promise<JsonRecord> {
+  try {
+    const data = await response.json()
+    return isJsonRecord(data) ? data : {}
+  } catch {
+    throw new Error("Search response was not valid JSON")
   }
+}
 
-  private finalizeCurrentResult() {
-    if (this.currentResult.title && this.currentResult.url) {
-      this.results.push({
-        title: this.currentResult.title,
-        url: this.currentResult.url,
-        description: this.currentResult.description || "",
-        breadcrumbs: [...this.currentBreadcrumbs],
-        tags: [...this.currentTags],
-        type: this.currentResult.type || "unknown",
-      })
-    }
-    this.resetCurrentResult()
+function extractSearchResults(items: unknown[]): SearchResult[] {
+  return items.flatMap((item) => {
+    const result = normalizeSearchResult(item)
+    return result ? [result] : []
+  })
+}
+
+function normalizeSearchResult(item: unknown): SearchResult | null {
+  if (!isJsonRecord(item)) {
+    return null
   }
 
-  element(element: Element) {
-    // Start of a search result
-    if (element.tagName === "li" && element.getAttribute("class")?.includes("search-result")) {
-      this.finalizeCurrentResult() // Finalize previous result if any
-
-      // Extract result type from class
-      const className = element.getAttribute("class") || ""
-      if (className.includes("documentation")) {
-        this.currentResult.type = "documentation"
-      } else if (className.includes("general")) {
-        this.currentResult.type = "general"
-      } else {
-        this.currentResult.type = "other"
-      }
+  const documentation = extractMetadataRecord(item.documentation)
+  if (documentation) {
+    const title = stringValue(documentation.title)
+    const url = stringValue(documentation.permalink)
+    if (!title || !url) {
+      return null
     }
 
-    // Result title link
-    if (
-      element.tagName === "a" &&
-      element.getAttribute("class")?.includes("click-analytics-result")
-    ) {
-      const href = element.getAttribute("href")
-      if (href) {
-        this.currentResult.url = href.startsWith("/") ? `https://developer.apple.com${href}` : href
-      }
-      this.isInResultTitle = true
+    return {
+      title,
+      url,
+      description: stringValue(documentation.description) ?? "",
+      breadcrumbs: splitHierarchy(stringValue(documentation.hierarchy)),
+      tags: compactStrings([stringValue(documentation.kind)]),
+      type: "documentation",
     }
+  }
 
-    // Result description
-    if (element.tagName === "p" && element.getAttribute("class")?.includes("result-description")) {
-      this.isInResultDescription = true
+  const developer = extractMetadataRecord(item.developer)
+  if (developer) {
+    const title = firstString(developer.titles)
+    const url = firstString(developer.permalinks)
+    if (!title || !url) {
+      return null
     }
 
-    // Breadcrumb items
-    if (
-      element.tagName === "li" &&
-      element.getAttribute("class")?.includes("breadcrumb-list-item")
-    ) {
-      this.isInBreadcrumb = true
+    return {
+      title,
+      url,
+      description: firstString(developer.descriptions) ?? "",
+      breadcrumbs: compactStrings([firstString(developer.projectNames)]),
+      tags: compactStrings([
+        firstString(developer.itemTypes),
+        firstString(developer.deliveryLanguageCodes),
+      ]),
+      type: (firstString(developer.itemTypes) ?? "developer").toLowerCase(),
     }
+  }
 
-    // Tag spans
-    if (
-      element.tagName === "span" &&
-      element.parentElement?.getAttribute("class")?.includes("result-tag")
-    ) {
-      this.isInTag = true
+  const devsite = extractMetadataRecord(item.devsite)
+  if (devsite) {
+    const title = stringValue(devsite.title)
+    const url = stringValue(devsite.sourceURL)
+    if (!title || !url) {
+      return null
     }
 
-    // Tag list items (for languages like "Swift", "Objective-C")
-    if (
-      element.tagName === "li" &&
-      element.getAttribute("class")?.includes("result-tag language")
-    ) {
-      this.isInTag = true
+    return {
+      title,
+      url,
+      description: stringValue(devsite.description) ?? "",
+      breadcrumbs: [],
+      tags: [],
+      type: "general",
     }
   }
 
-  text(text: Text) {
-    const content = text.text.trim()
-    if (!content) return
-
-    if (this.isInResultTitle && this.currentResult.url) {
-      this.currentResult.title = content
-      this.isInResultTitle = false
-    } else if (this.isInResultDescription) {
-      this.currentResult.description = content
-      this.isInResultDescription = false
-    } else if (this.isInBreadcrumb) {
-      this.currentBreadcrumbs.push(content)
-      this.isInBreadcrumb = false
-    } else if (this.isInTag) {
-      this.currentTags.push(content)
-      this.isInTag = false
+  const swiftdocs = extractMetadataRecord(item.swiftdocs)
+  if (swiftdocs) {
+    const title = stringValue(swiftdocs.title)
+    const url = stringValue(swiftdocs.sourceURL)
+    if (!title || !url) {
+      return null
+    }
+
+    return {
+      title,
+      url,
+      description: stringValue(swiftdocs.description) ?? "",
+      breadcrumbs: [],
+      tags: [],
+      type: "general",
     }
   }
 
-  end() {
-    this.finalizeCurrentResult() // Finalize the last result
+  return null
+}
+
+function extractMetadataRecord(container: unknown): JsonRecord | null {
+  if (!isJsonRecord(container)) {
+    return null
   }
+
+  const metadata = container.metadata
+  return isJsonRecord(metadata) ? metadata : null
 }
 
-export async function searchAppleDeveloperDocs(query: string): Promise<SearchResponse> {
-  const searchUrl = `https://developer.apple.com/search/?q=${encodeURIComponent(query)}`
-  const response = await fetch(searchUrl, {
-    headers: {
-      "User-Agent": getRandomUserAgent(),
-    },
-  })
+function isJsonRecord(value: unknown): value is JsonRecord {
+  return typeof value === "object" && value !== null
+}
 
-  if (!response.ok) {
-    throw new Error(`Search request failed: ${response.status}`)
-  }
+function stringValue(value: unknown): string | null {
+  return typeof value === "string" && value.length > 0 ? value : null
+}
 
-  const html = await response.text()
-  let results: SearchResult[] = []
-  if (typeof HTMLRewriter !== "undefined") {
-    const parser = new SearchResultParser()
-    const rewriter = new HTMLRewriter()
-      .on("li.search-result", parser)
-      .on("li.search-result a.click-analytics-result", parser)
-      .on("li.search-result p.result-description", parser)
-      .on("li.search-result li.breadcrumb-list-item", parser)
-      .on("li.search-result li.result-tag", parser)
-      .on("li.search-result li.result-tag span", parser)
-
-    // We need to consume the transformed response to trigger parsing callbacks.
-    await rewriter.transform(new Response(html)).text()
-    parser.end()
-    results = parser.getResults()
-  } else {
-    results = await parseSearchResultsWithCheerio(html)
+function firstString(value: unknown): string | null {
+  if (!Array.isArray(value)) {
+    return null
   }
 
-  return {
-    query,
-    results,
-  }
+  const first = value.find((item) => typeof item === "string" && item.length > 0)
+  return typeof first === "string" ? first : null
 }
 
-async function parseSearchResultsWithCheerio(html: string): Promise<SearchResult[]> {
-  const { load } = await import("cheerio")
-  const $ = load(html)
-  const results: SearchResult[] = []
+function splitHierarchy(hierarchy: string | null): string[] {
+  if (!hierarchy) {
+    return []
+  }
 
-  $("li.search-result").each((_, element) => {
-    const item = $(element)
-    const link = item.find("a.click-analytics-result").first()
-    const rawHref = link.attr("href")
-    const title = link.text().trim()
+  return hierarchy
+    .split(" > ")
+    .map((segment) => segment.trim())
+    .filter(Boolean)
+}
 
-    if (!rawHref || !title) {
-      return
-    }
+function compactStrings(values: Array<string | null>): string[] {
+  return values.filter((value): value is string => Boolean(value))
+}
 
-    const description = item.find("p.result-description").first().text().trim()
-    const breadcrumbs = item
-      .find("li.breadcrumb-list-item")
-      .toArray()
-      .map((breadcrumb) => $(breadcrumb).text().trim())
-      .filter(Boolean)
-
-    const tags = item
-      .find("li.result-tag span, li.result-tag.language")
-      .toArray()
-      .map((tag) => $(tag).text().trim())
-      .filter(Boolean)
-
-    const className = item.attr("class") ?? ""
-    const type = className.includes("documentation")
-      ? "documentation"
-      : className.includes("general")
-        ? "general"
-        : "other"
-
-    results.push({
-      title,
-      url: rawHref.startsWith("/") ? `https://developer.apple.com${rawHref}` : rawHref,
-      description,
-      breadcrumbs,
-      tags,
-      type,
-    })
-  })
+// Apple's MSC backend uses BCP-47 language tags ("en", "ja-JP", "zh-CN", etc.)
+// instead of POSIX locale codes ("en_US").
+// Mirror the mapping from
+// https://developer.apple.com/search/scripts/helpers.js
+function resolveTargetResultLocale(): string {
+  const locale = Intl.DateTimeFormat().resolvedOptions().locale
+  if (!locale) {
+    return DEFAULT_TARGET_RESULT_LOCALE
+  }
 
-  return results
+  try {
+    const normalized = new Intl.Locale(locale)
+    const lang = normalized.language
+    const region = normalized.region
+    const languageRegion = region ? `${lang}-${region}` : lang
+
+    return (
+      TARGET_RESULT_LOCALE_BY_BASE_NAME.get(normalized.baseName) ??
+      TARGET_RESULT_LOCALE_BY_BASE_NAME.get(languageRegion) ??
+      TARGET_RESULT_LOCALE_BY_BASE_NAME.get(lang) ??
+      DEFAULT_TARGET_RESULT_LOCALE
+    )
+  } catch {
+    return DEFAULT_TARGET_RESULT_LOCALE
+  }
 }

package/src/lib/skill.ts

@@ -0,0 +1,148 @@
+import { HTTPException } from "hono/http-exception"
+
+export const SKILL_NAME = "sosumi"
+
+export const skillHeaders = {
+  "Access-Control-Allow-Origin": "*",
+  "Cache-Control": "public, max-age=300, s-maxage=600",
+  "Content-Type": "text/markdown; charset=utf-8",
+}
+
+export const skillIndexHeaders = {
+  "Access-Control-Allow-Origin": "*",
+  "Cache-Control": "public, max-age=300, s-maxage=600",
+  "Content-Type": "application/json; charset=utf-8",
+}
+
+export interface SkillArtifact {
+  bytes: ArrayBuffer
+  description: string
+  name: string
+}
+
+export async function loadSkill(assets: Fetcher, baseUrl: string): Promise<SkillArtifact> {
+  const skillResponse = await assets.fetch(new Request(skillAssetUrl(baseUrl)))
+
+  if (!skillResponse.ok) {
+    throw new HTTPException(500, {
+      message: "Failed to load SKILL.md",
+    })
+  }
+
+  const bytes = await skillResponse.arrayBuffer()
+  const frontmatter = parseSkillFrontmatter(new TextDecoder().decode(bytes))
+
+  assertValidSkillFrontmatter(frontmatter)
+
+  return {
+    bytes,
+    description: frontmatter.description,
+    name: frontmatter.name,
+  }
+}
+
+export async function skillExists(assets: Fetcher, baseUrl: string): Promise<boolean> {
+  const response = await assets.fetch(new Request(skillAssetUrl(baseUrl), { method: "HEAD" }))
+  return response.ok
+}
+
+function skillAssetUrl(baseUrl: string): string {
+  return new URL("/SKILL.md", baseUrl).toString()
+}
+
+export async function createSkillIndex(skill: SkillArtifact) {
+  return {
+    $schema: "https://schemas.agentskills.io/discovery/0.2.0/schema.json",
+    skills: [
+      {
+        name: skill.name,
+        type: "skill-md",
+        description: skill.description,
+        url: `/.well-known/agent-skills/${skill.name}/SKILL.md`,
+        digest: `sha256:${await sha256Hex(skill.bytes)}`,
+        files: ["SKILL.md"],
+      },
+    ],
+  }
+}
+
+function parseSkillFrontmatter(markdown: string): Record<string, string> {
+  const match = markdown.match(/^---\r?\n([\s\S]*?)\r?\n---(?:\r?\n|$)/)
+
+  if (!match) {
+    throw new HTTPException(500, {
+      message: "SKILL.md must start with YAML frontmatter.",
+    })
+  }
+
+  const fields: Record<string, string> = {}
+
+  for (const line of match[1].split(/\r?\n/)) {
+    if (!line.trim()) {
+      continue
+    }
+
+    const separator = line.indexOf(":")
+
+    if (separator === -1) {
+      throw new HTTPException(500, {
+        message: `Invalid SKILL.md frontmatter line: ${line}`,
+      })
+    }
+
+    const key = line.slice(0, separator).trim()
+    const value = line.slice(separator + 1).trim()
+
+    fields[key] = stripQuotes(value)
+  }
+
+  return fields
+}
+
+function stripQuotes(value: string): string {
+  if (
+    (value.startsWith('"') && value.endsWith('"')) ||
+    (value.startsWith("'") && value.endsWith("'"))
+  ) {
+    return value.slice(1, -1)
+  }
+
+  return value
+}
+
+function assertValidSkillFrontmatter(frontmatter: Record<string, string>) {
+  if (typeof frontmatter.name !== "string" || frontmatter.name.length === 0) {
+    throw new HTTPException(500, {
+      message: "Skill name is required.",
+    })
+  }
+
+  if (frontmatter.name !== SKILL_NAME) {
+    throw new HTTPException(500, {
+      message: `Expected skill name "${SKILL_NAME}", got "${frontmatter.name}".`,
+    })
+  }
+
+  if (!/^[a-z0-9](?:-?[a-z0-9])*$/.test(frontmatter.name) || frontmatter.name.length > 64) {
+    throw new HTTPException(500, {
+      message: `Invalid skill name: ${frontmatter.name}`,
+    })
+  }
+
+  if (typeof frontmatter.description !== "string" || frontmatter.description.length === 0) {
+    throw new HTTPException(500, {
+      message: "Skill description is required.",
+    })
+  }
+
+  if (frontmatter.description.length > 1024) {
+    throw new HTTPException(500, {
+      message: "Skill description must be 1024 characters or fewer.",
+    })
+  }
+}
+
+async function sha256Hex(bytes: ArrayBuffer): Promise<string> {
+  const digest = await crypto.subtle.digest("SHA-256", bytes)
+  return [...new Uint8Array(digest)].map((byte) => byte.toString(16).padStart(2, "0")).join("")
+}

package/src/lib/types.ts

@@ -32,6 +32,14 @@ export interface Token {
   text?: string
 }
 
+/**
+ * One panel in Apple DocC `tabNavigator` JSON (e.g. a language tab with examples).
+ */
+export interface TabNavigatorTab {
+  title?: string
+  content?: ContentItem[]
+}
+
 /**
  * The main content item type used throughout the documentation structure.
  * Can represent text, code, lists, headings, and other content elements.
@@ -51,6 +59,9 @@ export interface ContentItem {
   inlineContent?: ContentItem[]
   items?: ContentItem[]
 
+  /** Tab panels for DocC `tabNavigator` content (e.g. Swift / Objective-C examples). */
+  tabs?: TabNavigatorTab[]
+
   // Code content
   code?: string | string[]
   syntax?: string
@@ -113,10 +124,12 @@ export interface SeeAlsoSection {
  */
 export interface PrimaryContentSection {
   kind: string
+  title?: string
   content?: ContentItem[]
   declarations?: Declaration[]
   parameters?: Parameter[]
   items?: PropertyItem[]
+  values?: PossibleValueItem[]
 }
 
 /**
@@ -137,6 +150,14 @@ export interface PropertyItem {
   }>
 }
 
+/**
+ * Represents a possible value item used in enum/string type pages.
+ */
+export interface PossibleValueItem {
+  name: string
+  content?: ContentItem[]
+}
+
 // ============================================================================
 // VARIANT TYPES
 // ============================================================================

package/wrangler.jsonc

@@ -16,7 +16,8 @@
   "compatibility_flags": ["nodejs_compat"],
   "assets": {
     "binding": "ASSETS",
-    "directory": "./public"
+    "directory": "./public",
+    "run_worker_first": ["/", "/.well-known/agent-skills/*"]
   },
   "observability": {
     "enabled": true