@nshipster/sosumi 1.0.6 → 1.0.8

package/src/lib/reference/render.ts

@@ -2,6 +2,18 @@
  * Apple Developer Reference documentation rendering functionality
  */
 
+import {
+  CONTENT_TOO_DEEP,
+  formatCallout,
+  formatCodeBlock,
+  formatList,
+  formatTable,
+  INLINE_CONTENT_TOO_DEEP,
+  MAX_CONTENT_DEPTH,
+  MAX_INLINE_DEPTH,
+  mapAsideStyleToCallout,
+} from "../markdown"
+import { extractTitleFromIdentifier } from "../url"
 import type {
   AppleDocJSON,
   ContentItem,
@@ -235,14 +247,6 @@ function generateBreadcrumbs(sourceUrl: string, externalOrigin?: string): string
   return `${breadcrumbs}\n\n`
 }
 
-/**
- * Map a DocC language code to a Markdown code-fence identifier. DocC uses
- * `occ` for Objective-C internally, but Markdown highlighters expect `objc`.
- */
-function normalizeFenceLanguage(syntax: string): string {
-  return syntax === "occ" ? "objc" : syntax
-}
-
 /**
  * Tab titles DocC uses for language-variant code examples. Used to detect
  * whether a `tabNavigator` is a language switch (so we can default to Swift)
@@ -399,9 +403,9 @@ function renderContentArray(
   externalOrigin?: string,
 ): string {
   // Prevent infinite recursion by limiting depth
-  if (depth > 50) {
+  if (depth > MAX_CONTENT_DEPTH) {
     console.warn("Maximum recursion depth reached in renderContentArray")
-    return "[Content too deeply nested]"
+    return CONTENT_TOO_DEEP
   }
 
   let markdown = ""
@@ -413,54 +417,29 @@ function renderContentArray(
       markdown += `${hashes} ${item.text}\n\n`
     } else if (item.type === "paragraph") {
       if (item.inlineContent) {
-        const text = renderInlineContent(item.inlineContent, references, depth, externalOrigin)
+        // Inline nesting is counted separately from block nesting, so a
+        // paragraph's inline content starts a fresh depth budget.
+        const text = renderInlineContent(item.inlineContent, references, 0, externalOrigin)
         markdown += `${text}\n\n`
       }
     } else if (item.type === "codeListing") {
-      let code = ""
-      if (Array.isArray(item.code)) {
-        code = item.code.join("\n")
-      } else {
-        code = String(item.code || "")
-      }
-      const syntax = normalizeFenceLanguage(item.syntax || "swift")
-
-      markdown += `\`\`\`${syntax}\n${code}\n\`\`\`\n\n`
-    } else if (item.type === "unorderedList") {
+      markdown += formatCodeBlock(item.code, item.syntax)
+    } else if (item.type === "unorderedList" || item.type === "orderedList") {
       if (item.items) {
-        for (const listItem of item.items) {
-          const itemText = renderContentArray(
-            listItem.content || [],
-            references,
-            depth + 1,
-            externalOrigin,
-          )
-          markdown += `- ${itemText.replace(/\n\n$/, "")}\n`
-        }
-        markdown += "\n"
-      }
-    } else if (item.type === "orderedList") {
-      if (item.items) {
-        item.items.forEach((listItem: ContentItem, index: number) => {
-          const itemText = renderContentArray(
-            listItem.content || [],
-            references,
-            depth + 1,
-            externalOrigin,
-          )
-          markdown += `${index + 1}. ${itemText.replace(/\n\n$/, "")}\n`
-        })
-        markdown += "\n"
+        markdown += formatList(
+          item.items.map((listItem) =>
+            renderContentArray(listItem.content || [], references, depth + 1, externalOrigin),
+          ),
+          item.type === "orderedList",
+        )
       }
     } else if (item.type === "aside") {
       const style = item.style || "note"
-      const calloutType = mapAsideStyleToCallout(style)
       const asideContent = item.content
         ? renderContentArray(item.content, references, depth + 1, externalOrigin)
         : ""
-      const cleanContent = asideContent.trim().replace(/\n/g, "\n> ")
-      const deprecatedLabel = style.toLowerCase() === "deprecated" ? "**Deprecated**\n>\n> " : ""
-      markdown += `> [!${calloutType}]\n> ${deprecatedLabel}${cleanContent}\n\n`
+      const lead = style.toLowerCase() === "deprecated" ? DEPRECATED_LEAD : undefined
+      markdown += formatCallout(mapAsideStyleToCallout(style), asideContent, lead)
     } else if (item.type === "table") {
       markdown += renderTable(item, references, depth, externalOrigin)
     } else if (item.type === "tabNavigator" && item.tabs?.length) {
@@ -507,27 +486,17 @@ function renderTable(
     header?: string
     rows?: ContentItem[][][] // rows[rowIndex][cellIndex] = ContentItem[]
   }
-  const rows = table.rows ?? []
-  if (rows.length === 0) return ""
-
-  const escapeCell = (s: string) => s.replace(/\|/g, "\\|").replace(/\n/g, " ").trim()
-  const renderCell = (cell: ContentItem | ContentItem[]) => {
-    const items = Array.isArray(cell) ? cell : [cell]
-    const s = renderContentArray(items, references, depth + 1, externalOrigin)
-    return escapeCell(s)
-  }
-
-  const firstRowIsHeader = table.header === "row"
-  let markdown = ""
-  rows.forEach((row, rowIndex) => {
-    const cells = row.map((c) => renderCell(c))
-    if (cells.length === 0) return
-    markdown += `| ${cells.join(" | ")} |\n`
-    if (firstRowIsHeader && rowIndex === 0) {
-      markdown += `| ${cells.map(() => "---").join(" | ")} |\n`
-    }
-  })
-  return markdown ? `${markdown}\n` : ""
+  const rows = (table.rows ?? []).map((row) =>
+    row.map((cell) =>
+      renderContentArray(
+        Array.isArray(cell) ? cell : [cell],
+        references,
+        depth + 1,
+        externalOrigin,
+      ),
+    ),
+  )
+  return formatTable(rows, table.header === "row")
 }
 
 /**
@@ -540,9 +509,9 @@ function renderInlineContent(
   externalOrigin?: string,
 ): string {
   // Prevent infinite recursion by limiting depth
-  if (depth > 20) {
+  if (depth > MAX_INLINE_DEPTH) {
     console.warn("Maximum recursion depth reached in renderInlineContent")
-    return "[Inline content too deeply nested]"
+    return INLINE_CONTENT_TOO_DEEP
   }
 
   return inlineContent
@@ -552,9 +521,11 @@ function renderInlineContent(
       } else if (item.type === "codeVoice") {
         return `\`${item.code}\``
       } else if (item.type === "reference") {
+        const reference = item.identifier ? references?.[item.identifier] : undefined
         const title =
           item.title ||
           item.text ||
+          resolveReferenceTitle(reference, references, depth, externalOrigin) ||
           (item.identifier ? extractTitleFromIdentifier(item.identifier) : "")
         const url = item.identifier
           ? convertIdentifierToURL(item.identifier, references, externalOrigin)
@@ -577,6 +548,56 @@ function renderInlineContent(
     .join("")
 }
 
+/**
+ * Resolve the display title for a reference, preferring its rich
+ * `titleInlineContent` (which preserves code spans) over the flat `title`.
+ * When there is no rich title, wrap symbol titles in backticks so older DocC
+ * pages that omit `titleInlineContent` still render code spans in link text.
+ */
+function resolveReferenceTitle(
+  reference: ContentItem | undefined,
+  references?: Record<string, ContentItem>,
+  depth: number = 0,
+  externalOrigin?: string,
+): string | undefined {
+  if (reference?.titleInlineContent) {
+    return renderInlineContent(reference.titleInlineContent, references, depth + 1, externalOrigin)
+  }
+  if (reference?.title && reference.role === "symbol") {
+    return `\`${reference.title}\``
+  }
+  return reference?.title
+}
+
+interface ResolvedLink {
+  title: string
+  url: string
+  abstract: string
+  deprecated: boolean
+}
+
+/**
+ * Resolve an identifier to a Markdown link, drawing the title and abstract from
+ * the page's variants first, then the shared references map, and finally the
+ * identifier itself.
+ */
+function resolveLink(
+  id: string,
+  variants?: Variant[],
+  references?: Record<string, ContentItem>,
+  externalOrigin?: string,
+): ResolvedLink {
+  const info = variants?.find((v) => v.identifier === id)
+  const reference = references?.[id]
+  const title =
+    info?.title ||
+    resolveReferenceTitle(reference, references, 0, externalOrigin) ||
+    extractTitleFromIdentifier(id)
+  const url = convertIdentifierToURL(id, references, externalOrigin)
+  const abstract = (info?.abstract ?? reference?.abstract)?.map((a) => a.text).join("") ?? ""
+  return { title, url, abstract, deprecated: reference?.deprecated === true }
+}
+
 /**
  * Render relationship sections
  */
@@ -592,10 +613,7 @@ function renderRelationships(
     if (rel.title && rel.identifiers) {
       markdown += `## ${rel.title}\n\n`
       for (const id of rel.identifiers) {
-        const info = variants?.find((v: Variant) => v.identifier === id)
-        const reference = references?.[id]
-        const title = info?.title || reference?.title || extractTitleFromIdentifier(id)
-        const url = convertIdentifierToURL(id, references, externalOrigin)
+        const { title, url } = resolveLink(id, variants, references, externalOrigin)
         markdown += `- [${title}](${url})\n`
       }
       markdown += "\n"
@@ -619,34 +637,24 @@ function renderTopicSections(
   for (const topic of topics) {
     if (topic.title) {
       markdown += `## ${topic.title}\n\n`
+    }
 
-      if (topic.identifiers) {
-        for (const id of topic.identifiers) {
-          const info = variants?.find((v: Variant) => v.identifier === id)
-          const reference = references?.[id]
-          if (info || reference) {
-            const title = info?.title || reference?.title || extractTitleFromIdentifier(id)
-            const url = convertIdentifierToURL(id, references, externalOrigin)
-            const abstract = info?.abstract
-              ? info.abstract.map((a: { text: string }) => a.text).join("")
-              : reference?.abstract
-                ? reference.abstract.map((a: { text: string }) => a.text).join("")
-                : ""
-
-            const deprecatedLabel = reference?.deprecated ? " *(Deprecated)*" : ""
-            markdown += `- [${title}](${url})${deprecatedLabel}`
-            if (abstract) {
-              markdown += ` ${abstract}`
-            }
-            markdown += "\n"
-          } else {
-            const title = extractTitleFromIdentifier(id)
-            const url = convertIdentifierToURL(id, references, externalOrigin)
-            markdown += `- [${title}](${url})\n`
-          }
+    if (topic.identifiers) {
+      for (const id of topic.identifiers) {
+        const { title, url, abstract, deprecated } = resolveLink(
+          id,
+          variants,
+          references,
+          externalOrigin,
+        )
+        const deprecatedLabel = deprecated ? " *(Deprecated)*" : ""
+        markdown += `- [${title}](${url})${deprecatedLabel}`
+        if (abstract) {
+          markdown += ` ${abstract}`
         }
         markdown += "\n"
       }
+      markdown += "\n"
     }
   }
 
@@ -721,10 +729,7 @@ function renderSeeAlso(
     if (section.title && section.identifiers) {
       markdown += `## ${section.title}\n\n`
       for (const id of section.identifiers) {
-        const info = variants?.find((v: Variant) => v.identifier === id)
-        const reference = references?.[id]
-        const title = info?.title || reference?.title || extractTitleFromIdentifier(id)
-        const url = convertIdentifierToURL(id, references, externalOrigin)
+        const { title, url } = resolveLink(id, variants, references, externalOrigin)
         markdown += `- [${title}](${url})\n`
       }
       markdown += "\n"
@@ -780,6 +785,9 @@ function getDeprecationMessageFromPlatforms(platforms?: Platform[]): string | nu
 
 const DEFAULT_DEPRECATION_MESSAGE = "This symbol is deprecated."
 
+/** Leading paragraph for callouts that flag deprecated APIs. */
+const DEPRECATED_LEAD = "**Deprecated**"
+
 /**
  * Resolve deprecation notice text from DocC JSON (without rendering the callout).
  */
@@ -826,28 +834,7 @@ function renderDeprecationNotice(
     return ""
   }
 
-  const quotedBody = body.replace(/\n/g, "\n> ")
-  return `> [!WARNING]\n> **Deprecated**\n>\n> ${quotedBody}\n\n`
-}
-
-/**
- * Map aside style to GitHub-style callout
- */
-function mapAsideStyleToCallout(style: string): string {
-  switch (style.toLowerCase()) {
-    case "warning":
-      return "WARNING"
-    case "important":
-      return "IMPORTANT"
-    case "caution":
-      return "CAUTION"
-    case "tip":
-      return "TIP"
-    case "deprecated":
-      return "WARNING"
-    default:
-      return "NOTE"
-  }
+  return formatCallout("WARNING", body, DEPRECATED_LEAD)
 }
 
 /**
@@ -893,34 +880,3 @@ function rewriteDocumentationPath(path: string | undefined, externalOrigin?: str
 
   return `/external/${externalOrigin}${path}`
 }
-
-/**
- * Extract title from identifier
- */
-export function extractTitleFromIdentifier(identifier: string): string {
-  const parts = identifier.split("/")
-  const lastPart = parts[parts.length - 1]
-
-  // Handle disambiguation suffixes (e.g., "body-8kl5o" -> "body", "init(exactly:)-63925" -> "init(exactly:)")
-  const disambiguationMatch = lastPart.match(/^(.+?)(?:-\w+)?$/)
-  if (disambiguationMatch) {
-    const baseName = disambiguationMatch[1]
-
-    // If it looks like a method signature (contains parentheses), preserve it
-    if (baseName.includes("(") && baseName.includes(")")) {
-      return baseName
-    }
-
-    // For simple identifiers, convert camelCase to readable format
-    return baseName
-      .replace(/([a-z])([A-Z])/g, "$1 $2")
-      .replace(/\s+/g, " ")
-      .trim()
-  }
-
-  // Fallback: convert camelCase to readable format
-  return lastPart
-    .replace(/([a-z])([A-Z])/g, "$1 $2")
-    .replace(/\s+/g, " ")
-    .trim()
-}

package/src/lib/search.ts

@@ -15,14 +15,24 @@ export interface SearchResponse {
 }
 
 // Apple's current search backend, discovered from
-// https://developer.apple.com/search/scripts/search.js (May 2026)
+// https://developer.apple.com/search/scripts/search.js (September 2026)
 //
 // Historical context:
 //   - The legacy /search/ HTML scraper broke when Apple switched to a JS-rendered SPA
 //   - PR #54 upstream (NSHipster/sosumi.ai) targeted /search/services/search.php with
 //     NDJSON-style streamed events; that endpoint is now also gone (404)
-//   - The current backend is a plain JSON POST API on Apple's MSC infrastructure
-const APPLE_SEARCH_SERVICE_URL = "https://devintserv.msc.sbz.apple.com/api/v1/search"
+//   - /api/v1/search replaced it, and now 404s in turn
+//   - The current backend is /api/v1/query, which answers only in JSONL:
+//     one event per line, each tagged with a `kind`
+const APPLE_SEARCH_SERVICE_URL = "https://devintserv.msc.sbz.apple.com/api/v1/query"
+
+// The backend rejects a request that does not name the response channels it wants.
+// `quickSearch` carries the top typeahead matches in a single event;
+// `search` streams the full ranked list.
+// The third channel Apple's own page requests, `ask`, is a generated answer,
+// which is not what this service reports.
+const INCLUDED_RESPONSES = ["quickSearch", "search"]
+
 const DEFAULT_TARGET_RESULT_LOCALE = "en"
 const TARGET_RESULT_LOCALE_BY_BASE_NAME = new Map([
   ["en", "en"],
@@ -49,7 +59,7 @@ async function searchAppleDeveloperDocsViaService(query: string): Promise<Search
     method: "POST",
     headers: {
       "Content-Type": "application/json",
-      Accept: "application/json",
+      Accept: "application/jsonl",
       // The MSC backend requires a browser-style Origin/Referer pair to accept the request.
       Origin: "https://developer.apple.com",
       Referer: "https://developer.apple.com/search/",
@@ -58,6 +68,7 @@ async function searchAppleDeveloperDocsViaService(query: string): Promise<Search
     body: JSON.stringify({
       text: query,
       targetResultLocale: resolveTargetResultLocale(),
+      includedResponses: INCLUDED_RESPONSES,
     }),
   })
 
@@ -65,36 +76,104 @@ async function searchAppleDeveloperDocsViaService(query: string): Promise<Search
     throw new Error(`Search request failed: ${response.status}`)
   }
 
-  const data = await readSearchResponseJson(response)
-  const rawResults = Array.isArray(data.results) ? (data.results as unknown[]) : []
-  return extractSearchResults(rawResults)
+  return extractSearchResults(await readSearchResponseItems(response))
+}
+
+// The response body is JSONL: one JSON object per line.
+// A `quickSearch` event carries its results inline,
+// while `search` events stream the full list as diffs against a JSON text buffer,
+// so those results can only be read once every line has been applied.
+// Events of any other kind — `quickSearchFinished`, `searchFinished`, `ask` —
+// are not part of the result set and are skipped.
+async function readSearchResponseItems(response: Response): Promise<unknown[]> {
+  const items: unknown[] = []
+  let streamedSearch = ""
+
+  for (const line of (await response.text()).split("\n")) {
+    const trimmed = line.trim()
+    if (!trimmed) {
+      continue
+    }
+
+    const event = parseJson(trimmed)
+    if (!isJsonRecord(event)) {
+      continue
+    }
+
+    if (event.kind === "quickSearch") {
+      items.push(...resultsOf(event.response))
+    } else if (event.kind === "search") {
+      streamedSearch = applySearchDiff(streamedSearch, event.diff)
+    }
+  }
+
+  if (streamedSearch) {
+    items.push(...resultsOf(parseJson(streamedSearch)))
+  }
+
+  return items
+}
+
+// Each `search` event appends to the buffer,
+// after dropping `removeLast` characters from the end of what came before.
+function applySearchDiff(buffer: string, diff: unknown): string {
+  if (!isJsonRecord(diff)) {
+    return buffer
+  }
+
+  const removeLast = typeof diff.removeLast === "number" ? diff.removeLast : 0
+  const append = typeof diff.append === "string" ? diff.append : ""
+
+  return buffer.slice(0, Math.max(0, buffer.length - removeLast)) + append
 }
 
-async function readSearchResponseJson(response: Response): Promise<JsonRecord> {
+function parseJson(text: string): unknown {
   try {
-    const data = await response.json()
-    return isJsonRecord(data) ? data : {}
+    return JSON.parse(text)
   } catch {
     throw new Error("Search response was not valid JSON")
   }
 }
 
+function resultsOf(container: unknown): unknown[] {
+  if (!isJsonRecord(container)) {
+    return []
+  }
+
+  return Array.isArray(container.results) ? container.results : []
+}
+
+// The same page can be reported by more than one channel,
+// so keep the first mention of each URL and drop the rest.
 function extractSearchResults(items: unknown[]): SearchResult[] {
+  const seen = new Set<string>()
+
   return items.flatMap((item) => {
     const result = normalizeSearchResult(item)
-    return result ? [result] : []
+    if (!result || seen.has(result.url)) {
+      return []
+    }
+
+    seen.add(result.url)
+    return [result]
   })
 }
 
+// A result is `{ metadata, origin }`, and its `metadata.metadataKind` says how to read it:
+//   - "documentation" for reference pages, with singular fields
+//   - "developer" for WWDC sessions and other media, with parallel arrays
+//   - "webPage" for marketing and swift.org pages, keyed by `sourceURL`
 function normalizeSearchResult(item: unknown): SearchResult | null {
-  if (!isJsonRecord(item)) {
+  const record = unwrapResultValue(item)
+  if (!record || !isJsonRecord(record.metadata)) {
     return null
   }
 
-  const documentation = extractMetadataRecord(item.documentation)
-  if (documentation) {
-    const title = stringValue(documentation.title)
-    const url = stringValue(documentation.permalink)
+  const metadata = record.metadata
+
+  if (metadata.metadataKind === "documentation") {
+    const title = stringValue(metadata.title)
+    const url = stringValue(metadata.permalink)
     if (!title || !url) {
       return null
     }
@@ -102,17 +181,16 @@ function normalizeSearchResult(item: unknown): SearchResult | null {
     return {
       title,
       url,
-      description: stringValue(documentation.description) ?? "",
-      breadcrumbs: splitHierarchy(stringValue(documentation.hierarchy)),
-      tags: compactStrings([stringValue(documentation.kind)]),
+      description: stringValue(metadata.description) ?? "",
+      breadcrumbs: splitHierarchy(stringValue(metadata.hierarchy)),
+      tags: compactStrings([stringValue(metadata.kind)]),
       type: "documentation",
     }
   }
 
-  const developer = extractMetadataRecord(item.developer)
-  if (developer) {
-    const title = firstString(developer.titles)
-    const url = firstString(developer.permalinks)
+  if (metadata.metadataKind === "developer") {
+    const title = firstString(metadata.titles)
+    const url = firstString(metadata.permalinks)
     if (!title || !url) {
       return null
     }
@@ -120,20 +198,19 @@ function normalizeSearchResult(item: unknown): SearchResult | null {
     return {
       title,
       url,
-      description: firstString(developer.descriptions) ?? "",
-      breadcrumbs: compactStrings([firstString(developer.projectNames)]),
+      description: firstString(metadata.descriptions) ?? "",
+      breadcrumbs: compactStrings([firstString(metadata.projectNames)]),
       tags: compactStrings([
-        firstString(developer.itemTypes),
-        firstString(developer.deliveryLanguageCodes),
+        firstString(metadata.itemTypes),
+        firstString(metadata.deliveryLanguageCodes),
       ]),
-      type: (firstString(developer.itemTypes) ?? "developer").toLowerCase(),
+      type: (firstString(metadata.itemTypes) ?? "developer").toLowerCase(),
     }
   }
 
-  const devsite = extractMetadataRecord(item.devsite)
-  if (devsite) {
-    const title = stringValue(devsite.title)
-    const url = stringValue(devsite.sourceURL)
+  if (metadata.metadataKind === "webPage") {
+    const title = stringValue(metadata.title)
+    const url = stringValue(metadata.sourceURL)
     if (!title || !url) {
       return null
     }
@@ -141,25 +218,7 @@ function normalizeSearchResult(item: unknown): SearchResult | null {
     return {
       title,
       url,
-      description: stringValue(devsite.description) ?? "",
-      breadcrumbs: [],
-      tags: [],
-      type: "general",
-    }
-  }
-
-  const swiftdocs = extractMetadataRecord(item.swiftdocs)
-  if (swiftdocs) {
-    const title = stringValue(swiftdocs.title)
-    const url = stringValue(swiftdocs.sourceURL)
-    if (!title || !url) {
-      return null
-    }
-
-    return {
-      title,
-      url,
-      description: stringValue(swiftdocs.description) ?? "",
+      description: stringValue(metadata.description) ?? "",
       breadcrumbs: [],
       tags: [],
       type: "general",
@@ -169,13 +228,14 @@ function normalizeSearchResult(item: unknown): SearchResult | null {
   return null
 }
 
-function extractMetadataRecord(container: unknown): JsonRecord | null {
-  if (!isJsonRecord(container)) {
+// Streamed `search` results wrap their payload in `value` alongside a match excerpt,
+// while `quickSearch` results carry it directly.
+function unwrapResultValue(item: unknown): JsonRecord | null {
+  if (!isJsonRecord(item)) {
     return null
   }
 
-  const metadata = container.metadata
-  return isJsonRecord(metadata) ? metadata : null
+  return isJsonRecord(item.value) ? item.value : item
 }
 
 function isJsonRecord(value: unknown): value is JsonRecord {

package/src/lib/types.ts

@@ -51,6 +51,13 @@ export interface ContentItem {
   title?: string
   name?: string
 
+  /**
+   * Rich inline rendering of a reference's title (e.g. a symbol name wrapped in
+   * a `codeVoice` span). Present on entries in the `references` map and preferred
+   * over `title` so code spans survive inside link text.
+   */
+  titleInlineContent?: ContentItem[]
+
   // Tokenized content
   tokens?: Token[]
 
@@ -69,6 +76,11 @@ export interface ContentItem {
   // Structural properties
   level?: number
   style?: string
+  /**
+   * DocC reference role (e.g. `"symbol"`, `"collection"`, `"article"`).
+   * Used to decide whether a plain title should be rendered as a code span.
+   */
+  role?: string
   identifier?: string
   identifiers?: string[]
   url?: string