@nshipster/sosumi 1.0.5 → 1.0.7

package/src/lib/search.ts

@@ -15,14 +15,24 @@ export interface SearchResponse {
 }
 
 // Apple's current search backend, discovered from
-// https://developer.apple.com/search/scripts/search.js (May 2026)
+// https://developer.apple.com/search/scripts/search.js (September 2026)
 //
 // Historical context:
 //   - The legacy /search/ HTML scraper broke when Apple switched to a JS-rendered SPA
 //   - PR #54 upstream (NSHipster/sosumi.ai) targeted /search/services/search.php with
 //     NDJSON-style streamed events; that endpoint is now also gone (404)
-//   - The current backend is a plain JSON POST API on Apple's MSC infrastructure
-const APPLE_SEARCH_SERVICE_URL = "https://devintserv.msc.sbz.apple.com/api/v1/search"
+//   - /api/v1/search replaced it, and now 404s in turn
+//   - The current backend is /api/v1/query, which answers only in JSONL:
+//     one event per line, each tagged with a `kind`
+const APPLE_SEARCH_SERVICE_URL = "https://devintserv.msc.sbz.apple.com/api/v1/query"
+
+// The backend rejects a request that does not name the response channels it wants.
+// `quickSearch` carries the top typeahead matches in a single event;
+// `search` streams the full ranked list.
+// The third channel Apple's own page requests, `ask`, is a generated answer,
+// which is not what this service reports.
+const INCLUDED_RESPONSES = ["quickSearch", "search"]
+
 const DEFAULT_TARGET_RESULT_LOCALE = "en"
 const TARGET_RESULT_LOCALE_BY_BASE_NAME = new Map([
   ["en", "en"],
@@ -49,7 +59,7 @@ async function searchAppleDeveloperDocsViaService(query: string): Promise<Search
     method: "POST",
     headers: {
       "Content-Type": "application/json",
-      Accept: "application/json",
+      Accept: "application/jsonl",
       // The MSC backend requires a browser-style Origin/Referer pair to accept the request.
       Origin: "https://developer.apple.com",
       Referer: "https://developer.apple.com/search/",
@@ -58,6 +68,7 @@ async function searchAppleDeveloperDocsViaService(query: string): Promise<Search
     body: JSON.stringify({
       text: query,
       targetResultLocale: resolveTargetResultLocale(),
+      includedResponses: INCLUDED_RESPONSES,
     }),
   })
 
@@ -65,36 +76,104 @@ async function searchAppleDeveloperDocsViaService(query: string): Promise<Search
     throw new Error(`Search request failed: ${response.status}`)
   }
 
-  const data = await readSearchResponseJson(response)
-  const rawResults = Array.isArray(data.results) ? (data.results as unknown[]) : []
-  return extractSearchResults(rawResults)
+  return extractSearchResults(await readSearchResponseItems(response))
+}
+
+// The response body is JSONL: one JSON object per line.
+// A `quickSearch` event carries its results inline,
+// while `search` events stream the full list as diffs against a JSON text buffer,
+// so those results can only be read once every line has been applied.
+// Events of any other kind — `quickSearchFinished`, `searchFinished`, `ask` —
+// are not part of the result set and are skipped.
+async function readSearchResponseItems(response: Response): Promise<unknown[]> {
+  const items: unknown[] = []
+  let streamedSearch = ""
+
+  for (const line of (await response.text()).split("\n")) {
+    const trimmed = line.trim()
+    if (!trimmed) {
+      continue
+    }
+
+    const event = parseJson(trimmed)
+    if (!isJsonRecord(event)) {
+      continue
+    }
+
+    if (event.kind === "quickSearch") {
+      items.push(...resultsOf(event.response))
+    } else if (event.kind === "search") {
+      streamedSearch = applySearchDiff(streamedSearch, event.diff)
+    }
+  }
+
+  if (streamedSearch) {
+    items.push(...resultsOf(parseJson(streamedSearch)))
+  }
+
+  return items
+}
+
+// Each `search` event appends to the buffer,
+// after dropping `removeLast` characters from the end of what came before.
+function applySearchDiff(buffer: string, diff: unknown): string {
+  if (!isJsonRecord(diff)) {
+    return buffer
+  }
+
+  const removeLast = typeof diff.removeLast === "number" ? diff.removeLast : 0
+  const append = typeof diff.append === "string" ? diff.append : ""
+
+  return buffer.slice(0, Math.max(0, buffer.length - removeLast)) + append
 }
 
-async function readSearchResponseJson(response: Response): Promise<JsonRecord> {
+function parseJson(text: string): unknown {
   try {
-    const data = await response.json()
-    return isJsonRecord(data) ? data : {}
+    return JSON.parse(text)
   } catch {
     throw new Error("Search response was not valid JSON")
   }
 }
 
+function resultsOf(container: unknown): unknown[] {
+  if (!isJsonRecord(container)) {
+    return []
+  }
+
+  return Array.isArray(container.results) ? container.results : []
+}
+
+// The same page can be reported by more than one channel,
+// so keep the first mention of each URL and drop the rest.
 function extractSearchResults(items: unknown[]): SearchResult[] {
+  const seen = new Set<string>()
+
   return items.flatMap((item) => {
     const result = normalizeSearchResult(item)
-    return result ? [result] : []
+    if (!result || seen.has(result.url)) {
+      return []
+    }
+
+    seen.add(result.url)
+    return [result]
   })
 }
 
+// A result is `{ metadata, origin }`, and its `metadata.metadataKind` says how to read it:
+//   - "documentation" for reference pages, with singular fields
+//   - "developer" for WWDC sessions and other media, with parallel arrays
+//   - "webPage" for marketing and swift.org pages, keyed by `sourceURL`
 function normalizeSearchResult(item: unknown): SearchResult | null {
-  if (!isJsonRecord(item)) {
+  const record = unwrapResultValue(item)
+  if (!record || !isJsonRecord(record.metadata)) {
     return null
   }
 
-  const documentation = extractMetadataRecord(item.documentation)
-  if (documentation) {
-    const title = stringValue(documentation.title)
-    const url = stringValue(documentation.permalink)
+  const metadata = record.metadata
+
+  if (metadata.metadataKind === "documentation") {
+    const title = stringValue(metadata.title)
+    const url = stringValue(metadata.permalink)
     if (!title || !url) {
       return null
     }
@@ -102,17 +181,16 @@ function normalizeSearchResult(item: unknown): SearchResult | null {
     return {
       title,
       url,
-      description: stringValue(documentation.description) ?? "",
-      breadcrumbs: splitHierarchy(stringValue(documentation.hierarchy)),
-      tags: compactStrings([stringValue(documentation.kind)]),
+      description: stringValue(metadata.description) ?? "",
+      breadcrumbs: splitHierarchy(stringValue(metadata.hierarchy)),
+      tags: compactStrings([stringValue(metadata.kind)]),
       type: "documentation",
     }
   }
 
-  const developer = extractMetadataRecord(item.developer)
-  if (developer) {
-    const title = firstString(developer.titles)
-    const url = firstString(developer.permalinks)
+  if (metadata.metadataKind === "developer") {
+    const title = firstString(metadata.titles)
+    const url = firstString(metadata.permalinks)
     if (!title || !url) {
       return null
     }
@@ -120,20 +198,19 @@ function normalizeSearchResult(item: unknown): SearchResult | null {
     return {
       title,
       url,
-      description: firstString(developer.descriptions) ?? "",
-      breadcrumbs: compactStrings([firstString(developer.projectNames)]),
+      description: firstString(metadata.descriptions) ?? "",
+      breadcrumbs: compactStrings([firstString(metadata.projectNames)]),
       tags: compactStrings([
-        firstString(developer.itemTypes),
-        firstString(developer.deliveryLanguageCodes),
+        firstString(metadata.itemTypes),
+        firstString(metadata.deliveryLanguageCodes),
       ]),
-      type: (firstString(developer.itemTypes) ?? "developer").toLowerCase(),
+      type: (firstString(metadata.itemTypes) ?? "developer").toLowerCase(),
     }
   }
 
-  const devsite = extractMetadataRecord(item.devsite)
-  if (devsite) {
-    const title = stringValue(devsite.title)
-    const url = stringValue(devsite.sourceURL)
+  if (metadata.metadataKind === "webPage") {
+    const title = stringValue(metadata.title)
+    const url = stringValue(metadata.sourceURL)
     if (!title || !url) {
       return null
     }
@@ -141,25 +218,7 @@ function normalizeSearchResult(item: unknown): SearchResult | null {
     return {
       title,
       url,
-      description: stringValue(devsite.description) ?? "",
-      breadcrumbs: [],
-      tags: [],
-      type: "general",
-    }
-  }
-
-  const swiftdocs = extractMetadataRecord(item.swiftdocs)
-  if (swiftdocs) {
-    const title = stringValue(swiftdocs.title)
-    const url = stringValue(swiftdocs.sourceURL)
-    if (!title || !url) {
-      return null
-    }
-
-    return {
-      title,
-      url,
-      description: stringValue(swiftdocs.description) ?? "",
+      description: stringValue(metadata.description) ?? "",
       breadcrumbs: [],
       tags: [],
       type: "general",
@@ -169,13 +228,14 @@ function normalizeSearchResult(item: unknown): SearchResult | null {
   return null
 }
 
-function extractMetadataRecord(container: unknown): JsonRecord | null {
-  if (!isJsonRecord(container)) {
+// Streamed `search` results wrap their payload in `value` alongside a match excerpt,
+// while `quickSearch` results carry it directly.
+function unwrapResultValue(item: unknown): JsonRecord | null {
+  if (!isJsonRecord(item)) {
     return null
   }
 
-  const metadata = container.metadata
-  return isJsonRecord(metadata) ? metadata : null
+  return isJsonRecord(item.value) ? item.value : item
 }
 
 function isJsonRecord(value: unknown): value is JsonRecord {

package/src/lib/types.ts

@@ -51,6 +51,13 @@ export interface ContentItem {
   title?: string
   name?: string
 
+  /**
+   * Rich inline rendering of a reference's title (e.g. a symbol name wrapped in
+   * a `codeVoice` span). Present on entries in the `references` map and preferred
+   * over `title` so code spans survive inside link text.
+   */
+  titleInlineContent?: ContentItem[]
+
   // Tokenized content
   tokens?: Token[]
 
@@ -69,12 +76,20 @@ export interface ContentItem {
   // Structural properties
   level?: number
   style?: string
+  /**
+   * DocC reference role (e.g. `"symbol"`, `"collection"`, `"article"`).
+   * Used to decide whether a plain title should be rendered as a code span.
+   */
+  role?: string
   identifier?: string
   identifiers?: string[]
   url?: string
 
   // Abstract content
   abstract?: TextFragment[]
+
+  /** Present on reference entries for deprecated symbols. */
+  deprecated?: boolean
 }
 
 // ============================================================================
@@ -261,6 +276,10 @@ export interface Platform {
   name: string
   introducedAt: string
   beta?: boolean
+  deprecated?: boolean
+  deprecatedAt?: string
+  message?: string
+  unavailable?: boolean
 }
 
 /**
@@ -297,6 +316,8 @@ export interface AppleDocJSON {
 
   // Content
   abstract?: Array<{ text: string; type: string }>
+  /** DocC deprecation notice shown at the top of deprecated symbol pages. */
+  deprecationSummary?: ContentItem[]
   sections?: ContentItem[]
 
   // Primary content sections

package/src/lib/url.ts

@@ -53,3 +53,38 @@ export function isValidAppleDocUrl(url: string): boolean {
 
   return url.startsWith(APPLE_DOC_BASE_URL)
 }
+
+/**
+ * Derives a human-readable title from the last component of a documentation
+ * path or `doc://` identifier.
+ *
+ * @param identifier - The identifier or path (e.g., `doc://com.apple.SwiftUI/documentation/SwiftUI/View/body-8kl5o`)
+ * @returns The title, with any disambiguation suffix removed (e.g., `body`)
+ */
+export function extractTitleFromIdentifier(identifier: string): string {
+  const parts = identifier.split("/")
+  const lastPart = parts[parts.length - 1]
+
+  // Handle disambiguation suffixes (e.g., "body-8kl5o" -> "body", "init(exactly:)-63925" -> "init(exactly:)")
+  const disambiguationMatch = lastPart.match(/^(.+?)(?:-\w+)?$/)
+  if (disambiguationMatch) {
+    const baseName = disambiguationMatch[1]
+
+    // If it looks like a method signature (contains parentheses), preserve it
+    if (baseName.includes("(") && baseName.includes(")")) {
+      return baseName
+    }
+
+    // For simple identifiers, convert camelCase to readable format
+    return baseName
+      .replace(/([a-z])([A-Z])/g, "$1 $2")
+      .replace(/\s+/g, " ")
+      .trim()
+  }
+
+  // Fallback: convert camelCase to readable format
+  return lastPart
+    .replace(/([a-z])([A-Z])/g, "$1 $2")
+    .replace(/\s+/g, " ")
+    .trim()
+}

package/src/lib/webbotauth.ts

@@ -0,0 +1,261 @@
+/**
+ * Web Bot Auth (IETF Web Bot Auth working group) support.
+ *
+ * Two responsibilities:
+ *
+ *  1. Publish a JWKS at `/.well-known/http-message-signatures-directory`
+ *     so receiving sites can look up the public key(s)
+ *     used to verify our signatures.
+ *  2. Sign the requests sosumi.ai makes to external Swift-DocC hosts
+ *     with RFC 9421 HTTP Message Signatures,
+ *     attaching `Signature-Agent`, `Signature-Input`, and `Signature` headers
+ *     so those sites can verify the traffic comes from us.
+ *
+ * The Ed25519 private key is provided out-of-band
+ * as the `WEB_BOT_AUTH_KEY` secret (a JSON Web Key).
+ * The matching public key served in the directory is derived from it,
+ * so no key material lives in the repository.
+ *
+ * See https://datatracker.ietf.org/wg/webbotauth/about/
+ */
+
+import {
+  directoryResponseHeaders,
+  HTTP_MESSAGE_SIGNATURES_DIRECTORY,
+  jwkToKeyID,
+  MediaType,
+  type Signer,
+  signatureHeaders,
+} from "web-bot-auth"
+import { helpers, signerFromJWK } from "web-bot-auth/crypto"
+
+export interface WebBotAuthEnv {
+  /**
+   * Ed25519 private key as a JSON Web Key (JSON string).
+   * Provided as a secret.
+   */
+  WEB_BOT_AUTH_KEY?: string
+  /** Origin advertised in the `Signature-Agent` header and hosting the directory. */
+  SIGNATURE_AGENT?: string
+}
+
+/** Path of the published key directory. */
+export const DIRECTORY_PATH = HTTP_MESSAGE_SIGNATURES_DIRECTORY
+
+/** Content type for the key directory response. */
+export const DIRECTORY_MEDIA_TYPE = MediaType.HTTP_MESSAGE_SIGNATURES_DIRECTORY
+
+const DEFAULT_SIGNATURE_AGENT = "https://sosumi.ai"
+
+/**
+ * How long an outbound request signature stays valid.
+ * Short, to limit replay.
+ */
+const OUTBOUND_SIGNATURE_TTL_MS = 5 * 60 * 1000
+
+/** How long the directory self-signature stays valid. */
+const DIRECTORY_SIGNATURE_TTL_MS = 60 * 60 * 1000
+
+/** A public JSON Web Key as published in the directory (RFC 8037 Ed25519). */
+export interface DirectoryKey {
+  kty?: string
+  crv?: string
+  x?: string
+  kid: string
+  use: string
+}
+
+interface WebBotAuthConfig {
+  signer: Signer
+  publicKey: DirectoryKey
+  agent: string
+}
+
+/**
+ * Cached, parsed configuration.
+ * The signing key is application-global (the same for every request),
+ * so it is memoized across requests in the isolate
+ * and only rebuilt when the underlying secret or agent changes.
+ */
+let cache: { key: string; agent: string; config: Promise<WebBotAuthConfig> } | null = null
+
+/**
+ * Prime the signing configuration from the environment.
+ * Call this once per request,
+ * before any outbound fetch or directory response is produced.
+ */
+export function configureWebBotAuth(env: WebBotAuthEnv): void {
+  const key = env.WEB_BOT_AUTH_KEY?.trim()
+  if (!key) {
+    cache = null
+    return
+  }
+
+  const agent = env.SIGNATURE_AGENT?.trim() || DEFAULT_SIGNATURE_AGENT
+  if (cache && cache.key === key && cache.agent === agent) {
+    return
+  }
+
+  cache = { key, agent, config: buildConfig(key, agent) }
+}
+
+interface Ed25519PrivateJwk {
+  kty: "OKP"
+  crv: "Ed25519"
+  x: string
+  d: string
+}
+
+/**
+ * Parse and validate the signing secret as an Ed25519 OKP private JWK.
+ * Fails closed on anything malformed
+ * so a misconfigured `WEB_BOT_AUTH_KEY` never yields a broken directory key
+ * or unverifiable signatures.
+ */
+function parseSigningJwk(key: string): Ed25519PrivateJwk {
+  let jwk: JsonWebKey
+  try {
+    jwk = JSON.parse(key) as JsonWebKey
+  } catch (error) {
+    throw new Error(`WEB_BOT_AUTH_KEY is not valid JSON: ${(error as Error).message}`)
+  }
+
+  if (
+    jwk.kty !== "OKP" ||
+    jwk.crv !== "Ed25519" ||
+    typeof jwk.x !== "string" ||
+    jwk.x.length === 0 ||
+    typeof jwk.d !== "string" ||
+    jwk.d.length === 0
+  ) {
+    throw new Error(
+      "WEB_BOT_AUTH_KEY must be an Ed25519 private JSON Web Key (kty=OKP, crv=Ed25519, with x and d).",
+    )
+  }
+
+  return { kty: jwk.kty, crv: jwk.crv, x: jwk.x, d: jwk.d }
+}
+
+async function buildConfig(key: string, agent: string): Promise<WebBotAuthConfig> {
+  const jwk = parseSigningJwk(key)
+  const signer = await signerFromJWK(jwk)
+  const kid = await jwkToKeyID(jwk, helpers.WEBCRYPTO_SHA256, helpers.BASE64URL_DECODE)
+
+  // Publish only the public half of the key (never the `d` parameter).
+  const publicKey: DirectoryKey = {
+    kty: jwk.kty,
+    crv: jwk.crv,
+    x: jwk.x,
+    kid,
+    use: "sig",
+  }
+
+  return { signer, publicKey, agent }
+}
+
+async function currentConfig(): Promise<WebBotAuthConfig | null> {
+  const entry = cache
+  if (!entry) {
+    return null
+  }
+
+  try {
+    return await entry.config
+  } catch (error) {
+    console.error("web-bot-auth: failed to load signing key", error)
+    // Drop the failed config so the next request rebuilds it,
+    // rather than serving the cached rejection until the isolate restarts.
+    // Guard against clobbering a newer config set by a concurrent reconfigure.
+    if (cache === entry) {
+      cache = null
+    }
+    return null
+  }
+}
+
+/** Serialize a value as an RFC 8941 structured-field string (a quoted string). */
+function structuredString(value: string): string {
+  return `"${value.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"`
+}
+
+export interface DirectoryResponse {
+  body: { keys: DirectoryKey[] }
+  headers: Record<string, string>
+}
+
+/**
+ * Build the signed key directory response,
+ * or `null` when no key is configured.
+ *
+ * The response body is a JWKS
+ * and carries `Signature`/`Signature-Input` headers
+ * signing over `@authority` with `tag="http-message-signatures-directory"`,
+ * demonstrating control of the published key.
+ */
+export async function webBotAuthDirectory(requestUrl: string): Promise<DirectoryResponse | null> {
+  const config = await currentConfig()
+  if (!config) {
+    return null
+  }
+
+  const created = new Date()
+  const expires = new Date(created.getTime() + DIRECTORY_SIGNATURE_TTL_MS)
+  const message = {
+    response: { status: 200, headers: {} as Record<string, string> },
+    request: { method: "GET", url: requestUrl, headers: {} as Record<string, string> },
+  }
+
+  const signature = await directoryResponseHeaders(message, [config.signer], { created, expires })
+
+  return {
+    body: { keys: [config.publicKey] },
+    headers: {
+      "Content-Type": DIRECTORY_MEDIA_TYPE,
+      Signature: signature.Signature,
+      "Signature-Input": signature["Signature-Input"],
+      "Cache-Control": "public, max-age=300, s-maxage=300",
+    },
+  }
+}
+
+/**
+ * Produce Web Bot Auth headers for an outbound request,
+ * or `{}` when no key is configured (e.g. local development or the CLI).
+ * Signing failures degrade to an unsigned request
+ * rather than breaking the fetch.
+ */
+export async function webBotAuthHeaders(
+  method: string,
+  url: string | URL,
+): Promise<Record<string, string>> {
+  const config = await currentConfig()
+  if (!config) {
+    return {}
+  }
+
+  try {
+    const agent = structuredString(config.agent)
+    // `signature-agent` must be present on the message
+    // so it is covered by the signature;
+    // the same value is returned for the outbound request.
+    const headers = new Headers()
+    headers.set("Signature-Agent", agent)
+
+    const created = new Date()
+    const expires = new Date(created.getTime() + OUTBOUND_SIGNATURE_TTL_MS)
+    const signature = await signatureHeaders(
+      { method, url: url.toString(), headers },
+      config.signer,
+      { created, expires },
+    )
+
+    return {
+      "Signature-Agent": agent,
+      "Signature-Input": signature["Signature-Input"],
+      Signature: signature.Signature,
+    }
+  } catch (error) {
+    console.error("web-bot-auth: failed to sign outbound request", error)
+    return {}
+  }
+}

package/src/lib/webmcp.ts

@@ -0,0 +1,38 @@
+import { z } from "zod"
+import { zodToJsonSchema } from "zod-to-json-schema"
+
+import { TOOL_DEFINITIONS } from "./mcp"
+
+export interface WebMcpToolManifest {
+  name: string
+  title: string
+  description: string
+  inputSchema: Record<string, unknown>
+  readOnly: boolean
+  http: {
+    path?: string
+    pathFrom?: string
+    pathPrefix?: string
+    query?: Record<string, string>
+  }
+}
+
+export function buildWebMcpManifest(): WebMcpToolManifest[] {
+  return Object.values(TOOL_DEFINITIONS).map((def) => {
+    const jsonSchema = zodToJsonSchema(z.object(def.inputSchema), {
+      $refStrategy: "none",
+    })
+    const { $schema: _, ...inputSchema } = jsonSchema as Record<string, unknown> & {
+      $schema?: unknown
+    }
+
+    return {
+      name: def.name,
+      title: def.title,
+      description: def.description,
+      inputSchema,
+      readOnly: def.annotations.readOnlyHint,
+      http: def.http,
+    }
+  })
+}

package/wrangler.jsonc

@@ -17,7 +17,7 @@
   "assets": {
     "binding": "ASSETS",
     "directory": "./public",
-    "run_worker_first": ["/", "/.well-known/agent-skills/*"]
+    "run_worker_first": ["/", "/.well-known/agent-card.json", "/.well-known/agent-skills/*"]
   },
   "observability": {
     "enabled": true