@huynle/hub 0.8.7 → 0.9.0

package/src/central/media-unfurl.ts

@@ -0,0 +1,119 @@
+// GET /api/media/unfurl?url=… (design §4, D6): a link's preview for the console, fetched by the hub through the safe
+// fetcher (./fetcher.ts) for signed-in users only. The fetcher asks for HTML, reads at most the first
+// UNFURL_MAX_BYTES (a longer page is cut, not refused) and only a 2xx text/html or application/xhtml+xml answer counts
+// (else 415 not_html). ./unfurl-parse.ts turns it into plain text; the answer is JSON {url (the final URL, after
+// redirects), title?, description?, siteName?, image?}, private and browser-cacheable for at most an hour. The image
+// URL is never fetched here: the console loads it through /api/media/remote, and only for an allowlisted host.
+// A tenant-scoped in-memory cache (per pod in cluster mode) keyed by the user and URL keeps previews for the site's
+// lifetime clamped to 1-24 h and failures for 5 min. Per user: ONE fetch budget with /api/media/remote (30 a minute,
+// 4 at once together, ./fetch-budget.ts); cache hits are free. Each fetch is audited with the user, the host and the
+// outcome, never the full URL. HUB_REMOTE_FETCH=off turns it off (403 disabled).
+import { createFetchBudget, type FetchBudget } from "./fetch-budget"
+import { checkURL, FetchError, type Fetcher } from "./fetcher"
+import { budgetMessage, failure, lifetime, type RemoteMediaAudit } from "./media-remote"
+import { decodeHtml, parseUnfurl, type UnfurlMeta } from "./unfurl-parse"
+
+/** Bytes of a page read at most (the rest is never downloaded). */
+export const UNFURL_MAX_BYTES = 512 * 1024
+/** The longest the browser may keep a preview (the hub's cache may keep it longer). */
+const BROWSER_MAX_AGE_S = 3600
+const FAILURE_TTL_MS = 5 * 60_000
+const MAX_CACHE_ENTRIES = 2048
+const ACCEPT = "text/html,application/xhtml+xml;q=0.9,*/*;q=0.1"
+const HTML_TYPES = new Set(["text/html", "application/xhtml+xml"])
+
+export interface Unfurled extends UnfurlMeta {
+  readonly url: string
+}
+
+export interface UnfurlOptions {
+  readonly fetcher: Fetcher
+  /** False: HUB_REMOTE_FETCH=off; every request is refused and nothing is fetched. */
+  readonly enabled: boolean
+  readonly now?: () => number
+  /** The hub's audit log (store.audit). */
+  readonly audit: (entry: RemoteMediaAudit) => unknown
+  /** The per-user fetch budget, shared with /api/media/remote (default: its own). */
+  readonly budget?: FetchBudget
+}
+
+export interface Unfurl {
+  /** Answers GET /api/media/unfurl for `userID` (already authenticated as a signed-in user). */
+  handle(userID: string, request: Request, url: URL): Promise<Response>
+}
+
+type Entry =
+  | { readonly expires: number; readonly ok: true; readonly body: string }
+  | { readonly expires: number; readonly ok: false; readonly status: number; readonly code: string; readonly message: string }
+
+export function createUnfurl(options: UnfurlOptions): Unfurl {
+  const now = options.now ?? Date.now
+  const budget = options.budget ?? createFetchBudget({ now })
+  const cache = new Map<string, Entry>()
+
+  function remember(key: string, entry: Entry): void {
+    cache.delete(key)
+    cache.set(key, entry)
+    while (cache.size > MAX_CACHE_ENTRIES) cache.delete(cache.keys().next().value!)
+  }
+  function lookup(key: string): Entry | undefined {
+    const entry = cache.get(key)
+    if (!entry) return undefined
+    cache.delete(key)
+    if (entry.expires <= now()) return undefined
+    cache.set(key, entry) // most recently used last
+    return entry
+  }
+
+  function answer(entry: Entry): Response {
+    if (!entry.ok) return failure(entry.status, entry.code, entry.message)
+    const maxAge = Math.max(0, Math.min(BROWSER_MAX_AGE_S, Math.floor((entry.expires - now()) / 1000)))
+    return new Response(entry.body, { headers: { "content-type": "application/json; charset=utf-8", "cache-control": `private, max-age=${maxAge}`, "x-content-type-options": "nosniff" } })
+  }
+
+  async function load(target: URL): Promise<{ entry: Entry; host: string; outcome: string }> {
+    const at = now()
+    const fail = (status: number, code: string, message: string): Entry => ({ expires: at + FAILURE_TTL_MS, ok: false, status, code, message })
+    try {
+      const got = await options.fetcher.fetch(target, { maxBytes: UNFURL_MAX_BYTES, truncate: true, accept: ACCEPT })
+      const host = got.url.hostname
+      const contentType = got.headers.get("content-type")
+      const mime = (contentType ?? "").split(";")[0]!.trim().toLowerCase()
+      if (!HTML_TYPES.has(mime)) return { entry: fail(415, "not_html", "The link is not a web page (HTML)"), host, outcome: "not_html" }
+      const meta = parseUnfurl(decodeHtml(got.body, contentType), got.url)
+      const body: Unfurled = { url: got.url.href, ...meta }
+      return { entry: { expires: at + lifetime(got.headers, at), ok: true, body: JSON.stringify(body) }, host, outcome: "ok" }
+    } catch (cause) {
+      if (!(cause instanceof FetchError)) throw cause
+      return { entry: fail(cause.status, cause.kind, cause.message), host: cause.host ?? target.hostname, outcome: cause.kind }
+    }
+  }
+
+  async function handle(userID: string, _request: Request, url: URL): Promise<Response> {
+    if (!options.enabled) return failure(403, "disabled", "Link previews are turned off on this hub (HUB_REMOTE_FETCH=off)")
+    const raw = url.searchParams.get("url")
+    if (!raw) return failure(400, "bad_url", "Expected a url query parameter")
+    let target: URL
+    try {
+      target = checkURL(raw)
+    } catch (cause) {
+      if (cause instanceof FetchError) return failure(cause.status, cause.kind, cause.message)
+      throw cause
+    }
+    const key = `${userID}\n${target.href}`
+    const hit = lookup(key)
+    if (hit) return answer(hit)
+    const slot = budget.take(userID)
+    if (!slot.ok) return failure(429, "rate_limited", budgetMessage(slot), { "retry-after": String(slot.retryAfter) })
+    try {
+      const { entry, host, outcome } = await load(target)
+      await Promise.resolve(options.audit({ actor: userID, action: "media.unfurl", target: host, detail: outcome })).catch(() => undefined)
+      remember(key, entry)
+      return answer(entry)
+    } finally {
+      slot.release()
+    }
+  }
+
+  return { handle }
+}

package/src/central/profiles/content.ts

@@ -80,41 +80,9 @@ export function renderTemplate(template: string, values: Readonly<Record<string,
   return out
 }
 
-/** Distinctive credential formats; prose about secrets ("set API_KEY=…") is not matched. */
-const CREDENTIALS: readonly (readonly [string, RegExp])[] = [
-  ["private key", /-----BEGIN [A-Z ]*PRIVATE KEY-----/],
-  ["hub app token", /\bhat_[0-9a-f]{16}_[A-Za-z0-9_-]{43,}/],
-  ["github token", /\bgh[pousr]_[A-Za-z0-9]{36,}/],
-  ["github token", /\bgithub_pat_[A-Za-z0-9_]{22,}/],
-  ["api key", /\bsk-[A-Za-z0-9_-]{20,}/],
-  ["api key", /\bAIza[0-9A-Za-z_-]{35}/],
-  ["aws access key", /\b(?:AKIA|ASIA)[0-9A-Z]{16}\b/],
-  ["slack token", /\bxox[abposr]-[A-Za-z0-9-]{10,}/],
-  ["jwt", /\beyJ[A-Za-z0-9_-]{10,}\.eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}/],
-  ["bearer token", /\b[Bb]earer\s+[A-Za-z0-9._~+/=-]{20,}/],
-]
-
-/** The kind of literal credential in a text, or undefined. */
-export function credentialFinding(text: string): string | undefined {
-  for (const [kind, pattern] of CREDENTIALS) if (pattern.test(text)) return kind
-  return undefined
-}
-
-/**
- * (REC-01) Replaces every literal credential in a text with `[redacted: <kind>]` (the CREDENTIALS table, applied
- * globally) and lists the kinds removed, each once, in table order. Voice recordings scrub every text field with it.
- */
-export function redactCredentials(text: string): { text: string; kinds: string[] } {
-  const kinds: string[] = []
-  let out = text
-  for (const [kind, pattern] of CREDENTIALS) {
-    const global = new RegExp(pattern.source, pattern.flags.includes("g") ? pattern.flags : `${pattern.flags}g`)
-    const next = out.replace(global, `[redacted: ${kind}]`)
-    if (next !== out && !kinds.includes(kind)) kinds.push(kind)
-    out = next
-  }
-  return { text: out, kinds }
-}
+// (REC-01) The credential table and scrubber live in the protocol package (the spoke scrubs voice-front wakes too).
+import { credentialFinding, redactCredentials } from "../../../packages/protocol/src/credentials.ts"
+export { credentialFinding, redactCredentials }
 
 /** Fields that would run code or install things on a machine: refused as unsupported. */
 const HOOK_FIELDS = new Set(["hooks", "hook", "setup", "install", "postinstall", "preinstall", "dependencies", "devdependencies", "requirements", "scripts"])

package/src/central/recording/routes.ts

@@ -2,7 +2,7 @@
 // is a console route with access "user": the user's cookie or device token only (the hub refuses session, app and
 // act tokens before these run), so no agent, not even the Concierge, can read or export a recording, and there is
 // no admin route. Every lookup is by (user.id, id); another user's, an unknown and a deleted ID answer 404 alike.
-import { RECORDING_AUDIO_TYPE, RECORDING_LIMITS, type RecordingClient } from "../../protocol"
+import { RECORDING_AUDIO_TYPE, RECORDING_FEATURES, RECORDING_LIMITS, type RecordingClient } from "../../protocol"
 import { readBounded } from "../agent/body"
 import { checkConsoleBatch } from "./schema"
 import { audioKey, blobTenant, isSegmentNumber, summaryOf, type AudioEntry } from "./store"
@@ -199,7 +199,7 @@ export function recordingRoutes(service: RecordingService, options: { readonly v
           return fail(429, `At most ${L.recordingsStored} recordings are kept; delete some in Settings → Voice`, "limit", { limit: L.recordingsStored })
       }
       const record = started.record
-      return Response.json({ recording: summaryOf(record), audio: record.audio, hubAt: options.now() }, { status: started.kind === "created" ? 201 : 200, headers: NO_STORE })
+      return Response.json({ recording: summaryOf(record), audio: record.audio, hubAt: options.now(), features: [...RECORDING_FEATURES] }, { status: started.kind === "created" ? 201 : 200, headers: NO_STORE })
     }),
     route("GET", "/api/recordings", async ({ url, user }) => {
       const limitRaw = url.searchParams.get("limit")

package/src/central/recording/schema.ts

@@ -45,7 +45,7 @@ export const CONSOLE_FIELDS: Readonly<Record<ConsoleEventKind, Fields>> = {
   "stt.received": { segment: int, "text?": text(L.textChars), "junk?": yes, "empty?": yes, "failed?": text(L.messageChars), "audio?": oneOf(...AUDIO_RESULTS) },
   "utterance.held": { text: text(L.utteranceChars) },
   "utterance.merged": { segments: list(int, 1000), pauseMs: num },
-  "utterance.sent": { text: text(L.utteranceChars), segments: list(int, 1000), reason: oneOf("finished", "hold", "max", "command", "typed"), holdMs: num },
+  "utterance.sent": { text: text(L.utteranceChars), segments: list(int, 1000), reason: oneOf("finished", "hold", "max", "command", "typed", "smart", "smart-fallback"), holdMs: num },
   "utterance.accepted": { inboxID: id },
   "utterance.failed": { "status?": int, message: text(L.messageChars) },
   "utterance.command": { command: oneOf("pause", "interrupt"), text: text(L.textChars) },
@@ -68,6 +68,8 @@ export const CONSOLE_FIELDS: Readonly<Record<ConsoleEventKind, Fields>> = {
   mic: { live: bool, "reason?": label },
   control: { action: oneOf("pause", "resume", "hush", "background", "cues"), "on?": bool },
   "turn.timing": { "speechEnd?": num, "transcript?": num, "accepted?": num, "firstSay?": num, "firstByte?": num, firstAudio: num },
+  // Feature "turn.predict" (Smart Turn): a score (none when the model failed or was slow) and what it decided.
+  "turn.predict": { "p?": num, ms: num, silenceMs: num, decision: oneOf("send", "hold", "fallback", "timer-fallback"), "error?": text(L.messageChars) },
   "decision.voice": { decisionID: id, option: id, outcome: oneOf("answered", "refused", "failed") },
   "clock.sync": { t0: num, t1: num, hubAt: num },
   error: { where: oneOf("stt", "tts", "send", "mic", "upload", "audio"), message: text(L.messageChars) },