dsh-thoughtdag 0.5.5 → 0.5.6

package/lib/index.js

@@ -293,7 +293,11 @@ function decompressZstdToText(raw) {
  *  `session.jsonl.zstd`, while session format V3 (0.1.5-rc.2 and later) uses
  *  `session.v3.jsonl.zstd`. Probe the known names, newest first, so a listing
  *  keeps working across both. */
-const SESSION_LOG_NAMES = ['session.v3.jsonl.zstd', 'session.jsonl.zstd']
+// `session.jsonl.zstd` (≤ 0.1.4), `session.v3.jsonl.zstd` (0.1.5), `session.v4.jsonl.zstd` (the 0.2
+// desktop) …: the session directory is listed and the highest version wins, so the next format
+// version is found before this file learns its name (the 0.2 desktop's own sessions were invisible
+// while only v3 was probed)
+const SESSION_LOG_RE = /^session(?:\.v(\d+))?\.jsonl\.zstd?$/
 async function findSessionFiles(dshHome) {
   const root = join(dshHome, 'sessions')
   const out = []
@@ -305,13 +309,13 @@ async function findSessionFiles(dshHome) {
     try { sessions = await readdir(join(root, ws.name), { withFileTypes: true }) } catch { continue }
     for (const s of sessions) {
       if (!s.isDirectory()) continue
-      let log
+      let names
+      try { names = await readdir(join(root, ws.name, s.name)) } catch { continue }
+      const best = names.map(n => ({ n, m: SESSION_LOG_RE.exec(n) })).filter(x => x.m !== null).sort((a, b) => Number(b.m[1] ?? 0) - Number(a.m[1] ?? 0))[0]
+      if (best === undefined) continue
+      const log = join(root, ws.name, s.name, best.n)
       let st
-      for (const name of SESSION_LOG_NAMES) {
-        const candidate = join(root, ws.name, s.name, name)
-        try { st = await stat(candidate); log = candidate; break } catch { /* try the next format */ }
-      }
-      if (log === undefined || st === undefined) continue
+      try { st = await stat(log) } catch { continue }
       out.push({ dir: ws.name, sessionDir: s.name, log, size: st.size, mtime: st.mtimeMs })
     }
   }
@@ -534,7 +538,7 @@ async function visionIdsOf(ctx, providerId, catalogModels) {
   return out
 }
 
-async function modelsPayload(ctx) {
+async function modelsPayload(ctx, ownProviders = []) {
   const cat = await ctx.sessionController.modelCatalog()
   // the harness itself, as an entry: the agent loop with tools, not a bare model
   // the harness's agent loop with tools, once per model it can run on —
@@ -549,15 +553,137 @@ async function modelsPayload(ctx) {
     }
   }
   models.push(...agents)
+  // the person's own interfaces follow, under their own names; a harness id wins a collision
+  const own = ownModelEntries(ownProviders, new Set(models.map(m => m.id)))
+  models.push(...own)
   const def = cat.default ? `${cat.default.provider}/${cat.default.model}` : null
   return {
     models,
     default: def && models.some(m => m.id === def) ? def : (models[0]?.id ?? null),
-    capabilities: { webSearch: false, searchEngine: 'none', scholarSearch: false, vision: models.some(m => m.vision) },
+    capabilities: { webSearch: ownProviders.some(p => isOpenRouterURL(p.baseURL)), searchEngine: 'none', scholarSearch: false, vision: models.some(m => m.vision) },
     harness: { routableProviders: cat.routableProviders ?? [], failures: (cat.failures ?? []).map(f => ({ provider: f.id, message: f.message })) },
   }
 }
 
+// ── the person's own interfaces ────────────────────────────────────────
+// The canvas keeps API interfaces in the browser (the same list the desktop
+// app has) and sends them along with every call and with a registration
+// at boot (POST /runtime-providers). Here they are spoken to directly over
+// the OpenAI-compatible protocol, so a key added inside the harness works
+// the way it does in the app; the harness's own models are untouched.
+const OWN_PROVIDER_CAP = 12
+const OWN_MODEL_CAP = 60
+const isOpenRouterURL = (u) => /openrouter\.ai/i.test(String(u ?? ''))
+
+/** The providers a request or a registration carries, sanitised. */
+function ownProvidersOf(list) {
+  const out = []
+  for (const p of (Array.isArray(list) ? list : []).slice(0, OWN_PROVIDER_CAP)) {
+    const baseURL = String(p?.baseURL ?? '').trim().replace(/\/+$/, '')
+    if (!/^https?:\/\//i.test(baseURL)) continue
+    const name = String(p?.name || 'Custom').slice(0, 40)
+    const models = (Array.isArray(p?.models) ? p.models : [])
+      .map(m => (typeof m === 'string' ? { id: m } : m))
+      .filter(m => m && typeof m.id === 'string' && m.id)
+      .slice(0, OWN_MODEL_CAP)
+      .map(m => ({ id: m.id, ...(typeof m.vision === 'boolean' ? { vision: m.vision } : {}) }))
+    if (models.length === 0) continue
+    out.push({ name, baseURL, apiKey: typeof p?.apiKey === 'string' ? p.apiKey : '', models })
+  }
+  return out
+}
+
+/** Model entries for the picker: the model's short id and its interface's name, like the app shows them. */
+function ownModelEntries(providers, taken) {
+  const entries = []
+  for (const p of providers) {
+    for (const m of p.models) {
+      if (taken.has(m.id)) continue
+      taken.add(m.id)
+      const shortId = m.id.includes('/') ? m.id.split('/').slice(1).join('/') : m.id
+      entries.push({ id: m.id, name: `${shortId} (${p.name})`, provider: p.name, vision: m.vision })
+    }
+  }
+  return entries
+}
+
+/** The interface a requested model belongs to, from the request's own list first, then the registered list. */
+function ownTargetOf(requested, fromBody, registered) {
+  if (typeof requested !== 'string' || !requested) return null
+  for (const p of [...fromBody, ...registered]) if (p.models.some(m => m.id === requested)) return { provider: p, model: requested }
+  return null
+}
+
+/** The canvas's { role, content } messages and images as OpenAI-compatible messages. */
+function ownMessagesOf(body) {
+  const messages = []
+  for (const m of Array.isArray(body?.messages) ? body.messages : []) {
+    if (!m || typeof m.content !== 'string') continue
+    messages.push({ role: m.role === 'system' ? 'system' : m.role === 'assistant' ? 'assistant' : 'user', content: m.content })
+  }
+  if (messages.length === 0) throw new HttpError(400, 'messages: nothing to send')
+  const images = Array.isArray(body?.images) ? body.images.filter(i => i && typeof i.data === 'string' && IMAGE_MEDIA.has(i.mimeType)) : []
+  if (images.length > 0) {
+    for (let i = messages.length - 1; i >= 0; i--) {
+      if (messages[i].role !== 'user') continue
+      messages[i] = { role: 'user', content: [{ type: 'text', text: messages[i].content }, ...images.map(img => ({ type: 'image_url', image_url: { url: `data:${img.mimeType};base64,${img.data}` } }))] }
+      break
+    }
+  }
+  return messages
+}
+
+/** One chat-completions request to the interface; the response as fetched (the caller reads it). */
+async function ownRequest(target, body, stream) {
+  const { provider, model } = target
+  // OpenRouter searches through its `:online` variant and takes a reasoning switch; a fast call never thinks out loud
+  const openrouter = isOpenRouterURL(provider.baseURL)
+  const id = openrouter && body?.webSearch && !model.endsWith(':online') ? `${model}:online` : model
+  const payload = { model: id, messages: ownMessagesOf(body), stream, ...(openrouter ? { reasoning: { enabled: !body?.fast } } : {}) }
+  const headers = { 'content-type': 'application/json', ...(provider.apiKey ? { authorization: `Bearer ${provider.apiKey}` } : {}), ...(openrouter ? { 'HTTP-Referer': 'https://github.com/chenxiachan/thoughtdag', 'X-Title': 'ThoughtDAG' } : {}) }
+  const res = await fetch(`${provider.baseURL}/chat/completions`, { method: 'POST', headers, body: JSON.stringify(payload), signal: AbortSignal.timeout(10 * 60 * 1000) })
+  if (!res.ok) {
+    const text = await res.text().catch(() => '')
+    let detail = text.slice(0, 300)
+    try { const j = JSON.parse(text); detail = j?.error?.message ?? j?.message ?? detail } catch { /* not json */ }
+    throw new HttpError(res.status === 401 || res.status === 403 ? 401 : 502, `${provider.name}: HTTP ${res.status}${detail ? ' · ' + detail : ''}`)
+  }
+  return res
+}
+
+/** Stream one answer from the interface as the canvas's frames: { text }, { reasoning }, [DONE]. */
+async function ownStream(res, target, body, isClosed) {
+  const upstream = await ownRequest(target, body, true)
+  const decoder = new TextDecoder()
+  let buffer = ''
+  for await (const chunk of upstream.body) {
+    if (isClosed()) break
+    buffer += decoder.decode(chunk, { stream: true })
+    let nl
+    while ((nl = buffer.indexOf('\n')) >= 0) {
+      const line = buffer.slice(0, nl).trim(); buffer = buffer.slice(nl + 1)
+      if (!line.startsWith('data:')) continue
+      const data = line.slice(5).trim()
+      if (data === '[DONE]') return
+      let parsed
+      try { parsed = JSON.parse(data) } catch { continue }
+      if (parsed?.error) { res.write(`data: ${JSON.stringify({ error: parsed.error.message ?? String(parsed.error) })}\n\n`); return }
+      const delta = parsed?.choices?.[0]?.delta ?? {}
+      const reasoning = delta.reasoning_content ?? delta.reasoning
+      if (typeof reasoning === 'string' && reasoning) res.write(`data: ${JSON.stringify({ reasoning })}\n\n`)
+      if (typeof delta.content === 'string' && delta.content) res.write(`data: ${JSON.stringify({ text: delta.content })}\n\n`)
+    }
+  }
+}
+
+/** One whole answer from the interface (the canvas's background calls). */
+async function ownCall(target, body) {
+  const upstream = await ownRequest(target, body, false)
+  const j = await upstream.json()
+  const c = j?.choices?.[0]?.message?.content
+  return typeof c === 'string' ? c : Array.isArray(c) ? c.map(x => (typeof x?.text === 'string' ? x.text : '')).join('') : ''
+}
+
 const IMAGE_MEDIA = new Set(['image/png', 'image/jpeg', 'image/webp', 'image/gif'])
 
 /** The SPA's { role, content } messages as harness Messages; system lines
@@ -834,7 +960,7 @@ async function visionModel(ctx) {
 
 /** Resolve the requested model against the catalog, falling back to its default. */
 async function targetOf(ctx, requested) {
-  const payload = await modelsPayload(ctx)
+  const payload = await modelsPayload(ctx)  // the harness's own; the person's interfaces are resolved before this is asked
   const id = requested && payload.models.some(m => m.id === requested) ? requested : payload.default
   const t = splitModelId(id)
   if (!t) throw new HttpError(503, 'no model is routable in this harness')
@@ -943,6 +1069,8 @@ export async function apply(ctx, config) {
     : '/thoughtdag'
   const trustedHosts = new Set(['localhost', '127.0.0.1', ...[...(config?.trustedHosts ?? [])].map(h => String(h).trim().toLowerCase()).filter(Boolean)])
 
+  // the interfaces the canvas registered (memory only: a restart forgets them, the canvas registers again at boot)
+  let registeredProviders = []
   const api = async (req, res) => {
     try {
       const hostname = (typeof req.headers.host === 'string' ? req.headers.host : '').replace(/:\d+$/, '').toLowerCase()
@@ -1010,6 +1138,7 @@ export async function apply(ctx, config) {
           const cwd = url.searchParams.get('cwd') ?? undefined
           return sendJson(res, 200, await whyLib.findJson(phrase, { scope, limit, ...(cwd ? { cwd } : {}) }))
         }
+        if (path === '/why/turns') return sendJson(res, 200, await whyLib.turnsJson({ offset: Number(url.searchParams.get('offset') ?? 0) || 0, ...(url.searchParams.get('limit') === null ? {} : { limit: Number(url.searchParams.get('limit')) || 0 }), head: Number(url.searchParams.get('head') ?? 300) || 300 }))
         if (path === '/why/recall') {
           try { return sendJson(res, 200, await whyLib.recallJson(url.searchParams.get('session') ?? '', Number(url.searchParams.get('turn') ?? 0))) }
           catch (e) { return sendJson(res, 404, { error: e instanceof Error ? e.message : String(e) }) }
@@ -1089,7 +1218,34 @@ export async function apply(ctx, config) {
         if (await agentsHttp().handle(req, res, path, body)) return
       }
       // ── model connection (the SPA's proxy protocol, on the harness's providers) ──
-      if (path === '/models' && req.method === 'GET') return sendJson(res, 200, await modelsPayload(ctx))
+      if (path === '/models' && req.method === 'GET') return sendJson(res, 200, await modelsPayload(ctx, registeredProviders))
+      // the canvas registers its browser-stored interfaces at boot (and after a change): the list answers with them in it
+      // the interface dialog asks an endpoint what it serves (the /models protocol standard), through this host
+      if (path === '/probe-models' && req.method === 'POST') {
+        const body = await readJson(req, MAX_WRITE_BODY_BYTES)
+        const baseURL = String(body?.baseURL ?? '').trim().replace(/\/+$/, '')
+        if (!/^https?:\/\//i.test(baseURL)) return sendJson(res, 400, { error: 'baseURL required' })
+        try {
+          const r = await fetch(`${baseURL}/models`, { headers: body?.apiKey ? { authorization: `Bearer ${body.apiKey}` } : {}, signal: AbortSignal.timeout(15000) })
+          if (!r.ok) return sendJson(res, r.status, { error: `endpoint answered HTTP ${r.status}` })
+          const j = await r.json()
+          const list = Array.isArray(j?.data) ? j.data : Array.isArray(j?.models) ? j.models : []
+          const models = list.map(m => ({
+            id: typeof (m?.id ?? m?.name) === 'string' ? (m.id ?? m.name).replace(/^models\//, '') : '',
+            ...(typeof m?.created === 'number' ? { created: m.created } : {}),
+            ...(Array.isArray(m?.architecture?.input_modalities) ? { vision: m.architecture.input_modalities.includes('image') } : {}),
+            ...(typeof m?.context_length === 'number' ? { contextLength: m.context_length } : {}),
+          })).filter(m => m.id)
+          return sendJson(res, 200, { models })
+        } catch (error) {
+          return sendJson(res, 502, { error: 'could not reach the endpoint: ' + (error instanceof Error ? error.message : String(error)) })
+        }
+      }
+      if (path === '/runtime-providers' && req.method === 'POST') {
+        const body = await readJson(req, MAX_CALL_BODY_BYTES)
+        registeredProviders = ownProvidersOf(body?.providers)
+        return sendJson(res, 200, await modelsPayload(ctx, registeredProviders))
+      }
       const approvalRoute = /^\/approvals\/([^/]+)$/.exec(path)
       if (approvalRoute !== null && req.method === 'POST') {
         const id = decodeURIComponent(approvalRoute[1])
@@ -1119,6 +1275,22 @@ export async function apply(ctx, config) {
           res.end()
           return
         }
+        // the person's own interface, when the model is one of theirs
+        const own = ownTargetOf(body?.model, ownProvidersOf(body?.providers), registeredProviders)
+        if (own) {
+          if (path === '/claude') return sendJson(res, 200, { text: await ownCall(own, body), model: own.model })
+          res.writeHead(200, { 'content-type': 'text/event-stream', 'cache-control': 'no-cache', connection: 'keep-alive' })
+          let closed = false
+          req.on('close', () => { closed = true })
+          try {
+            await ownStream(res, own, body, () => closed)
+            if (!closed) res.write('data: [DONE]\n\n')
+          } catch (error) {
+            if (!closed) res.write(`data: ${JSON.stringify({ error: error instanceof Error ? error.message : String(error) })}\n\n`)
+          }
+          res.end()
+          return
+        }
         const target = await targetOf(ctx, body?.model)
         const options = await callOf(ctx, body, target)
         if (path === '/claude') {

package/lib/why.mjs

@@ -3093,6 +3093,61 @@ async function findJson(phrase, opts = {}) {
     hits: shown.map((h) => ({ kind: h.kind, session: h.session.id, runner: h.session.runner, title: h.session.title, cwd: h.turn.cwd ?? h.session.cwd, file: h.session.file, turn: turnNumber(facts, h), at: h.turn.at ?? null, where: h.where, snippet: h.snippet, open: openLink(h) }))
   };
 }
+async function turnsJson(opts = {}) {
+  if (opts.limit === 0) {
+    const text = await loadText();
+    return { total: Object.values(text.sessions).reduce((a, b) => a + b, 0), turns: [] };
+  }
+  const facts = await ensureFresh();
+  const head = Math.max(40, Math.min(1e3, opts.head ?? 300));
+  const turnsOf = /* @__PURE__ */ new Map();
+  const lookup = (k, i) => {
+    let m = turnsOf.get(k);
+    if (!m) {
+      m = new Map(facts.sessions[k]?.turns.map((t) => [t.i, t]) ?? []);
+      turnsOf.set(k, m);
+    }
+    return m.get(i);
+  };
+  const clipHead = (text) => (text ?? "").replace(/\s+/g, " ").trim().slice(0, head);
+  const raw = [];
+  let lines = null;
+  try {
+    await fsp.access(TEXT_LINES);
+    lines = createInterface({ input: createReadStream(TEXT_LINES, { encoding: "utf8", highWaterMark: 1 << 20 }) });
+  } catch {
+    return { total: 0, turns: [] };
+  }
+  for await (const line of lines) {
+    let t;
+    try {
+      t = JSON.parse(line);
+    } catch {
+      continue;
+    }
+    const session = facts.sessions[t.k];
+    const turn = lookup(t.k, t.i);
+    if (!session || !turn) continue;
+    const memory = session.kind === "memory";
+    raw.push({ session, sourceKey: t.k, turn, q: memory ? "" : clipHead(t.q), a: clipHead(memory ? t.m : t.a), kind: memory ? "memory" : "turn" });
+  }
+  const seen = /* @__PURE__ */ new Set();
+  const all = [];
+  for (const h of raw.sort((a, b) => a.session.mtime - b.session.mtime)) {
+    if (h.turn.item) {
+      if (seen.has(h.turn.item)) continue;
+      seen.add(h.turn.item);
+    }
+    all.push(h);
+  }
+  all.sort((a, b) => (b.turn.at ?? "").localeCompare(a.turn.at ?? ""));
+  const offset = Math.max(0, opts.offset ?? 0);
+  const limit = Math.max(0, opts.limit ?? all.length);
+  return {
+    total: all.length,
+    turns: all.slice(offset, offset + limit).map((h) => ({ kind: h.kind, session: h.session.id, runner: h.session.runner, title: h.session.title, cwd: h.turn.cwd ?? h.session.cwd, file: h.session.file, turn: turnNumber(facts, h), at: h.turn.at ?? null, q: h.q, a: h.a, open: openLink(h) }))
+  };
+}
 async function recallJson(session, n) {
   return recallTurn(await ensureFresh(), session, n);
 }
@@ -3534,6 +3589,7 @@ export {
   suggestJson,
   summarize,
   topicsJson,
+  turnsJson,
   when,
   zstdSkipped
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dsh-thoughtdag",
-  "version": "0.5.5",
+  "version": "0.5.6",
   "description": "ThoughtDAG for DeepSeek Harness: turn the current DSH session into an editable thought graph on an infinite canvas.",
   "author": "Xia Chen",
   "repository": {