@threadcrumbs/common 0.1.2 → 0.1.4

package/README.md

@@ -39,21 +39,54 @@ repos/@threadcrumbs/
   threadnauts/
 ```
 
-Each app depends on it with `"@threadcrumbs/common": "link:../common"`, so an edit here shows up
-in both apps without a release. Clone this repo next to an app before running `pnpm install`
-there.
-
-An app that publishes to npm depends on a published version instead. This package is on npm as
+Each app depends on a published version, such as `"@threadcrumbs/common": "^0.1.2"`, because
+that's what npm installs for the app's users. Its `pnpm-workspace.yaml` also overrides that with
+`link:../common`, so an edit here shows up in both apps without a release. Clone this repo next to
+an app before running `pnpm install` there. The apps' CI strips the override and uses npm, since
+this repo is private. This package is on npm as
 [`@threadcrumbs/common`](https://www.npmjs.com/package/@threadcrumbs/common).
 
-## Publishing a version
+## Publishing an app, and this package with it
+
+From the app's folder, run:
+
+```
+pnpm auto-publish minor
+```
 
-1. Raise `version` in `package.json`, and commit it.
-2. Run `npm pack --dry-run` and check the list: only `package.json`, `README.md`, `LICENSE.md`,
-   `NOTICES.md`, and the folders in `files`.
-3. Tag the commit `v<version>`, and push the commit and the tag.
-4. Run `npm publish`. `publishConfig` makes it public, and npm asks for your second factor.
-5. Check it with `npm view @threadcrumbs/common version`.
+Use `patch`, `minor` or `major` for how far to bump the app. Leave it off to publish a version
+that's already in `package.json` but not on npm yet. Add `--dry-run` to see the plan and every
+problem without changing anything, or `--yes` to skip the final question.
+
+Before it changes anything, it checks:
+- that npm is logged in, and Cloudflare too for Threadnauts;
+- that the app, this repo and the app's community repo are on `master`, committed, and not behind
+  `origin`;
+- that the community repo's `CHANGELOG.md` has a `## <version>` entry.
+
+Then it shows the plan and waits for you to type `yes`. The plan, in order:
+1. **This package, if needed.** It's published first, when anything in its `files` changed since
+   the published version's commit (npm records that as `gitHead`). It gets a patch bump, its
+   tests run, then it's published, tagged and pushed.
+2. **The app's version.** The app is bumped, and its range on this package raised when this
+   package was just published. The lockfile is refreshed and the change committed.
+3. **Test and build.** The app's tests run and it builds once.
+4. **Pro content.** For Threadnauts, `pro-dist/` is checked against `dist/pro-manifest.json` and
+   uploaded to the production content bucket through the key-service repo.
+5. **Publish.** The app is published with `--ignore-scripts`, so npm ships the build that was
+   just tested and uploaded, rather than building again. npm may ask for your one-time code.
+6. **Push.** The app is tagged `v<version>` and pushed, which deploys its website. The community
+   repo is pushed if it has unpushed commits.
+7. **Check.** It waits for npm to list the version, then checks `npx` runs it.
+
+If a step fails, it stops there. Run it again with the same arguments and it carries on: a
+version that was bumped but never published is published as it is, not bumped twice. If it stops
+after npm has the new version, it prints the commands that are left to run by hand. The script
+is `tools/release.mjs`. It isn't in `files`, so it never ships.
+
+`pnpm auto-publish` always publishes the app as well. To publish only this package, do it by
+hand: raise `version`, commit, run `npm pack --dry-run` to check the file list, tag
+`v<version>`, push, `npm publish`, then `npm view @threadcrumbs/common version`.
 
 ## Settings it reads
 

package/lib/claude-transcript.mjs

@@ -1,5 +1,6 @@
 import fsp from 'node:fs/promises'
 import { requestCost } from './anthropic-prices.mjs'
+import { emptyFixed, emptySizes, setFixed, sizeCall, sizeCompaction, sizePrompt, sizeReply, sizeRequest, sizeResult, sizeResumed } from './context-breakdown.mjs'
 import { listFiles, parseJsonLine, readNewLines } from './fsutil.mjs'
 import {
   DETAIL_LIMITS,
@@ -9,19 +10,26 @@ import {
   clipBlock,
   diffHunk,
   emptyActivity,
+  emptySample,
   endCall,
   fileList,
   firstChange,
+  joinedText,
   keepNewest,
+  keepText,
   nextPlan,
+  openTool,
   patchLines,
   planItems,
+  samplePoint,
   settleCalls,
   startCall,
+  textList,
   toolFile,
   toolInput,
   toolTarget,
 } from './telemetry-fields.mjs'
+import { mcpServer } from './tool-kinds.mjs'
 
 const HEAD_BYTES = 192 * 1024
 const TAIL_BYTES = 64 * 1024
@@ -64,11 +72,11 @@ export async function readLiveSessions(dir, isAlive = pidAlive) {
   return live
 }
 
-export function transcriptReader({ skim = false, detail = false } = {}) {
+export function transcriptReader({ skim = false, detail = false, words = false, sizes = false } = {}) {
   const files = new Map()
 
   async function advance(entry, file, size) {
-    if (!entry.state || size < entry.state.offset) entry.state = emptyState(entry.sidechain, detail)
+    if (!entry.state || size < entry.state.offset) entry.state = emptyState(entry.sidechain, detail, words, sizes)
     const state = entry.state
     if (size === state.offset) return state
     try {
@@ -133,9 +141,12 @@ async function skimInto(state, file, size) {
   }
 }
 
-function emptyState(sidechain, detail) {
+function emptyState(sidechain, detail, words, sizes) {
   return {
     detail: detail ? emptyDetail() : null,
+    words: words ? { thoughts: [], replies: [] } : null,
+    sizes: sizes ? emptySizes() : null,
+    sent: sizes ? emptySent() : null,
     offset: 0,
     carry: Buffer.alloc(0),
     sidechain,
@@ -162,10 +173,12 @@ function emptyState(sidechain, detail) {
     open: new Map(),
     ending: null,
     handedBackAt: 0,
+    turnStartedAt: 0,
     awaiting: new Map(),
     cost: null,
     plan: null,
     prompts: 0,
+    contextSample: emptySample(),
     uuids: new Set(),
   }
 }
@@ -223,6 +236,7 @@ function applyRecord(state, record) {
     state.firstBranch ||= state.gitBranch
   }
   if (state.detail && record.type === 'system' && ownRecord(state, record)) applySystem(state.detail, record, at || state.lastAt)
+  if (state.sizes && ownRecord(state, record)) applySent(state, record)
   if (!isTurn(state, record)) return
   if (record.type === 'user') applyUser(state, record, at || state.lastAt)
   else applyAssistant(state, record, at || state.lastAt)
@@ -239,12 +253,88 @@ function applySystem(detail, record, at) {
   }
 }
 
+const emptySent = () => ({ system: 0, tools: [], deferred: new Set(), instructions: new Map(), skills: new Map(), skillsOther: 0, memory: new Map(), agents: 0 })
+
+function applySent(state, record) {
+  const sent = state.sent
+  const attachment = record.attachment
+  if (record.type === 'system' && record.subtype === 'compact_boundary') {
+    sizeCompaction(state.sizes)
+    state.sent = { ...emptySent(), system: sent.system, tools: sent.tools }
+    return
+  }
+  if (record.type === 'api-request-shape') {
+    const shape = record.shape || {}
+    sent.system = (Array.isArray(shape.system) ? shape.system : []).reduce((n, part) => n + text(part?.text).length, 0)
+    sent.tools = (Array.isArray(shape.tools) ? shape.tools : []).map((tool) => ({ name: text(tool?.name), chars: JSON.stringify(tool ?? null).length }))
+  } else if (attachment?.type === 'deferred_tools_delta') {
+    for (const name of [...listOf(attachment.addedNames), ...listOf(attachment.readdedNames)]) sent.deferred.add(text(name))
+    for (const name of listOf(attachment.removedNames)) sent.deferred.delete(text(name))
+  } else if (attachment?.type === 'mcp_instructions_delta') {
+    listOf(attachment.addedNames).forEach((name, i) => sent.instructions.set(text(name), text(listOf(attachment.addedBlocks)[i]).length))
+    for (const name of listOf(attachment.removedNames)) sent.instructions.delete(text(name))
+  } else if (attachment?.type === 'skill_listing') {
+    if (attachment.isInitial !== false) {
+      sent.skills = new Map()
+      sent.skillsOther = 0
+    }
+    const names = new Set(listOf(attachment.names).map(text))
+    for (const line of text(attachment.content).split('\n')) {
+      const name = /^\s*-\s*(\S+?):(?:\s|$)/.exec(line)?.[1]
+      if (name && names.has(name)) sent.skills.set(name, (sent.skills.get(name) || 0) + line.length + 1)
+      else sent.skillsOther += line.length + 1
+    }
+  } else if (attachment?.type === 'instructions') {
+    for (const file of listOf(attachment.files)) if (text(file?.path)) sent.memory.set(text(file.path), text(file.content).length)
+  } else if (attachment?.type === 'agent_listing_delta') {
+    sent.agents = (attachment.isInitial ? 0 : sent.agents) + listOf(attachment.addedLines).reduce((n, line) => n + text(line).length + 1, 0)
+  } else {
+    return
+  }
+  setFixed(state.sizes, fixedOf(sent))
+}
+
+const listOf = (value) => (Array.isArray(value) ? value : [])
+
+function fixedOf(sent) {
+  const fixed = emptyFixed()
+  fixed.system = sent.system + sent.agents + sent.skillsOther
+  const servers = new Map()
+  const server = (name) => {
+    if (!servers.has(name)) servers.set(name, { names: new Set(), chars: 0 })
+    return servers.get(name)
+  }
+  const tool = (name, chars) => {
+    const owner = mcpServer(name)
+    if (!owner) {
+      fixed.builtInTools += chars
+      return
+    }
+    server(owner).names.add(name)
+    server(owner).chars += chars
+  }
+  for (const loaded of sent.tools) tool(loaded.name, loaded.chars)
+  for (const name of sent.deferred) tool(name, name.length + 1)
+  for (const [name, chars] of sent.instructions) server(name).chars += chars
+  fixed.mcp = new Map([...servers].map(([name, s]) => [name, { tools: s.names.size || null, chars: s.chars }]))
+  fixed.skills = new Map(sent.skills)
+  fixed.memory = new Map(sent.memory)
+  return fixed
+}
+
+function resultChars(block) {
+  if (typeof block?.content === 'string') return block.content.length
+  return listOf(block?.content).reduce((n, part) => n + text(part?.text).length, 0)
+}
+
 function applyUser(state, record, at) {
   const content = record.message?.content
   let answered = false
+  if (state.sizes && record.isCompactSummary) sizeResumed(state.sizes)
   if (Array.isArray(content)) {
     for (const block of content) {
       if (block?.type !== 'tool_result') continue
+      if (state.sizes) sizeResult(state.sizes, block.tool_use_id, resultChars(block))
       state.open.delete(block.tool_use_id)
       answered = true
       const result = record.toolUseResult
@@ -254,6 +344,7 @@ function applyUser(state, record, at) {
       if (!failed) countEdits(state, result, call)
     }
   }
+  if (!state.ending || state.ending.handedBack) state.turnStartedAt = at
   state.ending = MID_TURN
   const notice = record.isMeta ? IDLE_NOTICE.exec(firstText(content)) : null
   if (notice) state.awaiting.delete(notice[1])
@@ -262,6 +353,7 @@ function applyUser(state, record, at) {
   if (!prompt || prompt.startsWith('<')) return
   state.open.clear()
   state.prompts++
+  if (state.sizes) sizePrompt(state.sizes, prompt.length)
   if (!state.firstPrompt) state.firstPrompt = prompt
   if (state.detail) {
     settleCalls(state.detail.activity)
@@ -347,9 +439,11 @@ function applyAssistant(state, record, at) {
       cacheWrite: number(usage.cache_creation_input_tokens),
     }
     const previous = state.usageById.get(message.id)
+    if (state.sizes && !previous) sizeRequest(state.sizes, next.input + next.cacheRead + next.cacheWrite, at)
     for (const key of Object.keys(next)) state.tokens[key] += next[key] - (previous?.[key] || 0)
     state.usageById.set(message.id, next)
     state.context = next.input + next.cacheRead + next.cacheWrite + next.output
+    samplePoint(state.contextSample, state.context, { update: Boolean(previous) })
     const cost = requestCost(message.model, usage)
     let added = null
     if (cost != null) {
@@ -365,11 +459,13 @@ function applyAssistant(state, record, at) {
   if (record.effort) state.effort = text(record.effort)
   const content = Array.isArray(message.content) ? message.content : []
   let calling = false
+  if (state.sizes) sizeReply(state.sizes, joinedText(content, 'text').length + joinedText(content, 'thinking').length)
   for (const block of content) {
     if (block?.type !== 'tool_use') continue
     calling = true
+    if (state.sizes) sizeCall(state.sizes, text(block.id), text(block.name), JSON.stringify(block.input ?? null).length)
     if (!state.open.has(block.id)) {
-      state.open.set(block.id, { name: text(block.name), startedAt: at })
+      state.open.set(block.id, openTool(text(block.name), at, block.input, state.cwd))
       if (state.detail) {
         startCall(state.detail.activity, {
           id: text(block.id),
@@ -390,6 +486,11 @@ function applyAssistant(state, record, at) {
     state.open.clear()
     state.handedBackAt = at
   }
+  if (state.words && record.isApiErrorMessage !== true && message.model !== '<synthetic>') {
+    const id = text(message.id) || null
+    keepText(state.words.thoughts, 'thoughts', { id, at, text: joinedText(content, 'thinking') })
+    keepText(state.words.replies, 'replies', { id, at, text: joinedText(content, 'text') })
+  }
   if (!state.detail) return
   if (handedBack) settleCalls(state.detail.activity)
   if (record.isApiErrorMessage === true) {
@@ -415,7 +516,7 @@ function recordUsage(detail, message, next, previous, context, at, cost) {
   detail.models.set(model, totals)
 }
 
-export function transcriptDetail(state) {
+export function transcriptDetail(state, words = state.words) {
   const detail = state.detail
   if (!detail) return null
   const models = [...detail.models].map(([model, m]) => ({ model, tokens: { ...m.tokens }, costUsd: m.unpriced ? null : m.usd }))
@@ -428,6 +529,8 @@ export function transcriptDetail(state) {
     plan: planItems(state.plan),
     files: fileList(detail.files, state.cwd),
     prompts: detail.prompts.map((p) => ({ ...p })),
+    thoughts: textList(words?.thoughts),
+    replies: textList(words?.replies),
     errors: detail.errors.map((e) => ({ ...e })),
   }
 }

package/lib/context-breakdown.mjs

@@ -0,0 +1,149 @@
+const CHARS_PER_TOKEN = 4
+
+const add = (map, key, n) => map.set(key, (map.get(key) || 0) + n)
+const sum = (values) => [...values].reduce((total, n) => total + n, 0)
+const noPending = () => ({ prompts: 0, replies: 0, tools: new Map() })
+
+export function emptyFixed() {
+  return { system: 0, builtInTools: 0, mcp: new Map(), skills: new Map(), memory: new Map() }
+}
+
+export function emptySizes() {
+  return {
+    fixed: emptyFixed(),
+    first: null,
+    firstPrompt: 0,
+    resumed: false,
+    lastInput: null,
+    lastAt: 0,
+    compacted: false,
+    pending: noPending(),
+    weights: { prompts: 0, replies: 0, summary: 0, tools: new Map() },
+    callNames: new Map(),
+  }
+}
+
+export function setFixed(sizes, fixed) {
+  sizes.fixed = fixed
+}
+
+export function sizePrompt(sizes, chars) {
+  if (!(chars > 0)) return
+  if (sizes.lastInput == null) sizes.firstPrompt += chars
+  sizes.pending.prompts += chars
+}
+
+export function sizeReply(sizes, chars) {
+  if (chars > 0) sizes.pending.replies += chars
+}
+
+export function sizeCall(sizes, id, name, chars) {
+  if (id) sizes.callNames.set(id, name)
+  if (chars > 0) add(sizes.pending.tools, name, chars)
+}
+
+export function sizeResult(sizes, id, chars) {
+  const name = sizes.callNames.get(id)
+  sizes.callNames.delete(id)
+  if (name && chars > 0) add(sizes.pending.tools, name, chars)
+}
+
+export function sizeResumed(sizes) {
+  if (sizes.lastInput == null) sizes.resumed = true
+}
+
+export function sizeCompaction(sizes) {
+  sizes.compacted = true
+  sizes.pending = noPending()
+  sizes.callNames.clear()
+}
+
+const fixedChars = (fixed) =>
+  fixed.system + fixed.builtInTools + sum([...fixed.mcp.values()].map((m) => m.chars)) + sum(fixed.skills.values()) + sum(fixed.memory.values())
+
+function spread(weights, pending, tokens) {
+  const chars = pending.prompts + pending.replies + sum(pending.tools.values())
+  if (!chars) {
+    weights.replies += tokens
+    return
+  }
+  weights.prompts += (tokens * pending.prompts) / chars
+  weights.replies += (tokens * pending.replies) / chars
+  for (const [name, n] of pending.tools) add(weights.tools, name, (tokens * n) / chars)
+}
+
+const counted = (sizes) => (sizes.first && !sizes.resumed ? sizes.first.input - sizes.first.prompt : 0)
+
+function ratio(sizes) {
+  const fixed = counted(sizes)
+  return fixed > 0 && sizes.first.chars ? fixed / sizes.first.chars : 1 / CHARS_PER_TOKEN
+}
+
+export function sizeRequest(sizes, input, at) {
+  if (!(input > 0) || input === sizes.lastInput) return
+  if (at) sizes.lastAt = at
+  if (sizes.lastInput == null) {
+    if (!sizes.firstPrompt) sizes.resumed = true
+    sizes.first = { input, prompt: Math.round(sizes.firstPrompt / CHARS_PER_TOKEN), chars: fixedChars(sizes.fixed) }
+    spread(sizes.weights, sizes.pending, Math.min(input, sizes.first.prompt))
+  } else if (sizes.compacted) {
+    const fixed = sizes.first.chars ? Math.round(fixedChars(sizes.fixed) * ratio(sizes)) : counted(sizes)
+    sizes.weights = { prompts: 0, replies: 0, summary: Math.max(0, input - fixed), tools: new Map() }
+  } else if (input > sizes.lastInput) {
+    spread(sizes.weights, sizes.pending, input - sizes.lastInput)
+  }
+  sizes.compacted = false
+  sizes.lastInput = input
+  sizes.pending = noPending()
+}
+
+function scaled(entries, total) {
+  const weight = sum(entries.map((e) => e.weight))
+  if (!weight || total <= 0) return []
+  return entries.map(({ weight: w, ...rest }) => ({ ...rest, tokens: Math.round((total * w) / weight) })).filter((e) => e.tokens > 0)
+}
+
+function messageItems(weights, tokens) {
+  const entries = [
+    { id: 'prompts', weight: weights.prompts },
+    { id: 'replies', weight: weights.replies },
+    { id: 'summary', weight: weights.summary },
+    ...[...weights.tools].map(([name, weight]) => ({ id: 'tool', name, weight })),
+  ]
+  return scaled(entries, tokens).sort((a, b) => b.tokens - a.tokens)
+}
+
+const twoParts = (base, weights, system) => ({
+  ...base,
+  parts: [
+    { id: 'messages', tokens: base.total - system, items: messageItems(weights, base.total - system) },
+    { id: 'system', tokens: system, items: [] },
+  ],
+})
+
+export function contextBreakdown(sizes, { total, limit = null, reserved = null }) {
+  if (!sizes || !(total > 0)) return null
+  const base = { at: sizes.lastAt, total, limit: limit || null, reserved }
+  const fixed = counted(sizes)
+  if (fixed <= 0 || fixed > total) {
+    const system = Math.round(fixedChars(sizes.fixed) / CHARS_PER_TOKEN)
+    return system > 0 && system < total ? twoParts(base, sizes.weights, system) : null
+  }
+  if (!sizes.first.chars) return twoParts(base, sizes.weights, fixed)
+  const tokens = (n) => Math.round(n * ratio(sizes))
+  const { system, builtInTools, mcp, skills, memory } = sizes.fixed
+  const servers = [...mcp].map(([name, m]) => ({ name, tools: m.tools ?? null, tokens: tokens(m.chars) })).filter((m) => m.tokens > 0)
+  const skillItems = [...skills].map(([name, n]) => ({ name, tokens: tokens(n) })).filter((s) => s.tokens > 0)
+  const memoryItems = [...memory].map(([path, n]) => ({ path, tokens: tokens(n) })).filter((m) => m.tokens > 0)
+  const parts = [
+    { id: 'system', tokens: tokens(system), items: [] },
+    { id: 'builtInTools', tokens: tokens(builtInTools), items: [] },
+    { id: 'mcpTools', tokens: sum(servers.map((m) => m.tokens)), items: servers.sort((a, b) => b.tokens - a.tokens) },
+    { id: 'skills', tokens: sum(skillItems.map((s) => s.tokens)), items: skillItems.sort((a, b) => b.tokens - a.tokens) },
+    { id: 'memory', tokens: sum(memoryItems.map((m) => m.tokens)), items: memoryItems },
+  ].filter((p) => p.tokens > 0)
+  const fixedNow = sum(parts.map((p) => p.tokens))
+  if (fixedNow >= total) return twoParts(base, sizes.weights, fixed)
+  const messages = total - fixedNow
+  return { ...base, parts: [{ id: 'messages', tokens: messages, items: messageItems(sizes.weights, messages) }, ...parts] }
+}

package/lib/jsonl-follower.mjs

@@ -43,3 +43,14 @@ export function jsonlFollower({ empty, apply }) {
 
   return { read, sweep }
 }
+
+export function recentReaders(make, max) {
+  const readers = new Map()
+  return (key) => {
+    const reader = readers.get(key) || make()
+    readers.delete(key)
+    readers.set(key, reader)
+    while (readers.size > max) readers.delete(readers.keys().next().value)
+    return reader
+  }
+}