@timqi/pier 0.0.8 → 0.0.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/README.md +26 -9
  2. package/dist/agent/events.js +53 -7
  3. package/dist/agent/listing.js +253 -0
  4. package/dist/agent/pi.js +279 -32
  5. package/dist/boards/boards.js +65 -16
  6. package/dist/boards/pier.css +1 -1
  7. package/dist/channels/attach.js +87 -0
  8. package/dist/channels/control.js +2 -2
  9. package/dist/channels/conversations.js +10 -0
  10. package/dist/channels/lark-api.js +38 -0
  11. package/dist/channels/lark-outbound.js +11 -2
  12. package/dist/channels/slack-api.js +36 -0
  13. package/dist/channels/slack-outbound.js +12 -2
  14. package/dist/channels/slack-tool.js +49 -9
  15. package/dist/channels/telegram-api.js +21 -2
  16. package/dist/channels/telegram.js +23 -8
  17. package/dist/cli.js +34 -0
  18. package/dist/core/identity.js +18 -0
  19. package/dist/core/inbound-file.js +3 -1
  20. package/dist/core/reply.js +2 -1
  21. package/dist/core/router.js +99 -11
  22. package/dist/db.js +87 -0
  23. package/dist/extensions/index.js +37 -0
  24. package/dist/extensions/web/anthropic.js +118 -0
  25. package/dist/extensions/web/artifacts.js +62 -0
  26. package/dist/extensions/web/content.js +130 -0
  27. package/dist/extensions/web/http.js +106 -0
  28. package/dist/extensions/web/index.js +9 -0
  29. package/dist/extensions/web/json.js +5 -0
  30. package/dist/extensions/web/language.js +47 -0
  31. package/dist/extensions/web/openai.js +112 -0
  32. package/dist/extensions/web/provider.js +121 -0
  33. package/dist/extensions/web/tools.js +304 -0
  34. package/dist/limits.js +14 -0
  35. package/dist/main.js +76 -10
  36. package/dist/paths.js +21 -1
  37. package/dist/settings.js +112 -13
  38. package/dist/tasks/agent.js +18 -4
  39. package/dist/tasks/callbacks.js +20 -1
  40. package/dist/tasks/definitions.js +56 -12
  41. package/dist/tasks/execution.js +5 -1
  42. package/dist/tasks/groups.js +4 -4
  43. package/dist/tasks/messages.js +4 -2
  44. package/dist/tasks/runs.js +2 -2
  45. package/dist/tasks/service.js +16 -6
  46. package/dist/tasks/tool.js +0 -12
  47. package/dist/tools-task.js +155 -0
  48. package/dist/tools.js +875 -0
  49. package/dist/web/auth.js +5 -3
  50. package/dist/web/explorer.js +15 -2
  51. package/dist/web/files.js +1 -1
  52. package/dist/web/instance.js +175 -22
  53. package/dist/web/providers.js +16 -0
  54. package/dist/web/public/assets/{ghostty-web-CcIc8O2I.js → ghostty-web-xcUrfRRs.js} +1 -1
  55. package/dist/web/public/assets/index-BWDlAMK2.js +93 -0
  56. package/dist/web/public/assets/index-DHqZnZr7.css +2 -0
  57. package/dist/web/public/index.html +5 -8
  58. package/dist/web/public/sw.js +4 -0
  59. package/dist/web/push.js +33 -9
  60. package/dist/web/repos.js +75 -0
  61. package/dist/web/server.js +170 -52
  62. package/dist/web/session-state.js +57 -44
  63. package/dist/web/terminal.js +34 -4
  64. package/dist/web/types.js +5 -0
  65. package/package.json +1 -1
  66. package/skills/pier-boards/SKILL.md +23 -13
  67. package/skills/pier-help/SKILL.md +1 -1
  68. package/skills/pier-slack/SKILL.md +21 -1
  69. package/skills/pier-tasks/SKILL.md +2 -2
  70. package/dist/web/public/assets/index-DmDJKOLH.js +0 -90
  71. package/dist/web/public/assets/index-gcSJ9QZ5.css +0 -2
@@ -17,6 +17,11 @@ const truncate = (message) => message.length > 600 ? `${message.slice(0, 600)}
17
17
  function keyOf(key) {
18
18
  return `${key.channelId}:${key.conversationId}`;
19
19
  }
20
+ /** `web:<id>` and `task:<id>` are two names for one session id, and neither is
21
+ * a chat — no Channel is registered under them. So they share a lock in
22
+ * `ensure`, and which of the two a session records costs nothing but the
23
+ * answer to "what is it answering". */
24
+ const isAlias = (key) => key.channelId === "web" || key.channelId === "task";
20
25
  export class Router {
21
26
  hub;
22
27
  resolve;
@@ -101,6 +106,9 @@ export class Router {
101
106
  */
102
107
  async evictIdle(ttlMs = IDLE_TTL_MS, now = Date.now(), { includeWatched = false } = {}) {
103
108
  let evicted = 0;
109
+ // Snapshot on purpose: this loop awaits dispose(), so another turn may
110
+ // attach or drop a session while it is suspended.
111
+ // oxlint-disable-next-line unicorn/no-useless-spread
104
112
  for (const [id, attached] of [...this.bySession]) {
105
113
  if (attached.session.state === "streaming")
106
114
  continue;
@@ -162,7 +170,7 @@ export class Router {
162
170
  const existing = this.bySession.get(session.id);
163
171
  if (existing?.session === session) {
164
172
  this.byKey.set(keyOf(key), session);
165
- existing.activeAt = Date.now();
173
+ this.reached(session, key);
166
174
  return;
167
175
  }
168
176
  if (existing) {
@@ -216,6 +224,15 @@ export class Router {
216
224
  });
217
225
  });
218
226
  }
227
+ // A queued message with no turn left to deliver it. `decide` reads the
228
+ // state once, so a steer chosen against a turn that ends before the call
229
+ // lands sits in Pi's queue until some *later* turn reads it — on IM that
230
+ // is indistinguishable from the message never arriving (§5b). Pi drains
231
+ // its own queues up to the agent_end handler, so a non-empty queue on an
232
+ // idle session is exactly the message that missed that window.
233
+ if (payload.type === "queue-state" && (payload.steering.length || payload.followUp.length)) {
234
+ this.promoteQueued(session, key);
235
+ }
219
236
  // Every turn-end reaches the channel, empty text included: an adapter's
220
237
  // per-turn UI (Telegram's 👀 receipts) is retired here, and a turn that
221
238
  // settled with nothing to say still has to settle.
@@ -237,9 +254,69 @@ export class Router {
237
254
  unsubscribe,
238
255
  });
239
256
  }
257
+ /** Sessions whose queue is being promoted right now. `clearQueue` and the
258
+ * prompt that follows it both emit queue-state of their own, so without
259
+ * this the handler would re-enter on its own effects. */
260
+ promoting = new Set();
261
+ /**
262
+ * Turn a stranded queue into the turn it was waiting for. Not routed through
263
+ * `dispatch`: the text was prefixed when it was first dispatched
264
+ * (identity.ts), and sending it back through would head it a second time.
265
+ *
266
+ * Only ever reached from a queue-state event, never from a turn ending: Pi
267
+ * leaves the queue alone on `abort()`, so promoting on idle would make /stop
268
+ * start the very turn it was asked to stop. Recovering *those* messages stays
269
+ * the web's recall route, which hands them back to the composer.
270
+ */
271
+ promoteQueued(session, key) {
272
+ if (session.state !== "idle" || this.promoting.has(session.id))
273
+ return;
274
+ this.promoting.add(session.id);
275
+ void (async () => {
276
+ try {
277
+ // Re-read: a turn may have started since the event, and it will drain
278
+ // the queue itself — clearing it here would take the messages out of it.
279
+ if (session.state !== "idle")
280
+ return;
281
+ const { steering, followUp } = await session.clearQueue();
282
+ const text = [...steering, ...followUp].join("\n").trim();
283
+ if (!text)
284
+ return;
285
+ // A drain is "no new turns", and this would be one. Told to the
286
+ // conversation rather than dropped, because the message is now out of
287
+ // the queue and nothing else would ever mention it (§5b).
288
+ if (this.draining) {
289
+ this.report(session.id, key, `queued message not taken — Pier is restarting; send it again: ${truncate(text)}`);
290
+ return;
291
+ }
292
+ log.info(`promoting ${String(steering.length + followUp.length)} queued message(s) → session ${session.id}`);
293
+ await session.prompt(text);
294
+ }
295
+ catch (err) {
296
+ this.report(session.id, key, `delivering the queued messages failed: ${String(err)}`);
297
+ }
298
+ finally {
299
+ this.promoting.delete(session.id);
300
+ }
301
+ })();
302
+ }
240
303
  async abort(sessionId) {
241
304
  await this.bySession.get(sessionId)?.session.abort();
242
305
  }
306
+ /**
307
+ * Drop what this session was told about who is speaking, so the next message
308
+ * carries a full header again.
309
+ *
310
+ * For the surfaces that take a prefixed message back *out* of the context it
311
+ * was counted into — a recalled queue, a rewound turn. The tracker's whole
312
+ * job is "the model has already been told" (identity.ts), and a header that
313
+ * never reached the model, or reached it and was then rewound away, makes
314
+ * every later message from that speaker unattributed in a group chat. One
315
+ * redundant header is the same price eviction already pays.
316
+ */
317
+ forgetSender(sessionId) {
318
+ this.senders.forget(sessionId);
319
+ }
243
320
  /** Refuse new work from every surface; in-flight turns keep running. */
244
321
  beginDrain() {
245
322
  this.draining = true;
@@ -288,16 +365,14 @@ export class Router {
288
365
  let session = this.byKey.get(keyOf(key));
289
366
  // Web and task conversation ids are session ids. Reuse an attached
290
367
  // instance so two surfaces never open the same Pi transcript twice.
291
- if (!session && (key.channelId === "web" || key.channelId === "task")) {
368
+ if (!session && isAlias(key)) {
292
369
  session = this.bySession.get(key.conversationId)?.session;
293
370
  if (session)
294
371
  this.byKey.set(keyOf(key), session);
295
372
  }
296
373
  if (!session) {
297
374
  // Aliases share one lock: web:<id> and task:<id> must not each open one.
298
- const lock = key.channelId === "web" || key.channelId === "task"
299
- ? `session:${key.conversationId}`
300
- : keyOf(key);
375
+ const lock = isAlias(key) ? `session:${key.conversationId}` : keyOf(key);
301
376
  const inflight = this.opening.get(lock);
302
377
  // A second caller rides the first one's resolve — which attaches before
303
378
  // this continuation runs, having awaited it first — and registers its own
@@ -305,7 +380,7 @@ export class Router {
305
380
  if (inflight) {
306
381
  session = await inflight;
307
382
  this.byKey.set(keyOf(key), session);
308
- return this.reached(session);
383
+ return this.reached(session, key);
309
384
  }
310
385
  try {
311
386
  // Inside the try: a resolver that throws synchronously is the same
@@ -323,7 +398,7 @@ export class Router {
323
398
  }
324
399
  this.attach(key, session);
325
400
  }
326
- return this.reached(session);
401
+ return this.reached(session, key);
327
402
  }
328
403
  /** A session that would not open has no event stream of its own to report on
329
404
  * — unless its id is what we were asked for, which is what a web or task key
@@ -343,11 +418,21 @@ export class Router {
343
418
  .catch((e) => log.error(`could not report it to ${key.channelId}`, e));
344
419
  }
345
420
  /** Reached for, so not idle — every surface that uses a session comes
346
- * through `ensure`, including the ones that only read it. */
347
- reached(session) {
421
+ * through `ensure`, including the ones that only read it.
422
+ *
423
+ * Also where a session learns which of its two aliases is current: a task
424
+ * callback (tasks/outbox.ts) opens a workbench session under `task:<id>`
425
+ * whenever nothing had it attached, and the key from that first attach used
426
+ * to stand forever — so the workbench's own next turn was still "a task",
427
+ * and the notification for it (web/push.ts) was never sent. A chat key is
428
+ * never overwritten: that one is also where turn-ends are delivered. */
429
+ reached(session, key) {
348
430
  const attached = this.bySession.get(session.id);
349
- if (attached)
350
- attached.activeAt = Date.now();
431
+ if (!attached)
432
+ return session;
433
+ attached.activeAt = Date.now();
434
+ if (isAlias(key) && isAlias(attached.key))
435
+ attached.key = key;
351
436
  return session;
352
437
  }
353
438
  async dispatch(msg) {
@@ -368,6 +453,9 @@ export class Router {
368
453
  // Turn outcomes flow through the event stream; a rejected call surfaces
369
454
  // there too, never as a thrown exception across the seam.
370
455
  session[action](prompt).catch((err) => {
456
+ // The header was counted as delivered a line above; this message never
457
+ // arrived, so the next one from this speaker must carry it again.
458
+ this.senders.forget(session.id);
371
459
  this.report(session.id, msg.key, String(err));
372
460
  });
373
461
  return { sessionId: session.id };
package/dist/db.js CHANGED
@@ -166,6 +166,93 @@ const MIGRATIONS = [
166
166
  private_key TEXT NOT NULL,
167
167
  created_at INTEGER NOT NULL
168
168
  );
169
+ `,
170
+ // 6 — Projects keeps the order the workbench was put in, by hand.
171
+ `
172
+ -- Manual order, both nullable: a row nobody has dragged sorts on top of the
173
+ -- list it belongs to, so a fresh database needs no backfill. sort places a
174
+ -- session inside its project; project_sort places the project, carried on
175
+ -- every one of its rows because a project is a cwd, not a table.
176
+ ALTER TABLE session_state ADD COLUMN sort INTEGER;
177
+ ALTER TABLE session_state ADD COLUMN project_sort INTEGER;
178
+ `,
179
+ // 7 — the session listing, so a transcript is read once (agent/listing.ts).
180
+ `
181
+ -- One row per session file. (size, mtime) is what makes the row usable
182
+ -- without opening the file; parsed_bytes is where reading resumes when it
183
+ -- grew, and is always a line boundary. Derived from disk and disposable: a
184
+ -- deleted row costs one re-read, never a fact.
185
+ CREATE TABLE session_index (
186
+ path TEXT PRIMARY KEY,
187
+ id TEXT NOT NULL,
188
+ cwd TEXT NOT NULL,
189
+ created_at INTEGER NOT NULL,
190
+ name TEXT,
191
+ first_message TEXT,
192
+ size INTEGER NOT NULL,
193
+ mtime INTEGER NOT NULL,
194
+ parsed_bytes INTEGER NOT NULL
195
+ );
196
+ `,
197
+ // 8 — Projects holds a working set: what is warm, plus what is kept.
198
+ `
199
+ -- Membership was permanent, so every throwaway session stayed in the rail
200
+ -- until someone removed it by hand. last_active is the lease: the end of a
201
+ -- turn renews it, and web/session-state.ts stops listing a row that ran out.
202
+ -- kept opts one row out of expiry entirely — what the pin control now means.
203
+ ALTER TABLE session_state ADD COLUMN kept INTEGER NOT NULL DEFAULT 0;
204
+ ALTER TABLE session_state ADD COLUMN last_active INTEGER;
205
+ -- Left NULL on purpose: the honest value is when the transcript was last
206
+ -- written, which only a listing knows. web/server.ts pays one for a database
207
+ -- carrying rows without it, the same gate the pin backfill already uses, so
208
+ -- the first rail after an upgrade is dated by use and not by creation.
209
+ `,
210
+ // 9 — the summary a transcript already carries is read, not mirrored.
211
+ `
212
+ -- Dropped rather than left unread: a column nobody writes still answers when
213
+ -- somebody selects it, and the next reader has no way to tell a stale title
214
+ -- from a current one. The pre-migration backup beside the database is the
215
+ -- way back, not a row of fossils. cwd stays — it is the key a project's
216
+ -- manual place is stamped on, and it never changes for a session.
217
+ ALTER TABLE session_state DROP COLUMN title;
218
+ ALTER TABLE session_state DROP COLUMN created_at;
219
+ ALTER TABLE session_state DROP COLUMN last_active;
220
+ `,
221
+ // 10 — taking a session into Projects is itself an act, and it is dated.
222
+ `
223
+ -- When a hand last put this session in Projects (pin, or a keep toggle).
224
+ -- Not the mirror migration 9 removed: last_active was a copy of a fact the
225
+ -- transcript owns, while this one exists nowhere else — pinning a cold
226
+ -- session back is a statement that it is warm again, and without a record of
227
+ -- *when* it was made the row is dropped by the same read that drew it.
228
+ -- NULL for every row that predates this: never pinned within a lease.
229
+ ALTER TABLE session_state ADD COLUMN pinned_at INTEGER;
230
+ `,
231
+ // 11 — Projects holds what a hand put there, for as long as the hand says.
232
+ `
233
+ -- The lease is gone, so both of its columns are. It expired nothing: a row
234
+ -- it dropped kept its transcript, its place and its ownership, and one more
235
+ -- turn brought it back — so what it actually did was hide rows nobody asked
236
+ -- it to hide, and kept existed only to opt out of that. On the instance this
237
+ -- was decided on, the lease had never dropped a row: 20 pinned sessions,
238
+ -- none past seven days, one kept. Removing a row from Projects is the ✓ on
239
+ -- the row, and it stays the only way out.
240
+ ALTER TABLE session_state DROP COLUMN kept;
241
+ ALTER TABLE session_state DROP COLUMN pinned_at;
242
+ `,
243
+ // 12 — one row is how two processes take turns (src/tools.ts).
244
+ `
245
+ -- The tools sync, held across processes: the token says who holds it, the
246
+ -- heartbeat says they are still alive. Both processes already open this
247
+ -- database, and BEGIN IMMEDIATE is real mutual exclusion — a lock file with
248
+ -- a pid in it is neither, which is what this replaces. One row, because
249
+ -- there is one thing to serialize; the second lock can bring its own table
250
+ -- and its own reason for existing.
251
+ CREATE TABLE tools_sync_lock (
252
+ id INTEGER PRIMARY KEY CHECK (id = 1),
253
+ token TEXT NOT NULL,
254
+ heartbeat_at INTEGER NOT NULL
255
+ );
169
256
  `,
170
257
  ];
171
258
  let shared;
@@ -0,0 +1,37 @@
1
+ // The extensions Pier ships with — the list, and nothing else.
2
+ //
3
+ // An extension is Pi-shaped by construction (it takes an ExtensionAPI), so
4
+ // this area is the second one allowed to import the Pi SDK. Nothing outside
5
+ // agent/ imports it: the Console sees names and summaries, which agent/ hands
6
+ // over as plain data through the ConfigStore seam.
7
+ //
8
+ // Bundled rather than dropped in <agentDir>/extensions because a copy on disk
9
+ // has an owner problem — an update either clobbers the user's edits or skips
10
+ // them forever. These ship inside the package, load as inline factories, and
11
+ // stand down when a copy on disk already registers the same tools.
12
+ import web from "./web/index.js";
13
+ export const BUNDLED = [
14
+ {
15
+ name: "web",
16
+ summary: "The public web through the provider's own hosted web tools — no extra " +
17
+ "key, no second service, no new dependency.",
18
+ tools: [
19
+ { name: "web_search", needs: "an authenticated Anthropic or OpenAI model" },
20
+ { name: "web_fetch", needs: "an authenticated Anthropic model — OpenAI hosts no fetch tool" },
21
+ ],
22
+ factory: web,
23
+ },
24
+ ];
25
+ /** The catalog a surface may show: no Pi types, nothing it cannot render.
26
+ * Half of one list; src/tools.ts has the other. */
27
+ export const bundledInfo = (enabled) => BUNDLED.map(({ name, summary, tools }) => ({
28
+ source: "bundled",
29
+ kind: "extension",
30
+ name,
31
+ summary,
32
+ adds: tools,
33
+ enabled: enabled.includes(name),
34
+ }));
35
+ /** The enabled ones as Pi inline extensions; unknown names are not ours. */
36
+ export const inlineExtensions = (enabled) => BUNDLED.filter((ext) => enabled.includes(ext.name))
37
+ .map(({ name, factory }) => ({ name, factory }));
@@ -0,0 +1,118 @@
1
+ import { postJson } from "./http.js";
2
+ import { isObject } from "./json.js";
3
+ const tokensFrom = (value) => {
4
+ const usage = isObject(value) ? value : {};
5
+ const count = (field) => (typeof field === "number" ? field : 0);
6
+ return { input: count(usage.input_tokens), output: count(usage.output_tokens) };
7
+ };
8
+ const NATIVE_TOOL_TYPES = {
9
+ web_search: "web_search_20250305",
10
+ web_fetch: "web_fetch_20250910",
11
+ };
12
+ const findCode = (value) => {
13
+ if (Array.isArray(value)) {
14
+ for (const item of value) {
15
+ const code = findCode(item);
16
+ if (code)
17
+ return code;
18
+ }
19
+ return undefined;
20
+ }
21
+ if (!isObject(value))
22
+ return undefined;
23
+ return typeof value.error_code === "string" ? value.error_code : findCode(value.content);
24
+ };
25
+ /**
26
+ * Every server-tool failure in the turn. A list, not the first one, and not a
27
+ * throw: these arrive per invocation — the third search can fail while the
28
+ * first two are in the transcript and the briefing is written from them. This
29
+ * used to abort the whole call on any of them, which threw away a good answer
30
+ * over `max_uses_exceeded`, a code we provoke ourselves by budgeting the
31
+ * searches the prompt then asks for.
32
+ */
33
+ function toolErrors(content) {
34
+ const codes = [];
35
+ for (const block of content) {
36
+ if (!isObject(block) || typeof block.type !== "string")
37
+ continue;
38
+ if (!block.type.endsWith("_tool_result"))
39
+ continue;
40
+ const code = findCode(block.content);
41
+ if (code)
42
+ codes.push(`${block.type}: ${code}`);
43
+ }
44
+ return [...new Set(codes)];
45
+ }
46
+ /** Whether anything usable came back at all: prose the model wrote, or a tool
47
+ * result that is not itself an error. This is what decides failure now. */
48
+ function hasContent(content) {
49
+ return content.some((block) => {
50
+ if (!isObject(block) || typeof block.type !== "string")
51
+ return false;
52
+ if (block.type === "text")
53
+ return typeof block.text === "string" && block.text.trim() !== "";
54
+ if (!block.type.endsWith("_tool_result"))
55
+ return false;
56
+ return findCode(block.content) === undefined;
57
+ });
58
+ }
59
+ export async function callNativeTool(request, name, prompt, options, signal,
60
+ /** Progress for the surface the call came from: a hosted search is tens of
61
+ * seconds of nothing otherwise (§5b). */
62
+ note) {
63
+ const tool = {
64
+ type: NATIVE_TOOL_TYPES[name],
65
+ name,
66
+ max_uses: options.maxUses,
67
+ };
68
+ if (options.allowedDomains?.length)
69
+ tool.allowed_domains = options.allowedDomains;
70
+ if (options.blockedDomains?.length)
71
+ tool.blocked_domains = options.blockedDomains;
72
+ if (name === "web_fetch") {
73
+ tool.citations = { enabled: true };
74
+ tool.max_content_tokens = options.maxContentTokens ?? 20_000;
75
+ }
76
+ const messages = [{ role: "user", content: prompt }];
77
+ const accumulated = [];
78
+ const spent = { input: 0, output: 0 };
79
+ for (let continuation = 0; continuation < 3; continuation++) {
80
+ if (continuation)
81
+ note?.(`still working — round ${continuation + 1}`);
82
+ const data = await postJson("Anthropic", request.url, request.headers, {
83
+ model: request.model,
84
+ max_tokens: request.maxTokens,
85
+ messages,
86
+ tools: [tool],
87
+ ...(continuation === 0 ? { tool_choice: { type: "tool", name } } : {}),
88
+ }, signal);
89
+ if (!Array.isArray(data.content)) {
90
+ throw new Error("Anthropic returned an invalid Messages response");
91
+ }
92
+ accumulated.push(...data.content);
93
+ const round = tokensFrom(data.usage);
94
+ spent.input += round.input;
95
+ spent.output += round.output;
96
+ if (data.stop_reason === "pause_turn") {
97
+ messages.push({ role: "assistant", content: data.content });
98
+ continue;
99
+ }
100
+ const errors = toolErrors(accumulated);
101
+ const used = accumulated.some((block) => isObject(block) &&
102
+ (block.type === "server_tool_use" || block.type === `${name}_tool_result`));
103
+ // Only now, with the whole turn in hand, is "this failed" answerable.
104
+ if (!used)
105
+ throw new Error(errors.join("; ") || `Claude did not invoke ${name}`);
106
+ if (!hasContent(accumulated)) {
107
+ throw new Error(errors.join("; ") || `${name} returned nothing usable`);
108
+ }
109
+ return {
110
+ content: accumulated,
111
+ model: request.model,
112
+ ...(typeof data.stop_reason === "string" ? { stopReason: data.stop_reason } : {}),
113
+ usage: spent,
114
+ errors,
115
+ };
116
+ }
117
+ throw new Error(`${name} exceeded the continuation limit`);
118
+ }
@@ -0,0 +1,62 @@
1
+ // A fetched page kept whole on disk, so the digest in the transcript is never
2
+ // the only copy: the model gets the distillate, the path gets the document.
3
+ import { createHash, randomUUID } from "node:crypto";
4
+ import { mkdir, readdir, rename, rm, stat, writeFile } from "node:fs/promises";
5
+ import { join } from "node:path";
6
+ import { logger } from "../../log.js";
7
+ import { pierPath } from "../../paths.js";
8
+ const log = logger("web");
9
+ const ARTIFACT_DIR = process.env.PIER_WEB_ARTIFACT_DIR?.trim() ||
10
+ pierPath("artifacts", "web");
11
+ const RETENTION_DAYS = Number(process.env.PIER_WEB_ARTIFACT_DAYS) || 30;
12
+ /** Retention is measured in days, so sweeping more than hourly is a directory
13
+ * walk per fetch buying nothing. Per process; a restart sweeps again. */
14
+ const PRUNE_EVERY_MS = 3_600_000;
15
+ let prunedAt = 0;
16
+ /** Drops expired artifacts and the temp files a crashed run left behind. */
17
+ async function prune() {
18
+ prunedAt = Date.now();
19
+ const cutoff = Date.now() - RETENTION_DAYS * 86_400_000;
20
+ const names = await readdir(ARTIFACT_DIR);
21
+ await Promise.all(names.map(async (name) => {
22
+ const file = join(ARTIFACT_DIR, name);
23
+ const info = await stat(file).catch(() => undefined);
24
+ if (info?.isFile() && info.mtimeMs < cutoff)
25
+ await rm(file, { force: true });
26
+ }));
27
+ }
28
+ function displayUrl(url) {
29
+ const redacted = new URL(url);
30
+ for (const key of redacted.searchParams.keys()) {
31
+ if (/token|key|secret|signature|credential|auth/i.test(key)) {
32
+ redacted.searchParams.set(key, "REDACTED");
33
+ }
34
+ }
35
+ return redacted.toString();
36
+ }
37
+ const digest = (value) => createHash("sha256").update(value).digest("hex").slice(0, 12);
38
+ export async function saveArtifact(url, text, retrievedAt) {
39
+ await mkdir(ARTIFACT_DIR, { recursive: true, mode: 0o700 });
40
+ const host = url.hostname.replace(/[^a-zA-Z0-9.-]+/g, "-").slice(0, 80) || "page";
41
+ // The URL alone is not the file's identity: a page fetched again is a
42
+ // different document, and keying on the URL overwrote the copy an older
43
+ // transcript's `artifactPath` still points at — the one promise this file
44
+ // makes. Content decides, so a refetch that changed writes a new file and one
45
+ // that did not costs nothing.
46
+ const path = join(ARTIFACT_DIR, `${host}-${digest(url.toString())}-${digest(text)}.md`);
47
+ const temporary = `${path}.${randomUUID()}.tmp`;
48
+ const header = [
49
+ `Source: ${displayUrl(url)}`,
50
+ `Retrieved: ${retrievedAt || new Date().toISOString()}`,
51
+ "",
52
+ ].join("\n");
53
+ await writeFile(temporary, `${header}${text}`, { encoding: "utf8", mode: 0o600 });
54
+ await rename(temporary, path);
55
+ // Housekeeping must not fail a successful fetch, but it must not vanish either.
56
+ if (Date.now() - prunedAt >= PRUNE_EVERY_MS) {
57
+ await prune().catch((error) => {
58
+ log.warn("artifact prune failed", error);
59
+ });
60
+ }
61
+ return path;
62
+ }
@@ -0,0 +1,130 @@
1
+ import { isObject } from "./json.js";
2
+ import { languageLabel } from "./language.js";
3
+ /**
4
+ * The one reader of a cited page, wherever it turns up: an Anthropic search
5
+ * result, a citation on a text block, a fetch result, an OpenAI action source.
6
+ * All four spell it `{url, title?}` (OpenAI sometimes as a bare string), all
7
+ * four had their own copy of this, and they disagreed about the fallback
8
+ * title. Keyed by url; the first real title wins over a url used as one.
9
+ */
10
+ export function putSource(into, value) {
11
+ const url = typeof value === "string"
12
+ ? value
13
+ : isObject(value) && typeof value.url === "string"
14
+ ? value.url
15
+ : undefined;
16
+ if (!url)
17
+ return;
18
+ const source = isObject(value) ? value : {};
19
+ const titled = typeof source.title === "string" && source.title;
20
+ const existing = into.get(url);
21
+ if (existing && (!titled || existing.title !== existing.url))
22
+ return;
23
+ into.set(url, {
24
+ title: titled || url,
25
+ url,
26
+ ...(typeof source.page_age === "string" ? { pageAge: source.page_age } : {}),
27
+ });
28
+ }
29
+ /** Everything the answer cited: what a fetch was allowed to say it read. */
30
+ export function sourcesFrom(content) {
31
+ const sources = new Map();
32
+ const add = (value) => putSource(sources, value);
33
+ for (const block of content) {
34
+ if (!isObject(block))
35
+ continue;
36
+ if (Array.isArray(block.citations))
37
+ block.citations.forEach(add);
38
+ if (block.type === "web_search_tool_result" && Array.isArray(block.content)) {
39
+ block.content.forEach(add);
40
+ }
41
+ if (block.type === "web_fetch_tool_result" && isObject(block.content))
42
+ add(block.content);
43
+ }
44
+ return [...sources.values()].map(({ title, url }) => ({ title, url }));
45
+ }
46
+ /** Only what the search itself returned, in the order it ranked them. */
47
+ export function searchResultsFrom(content) {
48
+ const results = new Map();
49
+ for (const block of content) {
50
+ if (!isObject(block) || block.type !== "web_search_tool_result")
51
+ continue;
52
+ if (!Array.isArray(block.content))
53
+ continue;
54
+ for (const item of block.content)
55
+ putSource(results, item);
56
+ }
57
+ return [...results.values()];
58
+ }
59
+ export function searchQueriesFrom(content) {
60
+ const queries = [];
61
+ for (const block of content) {
62
+ if (!isObject(block) || block.type !== "server_tool_use" || block.name !== "web_search") {
63
+ continue;
64
+ }
65
+ if (!isObject(block.input) || typeof block.input.query !== "string")
66
+ continue;
67
+ queries.push({ query: block.input.query, language: languageLabel(block.input.query) });
68
+ }
69
+ return queries;
70
+ }
71
+ export function searchOutcomeFrom(content, model, backend, spent) {
72
+ return {
73
+ text: textFrom(content),
74
+ queries: searchQueriesFrom(content),
75
+ results: searchResultsFrom(content),
76
+ model,
77
+ backend,
78
+ truncated: spent.stopReason === "max_tokens",
79
+ usage: spent.usage,
80
+ errors: spent.errors,
81
+ };
82
+ }
83
+ export function textFrom(content) {
84
+ return content
85
+ .filter((block) => isObject(block) && block.type === "text" && typeof block.text === "string")
86
+ .map((block) => block.text)
87
+ .join("\n\n")
88
+ .trim();
89
+ }
90
+ export function fetchedDocument(content) {
91
+ for (const block of content) {
92
+ if (!isObject(block) || block.type !== "web_fetch_tool_result")
93
+ continue;
94
+ if (!isObject(block.content))
95
+ continue;
96
+ const result = block.content;
97
+ if (result.type !== "web_fetch_result" || !isObject(result.content))
98
+ continue;
99
+ const source = isObject(result.content.source) ? result.content.source : undefined;
100
+ return {
101
+ url: typeof result.url === "string" ? result.url : undefined,
102
+ retrievedAt: typeof result.retrieved_at === "string" ? result.retrieved_at : undefined,
103
+ text: source?.type === "text" && typeof source.data === "string"
104
+ ? source.data
105
+ : undefined,
106
+ };
107
+ }
108
+ return {};
109
+ }
110
+ export function appendSources(text, sources) {
111
+ if (!sources.length)
112
+ return text;
113
+ return `${text}\n\nSources:\n${sources.map(({ title, url }) => `- [${title}](${url})`).join("\n")}`;
114
+ }
115
+ export function formatSearchResult(briefing, results, maxResults, queries) {
116
+ const queryList = queries
117
+ .map(({ query, language }, index) => `${index + 1}. [${language}] ${query}`)
118
+ .join("\n");
119
+ const listing = results
120
+ .slice(0, maxResults)
121
+ .map((result, index) => `${index + 1}. [${result.title}](${result.url})${result.pageAge ? ` — ${result.pageAge}` : ""}`)
122
+ .join("\n");
123
+ return [
124
+ briefing,
125
+ queryList ? `Queries used:\n${queryList}` : "",
126
+ listing ? `Search results:\n${listing}` : "",
127
+ ]
128
+ .filter(Boolean)
129
+ .join("\n\n");
130
+ }