@pmndre/pine 0.22.0 → 0.33.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. package/README.md +2 -1
  2. package/changes.json +265 -0
  3. package/docs/SKILL.md +31 -7
  4. package/docs/references/agents.md +245 -21
  5. package/docs/references/coding.md +1 -1
  6. package/docs/references/computer.md +1 -1
  7. package/docs/references/configuration.md +14 -5
  8. package/docs/references/mcp.md +9 -0
  9. package/docs/references/operations.md +20 -9
  10. package/docs/references/overview.md +3 -1
  11. package/docs/references/security.md +83 -24
  12. package/docs/references/setup.md +55 -9
  13. package/docs/references/source-types.md +1418 -0
  14. package/docs/references/sources.md +333 -19
  15. package/docs/references/stack.md +1 -0
  16. package/docs/references/tui.md +86 -12
  17. package/examples/home/AGENTS.md +4 -0
  18. package/examples/home/TOOLS.md +5 -0
  19. package/examples/home/agents/main/workspace/AGENTS.md +1 -1
  20. package/examples/home/agents/main/workspace/TOOLS.md +5 -0
  21. package/mcp/browser.d.ts +6 -0
  22. package/mcp/browser.js +19 -0
  23. package/mcp/render.d.ts +1 -0
  24. package/mcp/render.js +237 -0
  25. package/package.json +3 -1
  26. package/src/actions.d.ts +6 -1
  27. package/src/actions.js +17 -4
  28. package/src/admin.d.ts +4 -0
  29. package/src/admin.js +25 -6
  30. package/src/agents.d.ts +167 -2
  31. package/src/agents.js +912 -45
  32. package/src/catalogue.d.ts +60 -0
  33. package/src/catalogue.js +791 -0
  34. package/src/cli.js +10 -7
  35. package/src/config.d.ts +53 -4
  36. package/src/config.js +30 -5
  37. package/src/goals.d.ts +1 -1
  38. package/src/goals.js +14 -1
  39. package/src/manual.js +24 -2
  40. package/src/mime.d.ts +78 -0
  41. package/src/mime.js +342 -0
  42. package/src/net.d.ts +11 -0
  43. package/src/net.js +29 -2
  44. package/src/pine.js +10 -1
  45. package/src/router.js +3 -3
  46. package/src/sandbox-linux.d.ts +3 -1
  47. package/src/sandbox-linux.js +5 -0
  48. package/src/security.d.ts +22 -6
  49. package/src/security.js +44 -13
  50. package/src/slots.d.ts +106 -0
  51. package/src/slots.js +250 -0
  52. package/src/source-types.d.ts +20 -0
  53. package/src/sources/calendar.d.ts +24 -0
  54. package/src/sources/calendar.js +265 -0
  55. package/src/sources/command.js +109 -4
  56. package/src/sources/discord.d.ts +4 -0
  57. package/src/sources/discord.js +298 -0
  58. package/src/sources/feed.d.ts +19 -0
  59. package/src/sources/feed.js +257 -0
  60. package/src/sources/http.js +154 -103
  61. package/src/sources/mail.d.ts +51 -0
  62. package/src/sources/mail.js +1013 -0
  63. package/src/sources/matrix.d.ts +4 -0
  64. package/src/sources/matrix.js +232 -0
  65. package/src/sources/modbus.d.ts +15 -0
  66. package/src/sources/modbus.js +240 -0
  67. package/src/sources/onvif.d.ts +35 -0
  68. package/src/sources/onvif.js +326 -0
  69. package/src/sources/presence.d.ts +80 -0
  70. package/src/sources/presence.js +480 -0
  71. package/src/sources/serial.d.ts +21 -0
  72. package/src/sources/serial.js +220 -0
  73. package/src/sources/slack.d.ts +16 -0
  74. package/src/sources/slack.js +1062 -0
  75. package/src/sources/smtp.d.ts +4 -0
  76. package/src/sources/smtp.js +360 -0
  77. package/src/sources/stream.d.ts +68 -0
  78. package/src/sources/stream.js +534 -0
  79. package/src/sources/syslog.d.ts +32 -0
  80. package/src/sources/syslog.js +243 -0
  81. package/src/sources/system.d.ts +14 -0
  82. package/src/sources/system.js +236 -0
  83. package/src/sources/system.swift +102 -0
  84. package/src/sources/telegram.js +34 -1
  85. package/src/sources.d.ts +41 -1
  86. package/src/sources.js +147 -24
  87. package/src/tools.d.ts +1 -1
  88. package/src/tools.js +304 -23
  89. package/src/tui/app.d.ts +35 -2
  90. package/src/tui/app.js +306 -43
  91. package/src/tui/levels.d.ts +68 -0
  92. package/src/tui/levels.js +273 -0
  93. package/src/tui/picture.d.ts +17 -1
  94. package/src/tui/picture.js +72 -5
  95. package/src/types.d.ts +3 -0
  96. package/src/update.d.ts +9 -1
  97. package/src/update.js +75 -3
  98. package/src/watching.d.ts +156 -0
  99. package/src/watching.js +609 -4
package/src/security.js CHANGED
@@ -3,7 +3,7 @@ import crypto from "node:crypto";
3
3
  import fs from "node:fs";
4
4
  import os from "node:os";
5
5
  import path from "node:path";
6
- import { agentSkillsDir, conversationsDir, globalSkillsDir, mainAgentId, PINE_ROOT, unsandboxed, userFile, WORKSPACE_FILES } from "./config.js";
6
+ import { agentSkillsDir, conversationsDir, globalSkillsDir, mainAgentId, PINE_ROOT, sharedFiles, unsandboxed, WORKSPACE_FILES } from "./config.js";
7
7
  import { bwrapArgs, Guard, helperArgs, jailed, protectedNow, SESSIONS } from "./sandbox-linux.js";
8
8
  import { truncate } from "./util.js";
9
9
  /** realpath of the deepest existing ancestor, plus the part that does not exist yet. Defeats symlink escapes. */
@@ -34,9 +34,9 @@ export function rootsFor(cfg, agent) {
34
34
  const home = realish(cfg.home);
35
35
  const tmp = agent.sandbox.sharedTmp ? [...new Set([realish(os.tmpdir()), "/private/tmp"])] : [];
36
36
  // pine's manual is readable by every agent, so it can explain how its owner changes the setup.
37
- // The one file of pine's home that an agent may touch: what is known about the owner, and only for the main agent.
38
- const user = path.join(home, path.basename(userFile(cfg)));
39
- const keepsUser = agent.id === mainAgentId(cfg) ? [user] : [];
37
+ // The files of pine's home that an agent may touch: what holds for all agents, and only for the main agent.
38
+ const shared = sharedFiles(cfg).map((f) => path.join(home, path.basename(f)));
39
+ const keepsUser = agent.id === mainAgentId(cfg) ? shared : [];
40
40
  // Skills are read, never written, by the agent they are for: its own, its conversations', and those for all agents.
41
41
  const skills = [agentSkillsDir(agent), conversationsDir(agent), globalSkillsDir(cfg), ...cfg.skills, ...agent.skills].map(realish);
42
42
  const ownedRead = [...new Set([workspace, realish(sandboxHome), realish(path.join(PINE_ROOT, "docs")), ...skills, ...agent.sandbox.read.map(realish), ...keepsUser])];
@@ -50,14 +50,26 @@ export function rootsFor(cfg, agent) {
50
50
  write: [...ownedWrite, ...tmp],
51
51
  ownedRead,
52
52
  ownedWrite,
53
- userFile: user,
53
+ shared,
54
54
  secrets: [path.join(home, ".env"), path.join(home, "auth.json"), path.join(home, "models.json"), path.join(home, "pine.json"), realish(cfg.dataDir)],
55
55
  others: Object.values(cfg.agents)
56
56
  .filter((a) => a.id !== agent.id)
57
57
  .map((a) => realish(a.workspace))
58
58
  .filter((w) => ![...ownedRead, ...ownedWrite].some((own) => isInside(own, w) || isInside(w, own))),
59
+ sources: [],
59
60
  };
60
61
  }
62
+ /**
63
+ * Which of the folders that an agent's sources keep received files in the agent may read. A
64
+ * source of pine's keeps them in its own folder under `data/sources`, and that is the one part
65
+ * of pine's data that is opened. A folder elsewhere (a watched one) counts when it lies outside
66
+ * pine's home and is not another agent's: a subscription opens nothing that is closed for a reason.
67
+ */
68
+ export function received(cfg, roots, dirs) {
69
+ const kept = realish(path.join(cfg.dataDir, "sources"));
70
+ const about = (root, p) => isInside(root, p) || isInside(p, root);
71
+ return [...new Set(dirs.map(realish))].filter((d) => (isInside(kept, d) && d !== kept) || (!about(roots.home, d) && !roots.secrets.some((s) => about(s, d)) && !roots.others.some((o) => about(o, d))));
72
+ }
61
73
  export const isOpen = (v) => v === "open" || v === "open-restricted";
62
74
  /** pine's own program files. Never writable by an agent, in any mode. */
63
75
  const PINE_CODE = realish(PINE_ROOT);
@@ -75,8 +87,12 @@ export function checkPath(roots, kind, given, cwd, variant) {
75
87
  const target = tilde(given);
76
88
  const p = realish(path.isAbsolute(target) ? target : path.join(cwd, target));
77
89
  const restricted = variant === true || variant === "restricted" || variant === "open-restricted";
90
+ // What a source received for this agent is read, wherever it lies. It is not written: pine keeps and deletes it.
91
+ const fromSource = roots.sources.some((s) => isInside(s, p));
92
+ if (fromSource && kind === "read")
93
+ return undefined;
78
94
  if (roots.secrets.some((s) => isInside(s, p)))
79
- return `${target} is not accessible to agents`;
95
+ return fromSource ? `${target} is a file an event source received. You can read it, not change it: copy it into your workspace (${roots.workspace}) and work on the copy` : `${target} is not accessible to agents`;
80
96
  if (isOpen(variant)) {
81
97
  // Said as what it is: a model that reads "pine's own configuration" does not think of the tools that are for this.
82
98
  if (roots.others.some((o) => isInside(o, p)))
@@ -128,7 +144,7 @@ export function outsidePath(roots, kind, given, cwd, variant) {
128
144
  export function shellRefuses(roots, kind, p) {
129
145
  if (kind === "write")
130
146
  return !roots.write.some((r) => isInside(r, p));
131
- return isInside(realish(os.homedir()), p) && !roots.read.some((r) => isInside(r, p));
147
+ return isInside(realish(os.homedir()), p) && ![...roots.read, ...roots.sources].some((r) => isInside(r, p));
132
148
  }
133
149
  /**
134
150
  * What to ask the owner for when a command needs `p`. On Linux a path that does not exist yet
@@ -169,13 +185,13 @@ export function isInstructionPath(workspace, p) {
169
185
  const rel = path.relative(workspace, p);
170
186
  return INSTRUCTION_FILES.includes(path.basename(p)) || INSTRUCTION_DIRS.some((d) => rel === d || rel.startsWith(d + path.sep));
171
187
  }
172
- /** Whether a path is one of the files that are read into the agent's prompt from its workspace, or the owner's file in pine's home. */
188
+ /** Whether a path is one of the files that are read into the agent's prompt from its workspace, or one of those for all agents in pine's home. */
173
189
  export function isOwnFile(roots, p) {
174
- return p === roots.userFile || (path.dirname(p) === roots.workspace && WORKSPACE_FILES.includes(path.basename(p)));
190
+ return roots.shared.includes(p) || (path.dirname(p) === roots.workspace && WORKSPACE_FILES.includes(path.basename(p)));
175
191
  }
176
- /** The files an agent keeps itself: its memory and instructions at the top of its workspace, and for the main agent what is known about the owner. */
192
+ /** The files an agent keeps itself: its memory and instructions at the top of its workspace, and for the main agent what holds for all agents. */
177
193
  export function ownFiles(roots) {
178
- return [...WORKSPACE_FILES.map((f) => path.join(roots.workspace, f)), ...(roots.ownedWrite.includes(roots.userFile) ? [roots.userFile] : [])];
194
+ return [...WORKSPACE_FILES.map((f) => path.join(roots.workspace, f)), ...roots.shared.filter((f) => roots.ownedWrite.includes(f))];
179
195
  }
180
196
  /** The own file a path given to a file tool means, if it means one. */
181
197
  export function ownFileOf(roots, given, cwd) {
@@ -199,6 +215,13 @@ const GIT_RUNS_PROFILE = String.raw `(.*/)?\.git(/(modules/.+/)?(hooks(/.*)?|con
199
215
  const q = (s) => `"${s.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"`;
200
216
  const rx = (s) => s.replace(/[.*+?^${}()|[\]\\/]/g, "\\$&");
201
217
  const subpaths = (list) => list.map((p) => `(subpath ${q(p)})`).join(" ");
218
+ /** The folders that lie between `root` and `p`, neither of the two among them. */
219
+ const above = (root, p) => {
220
+ const out = [];
221
+ for (let d = path.dirname(p); d !== root && isInside(root, d); d = path.dirname(d))
222
+ out.push(d);
223
+ return out;
224
+ };
202
225
  /**
203
226
  * macOS sandbox profile for everything an agent does on the file system: its shell commands and
204
227
  * its file tools. Nothing is allowed unless a rule here allows it; for file rules the last match wins.
@@ -216,7 +239,7 @@ export function sandboxProfile(_cfg, agent, roots, variant, network = false, tag
216
239
  const restricted = variant === true || variant === "restricted";
217
240
  // What is read into the prompt cannot be rewritten by a run that outside content may steer.
218
241
  const frozen = () => {
219
- const out = [`(deny file-write* ${[roots.userFile, ...WORKSPACE_FILES.map((f) => path.join(roots.workspace, f))].map((f) => `(literal ${q(f)})`).join(" ")}${said})`];
242
+ const out = [`(deny file-write* ${[...roots.shared, ...WORKSPACE_FILES.map((f) => path.join(roots.workspace, f))].map((f) => `(literal ${q(f)})`).join(" ")}${said})`];
220
243
  for (const dir of [roots.workspace, ...(roots.folder ? [roots.folder] : [])]) {
221
244
  out.push(`(deny file-write* (regex #"^${rx(dir)}/(.*/)?(${INSTRUCTION_FILES.map(rx).join("|")})$") (regex #"^${rx(dir)}/${GIT_RUNS_PROFILE}") ${subpaths(INSTRUCTION_DIRS.map((d) => path.join(dir, d)))}${said})`);
222
245
  }
@@ -258,6 +281,10 @@ export function sandboxProfile(_cfg, agent, roots, variant, network = false, tag
258
281
  `(allow file-read* ${subpaths(roots.ownedRead)})`,
259
282
  `(allow file-write* ${subpaths(roots.ownedWrite)})`,
260
283
  `(deny file-read* file-write* ${subpaths([...roots.secrets, ...roots.others])}${said})`,
284
+ // What its event sources received: read, and nothing more. The folders on the way there can be
285
+ // seen to exist, which a program that resolves a path needs, and not listed.
286
+ ...(roots.sources.length ? [`(allow file-read* ${subpaths(roots.sources)})`] : []),
287
+ ...[...new Set(roots.sources.flatMap((s) => above(roots.home, s)))].map((d) => `(allow file-read-metadata (literal ${q(d)}))`),
261
288
  ];
262
289
  if (restricted)
263
290
  lines.push(...frozen());
@@ -377,7 +404,7 @@ export const sandboxMissing = () => (sandboxKind() ? "" : whyNot);
377
404
  /** What a run that has handled outside content must not change, as the Linux sandbox needs it said. */
378
405
  const protectedOf = (roots) => ({
379
406
  roots: [roots.workspace, ...(roots.folder ? [roots.folder] : [])],
380
- files: [roots.userFile, ...WORKSPACE_FILES.map((f) => path.join(roots.workspace, f))],
407
+ files: [...roots.shared, ...WORKSPACE_FILES.map((f) => path.join(roots.workspace, f))],
381
408
  names: INSTRUCTION_FILES,
382
409
  dirs: INSTRUCTION_DIRS.filter((d) => !d.includes("/")),
383
410
  open: roots.ownOpen ? ownFiles(roots) : [],
@@ -403,6 +430,7 @@ export function sandboxArgv(cfg, agent, roots, variant, network = false, tag) {
403
430
  ownedRead: roots.ownedRead,
404
431
  ownedWrite: roots.ownedWrite,
405
432
  closed: [...roots.secrets, ...roots.others, ...SESSIONS],
433
+ sources: roots.sources.filter(present),
406
434
  network: net,
407
435
  variant: v,
408
436
  frozen: v === "restricted" ? protectedNow(protectedOf(roots)) : undefined,
@@ -651,6 +679,9 @@ export function audit(cfg) {
651
679
  out.push(`${where}: the source "${name}" runs the module ${s.use} inside pine with your rights, outside any sandbox`);
652
680
  if (s.type === "webhook" && s.options.open === true)
653
681
  out.push(`${where}: the webhook source "${name}" has no secret; anyone who can reach the port can make it report`);
682
+ // syslog has no login: without a list of senders, whoever reaches the port writes lines an agent reads.
683
+ if (s.type === "syslog" && !(Array.isArray(s.options.allow) && s.options.allow.length))
684
+ out.push(`${where}: the syslog source "${name}" has no list of senders ("allow"); anyone who can reach the port can make it report`);
654
685
  }
655
686
  }
656
687
  for (const r of cfg.routes)
package/src/slots.d.ts ADDED
@@ -0,0 +1,106 @@
1
+ export interface Occupant {
2
+ /** The conversation whose text the slot holds: "agent:conversation". */
3
+ who: string;
4
+ /** How many tokens of it the slot holds: what it would cost to read again. */
5
+ tokens: number;
6
+ at: number;
7
+ /** For a slot pine filled itself (WARM): whose beginning it holds and which, as "agent:tools:instructions". */
8
+ what?: string;
9
+ }
10
+ /**
11
+ * A conversation that had to leave its slot, kept in a file of the model server (a "page"): a
12
+ * llama.cpp server that was started with a folder for it (--slot-save-path) writes what a slot
13
+ * holds into a file and reads it back, which takes about as long as copying the file and not as
14
+ * long as reading the conversation again. So more conversations than the server has slots keep
15
+ * what was read of them: the one that has to go is written out first, and read in again when it
16
+ * comes back. The files are a few, with fixed names, used again and again.
17
+ */
18
+ export interface Page {
19
+ who: string;
20
+ tokens: number;
21
+ at: number;
22
+ }
23
+ /** Who holds a slot that pine filled in the background with what every conversation begins with. */
24
+ export declare const WARM = "~warm";
25
+ /**
26
+ * Who holds a slot that somebody else uses: another client of the same model server (a second pine,
27
+ * another program). pine does not know whose conversation it is, only that the server says the slot
28
+ * holds one: it is not taken for empty and not filled, and it is taken last among slots of its size.
29
+ */
30
+ export declare const OTHER = "~other";
31
+ export declare class SlotBook {
32
+ /** Slots a server has, by server ("origin|model"). Zero: not a llama.cpp server, or not known yet. */
33
+ private count;
34
+ /** Who has a request under way in a slot, by server. */
35
+ private busy;
36
+ private held;
37
+ private save;
38
+ /** The files of a server, by number, and whose conversation each holds. */
39
+ private pages;
40
+ private savePages;
41
+ constructor(saved: Record<string, Array<Occupant | undefined>> | undefined, save: (all: Record<string, Array<Occupant | undefined>>) => void, pages?: Record<string, Array<Page | undefined>>, savePages?: (all: Record<string, Array<Page | undefined>>) => void);
42
+ known(server: string): boolean;
43
+ setCount(server: string, n: number): void;
44
+ /** The slot for a request of `who`, or undefined to let the server choose (no free slot, or no slots known). */
45
+ take(server: string, who: string): number | undefined;
46
+ /**
47
+ * As take(), and how much of the conversation the slot held before: nothing of that kept by the server means it
48
+ * has lost what it had read. `evicted` is whose place was taken, if it was somebody's.
49
+ */
50
+ takeFrom(server: string, who: string): {
51
+ slot: number;
52
+ held: number;
53
+ evicted?: Occupant;
54
+ } | undefined;
55
+ /** The request is over: the slot is free again, and holds this much of the conversation. */
56
+ give(server: string, slot: number | undefined, who: string, tokens?: number): void;
57
+ /**
58
+ * Slots to fill in the background with what an agent's conversations begin with (`what`:
59
+ * "agent:tools:instructions"): the empty ones, and the ones filled for this agent before whose
60
+ * content is no longer what a conversation begins with. One whose tools changed is read from
61
+ * its first token, so that waits until it is `old` milliseconds old: conversations of one agent
62
+ * that have different tools must not have the slot filled anew with every message.
63
+ */
64
+ toFill(server: string, what: string, old?: number): number[];
65
+ /** Whether no request of pine's is under way on the server. */
66
+ quiet(server: string): boolean;
67
+ /** Take a slot to fill it in the background. False when a request is under way in it. */
68
+ fill(server: string, slot: number, what: string): boolean;
69
+ /** The filling is over. One that failed leaves the slot empty; a conversation that took the slot meanwhile keeps it. */
70
+ filled(server: string, slot: number, ok: boolean, tokens?: number): void;
71
+ /** The server has lost what it had read (it was started anew): only the slot of the request that showed it holds something. */
72
+ lost(server: string, except?: number): void;
73
+ /**
74
+ * What the server says its slots hold, in tokens (undefined where it does not say: a slot that was
75
+ * never used, or one it read back from a file). pine's own notes are put right by it:
76
+ * - a slot pine has nothing in, and the server has something in, is somebody else's;
77
+ * - somebody else's that is empty now is empty;
78
+ * - a slot of one of pine's conversations, or one pine filled, that holds far less or far more than pine
79
+ * read into it, was taken by somebody else in between.
80
+ * A slot with a request of pine's under way is left as it is.
81
+ *
82
+ * `loose` is a conversation of pine's whose last request went out without a slot (the server's slots
83
+ * were not known yet: its model was being loaded) and how much was read of it: the slot that holds
84
+ * about that much, and that pine has nothing in, is where the server put it, and is that conversation's.
85
+ */
86
+ seen(server: string, real: Array<number | undefined>, loose?: {
87
+ who: string;
88
+ tokens: number;
89
+ }): void;
90
+ /** The file that holds a conversation that left its slot: its number, and how much of the conversation it holds. */
91
+ pageOf(server: string, who: string): {
92
+ page: number;
93
+ tokens: number;
94
+ } | undefined;
95
+ /** A file for a conversation that is about to leave its slot: its own, else a free one of `max`, else the one unused longest. */
96
+ pageFor(server: string, who: string, tokens: number, max: number): number | undefined;
97
+ /** The conversation is in a slot again, or its file could not be written or read: the file is anybody's. */
98
+ dropPage(server: string, who: string): void;
99
+ /** A conversation was read back into the slot it was just given: the slot holds that much of it again. */
100
+ restored(server: string, slot: number, who: string, tokens: number): void;
101
+ /** A conversation is gone (reset, removed, a sub-agent that is done): its slot is anybody's, and so is its file. */
102
+ forget(who: string): void;
103
+ /** An agent is gone: nothing of it is kept here either. */
104
+ forgetAgent(agent: string): void;
105
+ state(server: string): Array<Occupant | undefined>;
106
+ }
package/src/slots.js ADDED
@@ -0,0 +1,250 @@
1
+ // Which slot of a llama.cpp server a conversation's requests go to.
2
+ //
3
+ // A llama.cpp server keeps, per slot, what it has read of one conversation. Left to itself it
4
+ // gives a request to the slot whose content begins most like it. Conversations of one agent all
5
+ // begin alike (the tools, the instructions), so a new conversation, or a sub-agent, is given the
6
+ // slot of the conversation the owner is in the middle of, and that one is read again from its
7
+ // beginning when the owner comes back to it: for many models the server cannot go back part of
8
+ // the way (it can only return to a point it saved, and it saves few).
9
+ //
10
+ // So pine says which slot (`id_slot`): a conversation goes back to the slot it was in, and one
11
+ // that has none takes the slot that is cheapest to lose, an empty one first. Nothing here is
12
+ // needed for a correct answer; a server that knows no slots is never sent the word.
13
+ /** Who holds a slot that pine filled in the background with what every conversation begins with. */
14
+ export const WARM = "~warm";
15
+ /**
16
+ * Who holds a slot that somebody else uses: another client of the same model server (a second pine,
17
+ * another program). pine does not know whose conversation it is, only that the server says the slot
18
+ * holds one: it is not taken for empty and not filled, and it is taken last among slots of its size.
19
+ */
20
+ export const OTHER = "~other";
21
+ export class SlotBook {
22
+ /** Slots a server has, by server ("origin|model"). Zero: not a llama.cpp server, or not known yet. */
23
+ count = new Map();
24
+ /** Who has a request under way in a slot, by server. */
25
+ busy = new Map();
26
+ held;
27
+ save;
28
+ /** The files of a server, by number, and whose conversation each holds. */
29
+ pages;
30
+ savePages;
31
+ constructor(saved, save, pages, savePages = () => { }) {
32
+ this.held = new Map(Object.entries(saved ?? {}));
33
+ this.save = save;
34
+ this.pages = new Map(Object.entries(pages ?? {}));
35
+ this.savePages = savePages;
36
+ }
37
+ known(server) {
38
+ return this.count.has(server);
39
+ }
40
+ setCount(server, n) {
41
+ this.count.set(server, Math.max(0, Math.floor(n)));
42
+ }
43
+ /** The slot for a request of `who`, or undefined to let the server choose (no free slot, or no slots known). */
44
+ take(server, who) {
45
+ return this.takeFrom(server, who)?.slot;
46
+ }
47
+ /**
48
+ * As take(), and how much of the conversation the slot held before: nothing of that kept by the server means it
49
+ * has lost what it had read. `evicted` is whose place was taken, if it was somebody's.
50
+ */
51
+ takeFrom(server, who) {
52
+ const n = this.count.get(server) ?? 0;
53
+ if (n <= 0)
54
+ return undefined;
55
+ const busy = this.busy.get(server) ?? new Map();
56
+ this.busy.set(server, busy);
57
+ const held = this.held.get(server) ?? [];
58
+ this.held.set(server, held);
59
+ // A slot that is being filled in the background counts as free: the server takes the request when it is done
60
+ // with that, which beats taking the place of a conversation.
61
+ const free = Array.from({ length: n }, (_, i) => i).filter((i) => !busy.has(i) || busy.get(i) === WARM);
62
+ if (!free.length)
63
+ return undefined;
64
+ // Its own slot, if it still holds it; else one that was filled with what this agent's conversations begin with;
65
+ // else an empty one; else the one that is cheapest to read again, the longest unused first.
66
+ const mine = free.find((i) => held[i]?.who === who);
67
+ const agent = who.slice(0, who.indexOf(":") + 1);
68
+ const warm = free.find((i) => held[i]?.who === WARM && held[i].what?.startsWith(agent));
69
+ const empty = free.find((i) => !held[i]);
70
+ const cheapest = () => [...free].sort((a, b) => held[a].tokens - held[b].tokens || held[a].at - held[b].at)[0];
71
+ const slot = mine ?? warm ?? empty ?? cheapest();
72
+ const before = held[slot]?.who === who ? held[slot].tokens : 0;
73
+ const evicted = held[slot] && held[slot].who !== who ? held[slot] : undefined;
74
+ busy.set(slot, who);
75
+ if (held[slot]?.who !== who)
76
+ held[slot] = { who, tokens: 0, at: Date.now() };
77
+ return { slot, held: before, evicted };
78
+ }
79
+ /** The request is over: the slot is free again, and holds this much of the conversation. */
80
+ give(server, slot, who, tokens) {
81
+ if (slot === undefined)
82
+ return;
83
+ if (this.busy.get(server)?.get(slot) === who)
84
+ this.busy.get(server).delete(slot);
85
+ const held = this.held.get(server);
86
+ if (held && held[slot]?.who === who) {
87
+ held[slot] = { who, tokens: tokens ?? held[slot].tokens, at: Date.now() };
88
+ this.save(Object.fromEntries(this.held));
89
+ }
90
+ }
91
+ /**
92
+ * Slots to fill in the background with what an agent's conversations begin with (`what`:
93
+ * "agent:tools:instructions"): the empty ones, and the ones filled for this agent before whose
94
+ * content is no longer what a conversation begins with. One whose tools changed is read from
95
+ * its first token, so that waits until it is `old` milliseconds old: conversations of one agent
96
+ * that have different tools must not have the slot filled anew with every message.
97
+ */
98
+ toFill(server, what, old = 600_000) {
99
+ const n = this.count.get(server) ?? 0;
100
+ const held = this.held.get(server) ?? [];
101
+ const busy = this.busy.get(server);
102
+ const [agent, tools] = what.split(":");
103
+ return Array.from({ length: n }, (_, i) => i).filter((i) => {
104
+ if (busy?.has(i))
105
+ return false;
106
+ const h = held[i];
107
+ if (!h)
108
+ return true;
109
+ if (h.who !== WARM || !h.what || h.what === what)
110
+ return false;
111
+ const [a, t] = h.what.split(":");
112
+ return a === agent && (t === tools || h.at < Date.now() - old);
113
+ });
114
+ }
115
+ /** Whether no request of pine's is under way on the server. */
116
+ quiet(server) {
117
+ return !(this.busy.get(server)?.size ?? 0);
118
+ }
119
+ /** Take a slot to fill it in the background. False when a request is under way in it. */
120
+ fill(server, slot, what) {
121
+ const held = this.held.get(server) ?? [];
122
+ this.held.set(server, held);
123
+ const busy = this.busy.get(server) ?? new Map();
124
+ this.busy.set(server, busy);
125
+ if (busy.has(slot) || (held[slot] && held[slot].who !== WARM))
126
+ return false;
127
+ busy.set(slot, WARM);
128
+ // What every conversation begins with: nothing of anybody's is lost when the next conversation takes this slot.
129
+ held[slot] = { who: WARM, tokens: 0, at: Date.now(), what };
130
+ return true;
131
+ }
132
+ /** The filling is over. One that failed leaves the slot empty; a conversation that took the slot meanwhile keeps it. */
133
+ filled(server, slot, ok, tokens = 0) {
134
+ if (this.busy.get(server)?.get(slot) === WARM)
135
+ this.busy.get(server).delete(slot);
136
+ const held = this.held.get(server);
137
+ if (!held || held[slot]?.who !== WARM)
138
+ return;
139
+ held[slot] = ok ? { ...held[slot], tokens, at: Date.now() } : undefined;
140
+ this.save(Object.fromEntries(this.held));
141
+ }
142
+ /** The server has lost what it had read (it was started anew): only the slot of the request that showed it holds something. */
143
+ lost(server, except) {
144
+ const held = this.held.get(server);
145
+ if (!held)
146
+ return;
147
+ for (let i = 0; i < held.length; i++)
148
+ if (i !== except)
149
+ held[i] = undefined;
150
+ this.save(Object.fromEntries(this.held));
151
+ }
152
+ /**
153
+ * What the server says its slots hold, in tokens (undefined where it does not say: a slot that was
154
+ * never used, or one it read back from a file). pine's own notes are put right by it:
155
+ * - a slot pine has nothing in, and the server has something in, is somebody else's;
156
+ * - somebody else's that is empty now is empty;
157
+ * - a slot of one of pine's conversations, or one pine filled, that holds far less or far more than pine
158
+ * read into it, was taken by somebody else in between.
159
+ * A slot with a request of pine's under way is left as it is.
160
+ *
161
+ * `loose` is a conversation of pine's whose last request went out without a slot (the server's slots
162
+ * were not known yet: its model was being loaded) and how much was read of it: the slot that holds
163
+ * about that much, and that pine has nothing in, is where the server put it, and is that conversation's.
164
+ */
165
+ seen(server, real, loose) {
166
+ const held = this.held.get(server) ?? [];
167
+ this.held.set(server, held);
168
+ const busy = this.busy.get(server);
169
+ const least = 256;
170
+ const other = (tokens) => (tokens >= least ? { who: OTHER, tokens, at: Date.now() } : undefined);
171
+ const theirs = loose && !held.some((h) => h?.who === loose.who) ? real.findIndex((n, i) => n !== undefined && !busy?.has(i) && (!held[i] || held[i].who === OTHER) && Math.abs(n - loose.tokens) <= Math.max(64, loose.tokens * 0.03)) : -1;
172
+ for (let i = 0; i < real.length; i++) {
173
+ const n = real[i];
174
+ if (n === undefined || busy?.has(i))
175
+ continue;
176
+ const h = held[i];
177
+ if (i === theirs)
178
+ held[i] = { who: loose.who, tokens: n, at: Date.now() };
179
+ else if (!h || h.who === OTHER)
180
+ held[i] = other(n);
181
+ else if (h.tokens >= least && (n < h.tokens * 0.5 || n > h.tokens * 1.5 + 2048))
182
+ held[i] = other(n);
183
+ }
184
+ this.save(Object.fromEntries(this.held));
185
+ }
186
+ /** The file that holds a conversation that left its slot: its number, and how much of the conversation it holds. */
187
+ pageOf(server, who) {
188
+ const i = (this.pages.get(server) ?? []).findIndex((x) => x?.who === who);
189
+ return i < 0 ? undefined : { page: i, tokens: this.pages.get(server)[i].tokens };
190
+ }
191
+ /** A file for a conversation that is about to leave its slot: its own, else a free one of `max`, else the one unused longest. */
192
+ pageFor(server, who, tokens, max) {
193
+ if (max <= 0)
194
+ return undefined;
195
+ const pages = this.pages.get(server) ?? [];
196
+ this.pages.set(server, pages);
197
+ const all = Array.from({ length: max }, (_, i) => i);
198
+ const mine = all.find((i) => pages[i]?.who === who);
199
+ const free = all.find((i) => !pages[i]);
200
+ const page = mine ?? free ?? [...all].sort((a, b) => pages[a].at - pages[b].at)[0];
201
+ pages[page] = { who, tokens, at: Date.now() };
202
+ this.savePages(Object.fromEntries(this.pages));
203
+ return page;
204
+ }
205
+ /** The conversation is in a slot again, or its file could not be written or read: the file is anybody's. */
206
+ dropPage(server, who) {
207
+ const pages = this.pages.get(server);
208
+ if (!pages)
209
+ return;
210
+ for (let i = 0; i < pages.length; i++)
211
+ if (pages[i]?.who === who)
212
+ pages[i] = undefined;
213
+ this.savePages(Object.fromEntries(this.pages));
214
+ }
215
+ /** A conversation was read back into the slot it was just given: the slot holds that much of it again. */
216
+ restored(server, slot, who, tokens) {
217
+ const held = this.held.get(server);
218
+ if (held && held[slot]?.who === who)
219
+ held[slot] = { who, tokens, at: Date.now() };
220
+ }
221
+ /** A conversation is gone (reset, removed, a sub-agent that is done): its slot is anybody's, and so is its file. */
222
+ forget(who) {
223
+ for (const held of this.held.values())
224
+ for (let i = 0; i < held.length; i++)
225
+ if (held[i]?.who === who)
226
+ held[i] = undefined;
227
+ for (const pages of this.pages.values())
228
+ for (let i = 0; i < pages.length; i++)
229
+ if (pages[i]?.who === who)
230
+ pages[i] = undefined;
231
+ this.save(Object.fromEntries(this.held));
232
+ this.savePages(Object.fromEntries(this.pages));
233
+ }
234
+ /** An agent is gone: nothing of it is kept here either. */
235
+ forgetAgent(agent) {
236
+ for (const held of this.held.values())
237
+ for (let i = 0; i < held.length; i++)
238
+ if (held[i]?.who.startsWith(`${agent}:`) || held[i]?.what?.startsWith(`${agent}:`))
239
+ held[i] = undefined;
240
+ for (const pages of this.pages.values())
241
+ for (let i = 0; i < pages.length; i++)
242
+ if (pages[i]?.who.startsWith(`${agent}:`))
243
+ pages[i] = undefined;
244
+ this.save(Object.fromEntries(this.held));
245
+ this.savePages(Object.fromEntries(this.pages));
246
+ }
247
+ state(server) {
248
+ return [...(this.held.get(server) ?? [])];
249
+ }
250
+ }
@@ -28,6 +28,8 @@ export interface SourceSetup {
28
28
  workspace: string;
29
29
  /** pine's home folder, which no source may point into. */
30
30
  home: string;
31
+ /** The port of pine's own control API, if it has one: an address a source is never pointed at to act there. */
32
+ controlPort?: number;
31
33
  /** An absolute path for one an agent wrote: ~ is the user's home, anything relative is meant from the workspace. */
32
34
  resolve(p: string): string;
33
35
  /** A fresh secret of `bytes` random bytes, as letters and digits. */
@@ -44,6 +46,12 @@ export interface SourceType {
44
46
  * in .env and out of the configuration, the event log and what models see.
45
47
  */
46
48
  options: TSchema;
49
+ /**
50
+ * Optional secret options that a set-up with these options cannot do without (a password,
51
+ * unless the mailbox is signed in to another way). pine asks the owner for each that is
52
+ * missing, as it does for a required one, without the agent having to name it.
53
+ */
54
+ needs?(options: Record<string, any>): string[];
47
55
  /** Check the options and say what the source would do. Throw for what cannot work. Without it, a set-up is always put to the owner. */
48
56
  plan?(options: Record<string, any>, setup: SourceSetup): SourcePlan;
49
57
  /**
@@ -58,4 +66,16 @@ export interface SourceType {
58
66
  * offers every source that says this.
59
67
  */
60
68
  sends?: string;
69
+ /**
70
+ * What it sends reaches the owner and nobody else: the type itself sees to that (a Telegram bot
71
+ * that writes only to the chat the owner paired). A message to the owner is not information
72
+ * leaving, so the `send` tool is not asked about for such a source after outside content.
73
+ * Never set it for a type that can reach anybody else or act on a device.
74
+ */
75
+ toOwner?: boolean;
76
+ /**
77
+ * It can send a file along: a picture, a video or a sound file from the agent's workspace, handed to
78
+ * `onSend` as `file` ({ name, mime, bytes }). The daemon reads the file, never the type.
79
+ */
80
+ files?: boolean;
61
81
  }
@@ -0,0 +1,24 @@
1
+ import type { SourceType } from "../source-types.ts";
2
+ import type { SourceApi } from "../sources.ts";
3
+ export interface Item {
4
+ key: string;
5
+ uid: string;
6
+ summary: string;
7
+ start: string;
8
+ end?: string;
9
+ allDay?: boolean;
10
+ zone?: string;
11
+ location?: string;
12
+ description?: string;
13
+ repeats?: string;
14
+ /** When it begins and when it is over, as instants. */
15
+ at: number;
16
+ until: number;
17
+ }
18
+ export declare const type: SourceType;
19
+ /** A calendar's name and its entries. Throws for a document that is no calendar. */
20
+ export declare function parseCalendar(body: string): {
21
+ name: string;
22
+ items: Item[];
23
+ };
24
+ export default function calendar(source: SourceApi): void;