@pmndre/pine 0.0.0-stage → 0.22.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (151) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +226 -2
  3. package/bin/pine.js +27 -0
  4. package/docs/SKILL.md +234 -0
  5. package/docs/references/agents.md +454 -0
  6. package/docs/references/coding.md +311 -0
  7. package/docs/references/computer.md +212 -0
  8. package/docs/references/configuration.md +310 -0
  9. package/docs/references/events.md +267 -0
  10. package/docs/references/mcp.md +420 -0
  11. package/docs/references/operations.md +394 -0
  12. package/docs/references/overview.md +195 -0
  13. package/docs/references/security.md +896 -0
  14. package/docs/references/setup.md +385 -0
  15. package/docs/references/sources.md +772 -0
  16. package/docs/references/stack.md +121 -0
  17. package/docs/references/tui.md +575 -0
  18. package/examples/home/.env.example +7 -0
  19. package/examples/home/USER.md +4 -0
  20. package/examples/home/agents/main/agent.json +4 -0
  21. package/examples/home/agents/main/workspace/AGENTS.md +14 -0
  22. package/examples/home/agents/main/workspace/MEMORY.md +4 -0
  23. package/examples/home/agents/main/workspace/SOUL.md +4 -0
  24. package/examples/home/models.json +62 -0
  25. package/examples/home/pine.json +34 -0
  26. package/examples/home/skills/README.txt +7 -0
  27. package/examples/launchd/com.pine.daemon.plist +32 -0
  28. package/examples/systemd/pine.service +22 -0
  29. package/mcp/browser.d.ts +99 -0
  30. package/mcp/browser.js +496 -0
  31. package/mcp/commands.d.ts +20 -0
  32. package/mcp/commands.js +88 -0
  33. package/mcp/computer.d.ts +139 -0
  34. package/mcp/computer.js +698 -0
  35. package/mcp/computer.swift +989 -0
  36. package/mcp/lib.d.ts +23 -0
  37. package/mcp/lib.js +52 -0
  38. package/mcp/web.d.ts +6 -0
  39. package/mcp/web.js +140 -0
  40. package/package.json +32 -4
  41. package/src/access.d.ts +112 -0
  42. package/src/access.js +438 -0
  43. package/src/actions.d.ts +75 -0
  44. package/src/actions.js +167 -0
  45. package/src/admin.d.ts +167 -0
  46. package/src/admin.js +625 -0
  47. package/src/agent-admin.d.ts +24 -0
  48. package/src/agent-admin.js +55 -0
  49. package/src/agents.d.ts +540 -0
  50. package/src/agents.js +2776 -0
  51. package/src/background.d.ts +29 -0
  52. package/src/background.js +100 -0
  53. package/src/cli.d.ts +2 -0
  54. package/src/cli.js +777 -0
  55. package/src/computer.d.ts +30 -0
  56. package/src/computer.js +32 -0
  57. package/src/config.d.ts +396 -0
  58. package/src/config.js +549 -0
  59. package/src/dev/claude-model.d.ts +37 -0
  60. package/src/dev/claude-model.js +204 -0
  61. package/src/env-file.d.ts +4 -0
  62. package/src/env-file.js +40 -0
  63. package/src/goals.d.ts +133 -0
  64. package/src/goals.js +316 -0
  65. package/src/http.d.ts +46 -0
  66. package/src/http.js +509 -0
  67. package/src/limiter.d.ts +32 -0
  68. package/src/limiter.js +84 -0
  69. package/src/login.d.ts +23 -0
  70. package/src/login.js +130 -0
  71. package/src/manual.d.ts +61 -0
  72. package/src/manual.js +443 -0
  73. package/src/mcp-config.d.ts +49 -0
  74. package/src/mcp-config.js +182 -0
  75. package/src/models.d.ts +90 -0
  76. package/src/models.js +274 -0
  77. package/src/net.d.ts +50 -0
  78. package/src/net.js +150 -0
  79. package/src/pine.d.ts +32 -0
  80. package/src/pine.js +134 -0
  81. package/src/programs.d.ts +123 -0
  82. package/src/programs.js +412 -0
  83. package/src/router.d.ts +56 -0
  84. package/src/router.js +164 -0
  85. package/src/sandbox-fs.d.ts +34 -0
  86. package/src/sandbox-fs.js +186 -0
  87. package/src/sandbox-linux.d.ts +96 -0
  88. package/src/sandbox-linux.js +297 -0
  89. package/src/sandbox-setup.d.ts +70 -0
  90. package/src/sandbox-setup.js +216 -0
  91. package/src/security.d.ts +238 -0
  92. package/src/security.js +660 -0
  93. package/src/skills.d.ts +67 -0
  94. package/src/skills.js +163 -0
  95. package/src/snapshots.d.ts +68 -0
  96. package/src/snapshots.js +306 -0
  97. package/src/source-types.d.ts +61 -0
  98. package/src/source-types.js +1 -0
  99. package/src/sources/command.d.ts +4 -0
  100. package/src/sources/command.js +77 -0
  101. package/src/sources/folder.d.ts +6 -0
  102. package/src/sources/folder.js +116 -0
  103. package/src/sources/ftp.d.ts +4 -0
  104. package/src/sources/ftp.js +307 -0
  105. package/src/sources/http.d.ts +4 -0
  106. package/src/sources/http.js +201 -0
  107. package/src/sources/mqtt.d.ts +18 -0
  108. package/src/sources/mqtt.js +97 -0
  109. package/src/sources/telegram.d.ts +22 -0
  110. package/src/sources/telegram.js +283 -0
  111. package/src/sources/webhook.d.ts +4 -0
  112. package/src/sources/webhook.js +52 -0
  113. package/src/sources.d.ts +344 -0
  114. package/src/sources.js +966 -0
  115. package/src/store.d.ts +99 -0
  116. package/src/store.js +277 -0
  117. package/src/timers.d.ts +83 -0
  118. package/src/timers.js +318 -0
  119. package/src/tools.d.ts +11 -0
  120. package/src/tools.js +1297 -0
  121. package/src/tui/app.d.ts +305 -0
  122. package/src/tui/app.js +3195 -0
  123. package/src/tui/client.d.ts +236 -0
  124. package/src/tui/client.js +131 -0
  125. package/src/tui/code-pane.d.ts +68 -0
  126. package/src/tui/code-pane.js +535 -0
  127. package/src/tui/highlight.d.ts +8 -0
  128. package/src/tui/highlight.js +86 -0
  129. package/src/tui/picture.d.ts +206 -0
  130. package/src/tui/picture.js +801 -0
  131. package/src/tui/run.d.ts +8 -0
  132. package/src/tui/run.js +97 -0
  133. package/src/tui/theme.d.ts +32 -0
  134. package/src/tui/theme.js +128 -0
  135. package/src/tui/undo.d.ts +5 -0
  136. package/src/tui/undo.js +50 -0
  137. package/src/tui/voice.d.ts +49 -0
  138. package/src/tui/voice.js +155 -0
  139. package/src/tui/voice.swift +223 -0
  140. package/src/types.d.ts +139 -0
  141. package/src/types.js +1 -0
  142. package/src/undo.d.ts +224 -0
  143. package/src/undo.js +465 -0
  144. package/src/update.d.ts +35 -0
  145. package/src/update.js +130 -0
  146. package/src/util.d.ts +65 -0
  147. package/src/util.js +171 -0
  148. package/src/watching.d.ts +20 -0
  149. package/src/watching.js +68 -0
  150. package/src/workspace-view.d.ts +72 -0
  151. package/src/workspace-view.js +221 -0
@@ -0,0 +1,67 @@
1
+ import { type AgentConfig, type PineConfig } from "./config.ts";
2
+ /**
3
+ * Skills: what an agent knows how to do, written down. A skill is a folder with a SKILL.md (a
4
+ * name, a line on when to use it, and the instructions) and whatever files those instructions
5
+ * refer to. The agent sees name and line in its prompt and reads the rest when a task calls for it.
6
+ *
7
+ * Like event sources and MCP servers, a skill is for one conversation, for one agent or for all
8
+ * agents. Unlike them it needs no entry in a configuration file: where the folder lies says whom
9
+ * it is for.
10
+ *
11
+ * <home>/skills/<name>/ all agents
12
+ * <home>/agents/<id>/skills/<name>/ that agent, in all its conversations
13
+ * <home>/agents/<id>/conversations/<session>/skills/<name>/ that one conversation
14
+ *
15
+ * All three lie in pine's home, where an agent can read its own but write nothing: a skill is
16
+ * part of an agent's instructions, and is written by pine when a tool asks for it.
17
+ */
18
+ export type SkillScope = "local" | "agent" | "global";
19
+ export interface SkillInfo {
20
+ name: string;
21
+ description: string;
22
+ scope: SkillScope;
23
+ dir: string;
24
+ /** Put there by pine's tools, so the tools may also replace and remove it. The owner's own folders (`skills` in a configuration file) are not. */
25
+ managed: boolean;
26
+ }
27
+ export declare const localSkillsDir: (agent: AgentConfig, session: string) => string;
28
+ /** The folders an agent's session reads skills from, beyond what its configuration names. */
29
+ export declare function skillDirs(cfg: PineConfig, agent: AgentConfig, session: string): string[];
30
+ /** The skills one conversation of an agent has, the nearest first: its own, the agent's, those for all agents. */
31
+ export declare function listSkills(cfg: PineConfig, agent: AgentConfig, session: string): SkillInfo[];
32
+ /**
33
+ * Every skill that one of a scope would stand beside: for a conversation what it sees, for an
34
+ * agent also what its other conversations have, for all agents everything there is. A name is
35
+ * used once among those, so that no conversation ever has two skills of one name.
36
+ */
37
+ export declare function skillsBeside(cfg: PineConfig, agent: AgentConfig, scope: SkillScope, session: string): SkillInfo[];
38
+ /** Every skill there is, with whom it is for: for whoever looks at the whole setup. */
39
+ export declare function allSkills(cfg: PineConfig): Array<SkillInfo & {
40
+ agent?: string;
41
+ session?: string;
42
+ }>;
43
+ export interface SkillSpec {
44
+ name: string;
45
+ description: string;
46
+ instructions: string;
47
+ /** A folder whose files go along with the skill: scripts, references, templates. */
48
+ files?: string;
49
+ }
50
+ /** Check what a skill is to be before anybody is asked anything. Returns it tidied, with the files that would be copied. */
51
+ export declare function checkSkill(spec: SkillSpec, workspace: string): SkillSpec & {
52
+ copy: Array<{
53
+ from: string;
54
+ to: string;
55
+ }>;
56
+ workspace: string;
57
+ };
58
+ /** Where a skill of a scope lies. */
59
+ export declare function skillDir(cfg: PineConfig, agent: AgentConfig, scope: SkillScope, session: string, name: string): string;
60
+ /** Write a skill, in place of one of the same name in the same place. */
61
+ export declare function writeSkill(dir: string, skill: ReturnType<typeof checkSkill>): void;
62
+ /** Delete a skill's folder, and the folders above it that it leaves empty, up to the agent's or pine's own. */
63
+ export declare function deleteSkill(dir: string, stop: string): void;
64
+ /** A conversation is removed: the skills that were its own go with it. */
65
+ export declare function dropConversationSkills(agent: AgentConfig, session: string): string[];
66
+ /** A conversation got another name: its skills follow it. */
67
+ export declare function renameConversationSkills(agent: AgentConfig, from: string, to: string): void;
package/src/skills.js ADDED
@@ -0,0 +1,163 @@
1
+ import fs from "node:fs";
2
+ import path from "node:path";
3
+ import { loadSkillsFromDir } from "@earendil-works/pi-coding-agent";
4
+ import { agentSkillsDir, conversationsDir, globalSkillsDir } from "./config.js";
5
+ import { isInside, realish } from "./security.js";
6
+ import { safeName } from "./util.js";
7
+ const MAX_TEXT = 40_000;
8
+ const MAX_FILES = 200;
9
+ const MAX_BYTES = 4 * 1024 * 1024;
10
+ export const localSkillsDir = (agent, session) => path.join(conversationsDir(agent), safeName(session), "skills");
11
+ /** The folders an agent's session reads skills from, beyond what its configuration names. */
12
+ export function skillDirs(cfg, agent, session) {
13
+ return [globalSkillsDir(cfg), ...(session === "main" ? [] : [localSkillsDir(agent, session)])].filter((d) => fs.existsSync(d));
14
+ }
15
+ const read = (dir, scope, managed) => fs.existsSync(dir) ? loadSkillsFromDir({ dir, source: scope }).skills.map((s) => ({ name: s.name, description: s.description, scope, dir: s.baseDir, managed })) : [];
16
+ /** The skills one conversation of an agent has, the nearest first: its own, the agent's, those for all agents. */
17
+ export function listSkills(cfg, agent, session) {
18
+ const out = [
19
+ ...(session === "main" ? [] : read(localSkillsDir(agent, session), "local", true)),
20
+ ...read(agentSkillsDir(agent), "agent", true),
21
+ ...agent.skills.flatMap((d) => read(d, "agent", false)),
22
+ ...read(globalSkillsDir(cfg), "global", true),
23
+ ...cfg.skills.filter((d) => realish(d) !== realish(globalSkillsDir(cfg))).flatMap((d) => read(d, "global", false)),
24
+ ];
25
+ return out.filter((s, i) => out.findIndex((x) => x.name === s.name) === i);
26
+ }
27
+ /**
28
+ * Every skill that one of a scope would stand beside: for a conversation what it sees, for an
29
+ * agent also what its other conversations have, for all agents everything there is. A name is
30
+ * used once among those, so that no conversation ever has two skills of one name.
31
+ */
32
+ export function skillsBeside(cfg, agent, scope, session) {
33
+ const locals = (a) => (fs.existsSync(conversationsDir(a)) ? fs.readdirSync(conversationsDir(a)).flatMap((s) => read(path.join(conversationsDir(a), s, "skills"), "local", true)) : []);
34
+ if (scope === "local")
35
+ return listSkills(cfg, agent, session);
36
+ if (scope === "agent")
37
+ return [...locals(agent), ...listSkills(cfg, agent, "main")];
38
+ return Object.values(cfg.agents).flatMap((a) => [...locals(a), ...listSkills(cfg, a, "main")]);
39
+ }
40
+ /** Every skill there is, with whom it is for: for whoever looks at the whole setup. */
41
+ export function allSkills(cfg) {
42
+ const global = [...read(globalSkillsDir(cfg), "global", true), ...cfg.skills.filter((d) => realish(d) !== realish(globalSkillsDir(cfg))).flatMap((d) => read(d, "global", false))];
43
+ const perAgent = Object.values(cfg.agents).flatMap((a) => [
44
+ ...[...read(agentSkillsDir(a), "agent", true), ...a.skills.flatMap((d) => read(d, "agent", false))].map((s) => ({ ...s, agent: a.id })),
45
+ ...(fs.existsSync(conversationsDir(a)) ? fs.readdirSync(conversationsDir(a)) : []).flatMap((session) => read(path.join(conversationsDir(a), session, "skills"), "local", true).map((s) => ({ ...s, agent: a.id, session }))),
46
+ ]);
47
+ return [...global, ...perAgent];
48
+ }
49
+ /** Check what a skill is to be before anybody is asked anything. Returns it tidied, with the files that would be copied. */
50
+ export function checkSkill(spec, workspace) {
51
+ const name = String(spec.name ?? "").trim();
52
+ if (!/^[a-z0-9]+(-[a-z0-9]+)*$/.test(name) || name.length > 64)
53
+ throw new Error(`"${name}" is not a usable skill name: lowercase letters, digits and single hyphens, at most 64 characters (for example invoices, solar-yield)`);
54
+ if (name === "pine")
55
+ throw new Error('"pine" is the name of pine\'s own manual');
56
+ const description = String(spec.description ?? "").replace(/\s+/g, " ").trim();
57
+ if (!description)
58
+ throw new Error(`a skill needs a "description": one or two sentences on what it is for and when to use it. You decide by that line alone whether to read the skill`);
59
+ if (description.length > 1024)
60
+ throw new Error(`the description is too long (${description.length} characters, at most 1024). It says when to use the skill; how to do it goes into "instructions"`);
61
+ const instructions = String(spec.instructions ?? "").trim();
62
+ if (!instructions)
63
+ throw new Error(`a skill needs "instructions": what to do, step by step, written to yourself`);
64
+ if (instructions.length > MAX_TEXT)
65
+ throw new Error(`the instructions are too long (${instructions.length} characters, at most ${MAX_TEXT}). Put long material into files next to the skill ("files") and refer to them`);
66
+ const copy = [];
67
+ if (spec.files) {
68
+ const root = realish(path.resolve(workspace, spec.files));
69
+ if (!isInside(realish(workspace), root) || root === realish(workspace))
70
+ throw new Error(`"files" must be a folder inside your workspace`);
71
+ if (!fs.existsSync(root) || !fs.statSync(root).isDirectory())
72
+ throw new Error(`${root} is not a folder that exists`);
73
+ let bytes = 0;
74
+ const walk = (dir) => {
75
+ for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
76
+ const from = path.join(dir, entry.name);
77
+ // Only what is really there: a link could point at anything.
78
+ if (entry.isSymbolicLink())
79
+ throw new Error(`${from} is a link; a skill carries files, not links`);
80
+ if (entry.isDirectory()) {
81
+ if (entry.name === ".git" || entry.name === "node_modules")
82
+ continue;
83
+ walk(from);
84
+ }
85
+ else if (entry.isFile()) {
86
+ const to = path.relative(root, from);
87
+ if (to === "SKILL.md")
88
+ continue;
89
+ bytes += fs.statSync(from).size;
90
+ copy.push({ from, to });
91
+ if (copy.length > MAX_FILES || bytes > MAX_BYTES)
92
+ throw new Error(`${root} is too much for a skill (at most ${MAX_FILES} files and ${MAX_BYTES / 1024 / 1024} MB)`);
93
+ }
94
+ }
95
+ };
96
+ walk(root);
97
+ }
98
+ return { name, description, instructions, copy, workspace: realish(workspace) };
99
+ }
100
+ /** Where a skill of a scope lies. */
101
+ export function skillDir(cfg, agent, scope, session, name) {
102
+ return path.join(scope === "global" ? globalSkillsDir(cfg) : scope === "agent" ? agentSkillsDir(agent) : localSkillsDir(agent, session), name);
103
+ }
104
+ /** Write a skill, in place of one of the same name in the same place. */
105
+ export function writeSkill(dir, skill) {
106
+ // Built beside it and swapped in, so that a half-written skill is never read.
107
+ const fresh = `${dir}.new-${process.pid}`;
108
+ fs.rmSync(fresh, { recursive: true, force: true });
109
+ fs.mkdirSync(fresh, { recursive: true, mode: 0o700 });
110
+ for (const f of skill.copy) {
111
+ fs.mkdirSync(path.dirname(path.join(fresh, f.to)), { recursive: true });
112
+ // Between the check and now the owner may have been asked: what is copied is looked at again,
113
+ // and a file that has since become a link, or leads out of the workspace, is not followed.
114
+ let fd;
115
+ try {
116
+ if (!isInside(skill.workspace, realish(f.from)))
117
+ throw new Error("it leads out of your workspace");
118
+ fd = fs.openSync(f.from, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW);
119
+ if (!fs.fstatSync(fd).isFile())
120
+ throw new Error("it is not a file");
121
+ fs.writeFileSync(path.join(fresh, f.to), fs.readFileSync(fd));
122
+ }
123
+ catch (e) {
124
+ fs.rmSync(fresh, { recursive: true, force: true });
125
+ throw new Error(`${f.from} changed while the skill was being added and is not taken: ${e instanceof Error ? e.message : e}`);
126
+ }
127
+ finally {
128
+ if (fd !== undefined)
129
+ fs.closeSync(fd);
130
+ }
131
+ }
132
+ // The description as a quoted string, which is the same in JSON and in YAML: a colon in it cannot break the header.
133
+ fs.writeFileSync(path.join(fresh, "SKILL.md"), `---\nname: ${skill.name}\ndescription: ${JSON.stringify(skill.description)}\n---\n\n${skill.instructions}\n`);
134
+ fs.rmSync(dir, { recursive: true, force: true });
135
+ fs.renameSync(fresh, dir);
136
+ }
137
+ /** Delete a skill's folder, and the folders above it that it leaves empty, up to the agent's or pine's own. */
138
+ export function deleteSkill(dir, stop) {
139
+ fs.rmSync(dir, { recursive: true, force: true });
140
+ for (let up = path.dirname(dir); isInside(stop, up) && up !== stop; up = path.dirname(up)) {
141
+ if (!fs.existsSync(up) || fs.readdirSync(up).length)
142
+ break;
143
+ fs.rmdirSync(up);
144
+ }
145
+ }
146
+ /** A conversation is removed: the skills that were its own go with it. */
147
+ export function dropConversationSkills(agent, session) {
148
+ const dir = localSkillsDir(agent, session);
149
+ const names = read(dir, "local", true).map((s) => s.name);
150
+ fs.rmSync(path.dirname(dir), { recursive: true, force: true });
151
+ if (fs.existsSync(conversationsDir(agent)) && !fs.readdirSync(conversationsDir(agent)).length)
152
+ fs.rmdirSync(conversationsDir(agent));
153
+ return names;
154
+ }
155
+ /** A conversation got another name: its skills follow it. */
156
+ export function renameConversationSkills(agent, from, to) {
157
+ const a = path.dirname(localSkillsDir(agent, from));
158
+ const b = path.dirname(localSkillsDir(agent, to));
159
+ if (!fs.existsSync(a))
160
+ return;
161
+ fs.rmSync(b, { recursive: true, force: true });
162
+ fs.renameSync(a, b);
163
+ }
@@ -0,0 +1,68 @@
1
+ /** Files larger than this are not recorded. */
2
+ export declare const MAX_FILE: number;
3
+ export interface Snap {
4
+ tree: string;
5
+ /** Files left out for their size. They are never touched by a restore. */
6
+ skipped?: string[];
7
+ }
8
+ export interface FileChange {
9
+ /** From the folder. */
10
+ path: string;
11
+ /** "M" changed, "A" added, "D" deleted. */
12
+ status: "M" | "A" | "D";
13
+ added?: number;
14
+ removed?: number;
15
+ }
16
+ export declare class Snapshots {
17
+ private dir;
18
+ private places;
19
+ private lanes;
20
+ /** `dir`: where the git directories lie, one per folder. Inside pine's data folder. */
21
+ constructor(dir: string);
22
+ /** One thing at a time per folder: its states share one index. */
23
+ private alone;
24
+ private g;
25
+ private must;
26
+ private place;
27
+ /** Bring the index in line with the folder and write it as a tree. */
28
+ private record;
29
+ private entries;
30
+ private outward;
31
+ /** Record what is in a folder now. `root` is a real path. Throws with the reason when it cannot be recorded. */
32
+ take(root: string): Promise<Snap>;
33
+ /** What differs between two states of a folder, with line counts. */
34
+ changed(root: string, a: Snap, b: Snap): Promise<FileChange[]>;
35
+ /** The difference of one file between two states, as git prints it. */
36
+ diff(root: string, a: Snap, b: Snap, file: string): Promise<{
37
+ diff: string;
38
+ truncated: boolean;
39
+ } | undefined>;
40
+ /**
41
+ * Put files of a folder back to what they were in `target`. Which files: those that differ
42
+ * between the two states of `within` (what one message changed), or the paths in `only`; of
43
+ * these, the ones that are not as in `target` now. Nothing else in the folder is touched.
44
+ *
45
+ * The folder's present state is recorded first and returned (`cur`), so that the restore can
46
+ * itself be taken back. `paths` are the files that were set, for exactly that.
47
+ *
48
+ * The files are written by git, never by this process. Git does not write through a link: a
49
+ * folder on the way that has become a link is removed and made again as a folder. Whatever it
50
+ * did, the sandbox it runs in can write the folder and pine's own git directory and nothing else.
51
+ */
52
+ restore(root: string, target: Snap, o: {
53
+ within?: [Snap, Snap];
54
+ only?: string[];
55
+ avoid?: Snap[];
56
+ }): Promise<{
57
+ cur: Snap;
58
+ files: FileChange[];
59
+ paths: string[];
60
+ left: string[];
61
+ }>;
62
+ /**
63
+ * The tidying: states nobody refers to any more are dropped once they are a few days old, and
64
+ * the git directory of a folder that is gone, or that has no state left, is removed.
65
+ * `kept`: the trees still referred to, by folder.
66
+ */
67
+ prune(kept: Map<string, Set<string>>): Promise<void>;
68
+ }
@@ -0,0 +1,306 @@
1
+ import crypto from "node:crypto";
2
+ import fs from "node:fs";
3
+ import path from "node:path";
4
+ import { isInside, realish } from "./security.js";
5
+ import { git } from "./workspace-view.js";
6
+ // The state of a folder at one moment, kept so that it can be compared with another moment and
7
+ // gone back to: what undo is made of (src/undo.ts).
8
+ //
9
+ // A state is a git tree. It is written with git's plumbing into a git directory of pine's own
10
+ // under the data folder, with the folder as work tree: the owner's own repository, its index and
11
+ // its history are never touched, and a folder that is no repository works the same. Agents cannot
12
+ // write pine's data folder, so no agent can change what a state says.
13
+ //
14
+ // Three things are deliberately left out of a state, and so are never restored:
15
+ // - what the folder's ignore rules leave out (.gitignore, the repository's info/exclude);
16
+ // - files above MAX_FILE: one large file changed a few times would cost its size each time;
17
+ // - a repository inside the folder (a clone, a submodule): git leaves it to itself.
18
+ /** Files larger than this are not recorded. */
19
+ export const MAX_FILE = 2 * 1024 * 1024;
20
+ /** How much may be new or changed in one state. A folder beyond it is not recorded at all, and undo says so. */
21
+ const MAX_FILES = 50_000;
22
+ const MAX_BYTES = 512 * 1024 * 1024;
23
+ const MAX_SKIPPED = 1000;
24
+ /** A repository inside the folder appears as an entry of this kind. */
25
+ const LINK = "160000";
26
+ const NONE = "000000";
27
+ const DAY = 86_400_000;
28
+ /** A file in each git directory whose time says when it was last tidied. */
29
+ const TIDIED = "pine-tidied";
30
+ /**
31
+ * No setting and no file of the folder may make git run a program or rewrite what it stores: no
32
+ * hooks, no file system monitor, no line-ending conversion. The owner's own git settings
33
+ * (global, system) are not read at all, see `env` below: what is recorded depends on the folder
34
+ * alone, so the owner's global ignore file does not count either.
35
+ */
36
+ const CONFIG = ["-c", "core.autocrlf=false", "-c", "core.safecrlf=false", "-c", "core.untrackedCache=false", "-c", "gc.auto=0", "-c", "advice.addEmbeddedRepo=false"];
37
+ /**
38
+ * `info/attributes` of the git directory wins over every .gitattributes in the folder. A filter
39
+ * named there would be a program started by git; text conversion would give back a file that is
40
+ * not byte for byte what it was.
41
+ */
42
+ const ATTRIBUTES = "* -text -filter -ident -working-tree-encoding\n";
43
+ const split = (raw) => raw.toString("latin1").split("\0").filter(Boolean);
44
+ const bytes = (p) => Buffer.from(p, "latin1");
45
+ const shown = (p) => bytes(p).toString("utf8");
46
+ /** A path as a line of an exclude file: from the top, and nothing in it is a pattern. */
47
+ const pattern = (p) => `/${p.replace(/\/$/, "").replace(/[\\*?[\]!# ]/g, "\\$&")}${p.endsWith("/") ? "/" : ""}`;
48
+ export class Snapshots {
49
+ dir;
50
+ places = new Map();
51
+ lanes = new Map();
52
+ /** `dir`: where the git directories lie, one per folder. Inside pine's data folder. */
53
+ constructor(dir) {
54
+ this.dir = dir;
55
+ }
56
+ /** One thing at a time per folder: its states share one index. */
57
+ alone(root, fn) {
58
+ const run = (this.lanes.get(root) ?? Promise.resolve()).then(fn, fn);
59
+ const done = run.then(() => { }, () => { });
60
+ this.lanes.set(root, done);
61
+ void done.then(() => this.lanes.get(root) === done && this.lanes.delete(root));
62
+ return run;
63
+ }
64
+ g(at, args, opts = {}) {
65
+ return git(at.base, ["--git-dir", at.dir, "--work-tree", at.base, "--literal-pathspecs", ...CONFIG, "-c", `core.excludesFile=${at.excludes ?? "/dev/null"}`, ...args], {
66
+ timeoutMs: 120_000,
67
+ maxBytes: 64 * 1024 * 1024,
68
+ ...opts,
69
+ // Its own directory, and nothing else unless the call is a restore.
70
+ write: [at.dir, ...(opts.write ?? [])],
71
+ env: { GIT_CONFIG_GLOBAL: "/dev/null", GIT_CONFIG_SYSTEM: "/dev/null", GIT_CONFIG_NOSYSTEM: "1", ...opts.env },
72
+ });
73
+ }
74
+ async must(what, r) {
75
+ const got = await r;
76
+ if (got.code !== 0)
77
+ throw new Error(`${what}: ${got.err.trim().split("\n")[0] || `git ended with ${got.code}`}`);
78
+ return got;
79
+ }
80
+ async place(root) {
81
+ const known = this.places.get(root);
82
+ if (known && Date.now() - known.at < 10_000 && fs.existsSync(path.join(known.dir, "HEAD")))
83
+ return known;
84
+ const dir = path.join(this.dir, crypto.createHash("sha256").update(root).digest("hex").slice(0, 24));
85
+ if (!fs.existsSync(path.join(dir, "HEAD"))) {
86
+ fs.mkdirSync(this.dir, { recursive: true, mode: 0o700 });
87
+ // Without a template: nothing is copied into it that pine did not write.
88
+ await this.must("the place for this folder's states could not be made", git(this.dir, ["init", "-q", "--bare", "--template=", dir], { write: [this.dir] }));
89
+ fs.mkdirSync(path.join(dir, "info"), { recursive: true });
90
+ fs.writeFileSync(path.join(dir, "info", "attributes"), ATTRIBUTES);
91
+ fs.writeFileSync(path.join(dir, "pine-folder"), root);
92
+ fs.writeFileSync(path.join(dir, TIDIED), "");
93
+ }
94
+ const at = { dir, base: root, prefix: "", at: Date.now() };
95
+ // In a repository the rules above the folder count too (node_modules is usually ignored at the
96
+ // top). A folder its repository ignores altogether is not part of it: the home folder kept in
97
+ // git with everything ignored must not mean that nothing below it is ever recorded.
98
+ const top = await git(root, ["rev-parse", "--show-toplevel"]);
99
+ const found = top.code === 0 && top.out.trim() ? realish(top.out.trim()) : "";
100
+ if (found && (found === root || isInside(found, root)) && (found === root || (await git(found, ["check-ignore", "-q", "--", path.relative(found, root)])).code !== 0)) {
101
+ at.base = found;
102
+ at.prefix = path.relative(found, root);
103
+ const own = await git(root, ["rev-parse", "--path-format=absolute", "--git-path", "info/exclude"]);
104
+ const file = own.code === 0 ? own.out.trim() : "";
105
+ if (file && path.isAbsolute(file) && fs.existsSync(file))
106
+ at.excludes = file;
107
+ }
108
+ this.places.set(root, at);
109
+ return at;
110
+ }
111
+ /** Bring the index in line with the folder and write it as a tree. */
112
+ async record(at, root) {
113
+ const spec = at.prefix || ".";
114
+ const exclude = path.join(at.dir, "info", "exclude");
115
+ fs.writeFileSync(exclude, "");
116
+ // What is new or changed since the last state: only that is looked at and stored.
117
+ const listed = await this.must(`${root} could not be read`, this.g(at, ["ls-files", "-z", "-m", "-o", "--exclude-standard", "--", spec]));
118
+ const large = [];
119
+ const nested = [];
120
+ let files = 0;
121
+ let size = 0;
122
+ for (const p of new Set(split(listed.raw))) {
123
+ if (p.endsWith("/")) {
124
+ nested.push(p);
125
+ continue;
126
+ }
127
+ let stat;
128
+ try {
129
+ // Only its size and kind are asked for, and a link is not followed.
130
+ stat = fs.lstatSync(Buffer.concat([bytes(`${at.base}/`), bytes(p)]));
131
+ }
132
+ catch {
133
+ continue; // deleted
134
+ }
135
+ if (!stat.isFile())
136
+ continue;
137
+ if (stat.size > MAX_FILE)
138
+ large.push(p);
139
+ else
140
+ (files++, (size += stat.size));
141
+ }
142
+ if (files > MAX_FILES || size > MAX_BYTES)
143
+ throw new Error(`there is too much in ${root} to record (${files} files, ${Math.round(size / 1024 / 1024)} MB new or changed)`);
144
+ if (large.length > MAX_SKIPPED)
145
+ throw new Error(`there are too many large files in ${root} to record (${large.length})`);
146
+ const out = [...large, ...nested];
147
+ if (out.length) {
148
+ if (out.some((p) => p.includes("\n")))
149
+ throw new Error(`a file in ${root} that has to be left out has a line break in its name`);
150
+ fs.writeFileSync(exclude, `${out.map(pattern).join("\n")}\n`, "latin1");
151
+ // One that was small before is in the index: taken out, or the tree would keep its old content as if it were the present.
152
+ if (large.length)
153
+ await this.must("a large file could not be left out", this.g(at, ["update-index", "--force-remove", "-z", "--stdin"], { stdin: bytes(`${large.join("\0")}\0`) }));
154
+ }
155
+ if (listed.raw.length)
156
+ await this.must(`${root} could not be recorded`, this.g(at, ["add", "-A", "--", spec]));
157
+ const tree = (await this.must(`${root} could not be recorded`, this.g(at, ["write-tree"]))).out.trim();
158
+ if (!/^[0-9a-f]{40,64}$/.test(tree))
159
+ throw new Error(`${root} could not be recorded`);
160
+ return { tree, ...(large.length ? { skipped: large } : {}) };
161
+ }
162
+ async entries(at, a, b) {
163
+ const got = await this.must("two states could not be compared", this.g(at, ["diff-tree", "-r", "-z", "--no-renames", "--no-abbrev", a, b]));
164
+ const parts = split(got.raw);
165
+ const out = [];
166
+ for (let i = 0; i + 1 < parts.length; i += 2) {
167
+ const m = /^:(\d{6}) (\d{6}) ([0-9a-f]+) ([0-9a-f]+) \w/.exec(parts[i]);
168
+ // A repository inside the folder is not ours to compare or restore.
169
+ if (m && m[1] !== LINK && m[2] !== LINK)
170
+ out.push({ p: parts[i + 1], from: { mode: m[1], sha: m[3] }, to: { mode: m[2], sha: m[4] } });
171
+ }
172
+ return out;
173
+ }
174
+ outward(at, e) {
175
+ const full = shown(e.p);
176
+ return { path: at.prefix && full.startsWith(`${at.prefix}/`) ? full.slice(at.prefix.length + 1) : full, status: e.from.mode === NONE ? "A" : e.to.mode === NONE ? "D" : "M" };
177
+ }
178
+ /** Record what is in a folder now. `root` is a real path. Throws with the reason when it cannot be recorded. */
179
+ take(root) {
180
+ return this.alone(root, async () => this.record(await this.place(root), root));
181
+ }
182
+ /** What differs between two states of a folder, with line counts. */
183
+ changed(root, a, b) {
184
+ return this.alone(root, async () => {
185
+ if (a.tree === b.tree)
186
+ return [];
187
+ const at = await this.place(root);
188
+ const files = (await this.entries(at, a.tree, b.tree)).map((e) => ({ e, f: this.outward(at, e) }));
189
+ const counts = await this.g(at, ["diff-tree", "-r", "-z", "--no-renames", "--numstat", a.tree, b.tree]);
190
+ for (const line of split(counts.raw)) {
191
+ const m = /^(\d+)\t(\d+)\t([\s\S]+)$/.exec(line);
192
+ const hit = m && files.find((x) => x.e.p === m[3]);
193
+ if (m && hit)
194
+ Object.assign(hit.f, { added: Number(m[1]), removed: Number(m[2]) });
195
+ }
196
+ return files.map((x) => x.f);
197
+ });
198
+ }
199
+ /** The difference of one file between two states, as git prints it. */
200
+ diff(root, a, b, file) {
201
+ return this.alone(root, async () => {
202
+ if (!file || file.includes("\0"))
203
+ return undefined;
204
+ const at = await this.place(root);
205
+ const r = await this.g(at, ["diff", "--no-ext-diff", "--no-textconv", "--no-renames", a.tree, b.tree, "--", at.prefix ? `${at.prefix}/${file}` : file]);
206
+ if (r.code !== 0)
207
+ return undefined;
208
+ const max = 200_000;
209
+ return { diff: r.out.slice(0, max), truncated: r.out.length > max };
210
+ });
211
+ }
212
+ /**
213
+ * Put files of a folder back to what they were in `target`. Which files: those that differ
214
+ * between the two states of `within` (what one message changed), or the paths in `only`; of
215
+ * these, the ones that are not as in `target` now. Nothing else in the folder is touched.
216
+ *
217
+ * The folder's present state is recorded first and returned (`cur`), so that the restore can
218
+ * itself be taken back. `paths` are the files that were set, for exactly that.
219
+ *
220
+ * The files are written by git, never by this process. Git does not write through a link: a
221
+ * folder on the way that has become a link is removed and made again as a folder. Whatever it
222
+ * did, the sandbox it runs in can write the folder and pine's own git directory and nothing else.
223
+ */
224
+ restore(root, target, o) {
225
+ return this.alone(root, async () => {
226
+ const at = await this.place(root);
227
+ const cur = await this.record(at, root);
228
+ const allow = o.within ? new Set((await this.entries(at, o.within[0].tree, o.within[1].tree)).map((e) => e.p)) : new Set(o.only ?? []);
229
+ // A file that was too large in any of the states involved has no content on record on that
230
+ // side: setting it would put an old version over it, or delete it.
231
+ const skip = new Set([cur, target, ...(o.avoid ?? []), ...(o.within ?? [])].flatMap((s) => s.skipped ?? []));
232
+ const all = (await this.entries(at, cur.tree, target.tree)).filter((e) => allow.has(e.p));
233
+ const todo = all.filter((e) => !skip.has(e.p));
234
+ if (todo.length) {
235
+ // The tree to go to: the present one with exactly these files as they were. Built in an index of its own.
236
+ const index = path.join(at.dir, "restore-index");
237
+ fs.rmSync(index, { force: true });
238
+ const env = { GIT_INDEX_FILE: index };
239
+ await this.must("the files could not be restored", this.g(at, ["read-tree", cur.tree], { env }));
240
+ // What goes comes first: a file may give way to a folder of its name, or the other way round.
241
+ const lines = [...todo.filter((e) => e.to.mode === NONE), ...todo.filter((e) => e.to.mode !== NONE)].map((e) => `${e.to.mode === NONE ? `0 ${"0".repeat(e.to.sha.length)}` : `${e.to.mode} ${e.to.sha}`}\t${e.p}\0`);
242
+ await this.must("the files could not be restored", this.g(at, ["update-index", "-z", "--index-info"], { env, stdin: bytes(lines.join("")) }));
243
+ const want = (await this.must("the files could not be restored", this.g(at, ["write-tree"], { env }))).out.trim();
244
+ fs.rmSync(index, { force: true });
245
+ // The index is the present state, so only these files differ, and only they are written or removed.
246
+ await this.must("the files could not be restored", this.g(at, ["read-tree", "--reset", "-u", want], { write: [root] }));
247
+ }
248
+ return { cur, files: todo.map((e) => this.outward(at, e)), paths: todo.map((e) => e.p), left: all.filter((e) => skip.has(e.p)).map((e) => this.outward(at, e).path) };
249
+ });
250
+ }
251
+ /**
252
+ * The tidying: states nobody refers to any more are dropped once they are a few days old, and
253
+ * the git directory of a folder that is gone, or that has no state left, is removed.
254
+ * `kept`: the trees still referred to, by folder.
255
+ */
256
+ async prune(kept) {
257
+ let names = [];
258
+ try {
259
+ names = fs.readdirSync(this.dir).filter((n) => /^[0-9a-f]{24}$/.test(n));
260
+ }
261
+ catch {
262
+ return; // nothing recorded yet
263
+ }
264
+ for (const name of names) {
265
+ const dir = path.join(this.dir, name);
266
+ let root = "";
267
+ try {
268
+ root = fs.readFileSync(path.join(dir, "pine-folder"), "utf8");
269
+ }
270
+ catch {
271
+ // not one of ours, or half made
272
+ }
273
+ const trees = kept.get(root);
274
+ const age = (file) => {
275
+ try {
276
+ return Date.now() - fs.statSync(path.join(dir, file)).mtimeMs;
277
+ }
278
+ catch {
279
+ return Number.POSITIVE_INFINITY;
280
+ }
281
+ };
282
+ // The index is written with every state that is recorded. The directory's own time counts
283
+ // too: one that is being made this moment has neither file yet and must not be taken for an old one.
284
+ const idle = Math.min(age("."), age("index"), age(TIDIED)) > DAY;
285
+ if (!root || !fs.existsSync(root) || !trees?.size) {
286
+ if (idle)
287
+ fs.rmSync(dir, { recursive: true, force: true });
288
+ continue;
289
+ }
290
+ // Once a day is enough, and a folder in use is not held up by it more often.
291
+ if (age(TIDIED) < DAY)
292
+ continue;
293
+ fs.writeFileSync(path.join(dir, TIDIED), "");
294
+ await this.alone(root, async () => {
295
+ const at = { dir, base: root, prefix: "", at: 0 };
296
+ // A tree that is referred to gets a name, which is what keeps git from dropping it.
297
+ const have = split((await this.g(at, ["for-each-ref", "--format=%(refname)%00", "refs/pine/"])).raw).map((r) => r.trim()).filter(Boolean);
298
+ const want = [...(trees ?? [])].filter((t) => /^[0-9a-f]{40,64}$/.test(t));
299
+ const edits = [...have.filter((r) => !want.includes(r.slice("refs/pine/".length))).map((r) => `delete ${r}\n`), ...want.filter((t) => !have.includes(`refs/pine/${t}`)).map((t) => `update refs/pine/${t} ${t}\n`)];
300
+ if (edits.length)
301
+ await this.g(at, ["update-ref", "--stdin"], { stdin: edits.join("") });
302
+ await this.g(at, ["gc", "--quiet", "--prune=3.days.ago"], { timeoutMs: 300_000 });
303
+ }).catch(() => { });
304
+ }
305
+ }
306
+ }