@tenonai/yew 0.0.0-stage → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (133) hide show
  1. package/CHANGELOG.md +296 -0
  2. package/LICENSE +21 -0
  3. package/README.md +55 -2
  4. package/THIRD-PARTY-NOTICES.md +122 -0
  5. package/bin/yew.js +9 -0
  6. package/dist/cli/app.js +198 -0
  7. package/dist/cli/export.js +57 -0
  8. package/dist/cli/io.js +65 -0
  9. package/dist/cli/key.js +113 -0
  10. package/dist/cli/main.js +44 -0
  11. package/dist/cli/mcp.js +76 -0
  12. package/dist/cli/remote.js +116 -0
  13. package/dist/cli/run.js +72 -0
  14. package/dist/cli/schedule.js +123 -0
  15. package/dist/cli/send.js +126 -0
  16. package/dist/cli/serve.js +301 -0
  17. package/dist/cli/service.js +241 -0
  18. package/dist/cli/system.js +44 -0
  19. package/dist/remote/access.js +107 -0
  20. package/dist/remote/connector.js +278 -0
  21. package/dist/remote/identity.js +71 -0
  22. package/dist/remote/noise.js +287 -0
  23. package/dist/remote/protocol.js +51 -0
  24. package/dist/remote/push.js +37 -0
  25. package/dist/server/agent/compaction.js +242 -0
  26. package/dist/server/agent/context.js +119 -0
  27. package/dist/server/agent/held.js +48 -0
  28. package/dist/server/agent/helper.js +122 -0
  29. package/dist/server/agent/mailroom.js +510 -0
  30. package/dist/server/agent/prompt.js +268 -0
  31. package/dist/server/agent/runner.js +1159 -0
  32. package/dist/server/artifacts.js +216 -0
  33. package/dist/server/budget.js +88 -0
  34. package/dist/server/data-export.js +105 -0
  35. package/dist/server/data-usage.js +143 -0
  36. package/dist/server/db/database.js +49 -0
  37. package/dist/server/db/migrations.js +347 -0
  38. package/dist/server/errors.js +16 -0
  39. package/dist/server/events.js +17 -0
  40. package/dist/server/export.js +378 -0
  41. package/dist/server/folder-picker.js +32 -0
  42. package/dist/server/from-jig.js +98 -0
  43. package/dist/server/http/agents.js +78 -0
  44. package/dist/server/http/api.js +468 -0
  45. package/dist/server/http/app.js +111 -0
  46. package/dist/server/http/artifacts.js +53 -0
  47. package/dist/server/http/data.js +54 -0
  48. package/dist/server/http/devices.js +47 -0
  49. package/dist/server/http/events.js +134 -0
  50. package/dist/server/http/idempotency.js +53 -0
  51. package/dist/server/http/input.js +106 -0
  52. package/dist/server/http/mail.js +52 -0
  53. package/dist/server/http/mcp.js +134 -0
  54. package/dist/server/http/notes.js +91 -0
  55. package/dist/server/http/problem.js +55 -0
  56. package/dist/server/http/providers.js +131 -0
  57. package/dist/server/http/schedules.js +58 -0
  58. package/dist/server/http/search.js +42 -0
  59. package/dist/server/http/security.js +168 -0
  60. package/dist/server/http/skills.js +53 -0
  61. package/dist/server/http/system.js +26 -0
  62. package/dist/server/ids.js +30 -0
  63. package/dist/server/keys.js +51 -0
  64. package/dist/server/mail/envelope.js +152 -0
  65. package/dist/server/mail/post.js +603 -0
  66. package/dist/server/mail/store.js +279 -0
  67. package/dist/server/manage.js +601 -0
  68. package/dist/server/mcp-builtin.js +179 -0
  69. package/dist/server/mcp-config.js +305 -0
  70. package/dist/server/mcp-oauth.js +448 -0
  71. package/dist/server/mcp.js +668 -0
  72. package/dist/server/model/gateway.js +410 -0
  73. package/dist/server/paths.js +34 -0
  74. package/dist/server/permissions/engine.js +249 -0
  75. package/dist/server/permissions/paths.js +93 -0
  76. package/dist/server/permissions/shell.js +635 -0
  77. package/dist/server/push.js +345 -0
  78. package/dist/server/reveal.js +36 -0
  79. package/dist/server/runtime.js +17 -0
  80. package/dist/server/schedules.js +356 -0
  81. package/dist/server/settings.js +80 -0
  82. package/dist/server/skills.js +262 -0
  83. package/dist/server/store.js +1077 -0
  84. package/dist/server/tools/files.js +318 -0
  85. package/dist/server/tools/html.js +221 -0
  86. package/dist/server/tools/jobs.js +38 -0
  87. package/dist/server/tools/mail.js +680 -0
  88. package/dist/server/tools/match.js +81 -0
  89. package/dist/server/tools/misc.js +253 -0
  90. package/dist/server/tools/process.js +128 -0
  91. package/dist/server/tools/read-state.js +40 -0
  92. package/dist/server/tools/registry.js +43 -0
  93. package/dist/server/tools/search.js +175 -0
  94. package/dist/server/tools/shell.js +124 -0
  95. package/dist/server/tools/types.js +43 -0
  96. package/dist/server/tools/web-search.js +198 -0
  97. package/dist/server/tools/web.js +240 -0
  98. package/dist/shared/cron.js +105 -0
  99. package/dist/shared/mcp-presets.js +170 -0
  100. package/dist/shared/messages.js +45 -0
  101. package/dist/shared/policy.js +42 -0
  102. package/dist/shared/types.js +34 -0
  103. package/dist/shared/version.js +8 -0
  104. package/mcp-builtin/package-lock.json +3398 -0
  105. package/mcp-builtin/package.json +14 -0
  106. package/mcp-builtin/presets/gitee/package-lock.json +105 -0
  107. package/mcp-builtin/presets/gitee/package.json +10 -0
  108. package/mcp-builtin/presets/notion/package-lock.json +2243 -0
  109. package/mcp-builtin/presets/notion/package.json +10 -0
  110. package/mcp-builtin/presets/sentry/package-lock.json +28 -0
  111. package/mcp-builtin/presets/sentry/package.json +10 -0
  112. package/package.json +75 -5
  113. package/web/dist/THIRD-PARTY-NOTICES.txt +1944 -0
  114. package/web/dist/assets/index-B9oWPSod.css +1 -0
  115. package/web/dist/assets/index-C5qBMzFX.js +68 -0
  116. package/web/dist/icons/app-icon.png +0 -0
  117. package/web/dist/icons/apple-touch-icon.png +0 -0
  118. package/web/dist/icons/favicon-16.png +0 -0
  119. package/web/dist/icons/favicon-32.png +0 -0
  120. package/web/dist/icons/favicon-64.png +0 -0
  121. package/web/dist/icons/tenon-mark.svg +1 -0
  122. package/web/dist/icons/yew-mark.svg +1 -0
  123. package/web/dist/index.html +18 -0
  124. package/web/dist/mascot/gift.webp +0 -0
  125. package/web/dist/mascot/greeting.webp +0 -0
  126. package/web/dist/mascot/happy.webp +0 -0
  127. package/web/dist/mascot/lost.webp +0 -0
  128. package/web/dist/mascot/shush.webp +0 -0
  129. package/web/dist/mascot/sleepy.webp +0 -0
  130. package/web/dist/mascot/sorry.webp +0 -0
  131. package/web/dist/mascot/thinking.webp +0 -0
  132. package/web/dist/mascot/waking.webp +0 -0
  133. package/web/dist/mascot/writing.webp +0 -0
@@ -0,0 +1,216 @@
1
+ import { mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
2
+ import { join } from "node:path";
3
+ import { ARTIFACT_KINDS } from "../shared/types.js";
4
+ import { conflict, invalid, notFound } from "./errors.js";
5
+ import { newId } from "./ids.js";
6
+ import { errorResult, str, text } from "./tools/types.js";
7
+ /**
8
+ * Artifacts (blueprint 6.6; M4 plan, section 6): what a session makes for the person to keep and
9
+ * look at: a Markdown document, an HTML page, JSON data or a chart. Each belongs to the session
10
+ * that made it and has versions; saving under the same name again adds one. The files are in Yew's
11
+ * data folder, the list in the database.
12
+ */
13
+ const EXTENSION = {
14
+ markdown: "md",
15
+ html: "html",
16
+ json: "json",
17
+ chart: "json",
18
+ };
19
+ export const MAX_ARTIFACT_BYTES = 2 * 1024 * 1024;
20
+ const MAX_PER_SESSION = 200;
21
+ const toArtifact = (row) => ({
22
+ id: row.id,
23
+ sessionId: row.session_id,
24
+ name: row.name,
25
+ kind: row.kind,
26
+ version: row.version,
27
+ size: row.size,
28
+ createdAt: row.created_at,
29
+ updatedAt: row.updated_at,
30
+ });
31
+ /** A chart is data Yew draws itself: bars or lines over labels, a few series. */
32
+ export function checkChart(content) {
33
+ let chart;
34
+ try {
35
+ chart = JSON.parse(content);
36
+ }
37
+ catch {
38
+ throw invalid("A chart is JSON: { type, labels, series }.", "content");
39
+ }
40
+ const labels = chart?.labels;
41
+ const series = chart?.series;
42
+ if (chart?.type !== "bar" && chart?.type !== "line") {
43
+ throw invalid('A chart\'s type is "bar" or "line".', "content");
44
+ }
45
+ if (!Array.isArray(labels) || labels.length === 0 || labels.length > 200) {
46
+ throw invalid("A chart has 1 to 200 labels.", "content");
47
+ }
48
+ if (labels.some((label) => typeof label !== "string" && typeof label !== "number")) {
49
+ throw invalid("A chart's labels are text or numbers.", "content");
50
+ }
51
+ if (!Array.isArray(series) || series.length === 0 || series.length > 8) {
52
+ throw invalid("A chart has 1 to 8 series.", "content");
53
+ }
54
+ for (const one of series) {
55
+ const values = one?.values;
56
+ if (typeof one?.name !== "string" ||
57
+ !Array.isArray(values) ||
58
+ values.length !== labels.length ||
59
+ values.some((value) => typeof value !== "number" || !Number.isFinite(value))) {
60
+ throw invalid("Each series has a name and one number for every label: { name, values }.", "content");
61
+ }
62
+ }
63
+ }
64
+ export class ArtifactStore {
65
+ #store;
66
+ #folder;
67
+ #statements = new Map();
68
+ constructor(store, yewHome) {
69
+ this.#store = store;
70
+ this.#folder = join(yewHome, "artifacts");
71
+ }
72
+ #q(sql) {
73
+ let statement = this.#statements.get(sql);
74
+ if (!statement) {
75
+ statement = this.#store.db.prepare(sql);
76
+ this.#statements.set(sql, statement);
77
+ }
78
+ return statement;
79
+ }
80
+ #file(artifact, version) {
81
+ return join(this.#folder, artifact.sessionId, artifact.id, `v${version}.${EXTENSION[artifact.kind]}`);
82
+ }
83
+ /** Saves a version: the first under a new name, or the next of the artifact with that name. */
84
+ save(sessionId, input) {
85
+ const name = input.name.trim().replace(/\s+/g, " ");
86
+ if (!name || name.length > 120)
87
+ throw invalid("The name is 1 to 120 characters.", "name");
88
+ const size = Buffer.byteLength(input.content);
89
+ if (size === 0)
90
+ throw invalid("The content is empty.", "content");
91
+ if (size > MAX_ARTIFACT_BYTES)
92
+ throw invalid("An artifact can be at most 2 MB.", "content");
93
+ if (input.kind === "chart")
94
+ checkChart(input.content);
95
+ if (input.kind === "json") {
96
+ try {
97
+ JSON.parse(input.content);
98
+ }
99
+ catch {
100
+ throw invalid("The content is not valid JSON.", "content");
101
+ }
102
+ }
103
+ const artifact = this.#store.tx(() => {
104
+ const now = Date.now();
105
+ const existing = this.#q("SELECT * FROM artifacts WHERE session_id = ? AND name = ?").get(sessionId, name);
106
+ if (existing && existing.kind !== input.kind) {
107
+ throw conflict(`"${name}" is already a ${existing.kind} artifact; save this one under another name.`, undefined, "artifact-kind");
108
+ }
109
+ let id = existing?.id;
110
+ const version = (existing?.version ?? 0) + 1;
111
+ if (id) {
112
+ this.#q("UPDATE artifacts SET version = ?, size = ?, updated_at = ? WHERE id = ?").run(version, size, now, id);
113
+ }
114
+ else {
115
+ const count = this.#q("SELECT COUNT(*) AS n FROM artifacts WHERE session_id = ?").get(sessionId);
116
+ if (count.n >= MAX_PER_SESSION) {
117
+ throw conflict("This session already has 200 artifacts.", undefined, "artifact-limit");
118
+ }
119
+ id = newId();
120
+ this.#q(`INSERT INTO artifacts (id, session_id, name, kind, version, size, created_at, updated_at)
121
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?)`).run(id, sessionId, name, input.kind, version, size, now, now);
122
+ }
123
+ this.#q("INSERT INTO artifact_versions (artifact_id, version, size, created_at) VALUES (?, ?, ?, ?)").run(id, version, size, now);
124
+ return this.get(id);
125
+ });
126
+ const file = this.#file(artifact, artifact.version);
127
+ mkdirSync(join(file, ".."), { recursive: true, mode: 0o700 });
128
+ writeFileSync(file, input.content, { mode: 0o600 });
129
+ this.#store.publish({ type: "artifact", artifact });
130
+ return artifact;
131
+ }
132
+ get(id) {
133
+ const row = this.#q("SELECT * FROM artifacts WHERE id = ?").get(id);
134
+ if (!row)
135
+ throw notFound("Artifact", id);
136
+ return toArtifact(row);
137
+ }
138
+ /** Newest first; all of them, or one session's. */
139
+ list(options) {
140
+ const rows = (options.sessionId
141
+ ? this.#q("SELECT * FROM artifacts WHERE session_id = ? ORDER BY updated_at DESC, id LIMIT ?").all(options.sessionId, options.limit)
142
+ : this.#q("SELECT * FROM artifacts ORDER BY updated_at DESC, id LIMIT ?").all(options.limit));
143
+ return rows.map(toArtifact);
144
+ }
145
+ versions(id) {
146
+ const rows = this.#q("SELECT version, size, created_at FROM artifact_versions WHERE artifact_id = ? ORDER BY version DESC").all(id);
147
+ return rows.map((row) => ({ version: row.version, size: row.size, createdAt: row.created_at }));
148
+ }
149
+ content(id, version) {
150
+ const artifact = this.get(id);
151
+ if (!Number.isInteger(version) || version < 1 || version > artifact.version) {
152
+ throw notFound("Artifact version", `${id} v${version}`);
153
+ }
154
+ return { artifact, content: readFileSync(this.#file(artifact, version), "utf8") };
155
+ }
156
+ delete(id) {
157
+ const artifact = this.get(id);
158
+ this.#q("DELETE FROM artifacts WHERE id = ?").run(id);
159
+ rmSync(join(this.#folder, artifact.sessionId, id), { recursive: true, force: true });
160
+ this.#store.publish({ type: "artifactDeleted", id });
161
+ }
162
+ /** The files of sessions that were deleted for good; their rows went with the sessions. */
163
+ forgetSessions(sessionIds) {
164
+ for (const id of sessionIds)
165
+ rmSync(join(this.#folder, id), { recursive: true, force: true });
166
+ }
167
+ }
168
+ export function saveArtifactTool(artifacts) {
169
+ return {
170
+ name: "save_artifact",
171
+ description: 'Save something you made for the person to keep and look at in Yew: a document (markdown), a page (html, self-contained: inline styles and scripts, no network access), data (json) or a chart. A chart is JSON: {"type":"bar"|"line","title":"…","labels":[…],"series":[{"name":"…","values":[…]}]}. Saving under a name this session already used adds a new version. Use it for results worth keeping, not for every reply.',
172
+ parameters: {
173
+ type: "object",
174
+ properties: {
175
+ name: { type: "string", description: "A short title, such as 周报 or Pricing options." },
176
+ kind: { type: "string", enum: [...ARTIFACT_KINDS] },
177
+ content: { type: "string", description: "The whole content of this version." },
178
+ },
179
+ required: ["name", "kind", "content"],
180
+ additionalProperties: false,
181
+ },
182
+ readOnly: false,
183
+ needsFolder: false,
184
+ subject: () => ({ kind: "none" }),
185
+ async run(args, ctx) {
186
+ const kind = str(args, "kind", true);
187
+ if (!ARTIFACT_KINDS.includes(kind)) {
188
+ return errorResult(`kind is one of: ${ARTIFACT_KINDS.join(", ")}.`);
189
+ }
190
+ try {
191
+ const artifact = artifacts.save(ctx.session.id, {
192
+ name: str(args, "name", true),
193
+ kind,
194
+ content: str(args, "content", true),
195
+ });
196
+ return {
197
+ content: [
198
+ text(`Saved "${artifact.name}" as version ${artifact.version}. The person can open it from this conversation and from the artifacts page.`),
199
+ ],
200
+ detail: {
201
+ artifactId: artifact.id,
202
+ name: artifact.name,
203
+ kind: artifact.kind,
204
+ version: artifact.version,
205
+ },
206
+ };
207
+ }
208
+ catch (error) {
209
+ const status = error.status;
210
+ if (status === 400 || status === 409)
211
+ return errorResult(error.message);
212
+ throw error;
213
+ }
214
+ },
215
+ };
216
+ }
@@ -0,0 +1,88 @@
1
+ export const BUDGET_WARN_AT = 0.8;
2
+ export const MAX_DAILY_BUDGET = 100_000;
3
+ /** The calendar day of a moment in local time, as YYYY-MM-DD. */
4
+ export function localDay(at) {
5
+ const date = new Date(at);
6
+ const pad = (n) => String(n).padStart(2, "0");
7
+ return `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())}`;
8
+ }
9
+ /** The `count` local calendar days that end with the day of `now`, oldest first. */
10
+ export function lastDays(now, count) {
11
+ const end = new Date(now);
12
+ const days = [];
13
+ for (let back = count - 1; back >= 0; back--) {
14
+ // Noon keeps the step clear of the hour a clock change adds or removes.
15
+ days.push(localDay(new Date(end.getFullYear(), end.getMonth(), end.getDate() - back, 12).getTime()));
16
+ }
17
+ return days;
18
+ }
19
+ /** Whether text is a calendar day written as YYYY-MM-DD. */
20
+ export function isDay(text) {
21
+ const match = /^(\d{4})-(\d{2})-(\d{2})$/.exec(text);
22
+ if (!match)
23
+ return false;
24
+ const date = new Date(Number(match[1]), Number(match[2]) - 1, Number(match[3]), 12);
25
+ return localDay(date.getTime()) === text;
26
+ }
27
+ /** Checks a limit: dollars above zero, or null for none. */
28
+ export function isBudget(value) {
29
+ return (value === null ||
30
+ (typeof value === "number" && Number.isFinite(value) && value > 0 && value <= MAX_DAILY_BUDGET));
31
+ }
32
+ function extraOn(extra, day) {
33
+ return extra && extra.day === day ? extra.amount : 0;
34
+ }
35
+ /** Today's limits with what was spent against each; a scope without a limit is left out. */
36
+ export function budgets(store, session, now) {
37
+ const day = localDay(now);
38
+ const marks = [];
39
+ const own = session.options.dailyBudget;
40
+ if (own) {
41
+ marks.push({
42
+ scope: "session",
43
+ spent: store.spentOn(day, session.id),
44
+ limit: own + extraOn(session.options.budgetExtra, day),
45
+ });
46
+ }
47
+ const all = store.getSetting("dailyBudget", null);
48
+ if (all) {
49
+ marks.push({
50
+ scope: "all",
51
+ spent: store.spentOn(day),
52
+ limit: all + extraOn(store.getSetting("budgetExtra", null), day),
53
+ });
54
+ }
55
+ return marks;
56
+ }
57
+ /** The first limit that is used up, the session's own before the shared one. */
58
+ export function reached(store, session, now) {
59
+ return budgets(store, session, now).find((mark) => mark.spent >= mark.limit) ?? null;
60
+ }
61
+ /** Limits that the request which just cost `cost` took past 80%, without reaching them. */
62
+ export function crossedWarning(store, session, cost, now) {
63
+ if (cost <= 0)
64
+ return [];
65
+ return budgets(store, session, now).filter((mark) => {
66
+ const line = mark.limit * BUDGET_WARN_AT;
67
+ return mark.spent - cost < line && mark.spent >= line && mark.spent < mark.limit;
68
+ });
69
+ }
70
+ /**
71
+ * The person continues a session that stopped at a limit: each limit that is used up gets as much
72
+ * room again as the limit itself, for the rest of the day.
73
+ */
74
+ export function grantMore(store, session, now) {
75
+ const day = localDay(now);
76
+ for (const mark of budgets(store, session, now)) {
77
+ if (mark.spent < mark.limit)
78
+ continue;
79
+ const extra = { day, amount: mark.spent };
80
+ if (mark.scope === "session")
81
+ store.updateSession(session.id, { options: { budgetExtra: extra } });
82
+ else
83
+ store.setSetting("budgetExtra", extra);
84
+ }
85
+ }
86
+ export function dollars(amount) {
87
+ return `$${amount.toFixed(2)}`;
88
+ }
@@ -0,0 +1,105 @@
1
+ import { existsSync, mkdirSync, readdirSync, writeFileSync } from "node:fs";
2
+ import { join } from "node:path";
3
+ import { ArtifactStore } from "./artifacts.js";
4
+ import { invalid } from "./errors.js";
5
+ import { sessionMarkdown } from "./export.js";
6
+ import { MailStore } from "./mail/store.js";
7
+ import { ScheduleBook } from "./schedules.js";
8
+ import { readSettings } from "./settings.js";
9
+ const PAGE = 200;
10
+ function all(page) {
11
+ const items = [];
12
+ let cursor = null;
13
+ do {
14
+ const next = page(cursor);
15
+ items.push(...next.items);
16
+ cursor = next.nextCursor;
17
+ } while (cursor !== null);
18
+ return items;
19
+ }
20
+ /** A file name from a session's name: readable, and safe on every file system. */
21
+ function fileNameOf(name, taken) {
22
+ const base = name
23
+ .replace(/[\\/:*?"<>|\p{Cc}]/gu, " ")
24
+ .replace(/\s+/g, " ")
25
+ .trim()
26
+ .slice(0, 80) || "session";
27
+ let candidate = base;
28
+ for (let n = 2; taken.has(candidate.toLowerCase()); n++)
29
+ candidate = `${base} ${n}`;
30
+ taken.add(candidate.toLowerCase());
31
+ return `${candidate}.md`;
32
+ }
33
+ export function exportAll(store, folder, now = Date.now(), yewHome = null) {
34
+ if (existsSync(folder) && readdirSync(folder).length > 0) {
35
+ throw invalid(`${folder} is not empty. Export into a new or empty folder.`, "folder");
36
+ }
37
+ const settings = readSettings(store);
38
+ const language = settings.language === "en" ? "en" : "zh";
39
+ const mail = new MailStore(store);
40
+ let files = 0;
41
+ const write = (path, content) => {
42
+ writeFileSync(join(folder, path), content, { mode: 0o600 });
43
+ files += 1;
44
+ };
45
+ const json = (value) => `${JSON.stringify(value, null, "\t")}\n`;
46
+ mkdirSync(join(folder, "sessions"), { recursive: true, mode: 0o700 });
47
+ const sessions = [
48
+ ...all((cursor) => store.listSessions({ cursor, limit: PAGE })),
49
+ ...all((cursor) => store.listSessions({ archived: true, cursor, limit: PAGE })),
50
+ ];
51
+ const groups = new Map(store.listGroups().map((group) => [group.id, group.name]));
52
+ const taken = new Set();
53
+ const index = sessions.map((session) => {
54
+ const file = fileNameOf(session.name, taken);
55
+ write(join("sessions", file), sessionMarkdown({
56
+ session,
57
+ groupName: session.groupId ? (groups.get(session.groupId) ?? null) : null,
58
+ entries: store.allEntries(session.id),
59
+ toolCalls: store.toolCallsForSession(session.id),
60
+ language,
61
+ now,
62
+ }));
63
+ // The record is in the Markdown file; this lists what the session is.
64
+ const { rules, options, todos, ...summary } = session;
65
+ return {
66
+ ...summary,
67
+ team: session.groupId ? (groups.get(session.groupId) ?? null) : null,
68
+ file,
69
+ };
70
+ });
71
+ write("sessions.json", json(index));
72
+ const letters = all((cursor) => mail.listLetters({ cursor, limit: PAGE }));
73
+ write("letters.json", json(letters));
74
+ const cards = all((cursor) => mail.listCards({ cursor: cursor === null ? null : Number(cursor), limit: PAGE }));
75
+ write("cards.json", json(cards));
76
+ const schedules = new ScheduleBook(store).list();
77
+ write("schedules.json", json(schedules));
78
+ write("settings.json", json({ exportedAt: new Date(now).toISOString(), ...settings }));
79
+ // Artifacts: every version's file under the session's and the artifact's name, with a list.
80
+ let artifacts = 0;
81
+ if (yewHome) {
82
+ const kept = new ArtifactStore(store, yewHome);
83
+ const names = new Map(index.map((entry) => [entry.id, entry.file.replace(/\.md$/, "")]));
84
+ const used = new Set();
85
+ for (const artifact of kept.list({ limit: 100_000 })) {
86
+ const dir = join("artifacts", names.get(artifact.sessionId) ?? artifact.sessionId, fileNameOf(artifact.name, used).replace(/\.md$/, ""));
87
+ mkdirSync(join(folder, dir), { recursive: true, mode: 0o700 });
88
+ const versions = kept.versions(artifact.id);
89
+ for (const { version } of versions) {
90
+ const extension = { markdown: "md", html: "html", json: "json", chart: "json" }[artifact.kind];
91
+ write(join(dir, `v${version}.${extension}`), kept.content(artifact.id, version).content);
92
+ }
93
+ write(join(dir, "artifact.json"), json({ ...artifact, versions }));
94
+ artifacts += 1;
95
+ }
96
+ }
97
+ return {
98
+ artifacts,
99
+ sessions: sessions.length,
100
+ letters: letters.length,
101
+ cards: cards.length,
102
+ schedules: schedules.length,
103
+ files,
104
+ };
105
+ }
@@ -0,0 +1,143 @@
1
+ import { lstatSync, readdirSync, realpathSync, rmdirSync, unlinkSync } from "node:fs";
2
+ import { join } from "node:path";
3
+ import { conflict } from "./errors.js";
4
+ const DATABASE_FILES = new Set(["yew.db", "yew.db-wal", "yew.db-shm"]);
5
+ const FOLDERS = { artifacts: "artifacts", logs: "logs", tmp: "temp" };
6
+ function entries(dir) {
7
+ try {
8
+ return readdirSync(dir);
9
+ }
10
+ catch {
11
+ return [];
12
+ }
13
+ }
14
+ /** The bytes under a path, without following links; what cannot be read counts as nothing. */
15
+ function sizeOf(path) {
16
+ let stat;
17
+ try {
18
+ stat = lstatSync(path);
19
+ }
20
+ catch {
21
+ return 0;
22
+ }
23
+ if (!stat.isDirectory())
24
+ return stat.size;
25
+ let bytes = 0;
26
+ for (const name of entries(path))
27
+ bytes += sizeOf(join(path, name));
28
+ return bytes;
29
+ }
30
+ const EMPTY_SCAN = { removable: [], folders: [], keptFiles: 0, keptBytes: 0 };
31
+ /**
32
+ * Lists what is under `tmp`. The folder must really be `<data folder>/tmp`: when it is a link, or
33
+ * resolves to somewhere else, nothing is listed and the cleanup refuses. Below it only real folders
34
+ * are entered and only regular files are offered for deletion.
35
+ */
36
+ function scanTemp(home, inUse) {
37
+ const tmp = join(home, "tmp");
38
+ let real;
39
+ try {
40
+ if (lstatSync(tmp).isSymbolicLink())
41
+ throw outside();
42
+ real = realpathSync(tmp);
43
+ }
44
+ catch (error) {
45
+ if (error.code === "ENOENT")
46
+ return EMPTY_SCAN;
47
+ throw error;
48
+ }
49
+ if (real !== join(realpathSync(home), "tmp"))
50
+ throw outside();
51
+ const scan = { removable: [], folders: [], keptFiles: 0, keptBytes: 0 };
52
+ const walk = (dir, kept, top) => {
53
+ for (const name of entries(dir)) {
54
+ const path = join(dir, name);
55
+ let stat;
56
+ try {
57
+ stat = lstatSync(path);
58
+ }
59
+ catch {
60
+ continue;
61
+ }
62
+ // Directly under tmp a folder is a session's, named by its id.
63
+ const keep = kept || (top && stat.isDirectory() && inUse(name));
64
+ if (stat.isDirectory()) {
65
+ walk(path, keep, false);
66
+ if (!keep)
67
+ scan.folders.push(path);
68
+ }
69
+ else if (stat.isFile() && !keep) {
70
+ scan.removable.push({ path, bytes: stat.size });
71
+ }
72
+ else {
73
+ scan.keptFiles += 1;
74
+ scan.keptBytes += stat.size;
75
+ }
76
+ }
77
+ };
78
+ walk(real, false, true);
79
+ return scan;
80
+ }
81
+ const outside = () => conflict("The tmp folder in Yew's data folder is a link to somewhere else. Yew does not delete through it.", undefined, "temp-outside");
82
+ export function dataUsage(home, inUse = () => false, now = Date.now()) {
83
+ const bytes = {
84
+ database: 0,
85
+ artifacts: 0,
86
+ logs: 0,
87
+ temp: 0,
88
+ other: 0,
89
+ };
90
+ for (const name of entries(home)) {
91
+ const part = DATABASE_FILES.has(name) ? "database" : (FOLDERS[name] ?? "other");
92
+ bytes[part] += sizeOf(join(home, name));
93
+ }
94
+ let scan = EMPTY_SCAN;
95
+ try {
96
+ scan = scanTemp(home, inUse);
97
+ }
98
+ catch {
99
+ // A tmp folder that is a link: its size is the link's, and nothing in it is offered.
100
+ }
101
+ const removableBytes = scan.removable.reduce((sum, file) => sum + file.bytes, 0);
102
+ const parts = Object.keys(bytes).map((id) => ({ id, bytes: bytes[id] }));
103
+ return {
104
+ folder: home,
105
+ totalBytes: parts.reduce((sum, part) => sum + part.bytes, 0),
106
+ parts,
107
+ temp: {
108
+ files: scan.removable.length + scan.keptFiles,
109
+ bytes: removableBytes + scan.keptBytes,
110
+ removableFiles: scan.removable.length,
111
+ removableBytes,
112
+ },
113
+ readAt: now,
114
+ };
115
+ }
116
+ /**
117
+ * Deletes the files under `tmp`, except those of sessions with a command that may still be writing.
118
+ * It runs in one go without waiting, so no command can start in between. Emptied folders go too; the
119
+ * `tmp` folder itself stays.
120
+ */
121
+ export function cleanTemp(home, inUse = () => false) {
122
+ const scan = scanTemp(home, inUse);
123
+ const result = { deletedFiles: 0, freedBytes: 0, skippedFiles: scan.keptFiles };
124
+ for (const file of scan.removable) {
125
+ try {
126
+ unlinkSync(file.path);
127
+ result.deletedFiles += 1;
128
+ result.freedBytes += file.bytes;
129
+ }
130
+ catch {
131
+ result.skippedFiles += 1;
132
+ }
133
+ }
134
+ for (const folder of scan.folders) {
135
+ try {
136
+ rmdirSync(folder);
137
+ }
138
+ catch {
139
+ // Not empty: something in it was kept.
140
+ }
141
+ }
142
+ return result;
143
+ }
@@ -0,0 +1,49 @@
1
+ import { DatabaseSync } from "node:sqlite";
2
+ import { migrations } from "./migrations.js";
3
+ /** Opens (and creates or migrates) the database. Pass ":memory:" in tests. */
4
+ export function openDatabase(path) {
5
+ const db = new DatabaseSync(path, { timeout: 5000 });
6
+ db.exec("PRAGMA foreign_keys = ON");
7
+ // What is deleted is overwritten with zeros, not left in free pages of the file: a rewound
8
+ // conversation, a deleted session.
9
+ db.exec("PRAGMA secure_delete = ON");
10
+ if (path !== ":memory:") {
11
+ db.exec("PRAGMA journal_mode = WAL");
12
+ db.exec("PRAGMA synchronous = NORMAL");
13
+ }
14
+ migrate(db);
15
+ return db;
16
+ }
17
+ export function migrate(db) {
18
+ const row = db.prepare("PRAGMA user_version").get();
19
+ if (row.user_version > migrations.length) {
20
+ throw new Error(`The database was written by a newer version of Yew (schema ${row.user_version}); upgrade Yew.`);
21
+ }
22
+ for (let version = row.user_version; version < migrations.length; version++) {
23
+ transaction(db, () => {
24
+ db.exec(migrations[version]);
25
+ db.exec(`PRAGMA user_version = ${version + 1}`);
26
+ });
27
+ }
28
+ }
29
+ /** Runs `fn` in one write transaction; nested calls join the outer one. */
30
+ export function transaction(db, fn) {
31
+ if (db.isTransaction)
32
+ return fn();
33
+ db.exec("BEGIN IMMEDIATE");
34
+ try {
35
+ const result = fn();
36
+ db.exec("COMMIT");
37
+ return result;
38
+ }
39
+ catch (error) {
40
+ if (db.isTransaction)
41
+ db.exec("ROLLBACK");
42
+ throw error;
43
+ }
44
+ }
45
+ export function json(text, fallback) {
46
+ if (text == null)
47
+ return fallback;
48
+ return JSON.parse(text);
49
+ }