@dotdrelle/wiki-manager 0.6.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +58 -0
- package/LICENSE +108 -0
- package/README.md +693 -0
- package/agents.docker-compose.yml +96 -0
- package/bin/wiki-manager.js +34 -0
- package/bunfig.toml +1 -0
- package/docker-compose.yml +135 -0
- package/mcp.endpoints.example.json +28 -0
- package/package.json +57 -0
- package/src/agent/graph.js +638 -0
- package/src/agent/llm.js +239 -0
- package/src/cli/wiki-manager.js +389 -0
- package/src/commands/slash.js +1044 -0
- package/src/core/activity.js +236 -0
- package/src/core/activity.test.js +127 -0
- package/src/core/agentEvents.js +238 -0
- package/src/core/agentEvents.test.js +134 -0
- package/src/core/compose.js +310 -0
- package/src/core/documentIntake.js +311 -0
- package/src/core/documentIntake.test.js +121 -0
- package/src/core/env.js +57 -0
- package/src/core/jobQueue.js +197 -0
- package/src/core/mcp.js +402 -0
- package/src/core/mcp.test.js +228 -0
- package/src/core/plan.js +181 -0
- package/src/core/plan.test.js +168 -0
- package/src/core/skills.js +142 -0
- package/src/core/wikirc.js +65 -0
- package/src/core/workspaces.js +81 -0
- package/src/shell/FileEditorDialog.tsx +94 -0
- package/src/shell/LeftPane.tsx +680 -0
- package/src/shell/RightPane.tsx +291 -0
- package/src/shell/SlashDialog.tsx +39 -0
- package/src/shell/renderer.ts +69 -0
- package/src/shell/repl.js +1490 -0
- package/src/shell/tui.tsx +205 -0
- package/src/shell/useAgent.ts +47 -0
- package/src/shell/useSession.ts +370 -0
- package/tsconfig.json +16 -0
- package/wiki-workspace +773 -0
|
@@ -0,0 +1,311 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
import { existsSync } from 'node:fs';
|
|
3
|
+
import { copyFile, mkdir, readFile, rename, rm, stat, writeFile } from 'node:fs/promises';
|
|
4
|
+
import { basename, extname, isAbsolute, join, resolve } from 'node:path';
|
|
5
|
+
import { callMcpTool, formatMcpToolResult } from './mcp.js';
|
|
6
|
+
import { parseJsonText } from './activity.js';
|
|
7
|
+
import { userManagerDir } from './env.js';
|
|
8
|
+
|
|
9
|
+
const SUPPORTED_EXTENSIONS = new Set([
|
|
10
|
+
'.txt', '.md', '.markdown', '.csv', '.json', '.xml', '.yaml', '.yml', '.html', '.htm', '.rtf',
|
|
11
|
+
'.png', '.jpg', '.jpeg', '.tif', '.tiff', '.bmp', '.webp',
|
|
12
|
+
'.docx', '.pptx', '.xlsx', '.doc', '.ppt', '.xls', '.odt', '.ods', '.odp',
|
|
13
|
+
'.pdf',
|
|
14
|
+
]);
|
|
15
|
+
|
|
16
|
+
function agentsDataDir() {
|
|
17
|
+
const configured = process.env.AGENTS_DATA_DIR || '.agents-data';
|
|
18
|
+
return isAbsolute(configured) ? configured : resolve(userManagerDir(), configured);
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
function documentInputRoot() {
|
|
22
|
+
return resolve(agentsDataDir(), 'documents', 'input');
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function uploadsRoot() {
|
|
26
|
+
return resolve(agentsDataDir(), 'documents', 'uploads');
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function requireWorkspace(session) {
|
|
30
|
+
if (!session?.workspace) throw new Error('No workspace loaded. Use /use <workspace>.');
|
|
31
|
+
return session.workspace;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function maxUploadBytes() {
|
|
35
|
+
const value = Number(process.env.DOCUMENT_MAX_UPLOAD_BYTES || 50 * 1024 * 1024);
|
|
36
|
+
return Number.isFinite(value) && value > 0 ? value : 50 * 1024 * 1024;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function sanitizeFilename(filename) {
|
|
40
|
+
const name = basename(String(filename || '').trim())
|
|
41
|
+
.normalize('NFKD')
|
|
42
|
+
.replace(/[^\w.\- ]+/g, '_')
|
|
43
|
+
.replace(/\s+/g, '_')
|
|
44
|
+
.replace(/_+/g, '_')
|
|
45
|
+
.replace(/^\.+/g, '')
|
|
46
|
+
.slice(0, 120);
|
|
47
|
+
return name || 'upload.bin';
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function unquotePath(value) {
|
|
51
|
+
const text = String(value ?? '').trim();
|
|
52
|
+
if (text.length >= 2) {
|
|
53
|
+
const first = text[0];
|
|
54
|
+
const last = text.at(-1);
|
|
55
|
+
if ((first === "'" && last === "'") || (first === '"' && last === '"')) {
|
|
56
|
+
return text.slice(1, -1);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
return text;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function assertSupportedFile(filename, size) {
|
|
63
|
+
const ext = extname(filename).toLowerCase();
|
|
64
|
+
if (!SUPPORTED_EXTENSIONS.has(ext)) {
|
|
65
|
+
throw new Error(`Unsupported document type: ${ext || 'no extension'}`);
|
|
66
|
+
}
|
|
67
|
+
const max = maxUploadBytes();
|
|
68
|
+
if (size > max) {
|
|
69
|
+
throw new Error(`Document is too large: ${size} bytes (max ${max}).`);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
function manifestPath(workspace) {
|
|
74
|
+
return join(uploadsRoot(), `${workspace}.jsonl`);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
async function readManifest(workspace) {
|
|
78
|
+
const file = manifestPath(workspace);
|
|
79
|
+
if (!existsSync(file)) return [];
|
|
80
|
+
const raw = await readFile(file, 'utf8');
|
|
81
|
+
return raw
|
|
82
|
+
.split(/\r?\n/)
|
|
83
|
+
.filter(Boolean)
|
|
84
|
+
.flatMap((line) => {
|
|
85
|
+
try {
|
|
86
|
+
return [JSON.parse(line)];
|
|
87
|
+
} catch {
|
|
88
|
+
return [];
|
|
89
|
+
}
|
|
90
|
+
});
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
async function writeManifest(workspace, records) {
|
|
94
|
+
const file = manifestPath(workspace);
|
|
95
|
+
await mkdir(uploadsRoot(), { recursive: true });
|
|
96
|
+
const body = records.map((record) => JSON.stringify(record)).join('\n');
|
|
97
|
+
const tmp = `${file}.tmp.${process.pid}`;
|
|
98
|
+
await writeFile(tmp, body ? `${body}\n` : '', 'utf8');
|
|
99
|
+
await rename(tmp, file);
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
async function upsertUpload(record) {
|
|
103
|
+
const records = await readManifest(record.workspace);
|
|
104
|
+
const index = records.findIndex((item) => item.id === record.id);
|
|
105
|
+
if (index === -1) records.unshift(record);
|
|
106
|
+
else records[index] = { ...records[index], ...record };
|
|
107
|
+
await writeManifest(record.workspace, records);
|
|
108
|
+
return record;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
async function removeExistingUploadsForFilename(workspace, filename) {
|
|
112
|
+
const records = await readManifest(workspace);
|
|
113
|
+
const removed = records.filter((item) => item.filename === filename);
|
|
114
|
+
if (removed.length === 0) return [];
|
|
115
|
+
|
|
116
|
+
const keep = records.filter((item) => item.filename !== filename);
|
|
117
|
+
for (const record of removed) {
|
|
118
|
+
for (const file of [record.storedPath, record.outputPath]) {
|
|
119
|
+
if (file) await rm(file, { force: true }).catch(() => {});
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
await writeManifest(workspace, keep);
|
|
123
|
+
return removed;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
function documentsConverter(session) {
|
|
127
|
+
const endpoint = session?.mcp?.documents;
|
|
128
|
+
if (endpoint?.status !== 'connected') return null;
|
|
129
|
+
const hasTool = (endpoint.tools ?? []).some((tool) => tool.name === 'documents_convert_to_markdown');
|
|
130
|
+
return hasTool ? endpoint : null;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
function publicRecord(record) {
|
|
134
|
+
return {
|
|
135
|
+
id: record.id,
|
|
136
|
+
workspace: record.workspace,
|
|
137
|
+
filename: record.filename,
|
|
138
|
+
status: record.status,
|
|
139
|
+
provider: record.provider,
|
|
140
|
+
agentPath: record.agentPath,
|
|
141
|
+
outputPath: record.outputPath,
|
|
142
|
+
method: record.method,
|
|
143
|
+
bytes: record.bytes,
|
|
144
|
+
error: record.error,
|
|
145
|
+
createdAt: record.createdAt,
|
|
146
|
+
updatedAt: record.updatedAt,
|
|
147
|
+
};
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
function activityFromPayload(payload) {
|
|
151
|
+
return payload?._activity && typeof payload._activity === 'object' ? payload._activity : null;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
export async function storeDocumentUpload(session, sourcePath) {
|
|
155
|
+
const workspace = requireWorkspace(session);
|
|
156
|
+
const absolutePath = resolve(unquotePath(sourcePath));
|
|
157
|
+
const info = await stat(absolutePath);
|
|
158
|
+
if (!info.isFile()) throw new Error(`Not a file: ${sourcePath}`);
|
|
159
|
+
const filename = sanitizeFilename(absolutePath);
|
|
160
|
+
assertSupportedFile(filename, info.size);
|
|
161
|
+
|
|
162
|
+
await removeExistingUploadsForFilename(workspace, filename);
|
|
163
|
+
|
|
164
|
+
const id = randomUUID().slice(0, 8);
|
|
165
|
+
const storedFilename = `${id}-${filename}`;
|
|
166
|
+
const workspaceInput = join(documentInputRoot(), workspace);
|
|
167
|
+
await mkdir(workspaceInput, { recursive: true });
|
|
168
|
+
const storedPath = join(workspaceInput, storedFilename);
|
|
169
|
+
await copyFile(absolutePath, storedPath);
|
|
170
|
+
const now = new Date().toISOString();
|
|
171
|
+
const record = {
|
|
172
|
+
id,
|
|
173
|
+
workspace,
|
|
174
|
+
filename,
|
|
175
|
+
originalPath: absolutePath,
|
|
176
|
+
storedPath,
|
|
177
|
+
agentPath: `/documents/input/${workspace}/${storedFilename}`,
|
|
178
|
+
status: 'stored',
|
|
179
|
+
provider: null,
|
|
180
|
+
outputPath: null,
|
|
181
|
+
method: null,
|
|
182
|
+
bytes: info.size,
|
|
183
|
+
error: null,
|
|
184
|
+
createdAt: now,
|
|
185
|
+
updatedAt: now,
|
|
186
|
+
};
|
|
187
|
+
await upsertUpload(record);
|
|
188
|
+
return record;
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
export async function convertStoredDocument(session, id, options = {}) {
|
|
192
|
+
const workspace = requireWorkspace(session);
|
|
193
|
+
const records = await readManifest(workspace);
|
|
194
|
+
const record = records.find((item) => item.id === id);
|
|
195
|
+
if (!record) throw new Error(`Unknown upload id: ${id}`);
|
|
196
|
+
if (!existsSync(record.storedPath)) throw new Error(`Stored file is missing: ${record.storedPath}`);
|
|
197
|
+
if (!documentsConverter(session)) {
|
|
198
|
+
record.status = 'stored';
|
|
199
|
+
record.provider = null;
|
|
200
|
+
record.error = 'documents MCP is not connected';
|
|
201
|
+
record.updatedAt = new Date().toISOString();
|
|
202
|
+
await upsertUpload(record);
|
|
203
|
+
return { record, converted: false };
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
record.status = 'converting';
|
|
207
|
+
record.provider = 'documents';
|
|
208
|
+
record.error = null;
|
|
209
|
+
record.updatedAt = new Date().toISOString();
|
|
210
|
+
await upsertUpload(record);
|
|
211
|
+
|
|
212
|
+
try {
|
|
213
|
+
const toolArgs = {
|
|
214
|
+
workspace,
|
|
215
|
+
filePath: record.agentPath,
|
|
216
|
+
outputFilename: `${record.id}-${record.filename.replace(/\.[^.]+$/, '')}.md`,
|
|
217
|
+
};
|
|
218
|
+
const result = await callMcpTool(session.mcp, 'documents', 'documents_convert_to_markdown', toolArgs);
|
|
219
|
+
const payload = parseJsonText(formatMcpToolResult(result)) ?? {};
|
|
220
|
+
if (payload.ok === false) throw new Error(payload.error || 'documents conversion failed');
|
|
221
|
+
|
|
222
|
+
const activity = activityFromPayload(payload);
|
|
223
|
+
if (activity && !activity.terminal) {
|
|
224
|
+
record.jobId = payload.jobId ?? null;
|
|
225
|
+
record.updatedAt = new Date().toISOString();
|
|
226
|
+
await upsertUpload(record);
|
|
227
|
+
return { record, converted: false, activity };
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
record.status = 'converted';
|
|
231
|
+
record.outputPath = payload.outputPath ?? null;
|
|
232
|
+
record.method = payload.method ?? null;
|
|
233
|
+
record.error = null;
|
|
234
|
+
record.updatedAt = new Date().toISOString();
|
|
235
|
+
await upsertUpload(record);
|
|
236
|
+
return { record, converted: true, payload, activity };
|
|
237
|
+
} catch (err) {
|
|
238
|
+
record.status = 'failed';
|
|
239
|
+
record.error = err instanceof Error ? err.message : String(err);
|
|
240
|
+
record.updatedAt = new Date().toISOString();
|
|
241
|
+
await upsertUpload(record);
|
|
242
|
+
return { record, converted: false };
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
export async function storeAndMaybeConvertDocument(session, sourcePath, options = {}) {
|
|
247
|
+
const record = await storeDocumentUpload(session, sourcePath);
|
|
248
|
+
if (!documentsConverter(session)) {
|
|
249
|
+
record.error = 'documents MCP is not connected';
|
|
250
|
+
await upsertUpload(record);
|
|
251
|
+
return { record, converted: false };
|
|
252
|
+
}
|
|
253
|
+
return convertStoredDocument(session, record.id, options);
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
export async function listDocumentUploads(session) {
|
|
257
|
+
const workspace = requireWorkspace(session);
|
|
258
|
+
return (await readManifest(workspace)).map(publicRecord);
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
export async function convertPendingDocumentUploads(session, options = {}) {
|
|
262
|
+
const workspace = requireWorkspace(session);
|
|
263
|
+
const records = await readManifest(workspace);
|
|
264
|
+
const pending = records.filter((record) => ['stored', 'failed'].includes(record.status));
|
|
265
|
+
const results = [];
|
|
266
|
+
for (const record of pending) {
|
|
267
|
+
results.push(await convertStoredDocument(session, record.id, options));
|
|
268
|
+
}
|
|
269
|
+
return results;
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
function parseAgeMs(value = '30d') {
|
|
273
|
+
const match = String(value).trim().match(/^(\d+)([dhm])$/i);
|
|
274
|
+
if (!match) throw new Error('Invalid age. Use a value like 30d, 12h, or 90m.');
|
|
275
|
+
const amount = Number(match[1]);
|
|
276
|
+
const unit = match[2].toLowerCase();
|
|
277
|
+
const factor = unit === 'd' ? 24 * 60 * 60 * 1000 : unit === 'h' ? 60 * 60 * 1000 : 60 * 1000;
|
|
278
|
+
return amount * factor;
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
export async function cleanDocumentUploads(session, olderThan = '30d') {
|
|
282
|
+
const workspace = requireWorkspace(session);
|
|
283
|
+
const cutoff = Date.now() - parseAgeMs(olderThan);
|
|
284
|
+
const records = await readManifest(workspace);
|
|
285
|
+
const keep = [];
|
|
286
|
+
const removed = [];
|
|
287
|
+
for (const record of records) {
|
|
288
|
+
const created = Date.parse(record.createdAt || record.updatedAt || '');
|
|
289
|
+
if (Number.isFinite(created) && created < cutoff) {
|
|
290
|
+
removed.push(record);
|
|
291
|
+
if (record.storedPath) {
|
|
292
|
+
await rm(record.storedPath, { force: true }).catch(() => {});
|
|
293
|
+
}
|
|
294
|
+
} else {
|
|
295
|
+
keep.push(record);
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
if (removed.length > 0) await writeManifest(workspace, keep);
|
|
299
|
+
return { removed, kept: keep };
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
export function formatUploadRecord(record) {
|
|
303
|
+
const lines = [
|
|
304
|
+
`${record.id} ${record.status} ${record.filename}`,
|
|
305
|
+
`agentPath: ${record.agentPath}`,
|
|
306
|
+
];
|
|
307
|
+
if (record.outputPath) lines.push(`outputPath: ${record.outputPath}`);
|
|
308
|
+
if (record.method) lines.push(`method: ${record.method}`);
|
|
309
|
+
if (record.error) lines.push(`note: ${record.error}`);
|
|
310
|
+
return lines.join('\n');
|
|
311
|
+
}
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
import { test } from 'node:test';
|
|
2
|
+
import assert from 'node:assert/strict';
|
|
3
|
+
import { mkdir, mkdtemp, readFile, writeFile } from 'node:fs/promises';
|
|
4
|
+
import { existsSync } from 'node:fs';
|
|
5
|
+
import os from 'node:os';
|
|
6
|
+
import path from 'node:path';
|
|
7
|
+
import {
|
|
8
|
+
listDocumentUploads,
|
|
9
|
+
storeAndMaybeConvertDocument,
|
|
10
|
+
} from './documentIntake.js';
|
|
11
|
+
|
|
12
|
+
test('document intake stores uploads without requiring documents MCP', async () => {
|
|
13
|
+
const originalAgentsDataDir = process.env.AGENTS_DATA_DIR;
|
|
14
|
+
const root = await mkdtemp(path.join(os.tmpdir(), 'wiki-manager-doc-intake-'));
|
|
15
|
+
process.env.AGENTS_DATA_DIR = path.join(root, '.agents-data');
|
|
16
|
+
const source = path.join(root, 'rapport.pdf');
|
|
17
|
+
await writeFile(source, 'fake pdf content');
|
|
18
|
+
const session = {
|
|
19
|
+
workspace: 'juno',
|
|
20
|
+
mcp: {},
|
|
21
|
+
};
|
|
22
|
+
|
|
23
|
+
try {
|
|
24
|
+
const { record, converted } = await storeAndMaybeConvertDocument(session, source);
|
|
25
|
+
assert.equal(converted, false);
|
|
26
|
+
assert.equal(record.workspace, 'juno');
|
|
27
|
+
assert.equal(record.status, 'stored');
|
|
28
|
+
assert.equal(record.agentPath.startsWith('/documents/input/juno/'), true);
|
|
29
|
+
assert.equal(await readFile(record.storedPath, 'utf8'), 'fake pdf content');
|
|
30
|
+
|
|
31
|
+
const uploads = await listDocumentUploads(session);
|
|
32
|
+
assert.equal(uploads.length, 1);
|
|
33
|
+
assert.equal(uploads[0].id, record.id);
|
|
34
|
+
assert.equal(uploads[0].error, 'documents MCP is not connected');
|
|
35
|
+
} finally {
|
|
36
|
+
if (originalAgentsDataDir === undefined) delete process.env.AGENTS_DATA_DIR;
|
|
37
|
+
else process.env.AGENTS_DATA_DIR = originalAgentsDataDir;
|
|
38
|
+
}
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
test('document intake accepts quoted absolute paths with spaces', async () => {
|
|
42
|
+
const originalAgentsDataDir = process.env.AGENTS_DATA_DIR;
|
|
43
|
+
const root = await mkdtemp(path.join(os.tmpdir(), 'wiki-manager-doc-intake-'));
|
|
44
|
+
process.env.AGENTS_DATA_DIR = path.join(root, '.agents-data');
|
|
45
|
+
const source = path.join(root, 'Screenshot 2026-06-21 at 10.03.36.png');
|
|
46
|
+
await writeFile(source, 'fake png content');
|
|
47
|
+
const session = {
|
|
48
|
+
workspace: 'juno',
|
|
49
|
+
mcp: {},
|
|
50
|
+
};
|
|
51
|
+
|
|
52
|
+
try {
|
|
53
|
+
const { record } = await storeAndMaybeConvertDocument(session, `'${source}'`);
|
|
54
|
+
assert.equal(record.filename, 'Screenshot_2026-06-21_at_10.03.36.png');
|
|
55
|
+
assert.equal(await readFile(record.storedPath, 'utf8'), 'fake png content');
|
|
56
|
+
} finally {
|
|
57
|
+
if (originalAgentsDataDir === undefined) delete process.env.AGENTS_DATA_DIR;
|
|
58
|
+
else process.env.AGENTS_DATA_DIR = originalAgentsDataDir;
|
|
59
|
+
}
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
test('document intake accepts double-quoted absolute paths with spaces', async () => {
|
|
63
|
+
const originalAgentsDataDir = process.env.AGENTS_DATA_DIR;
|
|
64
|
+
const root = await mkdtemp(path.join(os.tmpdir(), 'wiki-manager-doc-intake-'));
|
|
65
|
+
process.env.AGENTS_DATA_DIR = path.join(root, '.agents-data');
|
|
66
|
+
const source = path.join(root, 'scan avec espace.pdf');
|
|
67
|
+
await writeFile(source, 'fake pdf content');
|
|
68
|
+
const session = {
|
|
69
|
+
workspace: 'juno',
|
|
70
|
+
mcp: {},
|
|
71
|
+
};
|
|
72
|
+
|
|
73
|
+
try {
|
|
74
|
+
const { record } = await storeAndMaybeConvertDocument(session, `"${source}"`);
|
|
75
|
+
assert.equal(record.filename, 'scan_avec_espace.pdf');
|
|
76
|
+
assert.equal(await readFile(record.storedPath, 'utf8'), 'fake pdf content');
|
|
77
|
+
} finally {
|
|
78
|
+
if (originalAgentsDataDir === undefined) delete process.env.AGENTS_DATA_DIR;
|
|
79
|
+
else process.env.AGENTS_DATA_DIR = originalAgentsDataDir;
|
|
80
|
+
}
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
test('document intake replaces an existing upload with the same original filename', async () => {
|
|
84
|
+
const originalAgentsDataDir = process.env.AGENTS_DATA_DIR;
|
|
85
|
+
const root = await mkdtemp(path.join(os.tmpdir(), 'wiki-manager-doc-intake-'));
|
|
86
|
+
const agentsDataDir = path.join(root, '.agents-data');
|
|
87
|
+
process.env.AGENTS_DATA_DIR = agentsDataDir;
|
|
88
|
+
const source = path.join(root, 'rapport.pdf');
|
|
89
|
+
const session = {
|
|
90
|
+
workspace: 'juno',
|
|
91
|
+
mcp: {},
|
|
92
|
+
};
|
|
93
|
+
|
|
94
|
+
try {
|
|
95
|
+
await writeFile(source, 'first pdf content');
|
|
96
|
+
const first = (await storeAndMaybeConvertDocument(session, source)).record;
|
|
97
|
+
const outputPath = path.join(root, 'workspace', 'raw', 'untracked', `${first.id}-rapport.md`);
|
|
98
|
+
await mkdir(path.dirname(outputPath), { recursive: true });
|
|
99
|
+
await writeFile(outputPath, 'old markdown');
|
|
100
|
+
|
|
101
|
+
const manifest = path.join(agentsDataDir, 'documents', 'uploads', 'juno.jsonl');
|
|
102
|
+
const firstRecord = JSON.parse(await readFile(manifest, 'utf8'));
|
|
103
|
+
await writeFile(manifest, `${JSON.stringify({ ...firstRecord, outputPath })}\n`, 'utf8');
|
|
104
|
+
|
|
105
|
+
await writeFile(source, 'second pdf content');
|
|
106
|
+
const second = (await storeAndMaybeConvertDocument(session, source)).record;
|
|
107
|
+
|
|
108
|
+
assert.notEqual(second.id, first.id);
|
|
109
|
+
assert.equal(existsSync(first.storedPath), false);
|
|
110
|
+
assert.equal(existsSync(outputPath), false);
|
|
111
|
+
assert.equal(await readFile(second.storedPath, 'utf8'), 'second pdf content');
|
|
112
|
+
|
|
113
|
+
const uploads = await listDocumentUploads(session);
|
|
114
|
+
assert.equal(uploads.length, 1);
|
|
115
|
+
assert.equal(uploads[0].id, second.id);
|
|
116
|
+
assert.equal(uploads[0].filename, 'rapport.pdf');
|
|
117
|
+
} finally {
|
|
118
|
+
if (originalAgentsDataDir === undefined) delete process.env.AGENTS_DATA_DIR;
|
|
119
|
+
else process.env.AGENTS_DATA_DIR = originalAgentsDataDir;
|
|
120
|
+
}
|
|
121
|
+
});
|
package/src/core/env.js
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import { existsSync, readFileSync } from 'node:fs';
|
|
2
|
+
import { join, resolve } from 'node:path';
|
|
3
|
+
|
|
4
|
+
export function userManagerDir() {
|
|
5
|
+
return process.cwd();
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
export function managerEnvFile() {
|
|
9
|
+
return process.env.WIKI_MANAGER_ENV_FILE
|
|
10
|
+
? resolve(process.env.WIKI_MANAGER_ENV_FILE)
|
|
11
|
+
: join(userManagerDir(), '.env');
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export function managerMcpEndpointsFile() {
|
|
15
|
+
return join(userManagerDir(), 'mcp.endpoints.json');
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
function parseEnvValue(value) {
|
|
19
|
+
if (value.startsWith('"') && value.endsWith('"')) {
|
|
20
|
+
return value
|
|
21
|
+
.slice(1, -1)
|
|
22
|
+
.replace(/\\(["\\nrt])/g, (_match, char) => {
|
|
23
|
+
if (char === 'n') return '\n';
|
|
24
|
+
if (char === 'r') return '\r';
|
|
25
|
+
if (char === 't') return '\t';
|
|
26
|
+
return char;
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
if (value.startsWith("'") && value.endsWith("'")) {
|
|
30
|
+
return value.slice(1, -1);
|
|
31
|
+
}
|
|
32
|
+
return value;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export function loadManagerEnv() {
|
|
36
|
+
const filePath = managerEnvFile();
|
|
37
|
+
if (!existsSync(filePath)) return;
|
|
38
|
+
const values = readEnvFile(filePath);
|
|
39
|
+
for (const [key, value] of Object.entries(values)) {
|
|
40
|
+
if (!(key in process.env)) process.env[key] = value;
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export function readEnvFile(filePath) {
|
|
45
|
+
const values = {};
|
|
46
|
+
const raw = readFileSync(filePath, 'utf8');
|
|
47
|
+
for (const sourceLine of raw.split(/\r?\n/)) {
|
|
48
|
+
const line = sourceLine.trim();
|
|
49
|
+
if (!line || line.startsWith('#')) continue;
|
|
50
|
+
const equalsIndex = line.indexOf('=');
|
|
51
|
+
if (equalsIndex === -1) continue;
|
|
52
|
+
const key = line.slice(0, equalsIndex).trim();
|
|
53
|
+
const value = line.slice(equalsIndex + 1).trim();
|
|
54
|
+
values[key] = parseEnvValue(value);
|
|
55
|
+
}
|
|
56
|
+
return values;
|
|
57
|
+
}
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
import { extractActivity, parseJsonText, sessionActivities } from './activity.js';
|
|
2
|
+
import { createAgentEvent, dispatchAgentEvent } from './agentEvents.js';
|
|
3
|
+
import { callMcpTool, formatMcpToolResult } from './mcp.js';
|
|
4
|
+
|
|
5
|
+
const TERMINAL = new Set(['done', 'failed', 'cancelled', 'canceled', 'complete', 'completed', 'success', 'error']);
|
|
6
|
+
|
|
7
|
+
function now() {
|
|
8
|
+
return new Date().toISOString();
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
function shortId() {
|
|
12
|
+
return `q-${Math.random().toString(16).slice(2, 6)}`;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
function terminalStatus(status) {
|
|
16
|
+
return TERMINAL.has(String(status ?? '').toLowerCase());
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export function ensureJobQueue(session) {
|
|
20
|
+
session.jobQueue ??= [];
|
|
21
|
+
return session.jobQueue;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export function productionLockBusy(session) {
|
|
25
|
+
return sessionActivities(session).some((activity) =>
|
|
26
|
+
activity.source === 'production' && !activity.terminal && activity.poll,
|
|
27
|
+
);
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export function enqueueProductionJob(session, args = {}, reason = 'waiting') {
|
|
31
|
+
const workspace = session.workspace ?? null;
|
|
32
|
+
const item = {
|
|
33
|
+
id: shortId(),
|
|
34
|
+
workspace,
|
|
35
|
+
server: 'production',
|
|
36
|
+
tool: 'production_start_job',
|
|
37
|
+
args: { ...args },
|
|
38
|
+
lockKey: workspace ? `production:${workspace}` : 'production',
|
|
39
|
+
status: 'waiting',
|
|
40
|
+
reason,
|
|
41
|
+
createdAt: now(),
|
|
42
|
+
};
|
|
43
|
+
ensureJobQueue(session).push(item);
|
|
44
|
+
return item;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export function queueSummary(args = {}) {
|
|
48
|
+
const parts = [
|
|
49
|
+
args.type,
|
|
50
|
+
Array.isArray(args.steps) && args.steps.length ? `steps=${args.steps.join(',')}` : null,
|
|
51
|
+
Array.isArray(args.templates) && args.templates.length ? `templates=${args.templates.length}` : null,
|
|
52
|
+
Array.isArray(args.deliverables) && args.deliverables.length ? `deliverables=${args.deliverables.length}` : null,
|
|
53
|
+
args.configPath ? `config=${args.configPath}` : null,
|
|
54
|
+
].filter(Boolean);
|
|
55
|
+
return parts.join(' ') || 'production job';
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export function queueCounts(session) {
|
|
59
|
+
let active = 0;
|
|
60
|
+
let current = 0;
|
|
61
|
+
let frozen = 0;
|
|
62
|
+
for (const item of ensureJobQueue(session)) {
|
|
63
|
+
if (['waiting', 'starting', 'running'].includes(item.status)) {
|
|
64
|
+
active += 1;
|
|
65
|
+
if (item.workspace === session.workspace) current += 1;
|
|
66
|
+
else frozen += 1;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
return { active, current, frozen };
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
export function formatQueue(session) {
|
|
73
|
+
const queue = ensureJobQueue(session);
|
|
74
|
+
if (queue.length === 0) return 'Queue is empty.';
|
|
75
|
+
return queue.map((item) => {
|
|
76
|
+
const frozen = item.workspace !== session.workspace && ['waiting', 'starting', 'running'].includes(item.status)
|
|
77
|
+
? ' frozen'
|
|
78
|
+
: '';
|
|
79
|
+
const job = item.jobId ? ` job=${item.jobId}` : '';
|
|
80
|
+
const error = item.error ? ` error=${item.error}` : '';
|
|
81
|
+
return `${item.id} ${item.status}${frozen} ${item.workspace ?? 'no-workspace'} ${queueSummary(item.args)}${job}${error}`;
|
|
82
|
+
}).join('\n');
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
export function clearFinishedQueueItems(session) {
|
|
86
|
+
const queue = ensureJobQueue(session);
|
|
87
|
+
const before = queue.length;
|
|
88
|
+
session.jobQueue = queue.filter((item) =>
|
|
89
|
+
item.workspace !== session.workspace || !terminalStatus(item.status),
|
|
90
|
+
);
|
|
91
|
+
return before - session.jobQueue.length;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
function findQueueItem(session, id) {
|
|
95
|
+
return ensureJobQueue(session).find((item) => item.id === id) ?? null;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
export async function cancelQueueItem(session, id) {
|
|
99
|
+
const item = findQueueItem(session, id);
|
|
100
|
+
if (!item) return { ok: false, message: `Unknown queue item: ${id}` };
|
|
101
|
+
if (item.status === 'waiting' || item.status === 'starting') {
|
|
102
|
+
const label = item.status === 'starting' ? 'starting' : 'queued';
|
|
103
|
+
item.status = 'cancelled';
|
|
104
|
+
item.finishedAt = now();
|
|
105
|
+
return { ok: true, message: `Cancelled ${label} job ${id}.` };
|
|
106
|
+
}
|
|
107
|
+
if (item.status === 'running') {
|
|
108
|
+
if (item.server === 'production' && item.jobId) {
|
|
109
|
+
await callMcpTool(session.mcp, 'production', 'production_cancel_job', { jobId: item.jobId });
|
|
110
|
+
item.status = 'cancelled';
|
|
111
|
+
item.finishedAt = now();
|
|
112
|
+
return { ok: true, message: `Cancellation requested for ${id} (${item.jobId}).` };
|
|
113
|
+
}
|
|
114
|
+
return { ok: false, message: `No cancel tool available for ${id}.` };
|
|
115
|
+
}
|
|
116
|
+
return { ok: false, message: `Queue item ${id} is already ${item.status}.` };
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
function activeCurrentWorkspaceProductionJob(session) {
|
|
120
|
+
return productionLockBusy(session)
|
|
121
|
+
|| ensureJobQueue(session).some((item) =>
|
|
122
|
+
item.workspace === session.workspace && ['starting', 'running'].includes(item.status),
|
|
123
|
+
);
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
function nextWaitingProductionItem(session) {
|
|
127
|
+
return ensureJobQueue(session).find((item) =>
|
|
128
|
+
item.workspace === session.workspace
|
|
129
|
+
&& item.server === 'production'
|
|
130
|
+
&& item.tool === 'production_start_job'
|
|
131
|
+
&& item.status === 'waiting',
|
|
132
|
+
) ?? null;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
export async function startNextQueuedJob(session, hooks = {}) {
|
|
136
|
+
if (!session.workspace) return null;
|
|
137
|
+
if (activeCurrentWorkspaceProductionJob(session)) return null;
|
|
138
|
+
const item = nextWaitingProductionItem(session);
|
|
139
|
+
if (!item) return null;
|
|
140
|
+
|
|
141
|
+
item.status = 'starting';
|
|
142
|
+
item.startedAt = now();
|
|
143
|
+
hooks.refresh?.();
|
|
144
|
+
hooks.addLog?.(`queue: starting ${item.id} ${queueSummary(item.args)}`);
|
|
145
|
+
|
|
146
|
+
try {
|
|
147
|
+
const args = session.workspace && !item.args.callerLabel
|
|
148
|
+
? { ...item.args, callerLabel: `${session.workspace}/wiki-manager` }
|
|
149
|
+
: item.args;
|
|
150
|
+
item.args = args;
|
|
151
|
+
const result = await callMcpTool(session.mcp, 'production', 'production_start_job', args);
|
|
152
|
+
const resultText = formatMcpToolResult(result);
|
|
153
|
+
const payload = parseJsonText(resultText);
|
|
154
|
+
if (payload?.ok === false && payload?.error === 'workspace_busy') {
|
|
155
|
+
item.status = 'waiting';
|
|
156
|
+
item.reason = 'workspace_busy';
|
|
157
|
+
hooks.addLog?.(`queue: ${item.id} still waiting, production lock busy`);
|
|
158
|
+
hooks.refresh?.();
|
|
159
|
+
return item;
|
|
160
|
+
}
|
|
161
|
+
const activity = extractActivity(payload, { server: 'production', tool: 'production_start_job' });
|
|
162
|
+
if (activity) {
|
|
163
|
+
item.status = activity.terminal ? activity.status : 'running';
|
|
164
|
+
item.jobId = activity.id;
|
|
165
|
+
item.activityKey = activity.key;
|
|
166
|
+
item.finishedAt = activity.terminal ? now() : undefined;
|
|
167
|
+
dispatchAgentEvent(session, createAgentEvent('activity_upserted', {
|
|
168
|
+
origin: 'queue',
|
|
169
|
+
payload: { activity },
|
|
170
|
+
}));
|
|
171
|
+
} else {
|
|
172
|
+
item.status = 'done';
|
|
173
|
+
item.finishedAt = now();
|
|
174
|
+
}
|
|
175
|
+
hooks.addLog?.(`queue: ${item.id} ${item.status}${item.jobId ? ` job=${item.jobId}` : ''}`);
|
|
176
|
+
hooks.refresh?.();
|
|
177
|
+
return item;
|
|
178
|
+
} catch (err) {
|
|
179
|
+
item.status = 'failed';
|
|
180
|
+
item.error = err instanceof Error ? err.message : String(err);
|
|
181
|
+
item.finishedAt = now();
|
|
182
|
+
hooks.addLog?.(`queue: ${item.id} failed · ${item.error}`);
|
|
183
|
+
hooks.refresh?.();
|
|
184
|
+
return item;
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
export function syncQueueWithActivity(session, activity) {
|
|
189
|
+
if (!activity?.id || activity.source !== 'production') return null;
|
|
190
|
+
const item = ensureJobQueue(session).find((entry) => entry.jobId === activity.id);
|
|
191
|
+
if (!item) return null;
|
|
192
|
+
item.status = activity.terminal ? activity.status : 'running';
|
|
193
|
+
item.activityKey = activity.key;
|
|
194
|
+
item.finishedAt = activity.terminal ? now() : item.finishedAt;
|
|
195
|
+
item.error = activity.error ?? item.error;
|
|
196
|
+
return item;
|
|
197
|
+
}
|