@opencraw/mcp 0.1.45 → 0.1.48
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -1
- package/dist/diff-tool/diff-tool.handler.d.ts +42 -0
- package/dist/diff-tool/index.d.ts +3 -0
- package/dist/docs-tool/docs-folder.repository.d.ts +24 -0
- package/dist/docs-tool/docs-root.client.d.ts +8 -0
- package/dist/docs-tool/docs-tool.handler.d.ts +21 -0
- package/dist/docs-tool/docs-topics.algorithm.d.ts +39 -0
- package/dist/docs-tool/index.d.ts +3 -0
- package/dist/index.d.ts +12 -1
- package/dist/index.esm.js +517 -1
- package/dist/list-tool/index.d.ts +3 -0
- package/dist/list-tool/list-recipes-tool.handler.d.ts +32 -0
- package/dist/main.d.ts +7 -0
- package/dist/probe-tool/index.d.ts +2 -0
- package/dist/probe-tool/probe-tool.handler.d.ts +27 -0
- package/dist/recipe-source/index.d.ts +3 -0
- package/dist/recipe-source/recipe-source.mapper.d.ts +21 -0
- package/dist/run-tool/index.d.ts +3 -0
- package/dist/run-tool/run-tool.handler.d.ts +51 -0
- package/dist/server/create-server.use-case.d.ts +11 -0
- package/dist/server/index.d.ts +2 -0
- package/dist/src/docs-tool/docs-folder.repository.d.ts +24 -0
- package/dist/src/docs-tool/docs-root.client.d.ts +8 -0
- package/dist/src/docs-tool/docs-tool.handler.d.ts +21 -0
- package/dist/src/docs-tool/docs-topics.algorithm.d.ts +39 -0
- package/dist/src/docs-tool/index.d.ts +3 -0
- package/dist/src/server/create-server.use-case.d.ts +1 -1
- package/dist/src/studio-tool/index.d.ts +2 -0
- package/dist/src/studio-tool/studio-command.client.d.ts +11 -0
- package/dist/src/studio-tool/studio-instance.repository.d.ts +22 -0
- package/dist/src/studio-tool/studio-tools.handler.d.ts +68 -0
- package/dist/studio-tool/index.d.ts +2 -0
- package/dist/studio-tool/studio-command.client.d.ts +11 -0
- package/dist/studio-tool/studio-instance.repository.d.ts +22 -0
- package/dist/studio-tool/studio-tools.handler.d.ts +68 -0
- package/dist/validate-tool/index.d.ts +3 -0
- package/dist/validate-tool/validate-tool.handler.d.ts +31 -0
- package/docs/how-it-works/01-recipes.md +698 -0
- package/docs/how-it-works/02-page-steps.md +672 -0
- package/docs/how-it-works/03-data-steps.md +813 -0
- package/docs/how-it-works/04-flow-steps.md +738 -0
- package/docs/how-it-works/05-templates-and-scope.md +428 -0
- package/docs/how-it-works/06-documents.md +1413 -0
- package/docs/how-it-works/07-mapping-and-transforms.md +699 -0
- package/docs/how-it-works/08-policies.md +405 -0
- package/docs/how-it-works/09-running.md +804 -0
- package/docs/how-it-works/README.md +225 -0
- package/docs/recipes/README.md +22 -0
- package/docs/recipes/access.md +440 -0
- package/docs/recipes/authoring.md +1503 -0
- package/docs/recipes/azure-durable.md +180 -0
- package/docs/recipes/captcha.md +204 -0
- package/docs/recipes/movies-example.md +12 -0
- package/docs/recipes/programmatic-authoring.md +101 -0
- package/docs/recipes/quick-guide.md +157 -0
- package/docs/recipes/vehicles-example.md +9 -0
- package/docs/recipes/worker-mode.md +221 -0
- package/docs/studio/manual.md +314 -0
- package/docs/studio/pdf-agreements.md +150 -0
- package/package.json +29 -4
package/dist/index.esm.js
CHANGED
|
@@ -3,6 +3,11 @@ import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
|
|
3
3
|
import { readRecordsFile, diffRecords, memorySink, fileSink, createCrawler, RecipeSet, RecipeValidationError, RecipeBindingError, parseOutputRecipe, parseInputRecipe, bindRecipeSet } from '@opencraw/core';
|
|
4
4
|
import { diffReport, readRecipeFiles, probeUrl, loadForRun, loadPlugins, resolveAccess } from '@opencraw/cli';
|
|
5
5
|
import { z } from 'zod';
|
|
6
|
+
import { readdir, readFile } from 'node:fs/promises';
|
|
7
|
+
import { join, dirname } from 'node:path';
|
|
8
|
+
import { existsSync } from 'node:fs';
|
|
9
|
+
import { fileURLToPath } from 'node:url';
|
|
10
|
+
import { homedir } from 'node:os';
|
|
6
11
|
|
|
7
12
|
/** Changes returned at most; the counts cover all of them. */
|
|
8
13
|
const CHANGE_LIMIT = 200;
|
|
@@ -56,6 +61,191 @@ async function diffTool(args) {
|
|
|
56
61
|
}
|
|
57
62
|
}
|
|
58
63
|
|
|
64
|
+
/**
|
|
65
|
+
* The topic id of a documentation file: its path under the docs folder, without the extension and with forward slashes, so `recipes/authoring.md` is
|
|
66
|
+
* `recipes/authoring`.
|
|
67
|
+
*
|
|
68
|
+
* @param relativePath - The file's path under the docs folder.
|
|
69
|
+
* @returns The id.
|
|
70
|
+
*/
|
|
71
|
+
function topicIdOf(relativePath) {
|
|
72
|
+
return relativePath.replaceAll('\\', '/').replace(/\.md$/i, '');
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* The title of a document: its first top-level heading, else its id.
|
|
76
|
+
*
|
|
77
|
+
* @param markdown - The document.
|
|
78
|
+
* @param fallback - The id, when there is no heading.
|
|
79
|
+
* @returns The title.
|
|
80
|
+
*/
|
|
81
|
+
function titleOf(markdown, fallback) {
|
|
82
|
+
const heading = outlineOf(markdown).find(candidate => candidate.level === 1);
|
|
83
|
+
return heading?.text ?? fallback;
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* Every heading of a Markdown document, in order. A line inside a fenced code block is not a heading (a shell comment starts with `#` too).
|
|
87
|
+
*
|
|
88
|
+
* @param markdown - The document.
|
|
89
|
+
* @returns The headings.
|
|
90
|
+
*/
|
|
91
|
+
function outlineOf(markdown) {
|
|
92
|
+
const headings = [];
|
|
93
|
+
let fence;
|
|
94
|
+
for (const [line, text] of markdown.split(/\r?\n/).entries()) {
|
|
95
|
+
const marker = /^\s*(`{3,}|~{3,})/.exec(text)?.[1];
|
|
96
|
+
if (marker !== undefined) {
|
|
97
|
+
if (fence === undefined) fence = marker[0];else if (marker[0] === fence) fence = undefined;
|
|
98
|
+
continue;
|
|
99
|
+
}
|
|
100
|
+
if (fence !== undefined) continue;
|
|
101
|
+
const match = /^(#{1,6}) (.+)$/.exec(text);
|
|
102
|
+
if (match !== null) headings.push({
|
|
103
|
+
level: match[1].length,
|
|
104
|
+
text: match[2].trim().replace(/ #+$/, ''),
|
|
105
|
+
line
|
|
106
|
+
});
|
|
107
|
+
}
|
|
108
|
+
return headings;
|
|
109
|
+
}
|
|
110
|
+
/**
|
|
111
|
+
* One section of a document: from the heading whose text contains `heading` (case-insensitive; an exact match wins over a partial one) to the next
|
|
112
|
+
* heading of the same or a higher level, so its subsections come with it.
|
|
113
|
+
*
|
|
114
|
+
* @param markdown - The document.
|
|
115
|
+
* @param heading - What the heading says, or part of it.
|
|
116
|
+
* @returns The section's text, or `undefined` when no heading matches.
|
|
117
|
+
*/
|
|
118
|
+
function sectionOf(markdown, heading) {
|
|
119
|
+
const wanted = heading.trim().toLowerCase();
|
|
120
|
+
const headings = outlineOf(markdown);
|
|
121
|
+
const start = headings.find(candidate => candidate.text.toLowerCase() === wanted) ?? headings.find(candidate => candidate.text.toLowerCase().includes(wanted));
|
|
122
|
+
if (start === undefined) return undefined;
|
|
123
|
+
const end = headings.find(candidate => candidate.line > start.line && candidate.level <= start.level);
|
|
124
|
+
const lines = markdown.split(/\r?\n/);
|
|
125
|
+
return lines.slice(start.line, end?.line ?? lines.length).join('\n').trim();
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/** The folders under the docs folder whose Markdown files are documentation for a person (or an agent) using OpenCraw. */
|
|
129
|
+
const FOLDERS = ['recipes', 'how-it-works', 'studio'];
|
|
130
|
+
/**
|
|
131
|
+
* The documents under a docs folder: the Markdown files directly inside `recipes`, `how-it-works` and `studio` (their pictures and capture scripts are
|
|
132
|
+
* not documentation to read).
|
|
133
|
+
*
|
|
134
|
+
* @param root - The docs folder.
|
|
135
|
+
* @returns The topics, ordered by id.
|
|
136
|
+
*/
|
|
137
|
+
async function listTopics(root) {
|
|
138
|
+
const topics = [];
|
|
139
|
+
for (const folder of FOLDERS) {
|
|
140
|
+
let names;
|
|
141
|
+
try {
|
|
142
|
+
names = await readdir(join(root, folder));
|
|
143
|
+
} catch {
|
|
144
|
+
continue;
|
|
145
|
+
}
|
|
146
|
+
const documents = names.filter(file => file.toLowerCase().endsWith('.md'));
|
|
147
|
+
for (const name of documents) {
|
|
148
|
+
const id = topicIdOf(`${folder}/${name}`);
|
|
149
|
+
const text = await readFile(join(root, folder, name), 'utf8');
|
|
150
|
+
topics.push({
|
|
151
|
+
id,
|
|
152
|
+
title: titleOf(text, id),
|
|
153
|
+
size: text.length
|
|
154
|
+
});
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
return topics.sort((a, b) => a.id.localeCompare(b.id));
|
|
158
|
+
}
|
|
159
|
+
/**
|
|
160
|
+
* One document's text.
|
|
161
|
+
*
|
|
162
|
+
* @param root - The docs folder.
|
|
163
|
+
* @param id - The topic id from `listTopics` (never a path: nothing outside the listed folders is read).
|
|
164
|
+
* @returns The Markdown, or `undefined` when there is no such topic.
|
|
165
|
+
*/
|
|
166
|
+
async function readTopic(root, id) {
|
|
167
|
+
const known = await listTopics(root);
|
|
168
|
+
if (known.every(topic => topic.id !== id)) return undefined;
|
|
169
|
+
return await readFile(join(root, `${id}.md`), 'utf8');
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/** Input schema for the `docs` tool. */
|
|
173
|
+
const docsInputShape = {
|
|
174
|
+
topic: z.string().optional().describe('A document id from the list this tool returns without a topic (`recipes/authoring`, `how-it-works/06-documents`, `studio/manual`…). Leave out to list every document.'),
|
|
175
|
+
section: z.string().optional().describe('A heading (or part of one) of that document: only that section comes back, subsections included. A long document answers with its outline until you ask for a section.')
|
|
176
|
+
};
|
|
177
|
+
/** A document longer than this many characters comes back as its outline, and the section asked for. */
|
|
178
|
+
const WHOLE_DOCUMENT_LIMIT = 30_000;
|
|
179
|
+
function text$1(value, isError = false) {
|
|
180
|
+
return {
|
|
181
|
+
content: [{
|
|
182
|
+
type: 'text',
|
|
183
|
+
text: typeof value === 'string' ? value : JSON.stringify(value)
|
|
184
|
+
}],
|
|
185
|
+
...(isError && {
|
|
186
|
+
isError
|
|
187
|
+
}),
|
|
188
|
+
...(typeof value === 'object' && value !== null && {
|
|
189
|
+
structuredContent: value
|
|
190
|
+
})
|
|
191
|
+
};
|
|
192
|
+
}
|
|
193
|
+
/**
|
|
194
|
+
* The `docs` tool: OpenCraw's own documentation, for an agent that has to write recipes right: the authoring guide (every step, selector, transform
|
|
195
|
+
* and policy), how it all works, the examples and the Studio manual. Without a topic it lists what there is; with one it gives the document, or just
|
|
196
|
+
* the section asked for, so a long guide costs only what is read.
|
|
197
|
+
*
|
|
198
|
+
* @param args - The tool's parsed input.
|
|
199
|
+
* @param root - The docs folder (the packaged copy, or the repository's).
|
|
200
|
+
* @returns The MCP tool result.
|
|
201
|
+
*/
|
|
202
|
+
async function docsTool(args, root) {
|
|
203
|
+
const topics = await listTopics(root);
|
|
204
|
+
if (topics.length === 0) return text$1('The documentation is not installed with this server.', true);
|
|
205
|
+
if (args.topic === undefined) return text$1({
|
|
206
|
+
topics,
|
|
207
|
+
hint: 'Ask for one with { topic }, and for part of a long one with { topic, section }.'
|
|
208
|
+
});
|
|
209
|
+
const document = await readTopic(root, args.topic);
|
|
210
|
+
if (document === undefined) {
|
|
211
|
+
const close = topics.filter(topic => topic.id.includes(args.topic?.toLowerCase() ?? '') || args.topic?.includes(topic.id) === true).map(topic => topic.id);
|
|
212
|
+
return text$1(`There is no document "${args.topic}". ${close.length > 0 ? `Did you mean: ${close.join(', ')}?` : `The documents are: ${topics.map(topic => topic.id).join(', ')}.`}`, true);
|
|
213
|
+
}
|
|
214
|
+
if (args.section !== undefined) {
|
|
215
|
+
const section = sectionOf(document, args.section);
|
|
216
|
+
return section === undefined ? text$1({
|
|
217
|
+
topic: args.topic,
|
|
218
|
+
error: `no heading contains "${args.section}"`,
|
|
219
|
+
outline: outlineOf(document).map(heading => `${'#'.repeat(heading.level)} ${heading.text}`)
|
|
220
|
+
}, true) : text$1({
|
|
221
|
+
topic: args.topic,
|
|
222
|
+
section: args.section,
|
|
223
|
+
text: section
|
|
224
|
+
});
|
|
225
|
+
}
|
|
226
|
+
if (document.length <= WHOLE_DOCUMENT_LIMIT) return text$1({
|
|
227
|
+
topic: args.topic,
|
|
228
|
+
text: document
|
|
229
|
+
});
|
|
230
|
+
return text$1({
|
|
231
|
+
topic: args.topic,
|
|
232
|
+
outline: outlineOf(document).map(heading => `${'#'.repeat(heading.level)} ${heading.text}`),
|
|
233
|
+
note: `This document is ${String(document.length)} characters: ask for a section with { topic, section } using a heading from the outline.`
|
|
234
|
+
});
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
/**
|
|
238
|
+
* Where the documentation is: a `docs` folder beside `dist` in the installed package (copied there at build), or the repository's own `docs` when the server runs
|
|
239
|
+
* from a checkout.
|
|
240
|
+
*
|
|
241
|
+
* @returns The docs folder.
|
|
242
|
+
*/
|
|
243
|
+
function docsRoot() {
|
|
244
|
+
const here = dirname(fileURLToPath(import.meta.url));
|
|
245
|
+
const candidates = [join(here, '..', 'docs'), join(here, '..', '..', '..', 'docs')];
|
|
246
|
+
return candidates.find(candidate => existsSync(join(candidate, 'recipes'))) ?? candidates[0];
|
|
247
|
+
}
|
|
248
|
+
|
|
59
249
|
/** Input schema for the `list_recipes` tool. */
|
|
60
250
|
const listRecipesInputShape = {
|
|
61
251
|
dir: z.string().describe('A directory of recipe files.')
|
|
@@ -308,6 +498,312 @@ function loadErrorText(error) {
|
|
|
308
498
|
return error instanceof Error ? error.message : String(error);
|
|
309
499
|
}
|
|
310
500
|
|
|
501
|
+
/**
|
|
502
|
+
* Sends one command to a running Studio (`POST /api/command`, the same door its own page uses) and reads the answer.
|
|
503
|
+
*
|
|
504
|
+
* @param instance - The Studio to talk to.
|
|
505
|
+
* @param command - A command `@opencraw/studio`'s `studioCommandSchema` accepts.
|
|
506
|
+
* @returns The JSON the Studio answered.
|
|
507
|
+
* @throws Error with the Studio's own message when it refuses.
|
|
508
|
+
*/
|
|
509
|
+
async function sendStudioCommand(instance, command) {
|
|
510
|
+
const response = await fetch(`${instance.url}/api/command`, {
|
|
511
|
+
method: 'POST',
|
|
512
|
+
headers: {
|
|
513
|
+
'content-type': 'application/json',
|
|
514
|
+
'x-opencraw-token': instance.token
|
|
515
|
+
},
|
|
516
|
+
body: JSON.stringify(command)
|
|
517
|
+
});
|
|
518
|
+
const body = await response.json();
|
|
519
|
+
if (!response.ok) throw new Error(body.error ?? `Studio answered ${String(response.status)}`);
|
|
520
|
+
return body;
|
|
521
|
+
}
|
|
522
|
+
|
|
523
|
+
/** Where running Studios announce themselves. */
|
|
524
|
+
function instancesDirectory() {
|
|
525
|
+
return join(homedir(), '.opencraw', 'studio');
|
|
526
|
+
}
|
|
527
|
+
function isAlive(pid) {
|
|
528
|
+
try {
|
|
529
|
+
process.kill(pid, 0);
|
|
530
|
+
return true;
|
|
531
|
+
} catch (error) {
|
|
532
|
+
// EPERM: the process exists but is not ours to signal.
|
|
533
|
+
return error.code === 'EPERM';
|
|
534
|
+
}
|
|
535
|
+
}
|
|
536
|
+
function instanceOf(text) {
|
|
537
|
+
try {
|
|
538
|
+
const value = JSON.parse(text);
|
|
539
|
+
return typeof value.pid === 'number' && typeof value.url === 'string' && typeof value.token === 'string' && typeof value.startedAt === 'string' ? {
|
|
540
|
+
pid: value.pid,
|
|
541
|
+
url: value.url,
|
|
542
|
+
token: value.token,
|
|
543
|
+
folder: value.folder,
|
|
544
|
+
startedAt: value.startedAt
|
|
545
|
+
} : undefined;
|
|
546
|
+
} catch {
|
|
547
|
+
return undefined;
|
|
548
|
+
}
|
|
549
|
+
}
|
|
550
|
+
/** The announcement files in a directory; none when it does not exist yet. */
|
|
551
|
+
async function namesIn(directory) {
|
|
552
|
+
try {
|
|
553
|
+
const names = await readdir(directory);
|
|
554
|
+
return names.filter(file => file.endsWith('.json'));
|
|
555
|
+
} catch {
|
|
556
|
+
return [];
|
|
557
|
+
}
|
|
558
|
+
}
|
|
559
|
+
/** A file's text; empty when it vanished (its Studio just stopped). */
|
|
560
|
+
async function textOf(file) {
|
|
561
|
+
try {
|
|
562
|
+
return await readFile(file, 'utf8');
|
|
563
|
+
} catch {
|
|
564
|
+
return '';
|
|
565
|
+
}
|
|
566
|
+
}
|
|
567
|
+
/**
|
|
568
|
+
* The Studios running on this machine, the newest first. A file whose process is gone (Studio was killed
|
|
569
|
+
* and could not withdraw it) is left out.
|
|
570
|
+
*
|
|
571
|
+
* @param directory - Where Studios announce themselves.
|
|
572
|
+
* @returns The live Studios.
|
|
573
|
+
*/
|
|
574
|
+
async function runningStudios(directory = instancesDirectory()) {
|
|
575
|
+
const found = [];
|
|
576
|
+
const names = await namesIn(directory);
|
|
577
|
+
for (const name of names) {
|
|
578
|
+
const instance = instanceOf(await textOf(join(directory, name)));
|
|
579
|
+
if (instance !== undefined && isAlive(instance.pid)) found.push(instance);
|
|
580
|
+
}
|
|
581
|
+
return found.sort((a, b) => b.startedAt.localeCompare(a.startedAt));
|
|
582
|
+
}
|
|
583
|
+
|
|
584
|
+
/** Input schema for `studio_status`: nothing. */
|
|
585
|
+
const studioStatusInputShape = {};
|
|
586
|
+
/** Input schema for `studio_open_recipe`. */
|
|
587
|
+
const studioOpenRecipeInputShape = {
|
|
588
|
+
path: z.string().min(1).describe('A recipe file (`*.input.json` / `*.ocw-in.json`), a folder of recipes, or a document, as an absolute path.'),
|
|
589
|
+
workspace: z.string().optional().describe('The folder a document belongs to, when `path` is a document.')
|
|
590
|
+
};
|
|
591
|
+
/** Input schema for `studio_pick`. */
|
|
592
|
+
const studioPickInputShape = {
|
|
593
|
+
recipeId: z.string().min(1).describe('The id of an input recipe whose start point is a web page, in the folder Studio has open.'),
|
|
594
|
+
selector: z.string().optional().describe('A CSS selector of the element to pick on the page of the recipe. Give this or `text`.'),
|
|
595
|
+
text: z.string().optional().describe('The text the element shows, when you do not know a selector: the smallest element showing it is picked.'),
|
|
596
|
+
add: z.boolean().optional().describe('Add the pick to the recipe straight away (default: leave it staged, for the person to choose a selector and press Add to recipe).')
|
|
597
|
+
};
|
|
598
|
+
/** Input schema for `studio_run_sample`. */
|
|
599
|
+
const studioRunSampleInputShape = {
|
|
600
|
+
recipeId: z.string().min(1).describe('The id of an input recipe in the folder Studio has open.'),
|
|
601
|
+
maxRecords: z.number().int().positive().optional().describe('Stop after this many records. Default 10.'),
|
|
602
|
+
waitSeconds: z.number().int().positive().max(300).optional().describe('How long to wait for the run to finish before answering that it is still running. Default 60.')
|
|
603
|
+
};
|
|
604
|
+
const NO_STUDIO = 'No Studio is running. Start one with `opencraw studio <folder>`, or open a recipe from the OpenCraw view in VS Code ("Open in Studio"), then ask again.';
|
|
605
|
+
const POLL_MS = 500;
|
|
606
|
+
/** How long a page gets to say how a request went. */
|
|
607
|
+
const REPORT_WAIT_MS = 30_000;
|
|
608
|
+
function text(value, isError = false) {
|
|
609
|
+
return {
|
|
610
|
+
content: [{
|
|
611
|
+
type: 'text',
|
|
612
|
+
text: typeof value === 'string' ? value : JSON.stringify(value)
|
|
613
|
+
}],
|
|
614
|
+
...(isError && {
|
|
615
|
+
isError
|
|
616
|
+
}),
|
|
617
|
+
...(typeof value === 'object' && value !== null && {
|
|
618
|
+
structuredContent: value
|
|
619
|
+
})
|
|
620
|
+
};
|
|
621
|
+
}
|
|
622
|
+
function messageOf(error) {
|
|
623
|
+
return error instanceof Error ? error.message : String(error);
|
|
624
|
+
}
|
|
625
|
+
async function studioOf(directory) {
|
|
626
|
+
const [newest] = await runningStudios(directory);
|
|
627
|
+
return newest;
|
|
628
|
+
}
|
|
629
|
+
/**
|
|
630
|
+
* `studio_status`: whether a Studio is running, which folder it has open, how many pages are watching and what it is running.
|
|
631
|
+
*
|
|
632
|
+
* @param directory - Where Studios announce themselves; the default is the user's own.
|
|
633
|
+
* @returns The MCP tool result.
|
|
634
|
+
*/
|
|
635
|
+
async function studioStatusTool(directory) {
|
|
636
|
+
const studio = await studioOf(directory);
|
|
637
|
+
if (studio === undefined) return text({
|
|
638
|
+
running: false,
|
|
639
|
+
hint: NO_STUDIO
|
|
640
|
+
});
|
|
641
|
+
try {
|
|
642
|
+
const status = await sendStudioCommand(studio, {
|
|
643
|
+
type: 'studio-status'
|
|
644
|
+
});
|
|
645
|
+
return text({
|
|
646
|
+
running: true,
|
|
647
|
+
url: studio.url,
|
|
648
|
+
...status
|
|
649
|
+
});
|
|
650
|
+
} catch (error) {
|
|
651
|
+
return text(`Studio did not answer: ${messageOf(error)}`, true);
|
|
652
|
+
}
|
|
653
|
+
}
|
|
654
|
+
function sleep(ms) {
|
|
655
|
+
return new Promise(resolve => {
|
|
656
|
+
setTimeout(resolve, ms);
|
|
657
|
+
});
|
|
658
|
+
}
|
|
659
|
+
/** Asks the open page to do something and waits for what it reports. */
|
|
660
|
+
async function askPage(studio, request, waitMs = REPORT_WAIT_MS) {
|
|
661
|
+
const {
|
|
662
|
+
pages,
|
|
663
|
+
requestId
|
|
664
|
+
} = await sendStudioCommand(studio, {
|
|
665
|
+
type: 'ask-ui',
|
|
666
|
+
request
|
|
667
|
+
});
|
|
668
|
+
if (pages === 0) return {
|
|
669
|
+
pages
|
|
670
|
+
};
|
|
671
|
+
const deadline = Date.now() + waitMs;
|
|
672
|
+
while (Date.now() < deadline) {
|
|
673
|
+
const outcome = await sendStudioCommand(studio, {
|
|
674
|
+
type: 'ui-outcome',
|
|
675
|
+
requestId
|
|
676
|
+
});
|
|
677
|
+
if (outcome.state === 'done') return {
|
|
678
|
+
pages,
|
|
679
|
+
outcome: {
|
|
680
|
+
ok: outcome.ok === true,
|
|
681
|
+
message: outcome.message ?? '',
|
|
682
|
+
detail: outcome.detail
|
|
683
|
+
}
|
|
684
|
+
};
|
|
685
|
+
await sleep(POLL_MS / 5);
|
|
686
|
+
}
|
|
687
|
+
return {
|
|
688
|
+
pages
|
|
689
|
+
};
|
|
690
|
+
}
|
|
691
|
+
const NO_PAGE = 'Studio is running but no page is open on it. Ask the person to open Studio in their browser.';
|
|
692
|
+
const NO_REPORT = 'The page did not answer in time. Is the recipe in the folder Studio has open?';
|
|
693
|
+
/**
|
|
694
|
+
* `studio_open_recipe`: opens a recipe, a folder or a document in the Studio the person is looking at, so they watch the work happen.
|
|
695
|
+
*
|
|
696
|
+
* @param args - What to open.
|
|
697
|
+
* @param directory - Where Studios announce themselves.
|
|
698
|
+
* @returns The MCP tool result.
|
|
699
|
+
*/
|
|
700
|
+
async function studioOpenRecipeTool(args, directory) {
|
|
701
|
+
const studio = await studioOf(directory);
|
|
702
|
+
if (studio === undefined) return text(NO_STUDIO, true);
|
|
703
|
+
try {
|
|
704
|
+
const answer = await askPage(studio, {
|
|
705
|
+
kind: 'open',
|
|
706
|
+
path: args.path,
|
|
707
|
+
workspace: args.workspace
|
|
708
|
+
});
|
|
709
|
+
if (answer.pages === 0) return text({
|
|
710
|
+
opened: false,
|
|
711
|
+
hint: NO_PAGE,
|
|
712
|
+
url: studio.url
|
|
713
|
+
});
|
|
714
|
+
if (answer.outcome === undefined) return text({
|
|
715
|
+
opened: false,
|
|
716
|
+
hint: NO_REPORT
|
|
717
|
+
});
|
|
718
|
+
return text({
|
|
719
|
+
opened: answer.outcome.ok,
|
|
720
|
+
message: answer.outcome.message
|
|
721
|
+
}, !answer.outcome.ok);
|
|
722
|
+
} catch (error) {
|
|
723
|
+
return text(`Studio refused: ${messageOf(error)}`, true);
|
|
724
|
+
}
|
|
725
|
+
}
|
|
726
|
+
/**
|
|
727
|
+
* `studio_pick`: picks a value on a recipe's web page in the Studio the person is looking at, as a click would: they see it selected and the
|
|
728
|
+
* ways to reach it. With `add` it is added to the recipe. Answers with the selector Studio chose and the others it offers.
|
|
729
|
+
*
|
|
730
|
+
* @param args - The recipe and what to pick (a selector or the text shown), and whether to add it.
|
|
731
|
+
* @param directory - Where Studios announce themselves.
|
|
732
|
+
* @returns The MCP tool result.
|
|
733
|
+
*/
|
|
734
|
+
async function studioPickTool(args, directory) {
|
|
735
|
+
if (args.selector === undefined && args.text === undefined) return text('Give a CSS selector or the text the element shows.', true);
|
|
736
|
+
const studio = await studioOf(directory);
|
|
737
|
+
if (studio === undefined) return text(NO_STUDIO, true);
|
|
738
|
+
try {
|
|
739
|
+
const answer = await askPage(studio, {
|
|
740
|
+
kind: 'pick',
|
|
741
|
+
recipeId: args.recipeId,
|
|
742
|
+
selector: args.selector,
|
|
743
|
+
text: args.text,
|
|
744
|
+
add: args.add
|
|
745
|
+
});
|
|
746
|
+
if (answer.pages === 0) return text({
|
|
747
|
+
picked: false,
|
|
748
|
+
hint: NO_PAGE,
|
|
749
|
+
url: studio.url
|
|
750
|
+
}, true);
|
|
751
|
+
if (answer.outcome === undefined) return text({
|
|
752
|
+
picked: false,
|
|
753
|
+
hint: NO_REPORT
|
|
754
|
+
}, true);
|
|
755
|
+
return text({
|
|
756
|
+
picked: answer.outcome.ok,
|
|
757
|
+
message: answer.outcome.message,
|
|
758
|
+
...answer.outcome.detail
|
|
759
|
+
}, !answer.outcome.ok);
|
|
760
|
+
} catch (error) {
|
|
761
|
+
return text(`Studio refused: ${messageOf(error)}`, true);
|
|
762
|
+
}
|
|
763
|
+
}
|
|
764
|
+
/**
|
|
765
|
+
* `studio_run_sample`: runs an input recipe as a sample in Studio (the person sees the records arrive), waits for it, and returns the
|
|
766
|
+
* counts and the first records.
|
|
767
|
+
*
|
|
768
|
+
* @param args - The recipe, how many records to stop at and how long to wait.
|
|
769
|
+
* @param directory - Where Studios announce themselves.
|
|
770
|
+
* @returns The MCP tool result.
|
|
771
|
+
*/
|
|
772
|
+
async function studioRunSampleTool(args, directory) {
|
|
773
|
+
const studio = await studioOf(directory);
|
|
774
|
+
if (studio === undefined) return text(NO_STUDIO, true);
|
|
775
|
+
try {
|
|
776
|
+
const {
|
|
777
|
+
folder
|
|
778
|
+
} = await sendStudioCommand(studio, {
|
|
779
|
+
type: 'studio-status'
|
|
780
|
+
});
|
|
781
|
+
if (folder === undefined) return text('Studio has no folder open yet. Call studio_open_recipe with a recipe file or a folder first.', true);
|
|
782
|
+
await sendStudioCommand(studio, {
|
|
783
|
+
type: 'run-sample',
|
|
784
|
+
recipeId: args.recipeId,
|
|
785
|
+
budget: {
|
|
786
|
+
maxRecords: args.maxRecords ?? 10
|
|
787
|
+
}
|
|
788
|
+
});
|
|
789
|
+
const deadline = Date.now() + (args.waitSeconds ?? 60) * 1000;
|
|
790
|
+
for (;;) {
|
|
791
|
+
const status = await sendStudioCommand(studio, {
|
|
792
|
+
type: 'run-status',
|
|
793
|
+
recipeId: args.recipeId
|
|
794
|
+
});
|
|
795
|
+
if (status.state !== 'running') return text(status);
|
|
796
|
+
if (Date.now() >= deadline) return text({
|
|
797
|
+
...status,
|
|
798
|
+
hint: 'Still running; ask again with studio_run_sample or look in Studio.'
|
|
799
|
+
});
|
|
800
|
+
await sleep(POLL_MS);
|
|
801
|
+
}
|
|
802
|
+
} catch (error) {
|
|
803
|
+
return text(`Studio could not run it: ${messageOf(error)}`, true);
|
|
804
|
+
}
|
|
805
|
+
}
|
|
806
|
+
|
|
311
807
|
/** Input schema for the `validate` tool. */
|
|
312
808
|
const validateInputShape = recipeSourceShape;
|
|
313
809
|
/**
|
|
@@ -408,7 +904,7 @@ const SERVER_INFO = {
|
|
|
408
904
|
};
|
|
409
905
|
/**
|
|
410
906
|
* Builds the MCP server: one tool per crawl primitive (`probe`, `validate`,
|
|
411
|
-
* `run`, `list_recipes`, `diff`), each a thin wrapper over `@opencraw/core` and
|
|
907
|
+
* `run`, `list_recipes`, `diff`), `docs` and the `studio_` tools that drive a running Studio, each a thin wrapper over `@opencraw/core` and
|
|
412
908
|
* `@opencraw/cli`'s structured helpers. No transport is attached; `main`
|
|
413
909
|
* connects it over stdio.
|
|
414
910
|
*
|
|
@@ -436,6 +932,26 @@ function createServer() {
|
|
|
436
932
|
description: 'Compare two runs\' JSON Lines files by record key: what was added, removed, and changed (each field before and after). Use to report what changed in a source since the last crawl.',
|
|
437
933
|
inputSchema: diffInputShape
|
|
438
934
|
}, args => diffTool(args));
|
|
935
|
+
server.registerTool('docs', {
|
|
936
|
+
description: 'OpenCraw\'s own documentation: the authoring guide (every step, selector, transform and policy), how it works, the examples and the Studio manual. Without a topic it lists the documents; with one it gives the document, or just a section of it. Read the guide before writing a recipe you are not sure of, instead of guessing.',
|
|
937
|
+
inputSchema: docsInputShape
|
|
938
|
+
}, args => docsTool(args, docsRoot()));
|
|
939
|
+
server.registerTool('studio_status', {
|
|
940
|
+
description: 'Whether OpenCraw Studio is running on this machine, which folder it has open, how many pages are watching and which recipe it is sampling. Call it before the other studio_ tools.',
|
|
941
|
+
inputSchema: studioStatusInputShape
|
|
942
|
+
}, () => studioStatusTool());
|
|
943
|
+
server.registerTool('studio_open_recipe', {
|
|
944
|
+
description: 'Open a recipe, a folder of recipes or a document in the Studio the person has open, so they watch the recipe being built. Does nothing to the files.',
|
|
945
|
+
inputSchema: studioOpenRecipeInputShape
|
|
946
|
+
}, args => studioOpenRecipeTool(args));
|
|
947
|
+
server.registerTool('studio_pick', {
|
|
948
|
+
description: 'Pick a value on the web page of an input recipe in Studio, as a click would: give a CSS selector or the text the element shows. The person sees it selected and the ways to reach it; with add it is added to the recipe. Returns the selector chosen and the alternatives, with how many elements each matches. Open the recipe first with studio_open_recipe.',
|
|
949
|
+
inputSchema: studioPickInputShape
|
|
950
|
+
}, args => studioPickTool(args));
|
|
951
|
+
server.registerTool('studio_run_sample', {
|
|
952
|
+
description: 'Run an input recipe as a sample in Studio (the person sees the records arrive) and get back the counts, the first records and what was rejected. Studio must have the folder of that recipe open: call studio_open_recipe first.',
|
|
953
|
+
inputSchema: studioRunSampleInputShape
|
|
954
|
+
}, args => studioRunSampleTool(args));
|
|
439
955
|
return server;
|
|
440
956
|
}
|
|
441
957
|
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import type { CallToolResult } from '@modelcontextprotocol/sdk/types.js';
|
|
2
|
+
import { z } from 'zod';
|
|
3
|
+
/** Input schema for the `list_recipes` tool. */
|
|
4
|
+
export declare const listRecipesInputShape: {
|
|
5
|
+
dir: z.ZodString;
|
|
6
|
+
};
|
|
7
|
+
/** What the `list_recipes` tool returns. */
|
|
8
|
+
export interface ListRecipesResult {
|
|
9
|
+
outputs: {
|
|
10
|
+
path: string;
|
|
11
|
+
id?: string;
|
|
12
|
+
}[];
|
|
13
|
+
inputs: {
|
|
14
|
+
path: string;
|
|
15
|
+
id?: string;
|
|
16
|
+
output?: string;
|
|
17
|
+
mode?: string;
|
|
18
|
+
}[];
|
|
19
|
+
others: string[];
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* The `list_recipes` tool: what recipes already exist in a directory, cheaper
|
|
23
|
+
* than a full `validate` call, so an agent can check before writing a new one
|
|
24
|
+
* or find what an existing output recipe already covers.
|
|
25
|
+
*
|
|
26
|
+
* @param args - The tool's parsed input.
|
|
27
|
+
* @returns The MCP tool result.
|
|
28
|
+
*/
|
|
29
|
+
export declare function listRecipesTool(args: {
|
|
30
|
+
dir: string;
|
|
31
|
+
}): Promise<CallToolResult>;
|
|
32
|
+
//# sourceMappingURL=list-recipes-tool.handler.d.ts.map
|
package/dist/main.d.ts
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Runs the server over stdio until the transport closes. This is what
|
|
3
|
+
* `bin/opencraw-mcp.mjs` calls; an MCP host launches it as a subprocess and
|
|
4
|
+
* talks JSON-RPC over stdin/stdout, so there is no argv to parse.
|
|
5
|
+
*/
|
|
6
|
+
export declare function main(): Promise<void>;
|
|
7
|
+
//# sourceMappingURL=main.d.ts.map
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import type { CallToolResult } from '@modelcontextprotocol/sdk/types.js';
|
|
2
|
+
import { z } from 'zod';
|
|
3
|
+
/** Input schema for the `probe` tool: a raw zod shape, as `McpServer.registerTool` expects. */
|
|
4
|
+
export declare const probeInputShape: {
|
|
5
|
+
url: z.ZodString;
|
|
6
|
+
browser: z.ZodOptional<z.ZodBoolean>;
|
|
7
|
+
browserPath: z.ZodOptional<z.ZodString>;
|
|
8
|
+
insecureTls: z.ZodOptional<z.ZodBoolean>;
|
|
9
|
+
userAgent: z.ZodOptional<z.ZodString>;
|
|
10
|
+
access: z.ZodOptional<z.ZodString>;
|
|
11
|
+
};
|
|
12
|
+
/**
|
|
13
|
+
* The `probe` tool: fetches a page and reports where its data lives, so an
|
|
14
|
+
* agent can write an extraction recipe without guessing selectors.
|
|
15
|
+
*
|
|
16
|
+
* @param args - The tool's parsed input.
|
|
17
|
+
* @returns The MCP tool result: the probe findings as JSON text.
|
|
18
|
+
*/
|
|
19
|
+
export declare function probeTool(args: {
|
|
20
|
+
url: string;
|
|
21
|
+
browser?: boolean;
|
|
22
|
+
browserPath?: string;
|
|
23
|
+
insecureTls?: boolean;
|
|
24
|
+
userAgent?: string;
|
|
25
|
+
access?: string;
|
|
26
|
+
}): Promise<CallToolResult>;
|
|
27
|
+
//# sourceMappingURL=probe-tool.handler.d.ts.map
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { RecipeSource } from '@opencraw/core';
|
|
2
|
+
import { z } from 'zod';
|
|
3
|
+
/** The two ways a tool takes recipes: files, or the recipes themselves. */
|
|
4
|
+
export declare const recipeSourceShape: {
|
|
5
|
+
paths: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
6
|
+
recipes: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodArray<z.ZodRecord<z.ZodString, z.ZodUnknown>>]>>;
|
|
7
|
+
};
|
|
8
|
+
/** A tool's recipe arguments. */
|
|
9
|
+
export interface RecipeSourceArgs {
|
|
10
|
+
paths?: string[];
|
|
11
|
+
recipes?: string | Record<string, unknown>[];
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* The recipe source a tool call names.
|
|
15
|
+
*
|
|
16
|
+
* @param args - The tool's parsed input.
|
|
17
|
+
* @returns What core reads: the paths, or the inline recipes.
|
|
18
|
+
* @throws Error when the call gives both or neither.
|
|
19
|
+
*/
|
|
20
|
+
export declare function recipeSourceOf(args: RecipeSourceArgs): RecipeSource;
|
|
21
|
+
//# sourceMappingURL=recipe-source.mapper.d.ts.map
|