@opencraw/mcp 0.1.44 → 0.1.47
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -1
- package/dist/diff-tool/diff-tool.handler.d.ts +42 -0
- package/dist/diff-tool/index.d.ts +3 -0
- package/dist/docs-tool/docs-folder.repository.d.ts +24 -0
- package/dist/docs-tool/docs-root.client.d.ts +8 -0
- package/dist/docs-tool/docs-tool.handler.d.ts +21 -0
- package/dist/docs-tool/docs-topics.algorithm.d.ts +39 -0
- package/dist/docs-tool/index.d.ts +3 -0
- package/dist/index.d.ts +12 -1
- package/dist/index.esm.js +449 -6
- package/dist/list-tool/index.d.ts +3 -0
- package/dist/list-tool/list-recipes-tool.handler.d.ts +32 -0
- package/dist/main.d.ts +7 -0
- package/dist/probe-tool/index.d.ts +2 -0
- package/dist/probe-tool/probe-tool.handler.d.ts +27 -0
- package/dist/recipe-source/index.d.ts +3 -0
- package/dist/recipe-source/recipe-source.mapper.d.ts +21 -0
- package/dist/run-tool/index.d.ts +3 -0
- package/dist/run-tool/run-tool.handler.d.ts +51 -0
- package/dist/server/create-server.use-case.d.ts +11 -0
- package/dist/server/index.d.ts +2 -0
- package/dist/src/docs-tool/docs-folder.repository.d.ts +24 -0
- package/dist/src/docs-tool/docs-root.client.d.ts +8 -0
- package/dist/src/docs-tool/docs-tool.handler.d.ts +21 -0
- package/dist/src/docs-tool/docs-topics.algorithm.d.ts +39 -0
- package/dist/src/docs-tool/index.d.ts +3 -0
- package/dist/src/server/create-server.use-case.d.ts +1 -1
- package/dist/src/studio-tool/index.d.ts +2 -0
- package/dist/src/studio-tool/studio-command.client.d.ts +11 -0
- package/dist/src/studio-tool/studio-instance.repository.d.ts +22 -0
- package/dist/src/studio-tool/studio-tools.handler.d.ts +47 -0
- package/dist/studio-tool/index.d.ts +2 -0
- package/dist/studio-tool/studio-command.client.d.ts +11 -0
- package/dist/studio-tool/studio-instance.repository.d.ts +22 -0
- package/dist/studio-tool/studio-tools.handler.d.ts +47 -0
- package/dist/validate-tool/index.d.ts +3 -0
- package/dist/validate-tool/validate-tool.handler.d.ts +31 -0
- package/docs/how-it-works/01-recipes.md +698 -0
- package/docs/how-it-works/02-page-steps.md +672 -0
- package/docs/how-it-works/03-data-steps.md +813 -0
- package/docs/how-it-works/04-flow-steps.md +738 -0
- package/docs/how-it-works/05-templates-and-scope.md +428 -0
- package/docs/how-it-works/06-documents.md +1413 -0
- package/docs/how-it-works/07-mapping-and-transforms.md +699 -0
- package/docs/how-it-works/08-policies.md +405 -0
- package/docs/how-it-works/09-running.md +804 -0
- package/docs/how-it-works/README.md +225 -0
- package/docs/recipes/README.md +22 -0
- package/docs/recipes/access.md +440 -0
- package/docs/recipes/authoring.md +1503 -0
- package/docs/recipes/azure-durable.md +180 -0
- package/docs/recipes/captcha.md +204 -0
- package/docs/recipes/movies-example.md +12 -0
- package/docs/recipes/programmatic-authoring.md +101 -0
- package/docs/recipes/quick-guide.md +157 -0
- package/docs/recipes/vehicles-example.md +9 -0
- package/docs/recipes/worker-mode.md +221 -0
- package/docs/studio/manual.md +312 -0
- package/docs/studio/pdf-agreements.md +150 -0
- package/package.json +29 -4
package/dist/index.esm.js
CHANGED
|
@@ -1,8 +1,13 @@
|
|
|
1
1
|
import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
|
|
2
2
|
import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
|
3
|
-
import { readRecordsFile, diffRecords, memorySink,
|
|
3
|
+
import { readRecordsFile, diffRecords, memorySink, fileSink, createCrawler, RecipeSet, RecipeValidationError, RecipeBindingError, parseOutputRecipe, parseInputRecipe, bindRecipeSet } from '@opencraw/core';
|
|
4
4
|
import { diffReport, readRecipeFiles, probeUrl, loadForRun, loadPlugins, resolveAccess } from '@opencraw/cli';
|
|
5
5
|
import { z } from 'zod';
|
|
6
|
+
import { readdir, readFile } from 'node:fs/promises';
|
|
7
|
+
import { join, dirname } from 'node:path';
|
|
8
|
+
import { existsSync } from 'node:fs';
|
|
9
|
+
import { fileURLToPath } from 'node:url';
|
|
10
|
+
import { homedir } from 'node:os';
|
|
6
11
|
|
|
7
12
|
/** Changes returned at most; the counts cover all of them. */
|
|
8
13
|
const CHANGE_LIMIT = 200;
|
|
@@ -56,6 +61,191 @@ async function diffTool(args) {
|
|
|
56
61
|
}
|
|
57
62
|
}
|
|
58
63
|
|
|
64
|
+
/**
|
|
65
|
+
* The topic id of a documentation file: its path under the docs folder, without the extension and with forward slashes, so `recipes/authoring.md` is
|
|
66
|
+
* `recipes/authoring`.
|
|
67
|
+
*
|
|
68
|
+
* @param relativePath - The file's path under the docs folder.
|
|
69
|
+
* @returns The id.
|
|
70
|
+
*/
|
|
71
|
+
function topicIdOf(relativePath) {
|
|
72
|
+
return relativePath.replaceAll('\\', '/').replace(/\.md$/i, '');
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* The title of a document: its first top-level heading, else its id.
|
|
76
|
+
*
|
|
77
|
+
* @param markdown - The document.
|
|
78
|
+
* @param fallback - The id, when there is no heading.
|
|
79
|
+
* @returns The title.
|
|
80
|
+
*/
|
|
81
|
+
function titleOf(markdown, fallback) {
|
|
82
|
+
const heading = outlineOf(markdown).find(candidate => candidate.level === 1);
|
|
83
|
+
return heading?.text ?? fallback;
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* Every heading of a Markdown document, in order. A line inside a fenced code block is not a heading (a shell comment starts with `#` too).
|
|
87
|
+
*
|
|
88
|
+
* @param markdown - The document.
|
|
89
|
+
* @returns The headings.
|
|
90
|
+
*/
|
|
91
|
+
function outlineOf(markdown) {
|
|
92
|
+
const headings = [];
|
|
93
|
+
let fence;
|
|
94
|
+
for (const [line, text] of markdown.split(/\r?\n/).entries()) {
|
|
95
|
+
const marker = /^\s*(`{3,}|~{3,})/.exec(text)?.[1];
|
|
96
|
+
if (marker !== undefined) {
|
|
97
|
+
if (fence === undefined) fence = marker[0];else if (marker[0] === fence) fence = undefined;
|
|
98
|
+
continue;
|
|
99
|
+
}
|
|
100
|
+
if (fence !== undefined) continue;
|
|
101
|
+
const match = /^(#{1,6}) (.+)$/.exec(text);
|
|
102
|
+
if (match !== null) headings.push({
|
|
103
|
+
level: match[1].length,
|
|
104
|
+
text: match[2].trim().replace(/ #+$/, ''),
|
|
105
|
+
line
|
|
106
|
+
});
|
|
107
|
+
}
|
|
108
|
+
return headings;
|
|
109
|
+
}
|
|
110
|
+
/**
|
|
111
|
+
* One section of a document: from the heading whose text contains `heading` (case-insensitive; an exact match wins over a partial one) to the next
|
|
112
|
+
* heading of the same or a higher level, so its subsections come with it.
|
|
113
|
+
*
|
|
114
|
+
* @param markdown - The document.
|
|
115
|
+
* @param heading - What the heading says, or part of it.
|
|
116
|
+
* @returns The section's text, or `undefined` when no heading matches.
|
|
117
|
+
*/
|
|
118
|
+
function sectionOf(markdown, heading) {
|
|
119
|
+
const wanted = heading.trim().toLowerCase();
|
|
120
|
+
const headings = outlineOf(markdown);
|
|
121
|
+
const start = headings.find(candidate => candidate.text.toLowerCase() === wanted) ?? headings.find(candidate => candidate.text.toLowerCase().includes(wanted));
|
|
122
|
+
if (start === undefined) return undefined;
|
|
123
|
+
const end = headings.find(candidate => candidate.line > start.line && candidate.level <= start.level);
|
|
124
|
+
const lines = markdown.split(/\r?\n/);
|
|
125
|
+
return lines.slice(start.line, end?.line ?? lines.length).join('\n').trim();
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/** The folders under the docs folder whose Markdown files are documentation for a person (or an agent) using OpenCraw. */
|
|
129
|
+
const FOLDERS = ['recipes', 'how-it-works', 'studio'];
|
|
130
|
+
/**
|
|
131
|
+
* The documents under a docs folder: the Markdown files directly inside `recipes`, `how-it-works` and `studio` (their pictures and capture scripts are
|
|
132
|
+
* not documentation to read).
|
|
133
|
+
*
|
|
134
|
+
* @param root - The docs folder.
|
|
135
|
+
* @returns The topics, ordered by id.
|
|
136
|
+
*/
|
|
137
|
+
async function listTopics(root) {
|
|
138
|
+
const topics = [];
|
|
139
|
+
for (const folder of FOLDERS) {
|
|
140
|
+
let names;
|
|
141
|
+
try {
|
|
142
|
+
names = await readdir(join(root, folder));
|
|
143
|
+
} catch {
|
|
144
|
+
continue;
|
|
145
|
+
}
|
|
146
|
+
const documents = names.filter(file => file.toLowerCase().endsWith('.md'));
|
|
147
|
+
for (const name of documents) {
|
|
148
|
+
const id = topicIdOf(`${folder}/${name}`);
|
|
149
|
+
const text = await readFile(join(root, folder, name), 'utf8');
|
|
150
|
+
topics.push({
|
|
151
|
+
id,
|
|
152
|
+
title: titleOf(text, id),
|
|
153
|
+
size: text.length
|
|
154
|
+
});
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
return topics.sort((a, b) => a.id.localeCompare(b.id));
|
|
158
|
+
}
|
|
159
|
+
/**
|
|
160
|
+
* One document's text.
|
|
161
|
+
*
|
|
162
|
+
* @param root - The docs folder.
|
|
163
|
+
* @param id - The topic id from `listTopics` (never a path: nothing outside the listed folders is read).
|
|
164
|
+
* @returns The Markdown, or `undefined` when there is no such topic.
|
|
165
|
+
*/
|
|
166
|
+
async function readTopic(root, id) {
|
|
167
|
+
const known = await listTopics(root);
|
|
168
|
+
if (known.every(topic => topic.id !== id)) return undefined;
|
|
169
|
+
return await readFile(join(root, `${id}.md`), 'utf8');
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/** Input schema for the `docs` tool. */
|
|
173
|
+
const docsInputShape = {
|
|
174
|
+
topic: z.string().optional().describe('A document id from the list this tool returns without a topic (`recipes/authoring`, `how-it-works/06-documents`, `studio/manual`…). Leave out to list every document.'),
|
|
175
|
+
section: z.string().optional().describe('A heading (or part of one) of that document: only that section comes back, subsections included. A long document answers with its outline until you ask for a section.')
|
|
176
|
+
};
|
|
177
|
+
/** A document longer than this many characters comes back as its outline, and the section asked for. */
|
|
178
|
+
const WHOLE_DOCUMENT_LIMIT = 30_000;
|
|
179
|
+
function text$1(value, isError = false) {
|
|
180
|
+
return {
|
|
181
|
+
content: [{
|
|
182
|
+
type: 'text',
|
|
183
|
+
text: typeof value === 'string' ? value : JSON.stringify(value)
|
|
184
|
+
}],
|
|
185
|
+
...(isError && {
|
|
186
|
+
isError
|
|
187
|
+
}),
|
|
188
|
+
...(typeof value === 'object' && value !== null && {
|
|
189
|
+
structuredContent: value
|
|
190
|
+
})
|
|
191
|
+
};
|
|
192
|
+
}
|
|
193
|
+
/**
|
|
194
|
+
* The `docs` tool: OpenCraw's own documentation, for an agent that has to write recipes right: the authoring guide (every step, selector, transform
|
|
195
|
+
* and policy), how it all works, the examples and the Studio manual. Without a topic it lists what there is; with one it gives the document, or just
|
|
196
|
+
* the section asked for, so a long guide costs only what is read.
|
|
197
|
+
*
|
|
198
|
+
* @param args - The tool's parsed input.
|
|
199
|
+
* @param root - The docs folder (the packaged copy, or the repository's).
|
|
200
|
+
* @returns The MCP tool result.
|
|
201
|
+
*/
|
|
202
|
+
async function docsTool(args, root) {
|
|
203
|
+
const topics = await listTopics(root);
|
|
204
|
+
if (topics.length === 0) return text$1('The documentation is not installed with this server.', true);
|
|
205
|
+
if (args.topic === undefined) return text$1({
|
|
206
|
+
topics,
|
|
207
|
+
hint: 'Ask for one with { topic }, and for part of a long one with { topic, section }.'
|
|
208
|
+
});
|
|
209
|
+
const document = await readTopic(root, args.topic);
|
|
210
|
+
if (document === undefined) {
|
|
211
|
+
const close = topics.filter(topic => topic.id.includes(args.topic?.toLowerCase() ?? '') || args.topic?.includes(topic.id) === true).map(topic => topic.id);
|
|
212
|
+
return text$1(`There is no document "${args.topic}". ${close.length > 0 ? `Did you mean: ${close.join(', ')}?` : `The documents are: ${topics.map(topic => topic.id).join(', ')}.`}`, true);
|
|
213
|
+
}
|
|
214
|
+
if (args.section !== undefined) {
|
|
215
|
+
const section = sectionOf(document, args.section);
|
|
216
|
+
return section === undefined ? text$1({
|
|
217
|
+
topic: args.topic,
|
|
218
|
+
error: `no heading contains "${args.section}"`,
|
|
219
|
+
outline: outlineOf(document).map(heading => `${'#'.repeat(heading.level)} ${heading.text}`)
|
|
220
|
+
}, true) : text$1({
|
|
221
|
+
topic: args.topic,
|
|
222
|
+
section: args.section,
|
|
223
|
+
text: section
|
|
224
|
+
});
|
|
225
|
+
}
|
|
226
|
+
if (document.length <= WHOLE_DOCUMENT_LIMIT) return text$1({
|
|
227
|
+
topic: args.topic,
|
|
228
|
+
text: document
|
|
229
|
+
});
|
|
230
|
+
return text$1({
|
|
231
|
+
topic: args.topic,
|
|
232
|
+
outline: outlineOf(document).map(heading => `${'#'.repeat(heading.level)} ${heading.text}`),
|
|
233
|
+
note: `This document is ${String(document.length)} characters: ask for a section with { topic, section } using a heading from the outline.`
|
|
234
|
+
});
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
/**
|
|
238
|
+
* Where the documentation is: a `docs` folder beside `dist` in the installed package (copied there at build), or the repository's own `docs` when the server runs
|
|
239
|
+
* from a checkout.
|
|
240
|
+
*
|
|
241
|
+
* @returns The docs folder.
|
|
242
|
+
*/
|
|
243
|
+
function docsRoot() {
|
|
244
|
+
const here = dirname(fileURLToPath(import.meta.url));
|
|
245
|
+
const candidates = [join(here, '..', 'docs'), join(here, '..', '..', '..', 'docs')];
|
|
246
|
+
return candidates.find(candidate => existsSync(join(candidate, 'recipes'))) ?? candidates[0];
|
|
247
|
+
}
|
|
248
|
+
|
|
59
249
|
/** Input schema for the `list_recipes` tool. */
|
|
60
250
|
const listRecipesInputShape = {
|
|
61
251
|
dir: z.string().describe('A directory of recipe files.')
|
|
@@ -173,7 +363,7 @@ const RECORD_CAP = 50;
|
|
|
173
363
|
/** Input schema for the `run` tool. */
|
|
174
364
|
const runInputShape = {
|
|
175
365
|
...recipeSourceShape,
|
|
176
|
-
out: z.string().optional().describe('Write records to this
|
|
366
|
+
out: z.string().optional().describe('Write records to this file instead of returning them inline; its extension is the format: .jsonl, .json, .csv, .xlsx, .yml/.yaml, .html or .pdf (any other extension is an error). Use for a crawl expected to produce more than a handful of records.'),
|
|
177
367
|
append: z.boolean().optional().describe('Keep what "out" holds and add to it; each line carries _key. Needs "out".'),
|
|
178
368
|
resume: z.boolean().optional().describe('Skip records "out" already has. Needs "append".'),
|
|
179
369
|
only: z.array(z.string()).optional().describe('Run only these input recipe ids.'),
|
|
@@ -216,9 +406,23 @@ async function runTool(args) {
|
|
|
216
406
|
};
|
|
217
407
|
}
|
|
218
408
|
let crawler;
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
409
|
+
let sink;
|
|
410
|
+
try {
|
|
411
|
+
sink = args.dryRun === true || args.out === undefined ? memorySink() : fileSink(args.out, {
|
|
412
|
+
append: args.append,
|
|
413
|
+
browser: {
|
|
414
|
+
executablePath: args.browserPath ?? process.env.OPENCRAW_CHROMIUM
|
|
415
|
+
}
|
|
416
|
+
});
|
|
417
|
+
} catch (error) {
|
|
418
|
+
return {
|
|
419
|
+
content: [{
|
|
420
|
+
type: 'text',
|
|
421
|
+
text: error instanceof Error ? error.message : String(error)
|
|
422
|
+
}],
|
|
423
|
+
isError: true
|
|
424
|
+
};
|
|
425
|
+
}
|
|
222
426
|
try {
|
|
223
427
|
const pluginsFile = serverFile('OPENCRAW_PLUGINS') ?? serverFile('OPENCRAW_HOOKS');
|
|
224
428
|
const plugins = pluginsFile === undefined ? undefined : await loadPlugins(pluginsFile);
|
|
@@ -294,6 +498,229 @@ function loadErrorText(error) {
|
|
|
294
498
|
return error instanceof Error ? error.message : String(error);
|
|
295
499
|
}
|
|
296
500
|
|
|
501
|
+
/**
|
|
502
|
+
* Sends one command to a running Studio (`POST /api/command`, the same door its own page uses) and reads the answer.
|
|
503
|
+
*
|
|
504
|
+
* @param instance - The Studio to talk to.
|
|
505
|
+
* @param command - A command `@opencraw/studio`'s `studioCommandSchema` accepts.
|
|
506
|
+
* @returns The JSON the Studio answered.
|
|
507
|
+
* @throws Error with the Studio's own message when it refuses.
|
|
508
|
+
*/
|
|
509
|
+
async function sendStudioCommand(instance, command) {
|
|
510
|
+
const response = await fetch(`${instance.url}/api/command`, {
|
|
511
|
+
method: 'POST',
|
|
512
|
+
headers: {
|
|
513
|
+
'content-type': 'application/json',
|
|
514
|
+
'x-opencraw-token': instance.token
|
|
515
|
+
},
|
|
516
|
+
body: JSON.stringify(command)
|
|
517
|
+
});
|
|
518
|
+
const body = await response.json();
|
|
519
|
+
if (!response.ok) throw new Error(body.error ?? `Studio answered ${String(response.status)}`);
|
|
520
|
+
return body;
|
|
521
|
+
}
|
|
522
|
+
|
|
523
|
+
/** Where running Studios announce themselves. */
|
|
524
|
+
function instancesDirectory() {
|
|
525
|
+
return join(homedir(), '.opencraw', 'studio');
|
|
526
|
+
}
|
|
527
|
+
function isAlive(pid) {
|
|
528
|
+
try {
|
|
529
|
+
process.kill(pid, 0);
|
|
530
|
+
return true;
|
|
531
|
+
} catch (error) {
|
|
532
|
+
// EPERM: the process exists but is not ours to signal.
|
|
533
|
+
return error.code === 'EPERM';
|
|
534
|
+
}
|
|
535
|
+
}
|
|
536
|
+
function instanceOf(text) {
|
|
537
|
+
try {
|
|
538
|
+
const value = JSON.parse(text);
|
|
539
|
+
return typeof value.pid === 'number' && typeof value.url === 'string' && typeof value.token === 'string' && typeof value.startedAt === 'string' ? {
|
|
540
|
+
pid: value.pid,
|
|
541
|
+
url: value.url,
|
|
542
|
+
token: value.token,
|
|
543
|
+
folder: value.folder,
|
|
544
|
+
startedAt: value.startedAt
|
|
545
|
+
} : undefined;
|
|
546
|
+
} catch {
|
|
547
|
+
return undefined;
|
|
548
|
+
}
|
|
549
|
+
}
|
|
550
|
+
/** The announcement files in a directory; none when it does not exist yet. */
|
|
551
|
+
async function namesIn(directory) {
|
|
552
|
+
try {
|
|
553
|
+
const names = await readdir(directory);
|
|
554
|
+
return names.filter(file => file.endsWith('.json'));
|
|
555
|
+
} catch {
|
|
556
|
+
return [];
|
|
557
|
+
}
|
|
558
|
+
}
|
|
559
|
+
/** A file's text; empty when it vanished (its Studio just stopped). */
|
|
560
|
+
async function textOf(file) {
|
|
561
|
+
try {
|
|
562
|
+
return await readFile(file, 'utf8');
|
|
563
|
+
} catch {
|
|
564
|
+
return '';
|
|
565
|
+
}
|
|
566
|
+
}
|
|
567
|
+
/**
|
|
568
|
+
* The Studios running on this machine, the newest first. A file whose process is gone (Studio was killed
|
|
569
|
+
* and could not withdraw it) is left out.
|
|
570
|
+
*
|
|
571
|
+
* @param directory - Where Studios announce themselves.
|
|
572
|
+
* @returns The live Studios.
|
|
573
|
+
*/
|
|
574
|
+
async function runningStudios(directory = instancesDirectory()) {
|
|
575
|
+
const found = [];
|
|
576
|
+
const names = await namesIn(directory);
|
|
577
|
+
for (const name of names) {
|
|
578
|
+
const instance = instanceOf(await textOf(join(directory, name)));
|
|
579
|
+
if (instance !== undefined && isAlive(instance.pid)) found.push(instance);
|
|
580
|
+
}
|
|
581
|
+
return found.sort((a, b) => b.startedAt.localeCompare(a.startedAt));
|
|
582
|
+
}
|
|
583
|
+
|
|
584
|
+
/** Input schema for `studio_status`: nothing. */
|
|
585
|
+
const studioStatusInputShape = {};
|
|
586
|
+
/** Input schema for `studio_open_recipe`. */
|
|
587
|
+
const studioOpenRecipeInputShape = {
|
|
588
|
+
path: z.string().min(1).describe('A recipe file (`*.input.json` / `*.ocw-in.json`), a folder of recipes, or a document, as an absolute path.'),
|
|
589
|
+
workspace: z.string().optional().describe('The folder a document belongs to, when `path` is a document.')
|
|
590
|
+
};
|
|
591
|
+
/** Input schema for `studio_run_sample`. */
|
|
592
|
+
const studioRunSampleInputShape = {
|
|
593
|
+
recipeId: z.string().min(1).describe('The id of an input recipe in the folder Studio has open.'),
|
|
594
|
+
maxRecords: z.number().int().positive().optional().describe('Stop after this many records. Default 10.'),
|
|
595
|
+
waitSeconds: z.number().int().positive().max(300).optional().describe('How long to wait for the run to finish before answering that it is still running. Default 60.')
|
|
596
|
+
};
|
|
597
|
+
const NO_STUDIO = 'No Studio is running. Start one with `opencraw studio <folder>`, or open a recipe from the OpenCraw view in VS Code ("Open in Studio"), then ask again.';
|
|
598
|
+
const POLL_MS = 500;
|
|
599
|
+
function text(value, isError = false) {
|
|
600
|
+
return {
|
|
601
|
+
content: [{
|
|
602
|
+
type: 'text',
|
|
603
|
+
text: typeof value === 'string' ? value : JSON.stringify(value)
|
|
604
|
+
}],
|
|
605
|
+
...(isError && {
|
|
606
|
+
isError
|
|
607
|
+
}),
|
|
608
|
+
...(typeof value === 'object' && value !== null && {
|
|
609
|
+
structuredContent: value
|
|
610
|
+
})
|
|
611
|
+
};
|
|
612
|
+
}
|
|
613
|
+
function messageOf(error) {
|
|
614
|
+
return error instanceof Error ? error.message : String(error);
|
|
615
|
+
}
|
|
616
|
+
async function studioOf(directory) {
|
|
617
|
+
const [newest] = await runningStudios(directory);
|
|
618
|
+
return newest;
|
|
619
|
+
}
|
|
620
|
+
/**
|
|
621
|
+
* `studio_status`: whether a Studio is running, which folder it has open, how many pages are watching and what it is running.
|
|
622
|
+
*
|
|
623
|
+
* @param directory - Where Studios announce themselves; the default is the user's own.
|
|
624
|
+
* @returns The MCP tool result.
|
|
625
|
+
*/
|
|
626
|
+
async function studioStatusTool(directory) {
|
|
627
|
+
const studio = await studioOf(directory);
|
|
628
|
+
if (studio === undefined) return text({
|
|
629
|
+
running: false,
|
|
630
|
+
hint: NO_STUDIO
|
|
631
|
+
});
|
|
632
|
+
try {
|
|
633
|
+
const status = await sendStudioCommand(studio, {
|
|
634
|
+
type: 'studio-status'
|
|
635
|
+
});
|
|
636
|
+
return text({
|
|
637
|
+
running: true,
|
|
638
|
+
url: studio.url,
|
|
639
|
+
...status
|
|
640
|
+
});
|
|
641
|
+
} catch (error) {
|
|
642
|
+
return text(`Studio did not answer: ${messageOf(error)}`, true);
|
|
643
|
+
}
|
|
644
|
+
}
|
|
645
|
+
/**
|
|
646
|
+
* `studio_open_recipe`: opens a recipe, a folder or a document in the Studio the person is looking at, so they watch the work happen.
|
|
647
|
+
*
|
|
648
|
+
* @param args - What to open.
|
|
649
|
+
* @param directory - Where Studios announce themselves.
|
|
650
|
+
* @returns The MCP tool result.
|
|
651
|
+
*/
|
|
652
|
+
async function studioOpenRecipeTool(args, directory) {
|
|
653
|
+
const studio = await studioOf(directory);
|
|
654
|
+
if (studio === undefined) return text(NO_STUDIO, true);
|
|
655
|
+
try {
|
|
656
|
+
const {
|
|
657
|
+
pages
|
|
658
|
+
} = await sendStudioCommand(studio, {
|
|
659
|
+
type: 'ask-ui',
|
|
660
|
+
request: {
|
|
661
|
+
kind: 'open',
|
|
662
|
+
path: args.path,
|
|
663
|
+
workspace: args.workspace
|
|
664
|
+
}
|
|
665
|
+
});
|
|
666
|
+
if (pages === 0) return text({
|
|
667
|
+
opened: false,
|
|
668
|
+
hint: 'Studio is running but no page is open on it. Ask the person to open Studio in their browser.',
|
|
669
|
+
url: studio.url
|
|
670
|
+
});
|
|
671
|
+
return text({
|
|
672
|
+
opened: true,
|
|
673
|
+
pages
|
|
674
|
+
});
|
|
675
|
+
} catch (error) {
|
|
676
|
+
return text(`Studio refused: ${messageOf(error)}`, true);
|
|
677
|
+
}
|
|
678
|
+
}
|
|
679
|
+
/**
|
|
680
|
+
* `studio_run_sample`: runs an input recipe as a sample in Studio (the person sees the records arrive), waits for it, and returns the
|
|
681
|
+
* counts and the first records.
|
|
682
|
+
*
|
|
683
|
+
* @param args - The recipe, how many records to stop at and how long to wait.
|
|
684
|
+
* @param directory - Where Studios announce themselves.
|
|
685
|
+
* @returns The MCP tool result.
|
|
686
|
+
*/
|
|
687
|
+
async function studioRunSampleTool(args, directory) {
|
|
688
|
+
const studio = await studioOf(directory);
|
|
689
|
+
if (studio === undefined) return text(NO_STUDIO, true);
|
|
690
|
+
try {
|
|
691
|
+
const {
|
|
692
|
+
folder
|
|
693
|
+
} = await sendStudioCommand(studio, {
|
|
694
|
+
type: 'studio-status'
|
|
695
|
+
});
|
|
696
|
+
if (folder === undefined) return text('Studio has no folder open yet. Call studio_open_recipe with a recipe file or a folder first.', true);
|
|
697
|
+
await sendStudioCommand(studio, {
|
|
698
|
+
type: 'run-sample',
|
|
699
|
+
recipeId: args.recipeId,
|
|
700
|
+
budget: {
|
|
701
|
+
maxRecords: args.maxRecords ?? 10
|
|
702
|
+
}
|
|
703
|
+
});
|
|
704
|
+
const deadline = Date.now() + (args.waitSeconds ?? 60) * 1000;
|
|
705
|
+
for (;;) {
|
|
706
|
+
const status = await sendStudioCommand(studio, {
|
|
707
|
+
type: 'run-status',
|
|
708
|
+
recipeId: args.recipeId
|
|
709
|
+
});
|
|
710
|
+
if (status.state !== 'running') return text(status);
|
|
711
|
+
if (Date.now() >= deadline) return text({
|
|
712
|
+
...status,
|
|
713
|
+
hint: 'Still running; ask again with studio_run_sample or look in Studio.'
|
|
714
|
+
});
|
|
715
|
+
await new Promise(resolve => {
|
|
716
|
+
setTimeout(resolve, POLL_MS);
|
|
717
|
+
});
|
|
718
|
+
}
|
|
719
|
+
} catch (error) {
|
|
720
|
+
return text(`Studio could not run it: ${messageOf(error)}`, true);
|
|
721
|
+
}
|
|
722
|
+
}
|
|
723
|
+
|
|
297
724
|
/** Input schema for the `validate` tool. */
|
|
298
725
|
const validateInputShape = recipeSourceShape;
|
|
299
726
|
/**
|
|
@@ -394,7 +821,7 @@ const SERVER_INFO = {
|
|
|
394
821
|
};
|
|
395
822
|
/**
|
|
396
823
|
* Builds the MCP server: one tool per crawl primitive (`probe`, `validate`,
|
|
397
|
-
* `run`, `list_recipes`, `diff`), each a thin wrapper over `@opencraw/core` and
|
|
824
|
+
* `run`, `list_recipes`, `diff`), `docs` and the `studio_` tools that drive a running Studio, each a thin wrapper over `@opencraw/core` and
|
|
398
825
|
* `@opencraw/cli`'s structured helpers. No transport is attached; `main`
|
|
399
826
|
* connects it over stdio.
|
|
400
827
|
*
|
|
@@ -422,6 +849,22 @@ function createServer() {
|
|
|
422
849
|
description: 'Compare two runs\' JSON Lines files by record key: what was added, removed, and changed (each field before and after). Use to report what changed in a source since the last crawl.',
|
|
423
850
|
inputSchema: diffInputShape
|
|
424
851
|
}, args => diffTool(args));
|
|
852
|
+
server.registerTool('docs', {
|
|
853
|
+
description: 'OpenCraw\'s own documentation: the authoring guide (every step, selector, transform and policy), how it works, the examples and the Studio manual. Without a topic it lists the documents; with one it gives the document, or just a section of it. Read the guide before writing a recipe you are not sure of, instead of guessing.',
|
|
854
|
+
inputSchema: docsInputShape
|
|
855
|
+
}, args => docsTool(args, docsRoot()));
|
|
856
|
+
server.registerTool('studio_status', {
|
|
857
|
+
description: 'Whether OpenCraw Studio is running on this machine, which folder it has open, how many pages are watching and which recipe it is sampling. Call it before the other studio_ tools.',
|
|
858
|
+
inputSchema: studioStatusInputShape
|
|
859
|
+
}, () => studioStatusTool());
|
|
860
|
+
server.registerTool('studio_open_recipe', {
|
|
861
|
+
description: 'Open a recipe, a folder of recipes or a document in the Studio the person has open, so they watch the recipe being built. Does nothing to the files.',
|
|
862
|
+
inputSchema: studioOpenRecipeInputShape
|
|
863
|
+
}, args => studioOpenRecipeTool(args));
|
|
864
|
+
server.registerTool('studio_run_sample', {
|
|
865
|
+
description: 'Run an input recipe as a sample in Studio (the person sees the records arrive) and get back the counts, the first records and what was rejected. Studio must have the folder of that recipe open: call studio_open_recipe first.',
|
|
866
|
+
inputSchema: studioRunSampleInputShape
|
|
867
|
+
}, args => studioRunSampleTool(args));
|
|
425
868
|
return server;
|
|
426
869
|
}
|
|
427
870
|
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import type { CallToolResult } from '@modelcontextprotocol/sdk/types.js';
|
|
2
|
+
import { z } from 'zod';
|
|
3
|
+
/** Input schema for the `list_recipes` tool. */
|
|
4
|
+
export declare const listRecipesInputShape: {
|
|
5
|
+
dir: z.ZodString;
|
|
6
|
+
};
|
|
7
|
+
/** What the `list_recipes` tool returns. */
|
|
8
|
+
export interface ListRecipesResult {
|
|
9
|
+
outputs: {
|
|
10
|
+
path: string;
|
|
11
|
+
id?: string;
|
|
12
|
+
}[];
|
|
13
|
+
inputs: {
|
|
14
|
+
path: string;
|
|
15
|
+
id?: string;
|
|
16
|
+
output?: string;
|
|
17
|
+
mode?: string;
|
|
18
|
+
}[];
|
|
19
|
+
others: string[];
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* The `list_recipes` tool: what recipes already exist in a directory, cheaper
|
|
23
|
+
* than a full `validate` call, so an agent can check before writing a new one
|
|
24
|
+
* or find what an existing output recipe already covers.
|
|
25
|
+
*
|
|
26
|
+
* @param args - The tool's parsed input.
|
|
27
|
+
* @returns The MCP tool result.
|
|
28
|
+
*/
|
|
29
|
+
export declare function listRecipesTool(args: {
|
|
30
|
+
dir: string;
|
|
31
|
+
}): Promise<CallToolResult>;
|
|
32
|
+
//# sourceMappingURL=list-recipes-tool.handler.d.ts.map
|
package/dist/main.d.ts
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Runs the server over stdio until the transport closes. This is what
|
|
3
|
+
* `bin/opencraw-mcp.mjs` calls; an MCP host launches it as a subprocess and
|
|
4
|
+
* talks JSON-RPC over stdin/stdout, so there is no argv to parse.
|
|
5
|
+
*/
|
|
6
|
+
export declare function main(): Promise<void>;
|
|
7
|
+
//# sourceMappingURL=main.d.ts.map
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import type { CallToolResult } from '@modelcontextprotocol/sdk/types.js';
|
|
2
|
+
import { z } from 'zod';
|
|
3
|
+
/** Input schema for the `probe` tool: a raw zod shape, as `McpServer.registerTool` expects. */
|
|
4
|
+
export declare const probeInputShape: {
|
|
5
|
+
url: z.ZodString;
|
|
6
|
+
browser: z.ZodOptional<z.ZodBoolean>;
|
|
7
|
+
browserPath: z.ZodOptional<z.ZodString>;
|
|
8
|
+
insecureTls: z.ZodOptional<z.ZodBoolean>;
|
|
9
|
+
userAgent: z.ZodOptional<z.ZodString>;
|
|
10
|
+
access: z.ZodOptional<z.ZodString>;
|
|
11
|
+
};
|
|
12
|
+
/**
|
|
13
|
+
* The `probe` tool: fetches a page and reports where its data lives, so an
|
|
14
|
+
* agent can write an extraction recipe without guessing selectors.
|
|
15
|
+
*
|
|
16
|
+
* @param args - The tool's parsed input.
|
|
17
|
+
* @returns The MCP tool result: the probe findings as JSON text.
|
|
18
|
+
*/
|
|
19
|
+
export declare function probeTool(args: {
|
|
20
|
+
url: string;
|
|
21
|
+
browser?: boolean;
|
|
22
|
+
browserPath?: string;
|
|
23
|
+
insecureTls?: boolean;
|
|
24
|
+
userAgent?: string;
|
|
25
|
+
access?: string;
|
|
26
|
+
}): Promise<CallToolResult>;
|
|
27
|
+
//# sourceMappingURL=probe-tool.handler.d.ts.map
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { RecipeSource } from '@opencraw/core';
|
|
2
|
+
import { z } from 'zod';
|
|
3
|
+
/** The two ways a tool takes recipes: files, or the recipes themselves. */
|
|
4
|
+
export declare const recipeSourceShape: {
|
|
5
|
+
paths: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
6
|
+
recipes: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodArray<z.ZodRecord<z.ZodString, z.ZodUnknown>>]>>;
|
|
7
|
+
};
|
|
8
|
+
/** A tool's recipe arguments. */
|
|
9
|
+
export interface RecipeSourceArgs {
|
|
10
|
+
paths?: string[];
|
|
11
|
+
recipes?: string | Record<string, unknown>[];
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* The recipe source a tool call names.
|
|
15
|
+
*
|
|
16
|
+
* @param args - The tool's parsed input.
|
|
17
|
+
* @returns What core reads: the paths, or the inline recipes.
|
|
18
|
+
* @throws Error when the call gives both or neither.
|
|
19
|
+
*/
|
|
20
|
+
export declare function recipeSourceOf(args: RecipeSourceArgs): RecipeSource;
|
|
21
|
+
//# sourceMappingURL=recipe-source.mapper.d.ts.map
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import type { CallToolResult } from '@modelcontextprotocol/sdk/types.js';
|
|
2
|
+
import type { CrawlReport } from '@opencraw/core';
|
|
3
|
+
import { z } from 'zod';
|
|
4
|
+
import type { RecipeSourceArgs } from '../recipe-source';
|
|
5
|
+
/** Input schema for the `run` tool. */
|
|
6
|
+
export declare const runInputShape: {
|
|
7
|
+
paths: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
8
|
+
recipes: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodArray<z.ZodRecord<z.ZodString, z.ZodUnknown>>]>>;
|
|
9
|
+
out: z.ZodOptional<z.ZodString>;
|
|
10
|
+
append: z.ZodOptional<z.ZodBoolean>;
|
|
11
|
+
resume: z.ZodOptional<z.ZodBoolean>;
|
|
12
|
+
only: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
13
|
+
dryRun: z.ZodOptional<z.ZodBoolean>;
|
|
14
|
+
headed: z.ZodOptional<z.ZodBoolean>;
|
|
15
|
+
browserPath: z.ZodOptional<z.ZodString>;
|
|
16
|
+
insecureTls: z.ZodOptional<z.ZodBoolean>;
|
|
17
|
+
userAgent: z.ZodOptional<z.ZodString>;
|
|
18
|
+
access: z.ZodOptional<z.ZodString>;
|
|
19
|
+
parallel: z.ZodOptional<z.ZodInt>;
|
|
20
|
+
};
|
|
21
|
+
interface RunArgs extends RecipeSourceArgs {
|
|
22
|
+
out?: string;
|
|
23
|
+
append?: boolean;
|
|
24
|
+
resume?: boolean;
|
|
25
|
+
only?: string[];
|
|
26
|
+
dryRun?: boolean;
|
|
27
|
+
headed?: boolean;
|
|
28
|
+
browserPath?: string;
|
|
29
|
+
insecureTls?: boolean;
|
|
30
|
+
userAgent?: string;
|
|
31
|
+
access?: string;
|
|
32
|
+
parallel?: number;
|
|
33
|
+
}
|
|
34
|
+
/** What the `run` tool returns. */
|
|
35
|
+
export interface RunResult {
|
|
36
|
+
report: CrawlReport;
|
|
37
|
+
/** Present when "out" was not given: the records this call produced, capped. */
|
|
38
|
+
records?: Record<string, unknown>[];
|
|
39
|
+
truncated?: boolean;
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* The `run` tool: crawls the recipes in `paths` or `recipes` (exactly one
|
|
43
|
+
* output recipe, any number of inputs) and returns what happened,
|
|
44
|
+
* structured, instead of the cli's printed summary.
|
|
45
|
+
*
|
|
46
|
+
* @param args - The tool's parsed input.
|
|
47
|
+
* @returns The MCP tool result.
|
|
48
|
+
*/
|
|
49
|
+
export declare function runTool(args: RunArgs): Promise<CallToolResult>;
|
|
50
|
+
export {};
|
|
51
|
+
//# sourceMappingURL=run-tool.handler.d.ts.map
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
|
2
|
+
/**
|
|
3
|
+
* Builds the MCP server: one tool per crawl primitive (`probe`, `validate`,
|
|
4
|
+
* `run`, `list_recipes`, `diff`), `docs` and the `studio_` tools that drive a running Studio, each a thin wrapper over `@opencraw/core` and
|
|
5
|
+
* `@opencraw/cli`'s structured helpers. No transport is attached; `main`
|
|
6
|
+
* connects it over stdio.
|
|
7
|
+
*
|
|
8
|
+
* @returns The server, ready to `connect`.
|
|
9
|
+
*/
|
|
10
|
+
export declare function createServer(): McpServer;
|
|
11
|
+
//# sourceMappingURL=create-server.use-case.d.ts.map
|