@opencraw/mcp 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -4
- package/dist/index.esm.js +12 -5
- package/package.json +3 -3
package/README.md
CHANGED
|
@@ -27,9 +27,11 @@ can write files passes their `paths`; one that can't (a chat-only host) passes t
|
|
|
27
27
|
`probe`/`run` call's `browserPath`/`insecureTls`, so a sandbox without Playwright's own bundled browser
|
|
28
28
|
does not need every tool call to repeat them.
|
|
29
29
|
|
|
30
|
-
`OPENCRAW_HOOKS` points at a
|
|
31
|
-
|
|
32
|
-
|
|
30
|
+
`OPENCRAW_PLUGINS` (or `OPENCRAW_HOOKS`) points at a plugins module: named exports `hooks` (the recipes'
|
|
31
|
+
`hook` steps and transforms), `accessPlugins` (for `{ kind: "plugin" }` access profiles) and `captchaSolvers`;
|
|
32
|
+
a module whose default export is `{ name: function }` is read as hooks alone. Only the server's environment
|
|
33
|
+
names it, never a tool call, so an agent can run recipes that use your plugins but can't make the server load
|
|
34
|
+
a module of its choosing.
|
|
33
35
|
|
|
34
36
|
`OPENCRAW_ACCESS` points at an access config ([access.md](../../docs/recipes/access.md)): proxy profiles, with
|
|
35
37
|
credentials as `{{env.NAME}}` read from the server's environment. The `probe` and `run` tools then take an
|
|
@@ -51,7 +53,15 @@ endpoints only a script fetches after load.
|
|
|
51
53
|
| `browserPath?`, `insecureTls?`, `userAgent?` | As in the cli. |
|
|
52
54
|
| `access?` | An access profile from the server's `OPENCRAW_ACCESS` config. |
|
|
53
55
|
|
|
54
|
-
Returns `{ url, status, findings: { jsonLd, inlineJson, jsonUrls, scriptHosts, apiLinks }, observed }`.
|
|
56
|
+
Returns `{ url, status, findings: { jsonLd, inlineJson, jsonUrls, scriptHosts, apiLinks }, observed }`. For a
|
|
57
|
+
PDF (served as `application/pdf`, or a local path) it adds `pdf: { pages, rows, headers }`: the first rows and
|
|
58
|
+
every likely table header with the `selector` a `table` extract needs. For a spreadsheet or a CSV (served as a
|
|
59
|
+
spreadsheet type or `text/csv`, or a local `.xlsx`, `.csv` or `.tsv` path) it adds
|
|
60
|
+
`workbook: { csv?: { encoding, delimiter }, sheets, rows, headers }`. For a presentation it adds
|
|
61
|
+
`deck: { width, height, slides, headers, charts, grids }`, and for JSON, JSON Lines or YAML
|
|
62
|
+
`json: { format, type, tree, lists }`: the structure, and every list of records with its path. An HTML page
|
|
63
|
+
adds `html: { tables, outline?, frontMatter? }`: its tables' header rows, and for Markdown its sections and
|
|
64
|
+
front matter keys.
|
|
55
65
|
|
|
56
66
|
### `validate`
|
|
57
67
|
|
package/dist/index.esm.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
|
|
2
2
|
import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
|
3
|
-
import { readRecipeFiles, probeUrl, loadForRun,
|
|
3
|
+
import { readRecipeFiles, probeUrl, loadForRun, loadPlugins, resolveAccess } from '@opencraw/cli';
|
|
4
4
|
import { z } from 'zod';
|
|
5
5
|
import { memorySink, jsonLinesSink, createCrawler, RecipeSet, RecipeValidationError, RecipeBindingError, parseOutputRecipe, parseInputRecipe, bindRecipeSet } from '@opencraw/core';
|
|
6
6
|
|
|
@@ -72,7 +72,8 @@ async function probeTool(args) {
|
|
|
72
72
|
insecureTls: args.insecureTls === true || process.env.OPENCRAW_INSECURE_TLS === '1',
|
|
73
73
|
userAgent: args.userAgent,
|
|
74
74
|
access: process.env.OPENCRAW_ACCESS === '' ? undefined : process.env.OPENCRAW_ACCESS,
|
|
75
|
-
accessProfile: args.access
|
|
75
|
+
accessProfile: args.access,
|
|
76
|
+
plugins: nonEmpty(process.env.OPENCRAW_PLUGINS) ?? nonEmpty(process.env.OPENCRAW_HOOKS)
|
|
76
77
|
});
|
|
77
78
|
return {
|
|
78
79
|
content: [{
|
|
@@ -91,6 +92,9 @@ async function probeTool(args) {
|
|
|
91
92
|
};
|
|
92
93
|
}
|
|
93
94
|
}
|
|
95
|
+
function nonEmpty(value) {
|
|
96
|
+
return value === '' ? undefined : value;
|
|
97
|
+
}
|
|
94
98
|
|
|
95
99
|
const recipeObject = z.record(z.string(), z.unknown());
|
|
96
100
|
const recipeObjects = z.array(recipeObject).min(1);
|
|
@@ -163,10 +167,13 @@ async function runTool(args) {
|
|
|
163
167
|
append: args.append
|
|
164
168
|
});
|
|
165
169
|
try {
|
|
166
|
-
const
|
|
170
|
+
const pluginsFile = serverFile('OPENCRAW_PLUGINS') ?? serverFile('OPENCRAW_HOOKS');
|
|
171
|
+
const plugins = pluginsFile === undefined ? undefined : await loadPlugins(pluginsFile);
|
|
167
172
|
crawler = createCrawler({
|
|
168
173
|
sink,
|
|
169
|
-
hooks:
|
|
174
|
+
hooks: plugins?.hooks,
|
|
175
|
+
accessPlugins: plugins?.accessPlugins,
|
|
176
|
+
captchaSolvers: plugins?.captchaSolvers,
|
|
170
177
|
access: await resolveAccess({
|
|
171
178
|
insecureTls: false,
|
|
172
179
|
access: serverFile('OPENCRAW_ACCESS'),
|
|
@@ -218,7 +225,7 @@ async function runTool(args) {
|
|
|
218
225
|
}
|
|
219
226
|
}
|
|
220
227
|
/**
|
|
221
|
-
* A file the server was started with (`OPENCRAW_ACCESS`, `OPENCRAW_HOOKS`).
|
|
228
|
+
* A file the server was started with (`OPENCRAW_ACCESS`, `OPENCRAW_PLUGINS` or `OPENCRAW_HOOKS`).
|
|
222
229
|
* Never a tool argument: a caller picks a profile, never a file, and never
|
|
223
230
|
* names a module for the server to run.
|
|
224
231
|
*/
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@opencraw/mcp",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.2",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"main": "./dist/index.esm.js",
|
|
6
6
|
"module": "./dist/index.esm.js",
|
|
@@ -24,8 +24,8 @@
|
|
|
24
24
|
"!**/*.js.map"
|
|
25
25
|
],
|
|
26
26
|
"dependencies": {
|
|
27
|
-
"@opencraw/core": "^0.1.
|
|
28
|
-
"@opencraw/cli": "^0.1.
|
|
27
|
+
"@opencraw/core": "^0.1.2",
|
|
28
|
+
"@opencraw/cli": "^0.1.2",
|
|
29
29
|
"@modelcontextprotocol/sdk": "^1.30.1",
|
|
30
30
|
"playwright": "^1.63.0",
|
|
31
31
|
"zod": "^4.6.5"
|