@opencraw/mcp 0.1.2 → 0.1.4

package/README.md

@@ -37,6 +37,9 @@ a module of its choosing.
 credentials as `{{env.NAME}}` read from the server's environment. The `probe` and `run` tools then take an
 `access` argument naming a profile. The file and the credentials never pass through a tool call.
 
+`OPENCRAW_PROFILES` is the directory of the persistent browser profiles recipes name in
+`session.browserProfile` (default `.opencraw/profiles` in the server's working directory).
+
 ## Tools
 
 ### `probe`
@@ -90,6 +93,7 @@ Crawls the recipes at the given paths, or passed inline.
 | `dryRun?` | One record per input recipe instead of a full crawl — for checking a recipe under construction. |
 | `headed?`, `browserPath?`, `insecureTls?`, `userAgent?` | As in the cli. |
 | `access?` | The access profile for recipes that name none, from the server's `OPENCRAW_ACCESS` config. |
+| `parallel?` | How many input recipes run at once (1 to 16, default 1). |
 
 Returns `{ report: CrawlReport, records?, truncated? }`. `records`/`truncated` are present only when `out`
 was not given, and `records` is capped at 50 even then.
@@ -105,6 +109,21 @@ call, for checking what already exists before writing a new recipe.
 
 Returns `{ outputs: [{ path, id }], inputs: [{ path, id, output, mode }], others: string[] }`.
 
+### `diff`
+
+Compares two runs' JSON Lines files by record key: what was added, what was removed, and which fields changed,
+before and after. For "what changed in this price list since last week" without re-reading it all.
+
+| Input | Meaning |
+|---|---|
+| `previous`, `current` | The two runs' files (a `run`'s `out`). |
+| `key?` | The fields that identify a record (the output recipe's key fields). Default: each line's `_key`, which a run with `append` writes. |
+| `ignore?` | Fields not compared, such as a `scrapedAt` the crawl stamps. |
+
+Returns `{ added, removed, changed, unchanged, shrunk?, summary, changes }`: `summary` is the report a person
+reads, `changes` the first 200 changes (`removed`, `changed` with `fields: [{ field, before, after }]`, `added`),
+and `shrunk` is set when the current run holds under half the records the previous one did.
+
 ## Building
 
 `nx build @opencraw/mcp`, `nx test @opencraw/mcp` (unit tests per tool), `nx run @opencraw/mcp:e2e`

package/dist/index.esm.js

@@ -1,8 +1,60 @@
 import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
 import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
-import { readRecipeFiles, probeUrl, loadForRun, loadPlugins, resolveAccess } from '@opencraw/cli';
+import { readRecordsFile, diffRecords, memorySink, jsonLinesSink, createCrawler, RecipeSet, RecipeValidationError, RecipeBindingError, parseOutputRecipe, parseInputRecipe, bindRecipeSet } from '@opencraw/core';
+import { diffReport, readRecipeFiles, probeUrl, loadForRun, loadPlugins, resolveAccess } from '@opencraw/cli';
 import { z } from 'zod';
-import { memorySink, jsonLinesSink, createCrawler, RecipeSet, RecipeValidationError, RecipeBindingError, parseOutputRecipe, parseInputRecipe, bindRecipeSet } from '@opencraw/core';
+
+/** Changes returned at most; the counts cover all of them. */
+const CHANGE_LIMIT = 200;
+/** Input schema for the `diff` tool. */
+const diffInputShape = {
+  previous: z.string().describe('The earlier run\'s JSON Lines file (a run\'s "out").'),
+  current: z.string().describe('The later run\'s JSON Lines file.'),
+  key: z.array(z.string()).optional().describe('The fields that identify a record (the output recipe\'s key fields). Default: each line\'s _key, which an appending run writes.'),
+  ignore: z.array(z.string()).optional().describe('Fields left out of the comparison, such as a scrapedAt the crawl stamps.')
+};
+/**
+ * The `diff` tool: compares two runs' records by key, so an agent watching a
+ * source can say what changed since last time instead of re-reading it all.
+ *
+ * @param args - The tool's parsed input.
+ * @returns The MCP tool result.
+ */
+async function diffTool(args) {
+  try {
+    const [previous, current] = await Promise.all([readRecordsFile(args.previous), readRecordsFile(args.current)]);
+    const diff = diffRecords(previous, current, {
+      key: args.key,
+      ignore: args.ignore
+    });
+    const result = {
+      added: diff.added,
+      removed: diff.removed,
+      changed: diff.changed,
+      unchanged: diff.unchanged,
+      ...(diff.shrunk !== undefined && {
+        shrunk: diff.shrunk
+      }),
+      summary: diffReport(diff),
+      changes: diff.changes.slice(0, CHANGE_LIMIT)
+    };
+    return {
+      content: [{
+        type: 'text',
+        text: JSON.stringify(result)
+      }],
+      structuredContent: result
+    };
+  } catch (error) {
+    return {
+      content: [{
+        type: 'text',
+        text: error instanceof Error ? error.message : String(error)
+      }],
+      isError: true
+    };
+  }
+}
 
 /** Input schema for the `list_recipes` tool. */
 const listRecipesInputShape = {
@@ -130,7 +182,8 @@ const runInputShape = {
   browserPath: z.string().optional().describe('A browser binary other than the one Playwright installed. Defaults to OPENCRAW_CHROMIUM in the server\'s environment.'),
   insecureTls: z.boolean().optional().describe("Accept an intercepting proxy's certificate. Defaults to OPENCRAW_INSECURE_TLS=1 in the server's environment."),
   userAgent: z.string().optional().describe('The user agent to send.'),
-  access: z.string().optional().describe('An access profile (a proxy) from the server\'s access config file, OPENCRAW_ACCESS, used by every recipe that names none. The error for an unknown name lists the profiles.')
+  access: z.string().optional().describe('An access profile (a proxy) from the server\'s access config file, OPENCRAW_ACCESS, used by every recipe that names none. The error for an unknown name lists the profiles.'),
+  parallel: z.int().min(1).max(16).optional().describe('How many input recipes run at once. Default 1. Iterations inside a recipe follow its limits.concurrency.')
 };
 /**
  * The `run` tool: crawls the recipes in `paths` or `recipes` (exactly one
@@ -179,6 +232,8 @@ async function runTool(args) {
         access: serverFile('OPENCRAW_ACCESS'),
         accessProfile: args.access
       }),
+      profilesDir: serverFile('OPENCRAW_PROFILES'),
+      parallel: args.parallel,
       resume: args.resume,
       browser: {
         headless: args.headed !== true,
@@ -339,7 +394,7 @@ const SERVER_INFO = {
 };
 /**
  * Builds the MCP server: one tool per crawl primitive (`probe`, `validate`,
- * `run`, `list_recipes`), each a thin wrapper over `@opencraw/core` and
+ * `run`, `list_recipes`, `diff`), each a thin wrapper over `@opencraw/core` and
  * `@opencraw/cli`'s structured helpers. No transport is attached; `main`
  * connects it over stdio.
  *
@@ -363,6 +418,10 @@ function createServer() {
     description: 'List the recipe files in a directory, split by kind, with their ids. Check before writing a new recipe, or to see what an existing output recipe covers.',
     inputSchema: listRecipesInputShape
   }, args => listRecipesTool(args));
+  server.registerTool('diff', {
+    description: 'Compare two runs\' JSON Lines files by record key: what was added, removed, and changed (each field before and after). Use to report what changed in a source since the last crawl.',
+    inputSchema: diffInputShape
+  }, args => diffTool(args));
   return server;
 }
 
@@ -376,5 +435,5 @@ async function main() {
   await server.connect(new StdioServerTransport());
 }
 
-export { createServer, listRecipesInputShape, listRecipesTool, main, probeInputShape, probeTool, runInputShape, runTool, validateInputShape, validateTool };
+export { createServer, diffInputShape, diffTool, listRecipesInputShape, listRecipesTool, main, probeInputShape, probeTool, runInputShape, runTool, validateInputShape, validateTool };
 //# sourceMappingURL=index.esm.js.map

package/dist/src/diff-tool/diff-tool.handler.d.ts

@@ -0,0 +1,42 @@
+import type { CallToolResult } from '@modelcontextprotocol/sdk/types.js';
+import type { RecordChange } from '@opencraw/core';
+import { z } from 'zod';
+/** Input schema for the `diff` tool. */
+export declare const diffInputShape: {
+    previous: z.ZodString;
+    current: z.ZodString;
+    key: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    ignore: z.ZodOptional<z.ZodArray<z.ZodString>>;
+};
+interface DiffArgs {
+    previous: string;
+    current: string;
+    key?: string[];
+    ignore?: string[];
+}
+/** What the `diff` tool returns. */
+export interface DiffResult {
+    added: number;
+    removed: number;
+    changed: number;
+    unchanged: number;
+    /** Set when the current run lost most of the previous run's records. */
+    shrunk?: {
+        previous: number;
+        current: number;
+    };
+    /** The report a person reads, one line per change. */
+    summary: string[];
+    /** The changes, up to 200: removed, changed (with each field's before and after), added. */
+    changes: RecordChange[];
+}
+/**
+ * The `diff` tool: compares two runs' records by key, so an agent watching a
+ * source can say what changed since last time instead of re-reading it all.
+ *
+ * @param args - The tool's parsed input.
+ * @returns The MCP tool result.
+ */
+export declare function diffTool(args: DiffArgs): Promise<CallToolResult>;
+export {};
+//# sourceMappingURL=diff-tool.handler.d.ts.map

package/dist/src/diff-tool/index.d.ts

@@ -0,0 +1,3 @@
+export { diffTool, diffInputShape } from './diff-tool.handler.js';
+export type { DiffResult } from './diff-tool.handler.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/src/index.d.ts

@@ -2,6 +2,8 @@ export { main } from './main.js';
 export { createServer } from './server/index.js';
 export { probeTool, probeInputShape } from './probe-tool/index.js';
 export { validateTool, validateInputShape } from './validate-tool/index.js';
+export { diffTool, diffInputShape } from './diff-tool/index.js';
+export type { DiffResult } from './diff-tool/index.js';
 export type { ValidateResult } from './validate-tool/index.js';
 export { runTool, runInputShape } from './run-tool/index.js';
 export type { RunResult } from './run-tool/index.js';

package/dist/src/run-tool/run-tool.handler.d.ts

@@ -14,6 +14,7 @@ export declare const runInputShape: {
     insecureTls: z.ZodOptional<z.ZodBoolean>;
     userAgent: z.ZodOptional<z.ZodString>;
     access: z.ZodOptional<z.ZodString>;
+    parallel: z.ZodOptional<z.ZodInt>;
     paths: z.ZodOptional<z.ZodArray<z.ZodString>>;
     recipes: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodArray<z.ZodRecord<z.ZodString, z.ZodUnknown>>]>>;
 };
@@ -28,6 +29,7 @@ interface RunArgs extends RecipeSourceArgs {
     insecureTls?: boolean;
     userAgent?: string;
     access?: string;
+    parallel?: number;
 }
 /** What the `run` tool returns. */
 export interface RunResult {

package/dist/src/server/create-server.use-case.d.ts

@@ -1,7 +1,7 @@
 import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 /**
  * Builds the MCP server: one tool per crawl primitive (`probe`, `validate`,
- * `run`, `list_recipes`), each a thin wrapper over `@opencraw/core` and
+ * `run`, `list_recipes`, `diff`), each a thin wrapper over `@opencraw/core` and
  * `@opencraw/cli`'s structured helpers. No transport is attached; `main`
  * connects it over stdio.
  *

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opencraw/mcp",
-  "version": "0.1.2",
+  "version": "0.1.4",
   "type": "module",
   "main": "./dist/index.esm.js",
   "module": "./dist/index.esm.js",
@@ -24,8 +24,8 @@
     "!**/*.js.map"
   ],
   "dependencies": {
-    "@opencraw/core": "^0.1.2",
-    "@opencraw/cli": "^0.1.2",
+    "@opencraw/core": "^0.1.4",
+    "@opencraw/cli": "^0.1.4",
     "@modelcontextprotocol/sdk": "^1.30.1",
     "playwright": "^1.63.0",
     "zod": "^4.6.5"