@opencraw/driver 0.1.0 → 0.2.0

package/README.md

@@ -17,8 +17,8 @@ npx opencraw-driver   # then write JSON-RPC lines to it
 ## The conversation
 
 ```jsonc
-→ {"jsonrpc":"2.0","id":1,"method":"initialize","params":{"clientName":"opencraw-python","clientVersion":"1.0.0","protocolVersion":"1.0.0"}}
-← {"jsonrpc":"2.0","id":1,"result":{"driverVersion":"0.1.0","protocolVersion":"1.0.0","methods":["initialize","validate","run","cancel","shutdown"]}}
+→ {"jsonrpc":"2.0","id":1,"method":"initialize","params":{"clientName":"opencraw-python","clientVersion":"1.0.0","protocolVersion":"1.1.0"}}
+← {"jsonrpc":"2.0","id":1,"result":{"driverVersion":"0.2.0","protocolVersion":"1.1.0","methods":["initialize","validate","run","cancel","probe","diff","export","shutdown"]}}
 → {"jsonrpc":"2.0","id":2,"method":"run","params":{"paths":["recipes/"],"hooks":["tidy"]}}
 ← {"jsonrpc":"2.0","id":2,"result":{"runId":"run-1"}}
 ← {"jsonrpc":"2.0","method":"event","params":{"runId":"run-1","event":{"type":"recipe:start",…}}}
@@ -33,13 +33,16 @@ npx opencraw-driver   # then write JSON-RPC lines to it
 |---|---|---|
 | `initialize` | client | Must come first. The driver refuses a protocol of another major. |
 | `validate` | client | Loads recipes (`paths` or inline `recipes`) without running them: the inputs that would run, every hook they call, and every problem with its path. |
-| `run` | client | Starts a run and answers its `runId` at once; the crawl goes on in the background. `hooks` names the hooks the client answers; `out` writes records to a file instead of sending them. |
+| `run` | client | Starts a run and answers its `runId` at once; the crawl goes on in the background. `hooks`, `captchaSolvers` and `accessPlugins` name what the client answers; `accessConfig` and `access` pick the access config and profile; `out` writes records to a file instead of sending them. |
 | `cancel` | client | Stops a run (its crawler is closed); it still ends with `finished`. |
+| `probe` | client | Fetches a page (or renders it, with `browser`) and says where its data lives: the CLI's and MCP's probe, through the same access fields as `run`. |
+| `diff` | client | Compares two runs' JSON Lines files by key: counts, the report a person reads, and the changes up to `limit`. |
+| `export` | client | Writes a run's JSON Lines file in another format (the extension of `out`), with an output recipe's columns. |
 | `shutdown` | client | Closes every run, answers, then exits. Closing stdin does the same. |
 | `event` | driver | Every crawl event, as core reports it. |
 | `record` | driver | Each record, unless the run has `out`. |
 | `finished` | driver | The run is over: its report, or why it could not run. |
-| `callout` | driver | A hook the client serves. The request and its answer are core's [callout contract](../core/README.md): `ok`, `error`, or `pending` with `retryAfterMs`, asked again with the same idempotency key. |
+| `callout` | driver | A hook, captcha solver or access plugin the client serves (`request.kind`: `hook`, `captcha`, `access`). The request and its answer are core's [callout contract](../core/README.md): `ok`, `error`, or `pending` with `retryAfterMs`, asked again with the same idempotency key. |
 
 Every answer comes back before the driver stops, even when a client writes its last requests and closes
 stdin at once. Answers may arrive out of order; match them by `id`.

package/dist/index.esm.js

@@ -4,8 +4,15 @@ import { once } from "node:events";
 // src/protocol/driver-message.contract.ts
 import { calloutRequestSchema, calloutResponseSchema } from "@opencraw/core";
 import { z } from "zod";
-var PROTOCOL_VERSION = "1.0.0";
+var PROTOCOL_VERSION = "1.1.0";
 var recipeObject = z.record(z.string(), z.unknown());
+var coreObject = z.record(z.string(), z.unknown());
+var accessFields = {
+  accessConfig: z.string().optional(),
+  access: z.string().optional(),
+  /** Access plugins the client serves: a profile `{ "kind": "plugin", "name": … }` naming one leases through a `callout` to the client. */
+  accessPlugins: z.array(z.string()).optional()
+};
 var recipeSource = {
   paths: z.array(z.string()).min(1).optional(),
   recipes: z.array(recipeObject).min(1).optional()
@@ -41,11 +48,58 @@ var runParamsSchema = z.strictObject({
   parallel: z.number().int().min(1).max(16).optional(),
   browser: z.strictObject({ headless: z.boolean().optional(), executablePath: z.string().optional(), ignoreHTTPSErrors: z.boolean().optional() }).optional(),
   /** The hooks the client answers itself: each becomes a `callout` request to the client when a recipe calls it. */
-  hooks: z.array(z.string()).optional()
+  hooks: z.array(z.string()).optional(),
+  /** Captcha solvers the client serves, by name: a challenge goes to the client as a `captcha` callout. */
+  captchaSolvers: z.array(z.string()).optional(),
+  ...accessFields
 });
 var runResultSchema = z.strictObject({ runId: z.string() });
 var cancelParamsSchema = z.strictObject({ runId: z.string() });
 var cancelResultSchema = z.strictObject({ cancelled: z.boolean() });
+var probeParamsSchema = z.strictObject({
+  url: z.string().min(1),
+  /** Render the page in Chromium first (for pages built by scripts). */
+  browser: z.boolean().optional(),
+  browserPath: z.string().optional(),
+  insecureTls: z.boolean().optional(),
+  userAgent: z.string().optional(),
+  ...accessFields
+});
+var probeResultSchema = coreObject;
+var diffParamsSchema = z.strictObject({
+  /** The earlier run's JSON Lines file. */
+  previous: z.string().min(1),
+  current: z.string().min(1),
+  /** The fields that identify a record; default: each line's `_key`. */
+  key: z.array(z.string()).optional(),
+  /** Fields left out of the comparison. */
+  ignore: z.array(z.string()).optional(),
+  /** Changes returned at most (default 200); the counts cover all of them. */
+  limit: z.number().int().min(0).optional()
+});
+var diffResultSchema = z.strictObject({
+  added: z.number(),
+  removed: z.number(),
+  changed: z.number(),
+  unchanged: z.number(),
+  /** Set when the current run lost most of the previous run's records. */
+  shrunk: z.strictObject({ previous: z.number(), current: z.number() }).optional(),
+  /** The report a person reads, one line per change. */
+  summary: z.array(z.string()),
+  /** The changes, up to `limit`: each `added`, `removed` or `changed` (with each field's before and after). */
+  changes: z.array(coreObject),
+  /** More changes than `limit` were found. */
+  truncated: z.boolean()
+});
+var exportParamsSchema = z.strictObject({
+  /** A run's JSON Lines file. */
+  records: z.string().min(1),
+  /** The output recipe that gives the columns. */
+  output: z.string().min(1),
+  /** The file to write; its extension names the format, as for `run`. */
+  out: z.string().min(1)
+});
+var exportResultSchema = z.strictObject({ written: z.number(), location: z.string() });
 var shutdownResultSchema = z.strictObject({});
 var eventParamsSchema = z.strictObject({ runId: z.string(), event: z.record(z.string(), z.unknown()) });
 var recordParamsSchema = z.strictObject({ runId: z.string(), recipeId: z.string(), key: z.string().nullable(), data: z.record(z.string(), z.unknown()) });
@@ -63,6 +117,9 @@ var MESSAGES = {
   validate: { direction: "client-to-driver", params: validateParamsSchema, result: validateResultSchema },
   run: { direction: "client-to-driver", params: runParamsSchema, result: runResultSchema },
   cancel: { direction: "client-to-driver", params: cancelParamsSchema, result: cancelResultSchema },
+  probe: { direction: "client-to-driver", params: probeParamsSchema, result: probeResultSchema },
+  diff: { direction: "client-to-driver", params: diffParamsSchema, result: diffResultSchema },
+  export: { direction: "client-to-driver", params: exportParamsSchema, result: exportResultSchema },
   shutdown: { direction: "client-to-driver", params: void 0, result: shutdownResultSchema },
   callout: { direction: "driver-to-client", params: calloutParamsSchema, result: calloutResultSchema },
   event: { direction: "driver-to-client", params: eventParamsSchema, result: void 0 },
@@ -99,19 +156,67 @@ function driverProtocolJsonSchema() {
   };
 }
 
+// src/record-files/diff-record-files.use-case.ts
+import { diffReport } from "@opencraw/cli";
+import { diffRecords, readRecordsFile } from "@opencraw/core";
+var CHANGE_LIMIT = 200;
+async function diffRecordFiles(params) {
+  const [previous, current] = await Promise.all([readRecordsFile(params.previous), readRecordsFile(params.current)]);
+  const diff = diffRecords(previous, current, { key: params.key, ignore: params.ignore });
+  const limit = params.limit ?? CHANGE_LIMIT;
+  return {
+    added: diff.added,
+    removed: diff.removed,
+    changed: diff.changed,
+    unchanged: diff.unchanged,
+    ...diff.shrunk !== void 0 && { shrunk: diff.shrunk },
+    summary: diffReport(diff),
+    changes: diff.changes.slice(0, limit).map((change) => ({ ...change })),
+    truncated: diff.changes.length > limit
+  };
+}
+
+// src/record-files/export-record-files.use-case.ts
+import { readFile } from "node:fs/promises";
+import { fileSink, outputRecordOf, parseOutputRecipe, readRecordsFile as readRecordsFile2 } from "@opencraw/core";
+async function exportRecordFiles(params) {
+  const output = parseOutputRecipe(JSON.parse(await readFile(params.output, "utf8")), params.output);
+  const stored = await readRecordsFile2(params.records);
+  const sink = fileSink(params.out, { browser: { executablePath: process.env.OPENCRAW_CHROMIUM } });
+  await sink.open(output);
+  for (const line of stored) await sink.write(outputRecordOf(line));
+  const summary = await sink.close();
+  return { written: summary.written, location: summary.location ?? params.out };
+}
+
 // src/session/run-session.use-case.ts
-import { createCrawler, fileSink, RecipeSet } from "@opencraw/core";
+import { createCrawler, fileSink as fileSink2, RecipeSet } from "@opencraw/core";
 
 // src/callout-bridge/client-hooks.client.ts
-import { calloutHook, calloutResponseSchema as calloutResponseSchema2 } from "@opencraw/core";
-function clientTransport(hookName, runId, ask) {
+import { accessPluginVia, calloutHook, calloutResponseSchema as calloutResponseSchema2, captchaSolverVia } from "@opencraw/core";
+function clientTransport(label, runId, ask) {
   return {
-    label: `client hook ${hookName}`,
+    label,
     call: async (request) => calloutResponseSchema2.parse(await ask("callout", { runId, request }))
   };
 }
 function clientHooks(names, runId, ask) {
-  return Object.fromEntries(names.map((name) => [name, calloutHook(name, clientTransport(name, runId, ask))]));
+  return Object.fromEntries(names.map((name) => [name, calloutHook(name, clientTransport(`client hook ${name}`, runId, ask))]));
+}
+function clientCaptchaSolvers(names, runId, ask) {
+  return names.map((name) => captchaSolverVia(name, clientTransport(`client captcha solver ${name}`, runId, ask)));
+}
+function clientAccessPlugins(names, runId, ask) {
+  return names.map((name) => accessPluginVia(name, clientTransport(`client access plugin ${name}`, runId, ask)));
+}
+
+// src/site-access/resolve-site-access.use-case.ts
+import { resolveAccess } from "@opencraw/cli";
+async function resolveSiteAccess(params, runId, ask, env = process.env) {
+  const file = params.accessConfig ?? env.OPENCRAW_ACCESS;
+  if (file === void 0 && params.access !== void 0) throw new Error(`access profile "${params.access}" needs an access config: give "accessConfig", or set OPENCRAW_ACCESS where the driver runs`);
+  const access = await resolveAccess({ insecureTls: false, access: file, accessProfile: params.access });
+  return { access, accessPlugins: clientAccessPlugins(params.accessPlugins ?? [], runId, ask) };
 }
 
 // src/session/recipe-loading.use-case.ts
@@ -171,9 +276,14 @@ function runSessions(notify, ask) {
       const loaded = await loadRecipes(params);
       const set = params.dryRun === true ? new RecipeSet(loaded.output, loaded.inputs.map((input) => ({ ...input, limits: { ...input.limits, maxRecords: 1 } }))) : loaded;
       const runId = `run-${next++}`;
+      const { access, accessPlugins } = await resolveSiteAccess(params, runId, ask);
       const crawler = createCrawler({
-        sink: params.out === void 0 ? streamingSink() : fileSink(params.out, { append: params.append, browser: { executablePath: params.browser?.executablePath } }),
+        sink: params.out === void 0 ? streamingSink() : fileSink2(params.out, { append: params.append, browser: { executablePath: params.browser?.executablePath } }),
         hooks: clientHooks(params.hooks ?? [], runId, ask),
+        captchaSolvers: clientCaptchaSolvers(params.captchaSolvers ?? [], runId, ask),
+        access,
+        accessPlugins,
+        profilesDir: process.env.OPENCRAW_PROFILES,
         parallel: params.parallel,
         resume: params.resume,
         debug: params.dryRun,
@@ -204,6 +314,21 @@ function runSessions(notify, ask) {
   };
 }
 
+// src/site-probe/probe-site.use-case.ts
+import { probeUrl } from "@opencraw/cli";
+async function probeSite(params, probeId, ask) {
+  const { access, accessPlugins } = await resolveSiteAccess(params, probeId, ask);
+  const result = await probeUrl(params.url, {
+    browser: params.browser ?? false,
+    browserPath: params.browserPath ?? process.env.OPENCRAW_CHROMIUM,
+    insecureTls: params.insecureTls ?? false,
+    userAgent: params.userAgent,
+    accessConfig: access,
+    accessPlugins
+  });
+  return { ...result };
+}
+
 // src/transport/line-channel.client.ts
 import { createInterface } from "node:readline";
 
@@ -310,7 +435,7 @@ function openLineChannel(input, output, handle) {
 }
 
 // src/server/serve-driver.use-case.ts
-var METHODS = ["initialize", "validate", "run", "cancel", "shutdown"];
+var METHODS = ["initialize", "validate", "run", "cancel", "probe", "diff", "export", "shutdown"];
 var major = (version) => version.split(".", 1)[0];
 function parsed(schema, params) {
   const result = schema.safeParse(params ?? {});
@@ -320,6 +445,7 @@ function parsed(schema, params) {
 async function serveDriver(input, output, identity) {
   let initialized = false;
   const stopping = new AbortController();
+  let probes = 0;
   const channel = openLineChannel(input, output, async (method, params) => {
     if (method === "initialize") {
       const hello = parsed(initializeParamsSchema, params);
@@ -333,6 +459,9 @@ async function serveDriver(input, output, identity) {
     if (method === "validate") return await validateRecipes(parsed(validateParamsSchema, params));
     if (method === "run") return { runId: await sessions.start(parsed(runParamsSchema, params)) };
     if (method === "cancel") return { cancelled: await sessions.cancel(parsed(cancelParamsSchema, params).runId) };
+    if (method === "probe") return await probeSite(parsed(probeParamsSchema, params), `probe-${++probes}`, channel.request);
+    if (method === "diff") return await diffRecordFiles(parsed(diffParamsSchema, params));
+    if (method === "export") return await exportRecordFiles(parsed(exportParamsSchema, params));
     await sessions.closeAll();
     setImmediate(() => {
       stopping.abort();
@@ -353,13 +482,24 @@ export {
   calloutResultSchema,
   cancelParamsSchema,
   cancelResultSchema,
+  clientAccessPlugins,
+  clientCaptchaSolvers,
   clientHooks,
+  diffParamsSchema,
+  diffRecordFiles,
+  diffResultSchema,
   driverProtocolJsonSchema,
   eventParamsSchema,
+  exportParamsSchema,
+  exportRecordFiles,
+  exportResultSchema,
   finishedParamsSchema,
   initializeParamsSchema,
   initializeResultSchema,
   openLineChannel,
+  probeParamsSchema,
+  probeResultSchema,
+  probeSite,
   recordParamsSchema,
   runParamsSchema,
   runResultSchema,

package/dist/main.esm.js

@@ -11,8 +11,15 @@ import { once } from "node:events";
 // src/protocol/driver-message.contract.ts
 import { calloutRequestSchema, calloutResponseSchema } from "@opencraw/core";
 import { z } from "zod";
-var PROTOCOL_VERSION = "1.0.0";
+var PROTOCOL_VERSION = "1.1.0";
 var recipeObject = z.record(z.string(), z.unknown());
+var coreObject = z.record(z.string(), z.unknown());
+var accessFields = {
+  accessConfig: z.string().optional(),
+  access: z.string().optional(),
+  /** Access plugins the client serves: a profile `{ "kind": "plugin", "name": … }` naming one leases through a `callout` to the client. */
+  accessPlugins: z.array(z.string()).optional()
+};
 var recipeSource = {
   paths: z.array(z.string()).min(1).optional(),
   recipes: z.array(recipeObject).min(1).optional()
@@ -48,11 +55,57 @@ var runParamsSchema = z.strictObject({
   parallel: z.number().int().min(1).max(16).optional(),
   browser: z.strictObject({ headless: z.boolean().optional(), executablePath: z.string().optional(), ignoreHTTPSErrors: z.boolean().optional() }).optional(),
   /** The hooks the client answers itself: each becomes a `callout` request to the client when a recipe calls it. */
-  hooks: z.array(z.string()).optional()
+  hooks: z.array(z.string()).optional(),
+  /** Captcha solvers the client serves, by name: a challenge goes to the client as a `captcha` callout. */
+  captchaSolvers: z.array(z.string()).optional(),
+  ...accessFields
 });
 var runResultSchema = z.strictObject({ runId: z.string() });
 var cancelParamsSchema = z.strictObject({ runId: z.string() });
 var cancelResultSchema = z.strictObject({ cancelled: z.boolean() });
+var probeParamsSchema = z.strictObject({
+  url: z.string().min(1),
+  /** Render the page in Chromium first (for pages built by scripts). */
+  browser: z.boolean().optional(),
+  browserPath: z.string().optional(),
+  insecureTls: z.boolean().optional(),
+  userAgent: z.string().optional(),
+  ...accessFields
+});
+var diffParamsSchema = z.strictObject({
+  /** The earlier run's JSON Lines file. */
+  previous: z.string().min(1),
+  current: z.string().min(1),
+  /** The fields that identify a record; default: each line's `_key`. */
+  key: z.array(z.string()).optional(),
+  /** Fields left out of the comparison. */
+  ignore: z.array(z.string()).optional(),
+  /** Changes returned at most (default 200); the counts cover all of them. */
+  limit: z.number().int().min(0).optional()
+});
+var diffResultSchema = z.strictObject({
+  added: z.number(),
+  removed: z.number(),
+  changed: z.number(),
+  unchanged: z.number(),
+  /** Set when the current run lost most of the previous run's records. */
+  shrunk: z.strictObject({ previous: z.number(), current: z.number() }).optional(),
+  /** The report a person reads, one line per change. */
+  summary: z.array(z.string()),
+  /** The changes, up to `limit`: each `added`, `removed` or `changed` (with each field's before and after). */
+  changes: z.array(coreObject),
+  /** More changes than `limit` were found. */
+  truncated: z.boolean()
+});
+var exportParamsSchema = z.strictObject({
+  /** A run's JSON Lines file. */
+  records: z.string().min(1),
+  /** The output recipe that gives the columns. */
+  output: z.string().min(1),
+  /** The file to write; its extension names the format, as for `run`. */
+  out: z.string().min(1)
+});
+var exportResultSchema = z.strictObject({ written: z.number(), location: z.string() });
 var shutdownResultSchema = z.strictObject({});
 var eventParamsSchema = z.strictObject({ runId: z.string(), event: z.record(z.string(), z.unknown()) });
 var recordParamsSchema = z.strictObject({ runId: z.string(), recipeId: z.string(), key: z.string().nullable(), data: z.record(z.string(), z.unknown()) });
@@ -68,19 +121,67 @@ var calloutParamsSchema = z.strictObject({ runId: z.string(), request: calloutRe
 // src/protocol/driver-json-schema.mapper.ts
 import { z as z2 } from "zod";
 
+// src/record-files/diff-record-files.use-case.ts
+import { diffReport } from "@opencraw/cli";
+import { diffRecords, readRecordsFile } from "@opencraw/core";
+var CHANGE_LIMIT = 200;
+async function diffRecordFiles(params) {
+  const [previous, current] = await Promise.all([readRecordsFile(params.previous), readRecordsFile(params.current)]);
+  const diff = diffRecords(previous, current, { key: params.key, ignore: params.ignore });
+  const limit = params.limit ?? CHANGE_LIMIT;
+  return {
+    added: diff.added,
+    removed: diff.removed,
+    changed: diff.changed,
+    unchanged: diff.unchanged,
+    ...diff.shrunk !== void 0 && { shrunk: diff.shrunk },
+    summary: diffReport(diff),
+    changes: diff.changes.slice(0, limit).map((change) => ({ ...change })),
+    truncated: diff.changes.length > limit
+  };
+}
+
+// src/record-files/export-record-files.use-case.ts
+import { readFile } from "node:fs/promises";
+import { fileSink, outputRecordOf, parseOutputRecipe, readRecordsFile as readRecordsFile2 } from "@opencraw/core";
+async function exportRecordFiles(params) {
+  const output = parseOutputRecipe(JSON.parse(await readFile(params.output, "utf8")), params.output);
+  const stored = await readRecordsFile2(params.records);
+  const sink = fileSink(params.out, { browser: { executablePath: process.env.OPENCRAW_CHROMIUM } });
+  await sink.open(output);
+  for (const line of stored) await sink.write(outputRecordOf(line));
+  const summary = await sink.close();
+  return { written: summary.written, location: summary.location ?? params.out };
+}
+
 // src/session/run-session.use-case.ts
-import { createCrawler, fileSink, RecipeSet } from "@opencraw/core";
+import { createCrawler, fileSink as fileSink2, RecipeSet } from "@opencraw/core";
 
 // src/callout-bridge/client-hooks.client.ts
-import { calloutHook, calloutResponseSchema as calloutResponseSchema2 } from "@opencraw/core";
-function clientTransport(hookName, runId, ask) {
+import { accessPluginVia, calloutHook, calloutResponseSchema as calloutResponseSchema2, captchaSolverVia } from "@opencraw/core";
+function clientTransport(label, runId, ask) {
   return {
-    label: `client hook ${hookName}`,
+    label,
     call: async (request) => calloutResponseSchema2.parse(await ask("callout", { runId, request }))
   };
 }
 function clientHooks(names, runId, ask) {
-  return Object.fromEntries(names.map((name) => [name, calloutHook(name, clientTransport(name, runId, ask))]));
+  return Object.fromEntries(names.map((name) => [name, calloutHook(name, clientTransport(`client hook ${name}`, runId, ask))]));
+}
+function clientCaptchaSolvers(names, runId, ask) {
+  return names.map((name) => captchaSolverVia(name, clientTransport(`client captcha solver ${name}`, runId, ask)));
+}
+function clientAccessPlugins(names, runId, ask) {
+  return names.map((name) => accessPluginVia(name, clientTransport(`client access plugin ${name}`, runId, ask)));
+}
+
+// src/site-access/resolve-site-access.use-case.ts
+import { resolveAccess } from "@opencraw/cli";
+async function resolveSiteAccess(params, runId, ask, env = process.env) {
+  const file = params.accessConfig ?? env.OPENCRAW_ACCESS;
+  if (file === void 0 && params.access !== void 0) throw new Error(`access profile "${params.access}" needs an access config: give "accessConfig", or set OPENCRAW_ACCESS where the driver runs`);
+  const access = await resolveAccess({ insecureTls: false, access: file, accessProfile: params.access });
+  return { access, accessPlugins: clientAccessPlugins(params.accessPlugins ?? [], runId, ask) };
 }
 
 // src/session/recipe-loading.use-case.ts
@@ -140,9 +241,14 @@ function runSessions(notify, ask) {
       const loaded = await loadRecipes(params);
       const set = params.dryRun === true ? new RecipeSet(loaded.output, loaded.inputs.map((input) => ({ ...input, limits: { ...input.limits, maxRecords: 1 } }))) : loaded;
       const runId = `run-${next++}`;
+      const { access, accessPlugins } = await resolveSiteAccess(params, runId, ask);
       const crawler = createCrawler({
-        sink: params.out === void 0 ? streamingSink() : fileSink(params.out, { append: params.append, browser: { executablePath: params.browser?.executablePath } }),
+        sink: params.out === void 0 ? streamingSink() : fileSink2(params.out, { append: params.append, browser: { executablePath: params.browser?.executablePath } }),
         hooks: clientHooks(params.hooks ?? [], runId, ask),
+        captchaSolvers: clientCaptchaSolvers(params.captchaSolvers ?? [], runId, ask),
+        access,
+        accessPlugins,
+        profilesDir: process.env.OPENCRAW_PROFILES,
         parallel: params.parallel,
         resume: params.resume,
         debug: params.dryRun,
@@ -173,6 +279,21 @@ function runSessions(notify, ask) {
   };
 }
 
+// src/site-probe/probe-site.use-case.ts
+import { probeUrl } from "@opencraw/cli";
+async function probeSite(params, probeId, ask) {
+  const { access, accessPlugins } = await resolveSiteAccess(params, probeId, ask);
+  const result = await probeUrl(params.url, {
+    browser: params.browser ?? false,
+    browserPath: params.browserPath ?? process.env.OPENCRAW_CHROMIUM,
+    insecureTls: params.insecureTls ?? false,
+    userAgent: params.userAgent,
+    accessConfig: access,
+    accessPlugins
+  });
+  return { ...result };
+}
+
 // src/transport/line-channel.client.ts
 import { createInterface } from "node:readline";
 
@@ -279,7 +400,7 @@ function openLineChannel(input, output, handle) {
 }
 
 // src/server/serve-driver.use-case.ts
-var METHODS = ["initialize", "validate", "run", "cancel", "shutdown"];
+var METHODS = ["initialize", "validate", "run", "cancel", "probe", "diff", "export", "shutdown"];
 var major = (version) => version.split(".", 1)[0];
 function parsed(schema, params) {
   const result = schema.safeParse(params ?? {});
@@ -289,6 +410,7 @@ function parsed(schema, params) {
 async function serveDriver(input, output, identity) {
   let initialized = false;
   const stopping = new AbortController();
+  let probes = 0;
   const channel = openLineChannel(input, output, async (method, params) => {
     if (method === "initialize") {
       const hello = parsed(initializeParamsSchema, params);
@@ -302,6 +424,9 @@ async function serveDriver(input, output, identity) {
     if (method === "validate") return await validateRecipes(parsed(validateParamsSchema, params));
     if (method === "run") return { runId: await sessions.start(parsed(runParamsSchema, params)) };
     if (method === "cancel") return { cancelled: await sessions.cancel(parsed(cancelParamsSchema, params).runId) };
+    if (method === "probe") return await probeSite(parsed(probeParamsSchema, params), `probe-${++probes}`, channel.request);
+    if (method === "diff") return await diffRecordFiles(parsed(diffParamsSchema, params));
+    if (method === "export") return await exportRecordFiles(parsed(exportParamsSchema, params));
     await sessions.closeAll();
     setImmediate(() => {
       stopping.abort();

package/dist/src/callout-bridge/client-hooks.client.d.ts

@@ -1,12 +1,17 @@
-import type { CalloutTransport, HookMap } from '@opencraw/core';
+import type { AccessPlugin, CalloutTransport, CaptchaSolver, HookMap } from '@opencraw/core';
 /** Sends one request to the client and waits for its answer: the channel's `request`. */
 export type AskClient = (method: string, params: unknown) => Promise<unknown>;
 /**
  * A transport whose handler is the client: each call is a `callout` request carrying core's own
  * `CalloutRequest`, and the client answers with a `CalloutResponse`, so `pending` and idempotency keys
  * work exactly as they do for a command or an HTTP hook.
+ *
+ * @param label - What the logs call it (`client hook tidy`).
+ * @param runId - The run it belongs to, sent with every callout.
+ * @param ask - The channel's request.
+ * @returns The transport.
  */
-export declare function clientTransport(hookName: string, runId: string, ask: AskClient): CalloutTransport;
+export declare function clientTransport(label: string, runId: string, ask: AskClient): CalloutTransport;
 /**
  * The hooks a run's client serves, each answered by the client over the channel.
  *
@@ -16,4 +21,25 @@ export declare function clientTransport(hookName: string, runId: string, ask: As
  * @returns A hook map for `createCrawler`.
  */
 export declare function clientHooks(names: readonly string[], runId: string, ask: AskClient): HookMap;
+/**
+ * The captcha solvers a run's client serves: a challenge goes to the client as a `captcha` callout (its
+ * input is core's `CaptchaCalloutInput`), and the answer's `token` or `text` is put on the page.
+ *
+ * @param names - The solvers the client said it serves.
+ * @param runId - The run they belong to.
+ * @param ask - The channel's request.
+ * @returns Solvers for `createCrawler`.
+ */
+export declare function clientCaptchaSolvers(names: readonly string[], runId: string, ask: AskClient): CaptchaSolver[];
+/**
+ * The access plugins a client serves: a profile `{ "kind": "plugin", "name": … }` leases through an
+ * `access` callout to the client (`phase: "lease"`), and releases through another (`phase: "release"`)
+ * when the client's lease asked for it.
+ *
+ * @param names - The plugins the client said it serves.
+ * @param runId - The run, or the probe, they belong to.
+ * @param ask - The channel's request.
+ * @returns Plugins for `createCrawler` or a probe.
+ */
+export declare function clientAccessPlugins(names: readonly string[], runId: string, ask: AskClient): AccessPlugin[];
 //# sourceMappingURL=client-hooks.client.d.ts.map

package/dist/src/callout-bridge/index.d.ts

@@ -1,3 +1,3 @@
-export { clientHooks, clientTransport } from './client-hooks.client.js';
+export { clientAccessPlugins, clientCaptchaSolvers, clientHooks, clientTransport } from './client-hooks.client.js';
 export type { AskClient } from './client-hooks.client.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/src/index.d.ts

@@ -1,10 +1,12 @@
 export { serveDriver } from './server/index.js';
 export type { DriverIdentity } from './server/index.js';
-export { PROTOCOL_VERSION, MESSAGES, driverProtocolJsonSchema, initializeParamsSchema, initializeResultSchema, validateParamsSchema, validateResultSchema, runParamsSchema, runResultSchema, cancelParamsSchema, cancelResultSchema, eventParamsSchema, recordParamsSchema, finishedParamsSchema, calloutParamsSchema, calloutResultSchema, } from './protocol/index.js';
-export type { InitializeParams, InitializeResult, ValidateParams, ValidateResult, RunParams, RunResult, CancelParams, CancelResult, EventParams, RecordParams, FinishedParams, CalloutParams, } from './protocol/index.js';
+export { PROTOCOL_VERSION, MESSAGES, driverProtocolJsonSchema, initializeParamsSchema, initializeResultSchema, validateParamsSchema, validateResultSchema, runParamsSchema, runResultSchema, cancelParamsSchema, cancelResultSchema, eventParamsSchema, recordParamsSchema, finishedParamsSchema, calloutParamsSchema, calloutResultSchema, probeParamsSchema, probeResultSchema, diffParamsSchema, diffResultSchema, exportParamsSchema, exportResultSchema, } from './protocol/index.js';
+export type { InitializeParams, InitializeResult, ValidateParams, ValidateResult, RunParams, RunResult, CancelParams, CancelResult, EventParams, RecordParams, FinishedParams, CalloutParams, ProbeParams, DiffParams, DiffResult, ExportParams, ExportResult, } from './protocol/index.js';
 export { openLineChannel, RPC_ERRORS, RpcFailure } from './transport/index.js';
 export type { LineChannel, RequestHandler, RpcError, RpcMessage } from './transport/index.js';
 export { runSessions, validateRecipes, streamingSink } from './session/index.js';
 export type { RunSessions, NotifyClient } from './session/index.js';
-export { clientHooks } from './callout-bridge/index.js';
+export { clientHooks, clientCaptchaSolvers, clientAccessPlugins } from './callout-bridge/index.js';
+export { probeSite } from './site-probe/index.js';
+export { diffRecordFiles, exportRecordFiles } from './record-files/index.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/src/protocol/driver-message.contract.d.ts

@@ -3,7 +3,7 @@ import { z } from 'zod';
  * The driver protocol's version, separate from the package's: a client pins a major and the driver refuses
  * another. Additions (methods, optional fields) raise the minor; anything a client could break on, the major.
  */
-export declare const PROTOCOL_VERSION = "1.0.0";
+export declare const PROTOCOL_VERSION = "1.1.0";
 export declare const initializeParamsSchema: z.ZodObject<{
     clientName: z.ZodString;
     clientVersion: z.ZodString;
@@ -35,6 +35,9 @@ export declare const validateResultSchema: z.ZodObject<{
 }, z.core.$strict>;
 export type ValidateResult = z.infer<typeof validateResultSchema>;
 export declare const runParamsSchema: z.ZodObject<{
+    accessConfig: z.ZodOptional<z.ZodString>;
+    access: z.ZodOptional<z.ZodString>;
+    accessPlugins: z.ZodOptional<z.ZodArray<z.ZodString>>;
     paths: z.ZodOptional<z.ZodArray<z.ZodString>>;
     recipes: z.ZodOptional<z.ZodArray<z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
     only: z.ZodOptional<z.ZodArray<z.ZodString>>;
@@ -49,6 +52,7 @@ export declare const runParamsSchema: z.ZodObject<{
         ignoreHTTPSErrors: z.ZodOptional<z.ZodBoolean>;
     }, z.core.$strict>>;
     hooks: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    captchaSolvers: z.ZodOptional<z.ZodArray<z.ZodString>>;
 }, z.core.$strict>;
 export type RunParams = z.infer<typeof runParamsSchema>;
 export declare const runResultSchema: z.ZodObject<{
@@ -63,6 +67,52 @@ export declare const cancelResultSchema: z.ZodObject<{
     cancelled: z.ZodBoolean;
 }, z.core.$strict>;
 export type CancelResult = z.infer<typeof cancelResultSchema>;
+export declare const probeParamsSchema: z.ZodObject<{
+    accessConfig: z.ZodOptional<z.ZodString>;
+    access: z.ZodOptional<z.ZodString>;
+    accessPlugins: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    url: z.ZodString;
+    browser: z.ZodOptional<z.ZodBoolean>;
+    browserPath: z.ZodOptional<z.ZodString>;
+    insecureTls: z.ZodOptional<z.ZodBoolean>;
+    userAgent: z.ZodOptional<z.ZodString>;
+}, z.core.$strict>;
+export type ProbeParams = z.infer<typeof probeParamsSchema>;
+/** What the page holds and how a recipe could reach it (`ProbeResult`); its shape is the probe's, versioned with it. */
+export declare const probeResultSchema: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+export declare const diffParamsSchema: z.ZodObject<{
+    previous: z.ZodString;
+    current: z.ZodString;
+    key: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    ignore: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    limit: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strict>;
+export type DiffParams = z.infer<typeof diffParamsSchema>;
+export declare const diffResultSchema: z.ZodObject<{
+    added: z.ZodNumber;
+    removed: z.ZodNumber;
+    changed: z.ZodNumber;
+    unchanged: z.ZodNumber;
+    shrunk: z.ZodOptional<z.ZodObject<{
+        previous: z.ZodNumber;
+        current: z.ZodNumber;
+    }, z.core.$strict>>;
+    summary: z.ZodArray<z.ZodString>;
+    changes: z.ZodArray<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+    truncated: z.ZodBoolean;
+}, z.core.$strict>;
+export type DiffResult = z.infer<typeof diffResultSchema>;
+export declare const exportParamsSchema: z.ZodObject<{
+    records: z.ZodString;
+    output: z.ZodString;
+    out: z.ZodString;
+}, z.core.$strict>;
+export type ExportParams = z.infer<typeof exportParamsSchema>;
+export declare const exportResultSchema: z.ZodObject<{
+    written: z.ZodNumber;
+    location: z.ZodString;
+}, z.core.$strict>;
+export type ExportResult = z.infer<typeof exportResultSchema>;
 export declare const shutdownResultSchema: z.ZodObject<{}, z.core.$strict>;
 /** Every crawl event, as core reports it (`CrawlEvent`); its shape is core's, versioned with core. */
 export declare const eventParamsSchema: z.ZodObject<{
@@ -83,7 +133,7 @@ export declare const finishedParamsSchema: z.ZodObject<{
     error: z.ZodOptional<z.ZodString>;
 }, z.core.$strict>;
 export type FinishedParams = z.infer<typeof finishedParamsSchema>;
-/** A hook the client serves: core's own callout request, answered with core's callout response. */
+/** A hook, captcha solver or access plugin the client serves (`request.kind`): core's own callout request, answered with core's callout response. */
 export declare const calloutParamsSchema: z.ZodObject<{
     runId: z.ZodString;
     request: z.ZodObject<{
@@ -153,6 +203,9 @@ export declare const MESSAGES: {
     readonly run: {
         readonly direction: 'client-to-driver';
         readonly params: z.ZodObject<{
+            accessConfig: z.ZodOptional<z.ZodString>;
+            access: z.ZodOptional<z.ZodString>;
+            accessPlugins: z.ZodOptional<z.ZodArray<z.ZodString>>;
             paths: z.ZodOptional<z.ZodArray<z.ZodString>>;
             recipes: z.ZodOptional<z.ZodArray<z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
             only: z.ZodOptional<z.ZodArray<z.ZodString>>;
@@ -167,6 +220,7 @@ export declare const MESSAGES: {
                 ignoreHTTPSErrors: z.ZodOptional<z.ZodBoolean>;
             }, z.core.$strict>>;
             hooks: z.ZodOptional<z.ZodArray<z.ZodString>>;
+            captchaSolvers: z.ZodOptional<z.ZodArray<z.ZodString>>;
         }, z.core.$strict>;
         readonly result: z.ZodObject<{
             runId: z.ZodString;
@@ -181,6 +235,55 @@ export declare const MESSAGES: {
             cancelled: z.ZodBoolean;
         }, z.core.$strict>;
     };
+    readonly probe: {
+        readonly direction: 'client-to-driver';
+        readonly params: z.ZodObject<{
+            accessConfig: z.ZodOptional<z.ZodString>;
+            access: z.ZodOptional<z.ZodString>;
+            accessPlugins: z.ZodOptional<z.ZodArray<z.ZodString>>;
+            url: z.ZodString;
+            browser: z.ZodOptional<z.ZodBoolean>;
+            browserPath: z.ZodOptional<z.ZodString>;
+            insecureTls: z.ZodOptional<z.ZodBoolean>;
+            userAgent: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>;
+        readonly result: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+    };
+    readonly diff: {
+        readonly direction: 'client-to-driver';
+        readonly params: z.ZodObject<{
+            previous: z.ZodString;
+            current: z.ZodString;
+            key: z.ZodOptional<z.ZodArray<z.ZodString>>;
+            ignore: z.ZodOptional<z.ZodArray<z.ZodString>>;
+            limit: z.ZodOptional<z.ZodNumber>;
+        }, z.core.$strict>;
+        readonly result: z.ZodObject<{
+            added: z.ZodNumber;
+            removed: z.ZodNumber;
+            changed: z.ZodNumber;
+            unchanged: z.ZodNumber;
+            shrunk: z.ZodOptional<z.ZodObject<{
+                previous: z.ZodNumber;
+                current: z.ZodNumber;
+            }, z.core.$strict>>;
+            summary: z.ZodArray<z.ZodString>;
+            changes: z.ZodArray<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+            truncated: z.ZodBoolean;
+        }, z.core.$strict>;
+    };
+    readonly export: {
+        readonly direction: 'client-to-driver';
+        readonly params: z.ZodObject<{
+            records: z.ZodString;
+            output: z.ZodString;
+            out: z.ZodString;
+        }, z.core.$strict>;
+        readonly result: z.ZodObject<{
+            written: z.ZodNumber;
+            location: z.ZodString;
+        }, z.core.$strict>;
+    };
     readonly shutdown: {
         readonly direction: 'client-to-driver';
         readonly params: undefined;

package/dist/src/protocol/index.d.ts

@@ -1,4 +1,4 @@
-export { PROTOCOL_VERSION, MESSAGES, initializeParamsSchema, initializeResultSchema, validateParamsSchema, validateResultSchema, runParamsSchema, runResultSchema, cancelParamsSchema, cancelResultSchema, shutdownResultSchema, eventParamsSchema, recordParamsSchema, finishedParamsSchema, calloutParamsSchema, calloutResultSchema, } from './driver-message.contract.js';
-export type { InitializeParams, InitializeResult, ValidateParams, ValidateResult, RunParams, RunResult, CancelParams, CancelResult, EventParams, RecordParams, FinishedParams, CalloutParams, } from './driver-message.contract.js';
+export { PROTOCOL_VERSION, MESSAGES, initializeParamsSchema, initializeResultSchema, validateParamsSchema, validateResultSchema, runParamsSchema, runResultSchema, cancelParamsSchema, cancelResultSchema, shutdownResultSchema, eventParamsSchema, recordParamsSchema, finishedParamsSchema, calloutParamsSchema, calloutResultSchema, probeParamsSchema, probeResultSchema, diffParamsSchema, diffResultSchema, exportParamsSchema, exportResultSchema, } from './driver-message.contract.js';
+export type { InitializeParams, InitializeResult, ValidateParams, ValidateResult, RunParams, RunResult, CancelParams, CancelResult, EventParams, RecordParams, FinishedParams, CalloutParams, ProbeParams, DiffParams, DiffResult, ExportParams, ExportResult, } from './driver-message.contract.js';
 export { driverProtocolJsonSchema } from './driver-json-schema.mapper.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/src/record-files/diff-record-files.use-case.d.ts

@@ -0,0 +1,10 @@
+import type { DiffParams, DiffResult } from '../protocol/index.js';
+/**
+ * Compares two runs' JSON Lines files by key: how many records were added, removed, changed and left
+ * alone, the report a person reads, and the changes themselves, up to `limit`.
+ *
+ * @param params - The two files, the key fields, the fields to ignore and the limit.
+ * @returns The comparison.
+ */
+export declare function diffRecordFiles(params: DiffParams): Promise<DiffResult>;
+//# sourceMappingURL=diff-record-files.use-case.d.ts.map

package/dist/src/record-files/export-record-files.use-case.d.ts

@@ -0,0 +1,11 @@
+import type { ExportParams, ExportResult } from '../protocol/index.js';
+/**
+ * Writes a finished run's JSON Lines file again in another format, the extension of `out` naming it, as
+ * the CLI's `export` does. The output recipe gives the columns: its fields, in order, with their types.
+ *
+ * @param params - The records, the output recipe and the file to write.
+ * @returns How many records were written, and where.
+ * @throws RecipeValidationError When `output` is not an output recipe.
+ */
+export declare function exportRecordFiles(params: ExportParams): Promise<ExportResult>;
+//# sourceMappingURL=export-record-files.use-case.d.ts.map

package/dist/src/record-files/index.d.ts

@@ -0,0 +1,3 @@
+export { diffRecordFiles } from './diff-record-files.use-case.js';
+export { exportRecordFiles } from './export-record-files.use-case.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/src/session/run-session.use-case.d.ts

@@ -13,7 +13,7 @@ export interface RunSessions {
  * itself, with its problems), answers with a run id at once, and crawls in the background: every crawl
  * event goes to the client as an `event` notification, every record as a `record` one unless `out`
  * names a file, and the end as `finished` with the report. A hook the client serves is asked back with
- * a `callout` request. `cancel` closes the run's crawler, which stops its recipes; the run still ends
+ * a `callout` request, and so are the captcha solvers and access plugins it serves. `cancel` closes the run's crawler, which stops its recipes; the run still ends
  * with `finished`.
  *
  * @param notify - The channel's notify.

package/dist/src/site-access/index.d.ts

@@ -0,0 +1,3 @@
+export { resolveSiteAccess } from './resolve-site-access.use-case.js';
+export type { SiteAccess, SiteAccessParams } from './resolve-site-access.use-case.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/src/site-access/resolve-site-access.use-case.d.ts

@@ -0,0 +1,26 @@
+import type { AccessConfig, AccessPlugin } from '@opencraw/core';
+import type { AskClient } from '../callout-bridge/index.js';
+/** What a run or a probe asks of access: the config file, the profile, and the plugins the client serves. */
+export interface SiteAccessParams {
+    accessConfig?: string;
+    access?: string;
+    accessPlugins?: string[];
+}
+/** How a run or a probe reaches sites. */
+export interface SiteAccess {
+    access?: AccessConfig;
+    accessPlugins: AccessPlugin[];
+}
+/**
+ * Loads the access config (the file the client names, else `OPENCRAW_ACCESS`), with `access` as its
+ * default profile, and builds a plugin for each one the client serves.
+ *
+ * @param params - The method's access fields.
+ * @param runId - The run, or the probe, the plugins' callouts belong to.
+ * @param ask - The channel's request.
+ * @param env - Where `OPENCRAW_ACCESS` is read.
+ * @returns The config and the plugins.
+ * @throws Error When a profile is named without a config, or the config has no such profile.
+ */
+export declare function resolveSiteAccess(params: SiteAccessParams, runId: string, ask: AskClient, env?: NodeJS.ProcessEnv): Promise<SiteAccess>;
+//# sourceMappingURL=resolve-site-access.use-case.d.ts.map

package/dist/src/site-probe/index.d.ts

@@ -0,0 +1,2 @@
+export { probeSite } from './probe-site.use-case.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/src/site-probe/probe-site.use-case.d.ts

@@ -0,0 +1,13 @@
+import type { AskClient } from '../callout-bridge/index.js';
+import type { ProbeParams } from '../protocol/index.js';
+/**
+ * Fetches a page, or renders it with `browser`, and says where its data lives: the probe the CLI and the
+ * MCP server use, through the access the client names (its plugins served by the client).
+ *
+ * @param params - The page and how to reach it.
+ * @param probeId - What the plugins' callouts carry as their `runId`.
+ * @param ask - The channel's request.
+ * @returns The probe's findings.
+ */
+export declare function probeSite(params: ProbeParams, probeId: string, ask: AskClient): Promise<Record<string, unknown>>;
+//# sourceMappingURL=probe-site.use-case.d.ts.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opencraw/driver",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "description": "OpenCraw's driver: the engine over JSON-RPC on stdio, for clients in other languages",
   "license": "MIT",
   "type": "module",
@@ -40,8 +40,8 @@
     "schemas": "node tools/emit-protocol-schema.mjs"
   },
   "dependencies": {
-    "@opencraw/cli": "^0.1.55",
-    "@opencraw/core": "^0.2.0",
+    "@opencraw/cli": "^0.1.56",
+    "@opencraw/core": "^0.3.0",
     "zod": "^4.6.5"
   }
 }

package/schemas/driver-protocol.schema.json

@@ -3,7 +3,7 @@
   "$id": "https://opencraw/schemas/driver-protocol.schema.json",
   "title": "OpenCraw driver protocol",
   "description": "JSON-RPC 2.0 over stdio, one message per line. Methods with a result are requests; the rest are notifications.",
-  "protocolVersion": "1.0.0",
+  "protocolVersion": "1.1.0",
   "methods": {
     "initialize": {
       "direction": "client-to-driver",
@@ -41,6 +41,33 @@
         "$ref": "#/$defs/cancel.result"
       }
     },
+    "probe": {
+      "direction": "client-to-driver",
+      "params": {
+        "$ref": "#/$defs/probe.params"
+      },
+      "result": {
+        "$ref": "#/$defs/probe.result"
+      }
+    },
+    "diff": {
+      "direction": "client-to-driver",
+      "params": {
+        "$ref": "#/$defs/diff.params"
+      },
+      "result": {
+        "$ref": "#/$defs/diff.result"
+      }
+    },
+    "export": {
+      "direction": "client-to-driver",
+      "params": {
+        "$ref": "#/$defs/export.params"
+      },
+      "result": {
+        "$ref": "#/$defs/export.result"
+      }
+    },
     "shutdown": {
       "direction": "client-to-driver",
       "result": {
@@ -273,6 +300,24 @@
           "items": {
             "type": "string"
           }
+        },
+        "captchaSolvers": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
+        "accessConfig": {
+          "type": "string"
+        },
+        "access": {
+          "type": "string"
+        },
+        "accessPlugins": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
         }
       },
       "additionalProperties": false
@@ -316,6 +361,192 @@
       ],
       "additionalProperties": false
     },
+    "probe.params": {
+      "$schema": "https://json-schema.org/draft/2020-12/schema",
+      "type": "object",
+      "properties": {
+        "url": {
+          "type": "string",
+          "minLength": 1
+        },
+        "browser": {
+          "type": "boolean"
+        },
+        "browserPath": {
+          "type": "string"
+        },
+        "insecureTls": {
+          "type": "boolean"
+        },
+        "userAgent": {
+          "type": "string"
+        },
+        "accessConfig": {
+          "type": "string"
+        },
+        "access": {
+          "type": "string"
+        },
+        "accessPlugins": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        }
+      },
+      "required": [
+        "url"
+      ],
+      "additionalProperties": false
+    },
+    "probe.result": {
+      "$schema": "https://json-schema.org/draft/2020-12/schema",
+      "type": "object",
+      "propertyNames": {
+        "type": "string"
+      },
+      "additionalProperties": {}
+    },
+    "diff.params": {
+      "$schema": "https://json-schema.org/draft/2020-12/schema",
+      "type": "object",
+      "properties": {
+        "previous": {
+          "type": "string",
+          "minLength": 1
+        },
+        "current": {
+          "type": "string",
+          "minLength": 1
+        },
+        "key": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
+        "ignore": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
+        "limit": {
+          "type": "integer",
+          "minimum": 0,
+          "maximum": 9007199254740991
+        }
+      },
+      "required": [
+        "previous",
+        "current"
+      ],
+      "additionalProperties": false
+    },
+    "diff.result": {
+      "$schema": "https://json-schema.org/draft/2020-12/schema",
+      "type": "object",
+      "properties": {
+        "added": {
+          "type": "number"
+        },
+        "removed": {
+          "type": "number"
+        },
+        "changed": {
+          "type": "number"
+        },
+        "unchanged": {
+          "type": "number"
+        },
+        "shrunk": {
+          "type": "object",
+          "properties": {
+            "previous": {
+              "type": "number"
+            },
+            "current": {
+              "type": "number"
+            }
+          },
+          "required": [
+            "previous",
+            "current"
+          ],
+          "additionalProperties": false
+        },
+        "summary": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
+        "changes": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "propertyNames": {
+              "type": "string"
+            },
+            "additionalProperties": {}
+          }
+        },
+        "truncated": {
+          "type": "boolean"
+        }
+      },
+      "required": [
+        "added",
+        "removed",
+        "changed",
+        "unchanged",
+        "summary",
+        "changes",
+        "truncated"
+      ],
+      "additionalProperties": false
+    },
+    "export.params": {
+      "$schema": "https://json-schema.org/draft/2020-12/schema",
+      "type": "object",
+      "properties": {
+        "records": {
+          "type": "string",
+          "minLength": 1
+        },
+        "output": {
+          "type": "string",
+          "minLength": 1
+        },
+        "out": {
+          "type": "string",
+          "minLength": 1
+        }
+      },
+      "required": [
+        "records",
+        "output",
+        "out"
+      ],
+      "additionalProperties": false
+    },
+    "export.result": {
+      "$schema": "https://json-schema.org/draft/2020-12/schema",
+      "type": "object",
+      "properties": {
+        "written": {
+          "type": "number"
+        },
+        "location": {
+          "type": "string"
+        }
+      },
+      "required": [
+        "written",
+        "location"
+      ],
+      "additionalProperties": false
+    },
     "shutdown.result": {
       "$schema": "https://json-schema.org/draft/2020-12/schema",
       "type": "object",