@opencraw/driver 0.3.0 → 0.5.0

package/README.md

@@ -47,7 +47,7 @@ npx opencraw-driver   # then write JSON-RPC lines to it
 | `work.done`, `work.failed` | driver | How an item ended: its report and records, or its outcome (`failure`, `neutral`). |
 | `callout` | driver | A hook, captcha solver or access plugin the client serves (`request.kind`: `hook`, `captcha`, `access`). The request and its answer are core's [callout contract](../core/README.md): `ok`, `error`, or `pending` with `retryAfterMs`, asked again with the same idempotency key. |
 
-Every answer comes back before the driver stops, even when a client writes its last requests and closes
+A run's answer (`run`, `work`) always comes before any of its events, records or callouts. Every answer comes back before the driver stops, even when a client writes its last requests and closes
 stdin at once. Answers may arrive out of order; match them by `id`.
 
 ## For client authors

package/dist/index.esm.js

@@ -240,8 +240,8 @@ function clientAccessPlugins(names, runId, ask) {
 // src/site-access/resolve-site-access.use-case.ts
 import { resolveAccess } from "@opencraw/cli";
 async function resolveSiteAccess(params, runId, ask, env = process.env) {
-  const file = params.accessConfig ?? env.OPENCRAW_ACCESS;
-  if (file === void 0 && params.access !== void 0) throw new Error(`access profile "${params.access}" needs an access config: give "accessConfig", or set OPENCRAW_ACCESS where the driver runs`);
+  const file = params.accessConfig ?? (env.OPENCRAW_ACCESS === "" ? void 0 : env.OPENCRAW_ACCESS);
+  if (file === void 0 && params.access !== void 0) throw new Error(`access profile "${params.access}" needs an access config, and there is none: no "accessConfig" was given, and OPENCRAW_ACCESS is not set where OpenCraw runs`);
   const access = await resolveAccess({ insecureTls: false, access: file, accessProfile: params.access });
   return { access, accessPlugins: clientAccessPlugins(params.accessPlugins ?? [], runId, ask) };
 }
@@ -344,17 +344,17 @@ function streamingSink() {
 }
 
 // src/session/run-session.use-case.ts
-function runSessions(notify, ask) {
+function runSessions(notify, ask, local = {}) {
   const running = /* @__PURE__ */ new Map();
   let next = 1;
   const crawlerFor = async (runId, params) => {
     const { access, accessPlugins } = await resolveSiteAccess(params, runId, ask);
     return createCrawler({
       sink: params.out === void 0 ? streamingSink() : fileSink2(params.out, { append: params.append, browser: { executablePath: params.browser?.executablePath } }),
-      hooks: clientHooks(params.hooks ?? [], runId, ask),
-      captchaSolvers: clientCaptchaSolvers(params.captchaSolvers ?? [], runId, ask),
+      hooks: { ...local.hooks, ...clientHooks(params.hooks ?? [], runId, ask) },
+      captchaSolvers: [...local.captchaSolvers ?? [], ...clientCaptchaSolvers(params.captchaSolvers ?? [], runId, ask)],
       access,
-      accessPlugins,
+      accessPlugins: [...local.accessPlugins ?? [], ...accessPlugins],
       profilesDir: process.env.OPENCRAW_PROFILES,
       parallel: params.parallel,
       resume: params.resume,
@@ -369,6 +369,7 @@ function runSessions(notify, ask) {
   const track = (runId, crawler, crawling) => {
     running.set(runId, crawler);
     void (async () => {
+      await new Promise((resolve) => setImmediate(resolve));
       try {
         notify("finished", { runId, report: await crawling() });
       } catch (error) {
@@ -379,15 +380,16 @@ function runSessions(notify, ask) {
       }
     })();
   };
+  const startLoaded = async (loaded, params) => {
+    const set = params.dryRun === true ? new RecipeSet(loaded.output, loaded.inputs.map((input) => ({ ...input, limits: { ...input.limits, maxRecords: 1 } }))) : loaded;
+    const runId = `run-${next++}`;
+    const crawler = await crawlerFor(runId, params);
+    track(runId, crawler, async () => await crawler.run(set));
+    return runId;
+  };
   return {
-    start: async (params) => {
-      const loaded = await loadRecipes(params);
-      const set = params.dryRun === true ? new RecipeSet(loaded.output, loaded.inputs.map((input) => ({ ...input, limits: { ...input.limits, maxRecords: 1 } }))) : loaded;
-      const runId = `run-${next++}`;
-      const crawler = await crawlerFor(runId, params);
-      track(runId, crawler, async () => await crawler.run(set));
-      return runId;
-    },
+    start: async (params) => await startLoaded(await loadRecipes(params), params),
+    startLoaded,
     startWork: async (params) => {
       const set = await loadRecipes(params);
       const runId = `run-${next++}`;

package/dist/main.esm.js

@@ -201,8 +201,8 @@ function clientAccessPlugins(names, runId, ask) {
 // src/site-access/resolve-site-access.use-case.ts
 import { resolveAccess } from "@opencraw/cli";
 async function resolveSiteAccess(params, runId, ask, env = process.env) {
-  const file = params.accessConfig ?? env.OPENCRAW_ACCESS;
-  if (file === void 0 && params.access !== void 0) throw new Error(`access profile "${params.access}" needs an access config: give "accessConfig", or set OPENCRAW_ACCESS where the driver runs`);
+  const file = params.accessConfig ?? (env.OPENCRAW_ACCESS === "" ? void 0 : env.OPENCRAW_ACCESS);
+  if (file === void 0 && params.access !== void 0) throw new Error(`access profile "${params.access}" needs an access config, and there is none: no "accessConfig" was given, and OPENCRAW_ACCESS is not set where OpenCraw runs`);
   const access = await resolveAccess({ insecureTls: false, access: file, accessProfile: params.access });
   return { access, accessPlugins: clientAccessPlugins(params.accessPlugins ?? [], runId, ask) };
 }
@@ -305,17 +305,17 @@ function streamingSink() {
 }
 
 // src/session/run-session.use-case.ts
-function runSessions(notify, ask) {
+function runSessions(notify, ask, local = {}) {
   const running = /* @__PURE__ */ new Map();
   let next = 1;
   const crawlerFor = async (runId, params) => {
     const { access, accessPlugins } = await resolveSiteAccess(params, runId, ask);
     return createCrawler({
       sink: params.out === void 0 ? streamingSink() : fileSink2(params.out, { append: params.append, browser: { executablePath: params.browser?.executablePath } }),
-      hooks: clientHooks(params.hooks ?? [], runId, ask),
-      captchaSolvers: clientCaptchaSolvers(params.captchaSolvers ?? [], runId, ask),
+      hooks: { ...local.hooks, ...clientHooks(params.hooks ?? [], runId, ask) },
+      captchaSolvers: [...local.captchaSolvers ?? [], ...clientCaptchaSolvers(params.captchaSolvers ?? [], runId, ask)],
       access,
-      accessPlugins,
+      accessPlugins: [...local.accessPlugins ?? [], ...accessPlugins],
       profilesDir: process.env.OPENCRAW_PROFILES,
       parallel: params.parallel,
       resume: params.resume,
@@ -330,6 +330,7 @@ function runSessions(notify, ask) {
   const track = (runId, crawler, crawling) => {
     running.set(runId, crawler);
     void (async () => {
+      await new Promise((resolve) => setImmediate(resolve));
       try {
         notify("finished", { runId, report: await crawling() });
       } catch (error) {
@@ -340,15 +341,16 @@ function runSessions(notify, ask) {
       }
     })();
   };
+  const startLoaded = async (loaded, params) => {
+    const set = params.dryRun === true ? new RecipeSet(loaded.output, loaded.inputs.map((input) => ({ ...input, limits: { ...input.limits, maxRecords: 1 } }))) : loaded;
+    const runId = `run-${next++}`;
+    const crawler = await crawlerFor(runId, params);
+    track(runId, crawler, async () => await crawler.run(set));
+    return runId;
+  };
   return {
-    start: async (params) => {
-      const loaded = await loadRecipes(params);
-      const set = params.dryRun === true ? new RecipeSet(loaded.output, loaded.inputs.map((input) => ({ ...input, limits: { ...input.limits, maxRecords: 1 } }))) : loaded;
-      const runId = `run-${next++}`;
-      const crawler = await crawlerFor(runId, params);
-      track(runId, crawler, async () => await crawler.run(set));
-      return runId;
-    },
+    start: async (params) => await startLoaded(await loadRecipes(params), params),
+    startLoaded,
     startWork: async (params) => {
       const set = await loadRecipes(params);
       const runId = `run-${next++}`;

package/dist/src/index.d.ts

@@ -5,7 +5,7 @@ export type { InitializeParams, InitializeResult, ValidateParams, ValidateResult
 export { openLineChannel, RPC_ERRORS, RpcFailure } from './transport/index.js';
 export type { LineChannel, RequestHandler, RpcError, RpcMessage } from './transport/index.js';
 export { runSessions, validateRecipes, streamingSink } from './session/index.js';
-export type { RunSessions } from './session/index.js';
+export type { LocalPlugins, RunSessions } from './session/index.js';
 export { clientHooks, clientCaptchaSolvers, clientAccessPlugins } from './callout-bridge/index.js';
 export type { AskClient, NotifyClient } from './callout-bridge/index.js';
 export { clientWorkSource } from './work-bridge/index.js';

package/dist/src/session/index.d.ts

@@ -1,5 +1,5 @@
 export { runSessions } from './run-session.use-case.js';
-export type { RunSessions } from './run-session.use-case.js';
+export type { LocalPlugins, RunSessions } from './run-session.use-case.js';
 export { validateRecipes, hooksOf, loadRecipes, sourceOf } from './recipe-loading.use-case.js';
 export { streamingSink } from './streaming-sink.repository.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/src/session/run-session.use-case.d.ts

@@ -1,12 +1,22 @@
+import { RecipeSet } from '@opencraw/core';
+import type { AccessPlugin, CaptchaSolver, HookMap } from '@opencraw/core';
 import type { AskClient, NotifyClient } from '../callout-bridge/index.js';
 import type { RunParams, WorkParams } from '../protocol/index.js';
 /** The runs a driver has going, by id, so `cancel` can reach them and `shutdown` close them all. */
 export interface RunSessions {
     start: (params: RunParams) => Promise<string>;
+    /** As `start`, for recipes the host has already loaded and bound (`params`' recipe source is ignored). */
+    startLoaded: (set: RecipeSet, params: Omit<RunParams, 'paths' | 'recipes'>) => Promise<string>;
     startWork: (params: WorkParams) => Promise<string>;
     cancel: (runId: string) => Promise<boolean>;
     closeAll: () => Promise<void>;
 }
+/** What the host serves itself, in process, besides what the client serves: an MCP server's plugins module. */
+export interface LocalPlugins {
+    hooks?: HookMap;
+    captchaSolvers?: CaptchaSolver[];
+    accessPlugins?: AccessPlugin[];
+}
 /**
  * Starts and tracks runs and worker pools. `start` and `startWork` load and bind the recipes (a bad
  * recipe fails the request itself, with its problems), answer with a run id at once, and crawl in the
@@ -18,7 +28,8 @@ export interface RunSessions {
  *
  * @param notify - The channel's notify.
  * @param ask - The channel's request, for callouts and work.
+ * @param local - Hooks, captcha solvers and access plugins the host serves itself, beside the client's.
  * @returns The sessions.
  */
-export declare function runSessions(notify: NotifyClient, ask: AskClient): RunSessions;
+export declare function runSessions(notify: NotifyClient, ask: AskClient, local?: LocalPlugins): RunSessions;
 //# sourceMappingURL=run-session.use-case.d.ts.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opencraw/driver",
-  "version": "0.3.0",
+  "version": "0.5.0",
   "description": "OpenCraw's driver: the engine over JSON-RPC on stdio, for clients in other languages",
   "license": "MIT",
   "type": "module",