@opencraw/driver 0.1.0 → 0.3.0

package/dist/main.esm.js

@@ -6,13 +6,20 @@ import { dirname, join } from "node:path";
 import { fileURLToPath } from "node:url";
 
 // src/server/serve-driver.use-case.ts
-import { once } from "node:events";
+import { once as once2 } from "node:events";
 
 // src/protocol/driver-message.contract.ts
 import { calloutRequestSchema, calloutResponseSchema } from "@opencraw/core";
 import { z } from "zod";
-var PROTOCOL_VERSION = "1.0.0";
+var PROTOCOL_VERSION = "1.2.0";
 var recipeObject = z.record(z.string(), z.unknown());
+var coreObject = z.record(z.string(), z.unknown());
+var accessFields = {
+  accessConfig: z.string().optional(),
+  access: z.string().optional(),
+  /** Access plugins the client serves: a profile `{ "kind": "plugin", "name": … }` naming one leases through a `callout` to the client. */
+  accessPlugins: z.array(z.string()).optional()
+};
 var recipeSource = {
   paths: z.array(z.string()).min(1).optional(),
   recipes: z.array(recipeObject).min(1).optional()
@@ -48,17 +55,86 @@ var runParamsSchema = z.strictObject({
   parallel: z.number().int().min(1).max(16).optional(),
   browser: z.strictObject({ headless: z.boolean().optional(), executablePath: z.string().optional(), ignoreHTTPSErrors: z.boolean().optional() }).optional(),
   /** The hooks the client answers itself: each becomes a `callout` request to the client when a recipe calls it. */
-  hooks: z.array(z.string()).optional()
+  hooks: z.array(z.string()).optional(),
+  /** Captcha solvers the client serves, by name: a challenge goes to the client as a `captcha` callout. */
+  captchaSolvers: z.array(z.string()).optional(),
+  ...accessFields
 });
 var runResultSchema = z.strictObject({ runId: z.string() });
+var windowsSchema = z.union([z.number().int().min(1).max(64), z.record(z.string(), z.unknown())]);
+var workParamsSchema = z.strictObject({
+  ...recipeSource,
+  only: z.array(z.string()).optional(),
+  out: z.string().optional(),
+  append: z.boolean().optional(),
+  browser: z.strictObject({ headless: z.boolean().optional(), executablePath: z.string().optional(), ignoreHTTPSErrors: z.boolean().optional() }).optional(),
+  hooks: z.array(z.string()).optional(),
+  captchaSolvers: z.array(z.string()).optional(),
+  ...accessFields,
+  windows: windowsSchema.optional()
+});
+var varValue = z.union([z.string(), z.number(), z.boolean()]);
+var workItemSchema = z.strictObject({
+  id: z.string().min(1),
+  vars: z.record(z.string(), varValue),
+  recipe: z.string().optional()
+});
 var cancelParamsSchema = z.strictObject({ runId: z.string() });
 var cancelResultSchema = z.strictObject({ cancelled: z.boolean() });
+var probeParamsSchema = z.strictObject({
+  url: z.string().min(1),
+  /** Render the page in Chromium first (for pages built by scripts). */
+  browser: z.boolean().optional(),
+  browserPath: z.string().optional(),
+  insecureTls: z.boolean().optional(),
+  userAgent: z.string().optional(),
+  ...accessFields
+});
+var diffParamsSchema = z.strictObject({
+  /** The earlier run's JSON Lines file. */
+  previous: z.string().min(1),
+  current: z.string().min(1),
+  /** The fields that identify a record; default: each line's `_key`. */
+  key: z.array(z.string()).optional(),
+  /** Fields left out of the comparison. */
+  ignore: z.array(z.string()).optional(),
+  /** Changes returned at most (default 200); the counts cover all of them. */
+  limit: z.number().int().min(0).optional()
+});
+var diffResultSchema = z.strictObject({
+  added: z.number(),
+  removed: z.number(),
+  changed: z.number(),
+  unchanged: z.number(),
+  /** Set when the current run lost most of the previous run's records. */
+  shrunk: z.strictObject({ previous: z.number(), current: z.number() }).optional(),
+  /** The report a person reads, one line per change. */
+  summary: z.array(z.string()),
+  /** The changes, up to `limit`: each `added`, `removed` or `changed` (with each field's before and after). */
+  changes: z.array(coreObject),
+  /** More changes than `limit` were found. */
+  truncated: z.boolean()
+});
+var exportParamsSchema = z.strictObject({
+  /** A run's JSON Lines file. */
+  records: z.string().min(1),
+  /** The output recipe that gives the columns. */
+  output: z.string().min(1),
+  /** The file to write; its extension names the format, as for `run`. */
+  out: z.string().min(1)
+});
+var exportResultSchema = z.strictObject({ written: z.number(), location: z.string() });
 var shutdownResultSchema = z.strictObject({});
+var workNextParamsSchema = z.strictObject({ runId: z.string() });
+var workNextResultSchema = z.strictObject({ item: workItemSchema.nullable() });
+var workRecordSchema = z.strictObject({ key: z.string().nullable(), data: z.record(z.string(), z.unknown()) });
+var workDoneParamsSchema = z.strictObject({ runId: z.string(), item: workItemSchema, report: coreObject, records: z.array(workRecordSchema) });
+var workFailedParamsSchema = z.strictObject({ runId: z.string(), item: workItemSchema, report: coreObject, outcome: z.enum(["failure", "neutral"]) });
 var eventParamsSchema = z.strictObject({ runId: z.string(), event: z.record(z.string(), z.unknown()) });
 var recordParamsSchema = z.strictObject({ runId: z.string(), recipeId: z.string(), key: z.string().nullable(), data: z.record(z.string(), z.unknown()) });
 var finishedParamsSchema = z.strictObject({
   runId: z.string(),
-  /** The crawl report (`CrawlReport`), absent when the run could not start. */
+  /** The crawl report (`CrawlReport`), or for `work` the work report (`WorkReport`); absent when the run could not start. */
   report: z.record(z.string(), z.unknown()).optional(),
   /** Why the run could not start, or stopped as a whole. */
   error: z.string().optional()
@@ -68,19 +144,127 @@ var calloutParamsSchema = z.strictObject({ runId: z.string(), request: calloutRe
 // src/protocol/driver-json-schema.mapper.ts
 import { z as z2 } from "zod";
 
+// src/record-files/diff-record-files.use-case.ts
+import { diffReport } from "@opencraw/cli";
+import { diffRecords, readRecordsFile } from "@opencraw/core";
+var CHANGE_LIMIT = 200;
+async function diffRecordFiles(params) {
+  const [previous, current] = await Promise.all([readRecordsFile(params.previous), readRecordsFile(params.current)]);
+  const diff = diffRecords(previous, current, { key: params.key, ignore: params.ignore });
+  const limit = params.limit ?? CHANGE_LIMIT;
+  return {
+    added: diff.added,
+    removed: diff.removed,
+    changed: diff.changed,
+    unchanged: diff.unchanged,
+    ...diff.shrunk !== void 0 && { shrunk: diff.shrunk },
+    summary: diffReport(diff),
+    changes: diff.changes.slice(0, limit).map((change) => ({ ...change })),
+    truncated: diff.changes.length > limit
+  };
+}
+
+// src/record-files/export-record-files.use-case.ts
+import { readFile } from "node:fs/promises";
+import { fileSink, outputRecordOf, parseOutputRecipe, readRecordsFile as readRecordsFile2 } from "@opencraw/core";
+async function exportRecordFiles(params) {
+  const output = parseOutputRecipe(JSON.parse(await readFile(params.output, "utf8")), params.output);
+  const stored = await readRecordsFile2(params.records);
+  const sink = fileSink(params.out, { browser: { executablePath: process.env.OPENCRAW_CHROMIUM } });
+  await sink.open(output);
+  for (const line of stored) await sink.write(outputRecordOf(line));
+  const summary = await sink.close();
+  return { written: summary.written, location: summary.location ?? params.out };
+}
+
 // src/session/run-session.use-case.ts
-import { createCrawler, fileSink, RecipeSet } from "@opencraw/core";
+import { createCrawler, fileSink as fileSink2, RecipeSet } from "@opencraw/core";
 
 // src/callout-bridge/client-hooks.client.ts
-import { calloutHook, calloutResponseSchema as calloutResponseSchema2 } from "@opencraw/core";
-function clientTransport(hookName, runId, ask) {
+import { accessPluginVia, calloutHook, calloutResponseSchema as calloutResponseSchema2, captchaSolverVia } from "@opencraw/core";
+function clientTransport(label, runId, ask) {
   return {
-    label: `client hook ${hookName}`,
+    label,
     call: async (request) => calloutResponseSchema2.parse(await ask("callout", { runId, request }))
   };
 }
 function clientHooks(names, runId, ask) {
-  return Object.fromEntries(names.map((name) => [name, calloutHook(name, clientTransport(name, runId, ask))]));
+  return Object.fromEntries(names.map((name) => [name, calloutHook(name, clientTransport(`client hook ${name}`, runId, ask))]));
+}
+function clientCaptchaSolvers(names, runId, ask) {
+  return names.map((name) => captchaSolverVia(name, clientTransport(`client captcha solver ${name}`, runId, ask)));
+}
+function clientAccessPlugins(names, runId, ask) {
+  return names.map((name) => accessPluginVia(name, clientTransport(`client access plugin ${name}`, runId, ask)));
+}
+
+// src/site-access/resolve-site-access.use-case.ts
+import { resolveAccess } from "@opencraw/cli";
+async function resolveSiteAccess(params, runId, ask, env = process.env) {
+  const file = params.accessConfig ?? env.OPENCRAW_ACCESS;
+  if (file === void 0 && params.access !== void 0) throw new Error(`access profile "${params.access}" needs an access config: give "accessConfig", or set OPENCRAW_ACCESS where the driver runs`);
+  const access = await resolveAccess({ insecureTls: false, access: file, accessProfile: params.access });
+  return { access, accessPlugins: clientAccessPlugins(params.accessPlugins ?? [], runId, ask) };
+}
+
+// src/work-bridge/client-work-source.client.ts
+import { once } from "node:events";
+function clientWorkSource(runId, ask, notify) {
+  const kept = [];
+  let ended = false;
+  const askClient = async () => {
+    const { item } = workNextResultSchema.parse(await ask("work.next", { runId }));
+    if (item === null) {
+      ended = true;
+      return void 0;
+    }
+    return item;
+  };
+  const keepLate = async (answer) => {
+    try {
+      const item = await answer;
+      if (item !== void 0) kept.push(item);
+    } catch {
+    }
+  };
+  return {
+    next: async (options) => {
+      const waiting = kept.shift();
+      if (waiting !== void 0) return waiting;
+      if (ended) return;
+      const answer = askClient();
+      const signal = options?.signal;
+      if (signal === void 0) return await answer;
+      signal.throwIfAborted();
+      const settled = new AbortController();
+      const answered = (async () => {
+        const item = await answer;
+        return { item };
+      })();
+      const retired = (async () => {
+        try {
+          await once(signal, "abort", { signal: settled.signal });
+          return "retired";
+        } catch {
+          return "answered";
+        }
+      })();
+      try {
+        const first = await Promise.race([answered, retired]);
+        if (first !== "retired" && first !== "answered") return first.item;
+        void keepLate(answer);
+        throw signal.reason instanceof Error ? signal.reason : new Error(String(signal.reason));
+      } finally {
+        settled.abort();
+      }
+    },
+    done: (item, report, records) => {
+      notify("work.done", { runId, item, report, records: records.map((record) => ({ key: record.key, data: record.data })) });
+    },
+    failed: (item, report, outcome) => {
+      notify("work.failed", { runId, item, report, outcome });
+    }
+  };
 }
 
 // src/session/recipe-loading.use-case.ts
@@ -124,36 +308,53 @@ function streamingSink() {
 function runSessions(notify, ask) {
   const running = /* @__PURE__ */ new Map();
   let next = 1;
-  const crawl = async (runId, crawler, set) => {
-    try {
-      const report = await crawler.run(set);
-      notify("finished", { runId, report });
-    } catch (error) {
-      notify("finished", { runId, error: error instanceof Error ? error.message : String(error) });
-    } finally {
-      running.delete(runId);
-      await crawler.close();
-    }
+  const crawlerFor = async (runId, params) => {
+    const { access, accessPlugins } = await resolveSiteAccess(params, runId, ask);
+    return createCrawler({
+      sink: params.out === void 0 ? streamingSink() : fileSink2(params.out, { append: params.append, browser: { executablePath: params.browser?.executablePath } }),
+      hooks: clientHooks(params.hooks ?? [], runId, ask),
+      captchaSolvers: clientCaptchaSolvers(params.captchaSolvers ?? [], runId, ask),
+      access,
+      accessPlugins,
+      profilesDir: process.env.OPENCRAW_PROFILES,
+      parallel: params.parallel,
+      resume: params.resume,
+      debug: params.dryRun,
+      browser: { headless: params.browser?.headless ?? true, executablePath: params.browser?.executablePath, ignoreHTTPSErrors: params.browser?.ignoreHTTPSErrors },
+      onEvent: (event) => {
+        notify("event", { runId, event });
+        if (event.type === "record:emit" && params.out === void 0) notify("record", { runId, recipeId: event.recipeId, key: event.key, data: event.data });
+      }
+    });
+  };
+  const track = (runId, crawler, crawling) => {
+    running.set(runId, crawler);
+    void (async () => {
+      try {
+        notify("finished", { runId, report: await crawling() });
+      } catch (error) {
+        notify("finished", { runId, error: error instanceof Error ? error.message : String(error) });
+      } finally {
+        running.delete(runId);
+        await crawler.close();
+      }
+    })();
   };
   return {
     start: async (params) => {
       const loaded = await loadRecipes(params);
       const set = params.dryRun === true ? new RecipeSet(loaded.output, loaded.inputs.map((input) => ({ ...input, limits: { ...input.limits, maxRecords: 1 } }))) : loaded;
       const runId = `run-${next++}`;
-      const crawler = createCrawler({
-        sink: params.out === void 0 ? streamingSink() : fileSink(params.out, { append: params.append, browser: { executablePath: params.browser?.executablePath } }),
-        hooks: clientHooks(params.hooks ?? [], runId, ask),
-        parallel: params.parallel,
-        resume: params.resume,
-        debug: params.dryRun,
-        browser: { headless: params.browser?.headless ?? true, executablePath: params.browser?.executablePath, ignoreHTTPSErrors: params.browser?.ignoreHTTPSErrors },
-        onEvent: (event) => {
-          notify("event", { runId, event });
-          if (event.type === "record:emit" && params.out === void 0) notify("record", { runId, recipeId: event.recipeId, key: event.key, data: event.data });
-        }
-      });
-      running.set(runId, crawler);
-      void crawl(runId, crawler, set);
+      const crawler = await crawlerFor(runId, params);
+      track(runId, crawler, async () => await crawler.run(set));
+      return runId;
+    },
+    startWork: async (params) => {
+      const set = await loadRecipes(params);
+      const runId = `run-${next++}`;
+      const crawler = await crawlerFor(runId, params);
+      const windows = params.windows;
+      track(runId, crawler, async () => await crawler.work(set, clientWorkSource(runId, ask, notify), { windows }));
       return runId;
     },
     cancel: async (runId) => {
@@ -173,6 +374,21 @@ function runSessions(notify, ask) {
   };
 }
 
+// src/site-probe/probe-site.use-case.ts
+import { probeUrl } from "@opencraw/cli";
+async function probeSite(params, probeId, ask) {
+  const { access, accessPlugins } = await resolveSiteAccess(params, probeId, ask);
+  const result = await probeUrl(params.url, {
+    browser: params.browser ?? false,
+    browserPath: params.browserPath ?? process.env.OPENCRAW_CHROMIUM,
+    insecureTls: params.insecureTls ?? false,
+    userAgent: params.userAgent,
+    accessConfig: access,
+    accessPlugins
+  });
+  return { ...result };
+}
+
 // src/transport/line-channel.client.ts
 import { createInterface } from "node:readline";
 
@@ -279,7 +495,7 @@ function openLineChannel(input, output, handle) {
 }
 
 // src/server/serve-driver.use-case.ts
-var METHODS = ["initialize", "validate", "run", "cancel", "shutdown"];
+var METHODS = ["initialize", "validate", "run", "work", "cancel", "probe", "diff", "export", "shutdown"];
 var major = (version) => version.split(".", 1)[0];
 function parsed(schema, params) {
   const result = schema.safeParse(params ?? {});
@@ -289,6 +505,7 @@ function parsed(schema, params) {
 async function serveDriver(input, output, identity) {
   let initialized = false;
   const stopping = new AbortController();
+  let probes = 0;
   const channel = openLineChannel(input, output, async (method, params) => {
     if (method === "initialize") {
       const hello = parsed(initializeParamsSchema, params);
@@ -301,7 +518,11 @@ async function serveDriver(input, output, identity) {
     if (!initialized) throw new RpcFailure(RPC_ERRORS.server, "call initialize first");
     if (method === "validate") return await validateRecipes(parsed(validateParamsSchema, params));
     if (method === "run") return { runId: await sessions.start(parsed(runParamsSchema, params)) };
+    if (method === "work") return { runId: await sessions.startWork(parsed(workParamsSchema, params)) };
     if (method === "cancel") return { cancelled: await sessions.cancel(parsed(cancelParamsSchema, params).runId) };
+    if (method === "probe") return await probeSite(parsed(probeParamsSchema, params), `probe-${++probes}`, channel.request);
+    if (method === "diff") return await diffRecordFiles(parsed(diffParamsSchema, params));
+    if (method === "export") return await exportRecordFiles(parsed(exportParamsSchema, params));
     await sessions.closeAll();
     setImmediate(() => {
       stopping.abort();
@@ -309,7 +530,7 @@ async function serveDriver(input, output, identity) {
     return {};
   });
   const sessions = runSessions(channel.notify, channel.request);
-  await Promise.race([channel.closed, once(stopping.signal, "abort")]);
+  await Promise.race([channel.closed, once2(stopping.signal, "abort")]);
   await channel.drained();
   await sessions.closeAll();
 }

package/dist/src/callout-bridge/client-hooks.client.d.ts

@@ -1,12 +1,19 @@
-import type { CalloutTransport, HookMap } from '@opencraw/core';
+import type { AccessPlugin, CalloutTransport, CaptchaSolver, HookMap } from '@opencraw/core';
 /** Sends one request to the client and waits for its answer: the channel's `request`. */
 export type AskClient = (method: string, params: unknown) => Promise<unknown>;
+/** Sends a notification to the client: the channel's `notify`. */
+export type NotifyClient = (method: string, params: unknown) => void;
 /**
  * A transport whose handler is the client: each call is a `callout` request carrying core's own
  * `CalloutRequest`, and the client answers with a `CalloutResponse`, so `pending` and idempotency keys
  * work exactly as they do for a command or an HTTP hook.
+ *
+ * @param label - What the logs call it (`client hook tidy`).
+ * @param runId - The run it belongs to, sent with every callout.
+ * @param ask - The channel's request.
+ * @returns The transport.
  */
-export declare function clientTransport(hookName: string, runId: string, ask: AskClient): CalloutTransport;
+export declare function clientTransport(label: string, runId: string, ask: AskClient): CalloutTransport;
 /**
  * The hooks a run's client serves, each answered by the client over the channel.
  *
@@ -16,4 +23,25 @@ export declare function clientTransport(hookName: string, runId: string, ask: As
  * @returns A hook map for `createCrawler`.
  */
 export declare function clientHooks(names: readonly string[], runId: string, ask: AskClient): HookMap;
+/**
+ * The captcha solvers a run's client serves: a challenge goes to the client as a `captcha` callout (its
+ * input is core's `CaptchaCalloutInput`), and the answer's `token` or `text` is put on the page.
+ *
+ * @param names - The solvers the client said it serves.
+ * @param runId - The run they belong to.
+ * @param ask - The channel's request.
+ * @returns Solvers for `createCrawler`.
+ */
+export declare function clientCaptchaSolvers(names: readonly string[], runId: string, ask: AskClient): CaptchaSolver[];
+/**
+ * The access plugins a client serves: a profile `{ "kind": "plugin", "name": … }` leases through an
+ * `access` callout to the client (`phase: "lease"`), and releases through another (`phase: "release"`)
+ * when the client's lease asked for it.
+ *
+ * @param names - The plugins the client said it serves.
+ * @param runId - The run, or the probe, they belong to.
+ * @param ask - The channel's request.
+ * @returns Plugins for `createCrawler` or a probe.
+ */
+export declare function clientAccessPlugins(names: readonly string[], runId: string, ask: AskClient): AccessPlugin[];
 //# sourceMappingURL=client-hooks.client.d.ts.map

package/dist/src/callout-bridge/index.d.ts

@@ -1,3 +1,3 @@
-export { clientHooks, clientTransport } from './client-hooks.client.js';
-export type { AskClient } from './client-hooks.client.js';
+export { clientAccessPlugins, clientCaptchaSolvers, clientHooks, clientTransport } from './client-hooks.client.js';
+export type { AskClient, NotifyClient } from './client-hooks.client.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/src/index.d.ts

@@ -1,10 +1,14 @@
 export { serveDriver } from './server/index.js';
 export type { DriverIdentity } from './server/index.js';
-export { PROTOCOL_VERSION, MESSAGES, driverProtocolJsonSchema, initializeParamsSchema, initializeResultSchema, validateParamsSchema, validateResultSchema, runParamsSchema, runResultSchema, cancelParamsSchema, cancelResultSchema, eventParamsSchema, recordParamsSchema, finishedParamsSchema, calloutParamsSchema, calloutResultSchema, } from './protocol/index.js';
-export type { InitializeParams, InitializeResult, ValidateParams, ValidateResult, RunParams, RunResult, CancelParams, CancelResult, EventParams, RecordParams, FinishedParams, CalloutParams, } from './protocol/index.js';
+export { PROTOCOL_VERSION, MESSAGES, driverProtocolJsonSchema, initializeParamsSchema, initializeResultSchema, validateParamsSchema, validateResultSchema, runParamsSchema, runResultSchema, cancelParamsSchema, cancelResultSchema, eventParamsSchema, recordParamsSchema, finishedParamsSchema, calloutParamsSchema, calloutResultSchema, probeParamsSchema, probeResultSchema, diffParamsSchema, diffResultSchema, exportParamsSchema, exportResultSchema, workParamsSchema, workItemSchema, workNextParamsSchema, workNextResultSchema, workDoneParamsSchema, workFailedParamsSchema, } from './protocol/index.js';
+export type { InitializeParams, InitializeResult, ValidateParams, ValidateResult, RunParams, RunResult, CancelParams, CancelResult, EventParams, RecordParams, FinishedParams, CalloutParams, ProbeParams, DiffParams, DiffResult, ExportParams, ExportResult, WorkParams, WorkItemMessage, WorkNextResult, WorkDoneParams, WorkFailedParams, } from './protocol/index.js';
 export { openLineChannel, RPC_ERRORS, RpcFailure } from './transport/index.js';
 export type { LineChannel, RequestHandler, RpcError, RpcMessage } from './transport/index.js';
 export { runSessions, validateRecipes, streamingSink } from './session/index.js';
-export type { RunSessions, NotifyClient } from './session/index.js';
-export { clientHooks } from './callout-bridge/index.js';
+export type { RunSessions } from './session/index.js';
+export { clientHooks, clientCaptchaSolvers, clientAccessPlugins } from './callout-bridge/index.js';
+export type { AskClient, NotifyClient } from './callout-bridge/index.js';
+export { clientWorkSource } from './work-bridge/index.js';
+export { probeSite } from './site-probe/index.js';
+export { diffRecordFiles, exportRecordFiles } from './record-files/index.js';
 //# sourceMappingURL=index.d.ts.map