@opencraw/cli 0.1.7 → 0.1.9

package/dist/index.esm.js

@@ -1,6 +1,6 @@
 import { parseArgs } from 'node:util';
 import { writeFile } from 'node:fs/promises';
-import { readableKey, readRecordsFile, diffRecords, loadAccessConfig, htmlTableSheets, AccessBroker, HttpClient, parseXml, BrowserClient, readRecipeSource, parseOutputRecipe, parseInputRecipe, bindRecipeSet, memorySink, jsonLinesSink, createCrawler, RecipeSet, diffOptionsFor, RecipeValidationError, RecipeBindingError, traceLine } from '@opencraw/core';
+import { readableKey, readRecordsFile, diffRecords, loadAccessConfig, htmlTableSheets, HostAllowlist, AccessBroker, HttpClient, parseXml, BrowserClient, readRecipeSource, parseOutputRecipe, parseInputRecipe, bindRecipeSet, memorySink, jsonLinesSink, createCrawler, RecipeSet, variantLabel, diffOptionsFor, RecipeValidationError, RecipeBindingError, traceLine } from '@opencraw/core';
 import { resolve } from 'node:path';
 import { pathToFileURL } from 'node:url';
 import { readFileSync } from 'node:fs';
@@ -1046,8 +1046,10 @@ const OBSERVE_MS = 4000;
  * @throws Error when the fetch itself fails.
  */
 async function probeUrl(url, options) {
-  const plugins = options.plugins === undefined ? undefined : await loadPlugins(options.plugins);
-  const lease = await new AccessBroker(await resolveAccess(options), plugins?.accessPlugins).lease({
+  const allowedHosts = HostAllowlist.of(options.allowedHosts);
+  const plugins = options.accessPlugins === undefined && options.plugins !== undefined ? await loadPlugins(options.plugins) : undefined;
+  const access = options.accessConfig === undefined ? await resolveAccess(options) : withDefault(options.accessConfig, options.accessProfile);
+  const lease = await new AccessBroker(access, options.accessPlugins ?? plugins?.accessPlugins).lease({
     recipeId: 'probe'
   });
   if (lease.cdp !== undefined) throw new Error(`access profile "${lease.profile}" is a remote browser; probe fetches over HTTP and needs a proxy profile`);
@@ -1055,7 +1057,8 @@ async function probeUrl(url, options) {
     userAgent: options.userAgent ?? BROWSER_USER_AGENT,
     ignoreHTTPSErrors: options.insecureTls || lease.ignoreHTTPSErrors === true,
     proxy: lease.proxy,
-    headers: lease.headers
+    headers: lease.headers,
+    allowedHosts
   });
   try {
     const target = /^[a-z][\w+.-]+:/i.test(url) ? url : pathToFileURL(resolve(url)).href;
@@ -1106,7 +1109,7 @@ async function probeUrl(url, options) {
       xml: describeXml(parseXml(body.xml, response.url))
     };
     const text = body.kind === 'html' ? body.html : body.text;
-    const observed = options.browser && !target.startsWith('file:') ? await observeBrowserJson(url, options, lease) : [];
+    const observed = options.browser && !target.startsWith('file:') ? await observeBrowserJson(url, options, lease, allowedHosts) : [];
     const html = body.kind === 'html' ? {
       html: describeHtml(body.html, response.format === 'markdown' || response.format === 'docx')
     } : {};
@@ -1139,7 +1142,7 @@ async function probePage(url, options, terminal) {
     return 1;
   }
 }
-async function observeBrowserJson(url, options, lease) {
+async function observeBrowserJson(url, options, lease, allowedHosts) {
   const browser = await BrowserClient.launch({
     executablePath: options.browserPath,
     ignoreHTTPSErrors: options.insecureTls
@@ -1149,7 +1152,8 @@ async function observeBrowserJson(url, options, lease) {
       userAgent: options.userAgent,
       proxy: lease.proxy,
       headers: lease.headers,
-      ignoreHTTPSErrors: lease.ignoreHTTPSErrors
+      ignoreHTTPSErrors: lease.ignoreHTTPSErrors,
+      allowedHosts
     });
     const seen = [];
     session.page.on('response', response => {
@@ -1169,6 +1173,15 @@ async function observeBrowserJson(url, options, lease) {
     await browser.close();
   }
 }
+/** An access config with its default profile overridden, when one is named. */
+function withDefault(config, profile) {
+  if (profile === undefined) return config;
+  if (!Object.hasOwn(config.profiles, profile)) throw new Error(`access profile "${profile}" is not in the access config (profiles: ${Object.keys(config.profiles).join(', ') || 'none'})`);
+  return {
+    ...config,
+    default: profile
+  };
+}
 function reportOf(result) {
   if (result.pdf !== undefined) return pdfReport(result.url, result.pdf);
   if (result.workbook !== undefined) return workbookReport(result.url, result.workbook);
@@ -1299,7 +1312,7 @@ async function runRecipes(command, terminal) {
     const report = await crawler.run(new RecipeSet(set.output, inputs));
     if (command.out === undefined && !command.dryRun) for (const record of sink.records) terminal.out(JSON.stringify(record.data));
     for (const recipe of report.recipes) {
-      terminal.err(`${recipe.recipeId}: ${recipe.emitted} emitted, ${recipe.rejected} rejected, ${recipe.duplicates} duplicates, ${recipe.skipped} skipped, ${recipe.stepsSkipped > 0 ? `${recipe.stepsSkipped} steps skipped (see --trace), ` : ''}${recipe.pages} pages, ${recipe.durationMs} ms${recipe.error === undefined ? '' : `, stopped: ${recipe.error}`}`);
+      terminal.err(`${recipe.recipeId}${recipe.variant === undefined ? '' : ` [${variantLabel(recipe.variant)}]`}: ${recipe.emitted} emitted, ${recipe.rejected} rejected, ${recipe.duplicates} duplicates, ${recipe.skipped} skipped, ${recipe.stepsSkipped > 0 ? `${recipe.stepsSkipped} steps skipped (see --trace), ` : ''}${recipe.pages} pages, ${recipe.durationMs} ms${recipe.error === undefined ? '' : `, stopped: ${recipe.error}`}`);
     }
     if (!command.dryRun) terminal.err(`${report.records} records${report.sink.location === undefined ? '' : ` written to ${report.sink.location}`}`);
     if (previous !== undefined) {

package/dist/src/index.d.ts

@@ -4,7 +4,7 @@ export type { Command } from './arguments/index.js';
 export { processTerminal } from './terminal/index.js';
 export type { Terminal } from './terminal/index.js';
 export { findData, probeReport, probeUrl } from './probe/index.js';
-export type { ProbeFindings, ProbeResult } from './probe/index.js';
+export type { ProbeFindings, ProbeResult, ProbeOptions } from './probe/index.js';
 export { readRecipeFiles } from './recipe-files/index.js';
 export type { RecipeFiles } from './recipe-files/index.js';
 export { loadForRun } from './run/index.js';

package/dist/src/probe/index.d.ts

@@ -1,5 +1,5 @@
 export { probePage, probeUrl } from './probe-page.use-case.js';
-export type { ProbeResult } from './probe-page.use-case.js';
+export type { ProbeResult, ProbeOptions } from './probe-page.use-case.js';
 export { findData } from './find-data.algorithm.js';
 export type { ProbeFindings, JsonLdFinding, InlineJsonFinding } from './find-data.algorithm.js';
 export { probeReport, pdfReport, workbookReport, deckReport, jsonReport, xmlReport } from './probe-report.mapper.js';

package/dist/src/probe/probe-page.use-case.d.ts

@@ -1,3 +1,4 @@
+import type { AccessConfig, AccessPlugin } from '@opencraw/core';
 import type { CommonOptions } from '../arguments/index.js';
 import type { Terminal } from '../terminal/index.js';
 import { findData } from './find-data.algorithm.js';
@@ -27,6 +28,17 @@ export interface ProbeResult {
     /** Present for XML: its namespaces, repeated elements and structure. */
     xml?: XmlFindings;
 }
+/** How to probe: the cli's options, plus what a service probing for others sets. */
+export interface ProbeOptions extends CommonOptions {
+    /** Also render the page and list the JSON it fetches. */
+    browser: boolean;
+    /** The only hosts the probe may reach, as `CrawlOptions.allowedHosts`; local paths and `file:` are refused. */
+    allowedHosts?: string[];
+    /** An access config already loaded, instead of `access` (a file). `accessProfile` still picks the default. */
+    accessConfig?: AccessConfig;
+    /** Access plugins already loaded, instead of those in `plugins` (a module). */
+    accessPlugins?: AccessPlugin[];
+}
 /**
  * Fetches a page and finds where its data lives, through the access profile the
  * options name (direct without one). With `browser: true` it also
@@ -43,9 +55,7 @@ export interface ProbeResult {
  * @returns What was found.
  * @throws Error when the fetch itself fails.
  */
-export declare function probeUrl(url: string, options: {
-    browser: boolean;
-} & CommonOptions): Promise<ProbeResult>;
+export declare function probeUrl(url: string, options: ProbeOptions): Promise<ProbeResult>;
 /**
  * Fetches a page and reports where its data lives, to a `Terminal`.
  *

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opencraw/cli",
-  "version": "0.1.7",
+  "version": "0.1.9",
   "type": "module",
   "main": "./dist/index.esm.js",
   "module": "./dist/index.esm.js",
@@ -24,7 +24,7 @@
     "!**/*.js.map"
   ],
   "dependencies": {
-    "@opencraw/core": "^0.1.7",
+    "@opencraw/core": "^0.1.9",
     "playwright": "^1.63.0"
   },
   "publishConfig": {