@opencraw/cli 0.1.0 → 0.1.1

package/README.md

@@ -62,6 +62,14 @@ opencraw probe https://example.com/product/1
 opencraw probe https://example.com/configurator --browser
 ```
 
+A PDF (a URL served as `application/pdf`, or a local `.pdf` path) is read instead: `probe` lists its first
+rows and every row that looks like a table header, with the `selector` a `table` extract needs.
+
+```sh
+opencraw probe https://example.com/price-list.pdf
+opencraw probe ./sheets/september.pdf
+```
+
 Use it before writing an input recipe, to find the shape a site's data actually takes (§9 of
 [the authoring guide](../../docs/recipes/authoring.md)).
 

package/dist/index.esm.js

@@ -1,8 +1,8 @@
 import { parseArgs } from 'node:util';
-import { loadAccessConfig, AccessBroker, HttpClient, BrowserClient, readRecipeSource, parseOutputRecipe, parseInputRecipe, bindRecipeSet, memorySink, jsonLinesSink, createCrawler, RecipeSet, RecipeValidationError, RecipeBindingError, traceLine } from '@opencraw/core';
-import { readFileSync } from 'node:fs';
 import { resolve } from 'node:path';
 import { pathToFileURL } from 'node:url';
+import { loadAccessConfig, AccessBroker, HttpClient, BrowserClient, readRecipeSource, parseOutputRecipe, parseInputRecipe, bindRecipeSet, memorySink, jsonLinesSink, createCrawler, RecipeSet, RecipeValidationError, RecipeBindingError, traceLine } from '@opencraw/core';
+import { readFileSync } from 'node:fs';
 
 const USAGE = `opencraw <command> [options]
 
@@ -304,6 +304,53 @@ function tryParse(text) {
   }
 }
 
+/** Rows shown in a probe, at most. */
+const ROW_LIMIT = 80;
+/**
+ * Summarises a PDF for someone writing a recipe: what the rows look like, and
+ * where the tables start, so a `table` extract's selector and columns can be
+ * read off the result instead of guessed.
+ *
+ * @param document - The read PDF.
+ * @returns The findings.
+ */
+function describePdf(document) {
+  const rows = document.pages.flatMap(page => page.rows.map((row, index) => ({
+    page: page.number,
+    row,
+    next: page.rows.slice(index + 1, index + 3)
+  })));
+  const headers = rows.filter(({
+    row,
+    next
+  }) => row.cells.length >= 2 && row.cells.every(cell => !/\d/.test(cell.text)) && next.some(candidate => dataRow(candidate)));
+  return {
+    pages: document.pages.length,
+    rows: rows.slice(0, ROW_LIMIT).map(({
+      page,
+      row
+    }) => ({
+      page,
+      text: row.cells.map(cell => cell.text).join(' | ')
+    })),
+    headers: headers.map(({
+      page,
+      row
+    }) => ({
+      page,
+      text: row.cells.map(cell => cell.text).join(' | '),
+      selector: `^${escape(row.cells[0].text)}`
+    }))
+  };
+}
+/** A row of at least two cells, one with a digit: data, not a heading. */
+function dataRow(row) {
+  return row.cells.length >= 2 && row.cells.some(cell => /\d/.test(cell.text));
+}
+function escape(text) {
+  return text.replaceAll(/[$()*+.?[\\\]^{|}]/g, String.raw`\$&`);
+}
+
 /**
  * Renders a probe's findings as text.
  *
@@ -317,6 +364,16 @@ function probeReport(url, status, findings, observed = []) {
   const sections = [section('JSON-LD blocks', findings.jsonLd.map(block => `  type=${block.types}, ${block.keys} keys`)), section('Inline JSON (candidates)', findings.inlineJson.map(block => `  ${block.where}, ${block.size} chars, keys: ${block.keys.slice(0, 8).join(', ')}`)), section('.json URLs referenced', findings.jsonUrls.map(url_ => `  ${url_}`)), section('Script hosts', findings.scriptHosts.map(host => `  ${host}`)), section('Links that look like an API', findings.apiLinks.map(link => `  ${link}`)), observed.length > 0 ? section('JSON responses observed in the browser', observed.map(entry => `  ${entry}`)) : ''];
   return [`${url} (HTTP ${status})`, '', ...sections].filter(line => line !== '').join('\n');
 }
+/**
+ * Renders what a probe found in a PDF as text.
+ *
+ * @param url - The URL or file probed.
+ * @param pdf - What `describePdf` found.
+ * @returns The report.
+ */
+function pdfReport(url, pdf) {
+  return [`${url} (PDF, ${pdf.pages} page${pdf.pages === 1 ? '' : 's'})`, '', section('Likely table headers (a "table" extract selector for each)', pdf.headers.map(header => `  p${header.page}  ${header.selector}\n        ${header.text}`)), section('First rows (cells separated by " | ")', pdf.rows.map(row => `  p${row.page}  ${row.text}`))].filter(line => line !== '').join('\n');
+}
 function section(title, rows) {
   if (rows.length === 0) return '';
   return [`${title}:`, ...rows, ''].join('\n');
@@ -332,7 +389,10 @@ const OBSERVE_MS = 4000;
  * involved: the cli's `probePage` and `@opencraw/mcp`'s probe tool both
  * build on this, one printing the result, the other returning it as data.
  *
- * @param url - The page to probe.
+ * A PDF (by content type, or a local `.pdf` path) is read instead: its rows,
+ * and the rows that look like table headers.
+ *
+ * @param url - The page to probe, or a local file path.
  * @param options - Browser path, TLS and user agent, plus whether to render.
  * @returns What was found.
  * @throws Error when the fetch itself fails.
@@ -349,16 +409,26 @@ async function probeUrl(url, options) {
     headers: lease.headers
   });
   try {
+    const target = /^[a-z][\w+.-]+:/i.test(url) ? url : pathToFileURL(resolve(url)).href;
     const response = await client.send({
-      url,
-      as: 'html'
+      url: target
     });
-    const html = response.body.kind === 'html' ? response.body.html : '';
-    const observed = options.browser ? await observeBrowserJson(url, options, lease) : [];
+    const {
+      body
+    } = response;
+    if (body.kind === 'pdf') return {
+      url: response.url,
+      status: response.status,
+      findings: findData(''),
+      observed: [],
+      pdf: describePdf(body)
+    };
+    const text = body.kind === 'html' ? body.html : body.kind === 'text' ? body.text : JSON.stringify(body.data);
+    const observed = options.browser && !target.startsWith('file:') ? await observeBrowserJson(url, options, lease) : [];
     return {
       url: response.url,
       status: response.status,
-      findings: findData(html),
+      findings: findData(text),
       observed
     };
   } finally {
@@ -376,7 +446,7 @@ async function probeUrl(url, options) {
 async function probePage(url, options, terminal) {
   try {
     const result = await probeUrl(url, options);
-    terminal.out(probeReport(result.url, result.status, result.findings, result.observed));
+    terminal.out(result.pdf === undefined ? probeReport(result.url, result.status, result.findings, result.observed) : pdfReport(result.url, result.pdf));
     return 0;
   } catch (error) {
     terminal.err(`probe failed: ${error instanceof Error ? error.message : String(error)}`);
@@ -565,7 +635,7 @@ async function runRecipes(command, terminal) {
     const report = await crawler.run(new RecipeSet(set.output, inputs));
     if (command.out === undefined && !command.dryRun) for (const record of sink.records) terminal.out(JSON.stringify(record.data));
     for (const recipe of report.recipes) {
-      terminal.err(`${recipe.recipeId}: ${recipe.emitted} emitted, ${recipe.rejected} rejected, ${recipe.duplicates} duplicates, ${recipe.skipped} skipped, ${recipe.pages} pages, ${recipe.durationMs} ms${recipe.error === undefined ? '' : `, stopped: ${recipe.error}`}`);
+      terminal.err(`${recipe.recipeId}: ${recipe.emitted} emitted, ${recipe.rejected} rejected, ${recipe.duplicates} duplicates, ${recipe.skipped} skipped, ${recipe.stepsSkipped > 0 ? `${recipe.stepsSkipped} steps skipped (see --trace), ` : ''}${recipe.pages} pages, ${recipe.durationMs} ms${recipe.error === undefined ? '' : `, stopped: ${recipe.error}`}`);
     }
     if (!command.dryRun) terminal.err(`${report.records} records${report.sink.location === undefined ? '' : ` written to ${report.sink.location}`}`);
     return report.recipes.some(recipe => recipe.error !== undefined) ? 1 : 0;

package/dist/src/probe/index.d.ts

@@ -2,5 +2,7 @@ export { probePage, probeUrl } from './probe-page.use-case.js';
 export type { ProbeResult } from './probe-page.use-case.js';
 export { findData } from './find-data.algorithm.js';
 export type { ProbeFindings, JsonLdFinding, InlineJsonFinding } from './find-data.algorithm.js';
-export { probeReport } from './probe-report.mapper.js';
+export { probeReport, pdfReport } from './probe-report.mapper.js';
+export { describePdf } from './pdf-findings.mapper.js';
+export type { PdfFindings } from './pdf-findings.mapper.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/src/probe/pdf-findings.mapper.d.ts

@@ -0,0 +1,26 @@
+import type { PdfDocument } from '@opencraw/core';
+/** What a probe shows of a PDF: its rows, and the rows that look like table headers. */
+export interface PdfFindings {
+    pages: number;
+    /** The first rows, cells separated by " | ". */
+    rows: {
+        page: number;
+        text: string;
+    }[];
+    /** Rows of text cells (no digit) with a row of data (a digit) right under them: likely table headers, with a selector that finds them. */
+    headers: {
+        page: number;
+        text: string;
+        selector: string;
+    }[];
+}
+/**
+ * Summarises a PDF for someone writing a recipe: what the rows look like, and
+ * where the tables start, so a `table` extract's selector and columns can be
+ * read off the result instead of guessed.
+ *
+ * @param document - The read PDF.
+ * @returns The findings.
+ */
+export declare function describePdf(document: PdfDocument): PdfFindings;
+//# sourceMappingURL=pdf-findings.mapper.d.ts.map

package/dist/src/probe/probe-page.use-case.d.ts

@@ -1,6 +1,7 @@
 import type { CommonOptions } from '../arguments/index.js';
 import type { Terminal } from '../terminal/index.js';
 import { findData } from './find-data.algorithm.js';
+import type { PdfFindings } from './pdf-findings.mapper.js';
 /** What fetching and, optionally, rendering a page found. */
 export interface ProbeResult {
     url: string;
@@ -8,6 +9,8 @@ export interface ProbeResult {
     findings: ReturnType<typeof findData>;
     /** JSON responses observed while the page rendered; empty unless `options.browser` was set. */
     observed: string[];
+    /** Present when the URL is a PDF: its rows and likely table headers. */
+    pdf?: PdfFindings;
 }
 /**
  * Fetches a page and finds where its data lives, through the access profile the
@@ -17,7 +20,10 @@ export interface ProbeResult {
  * involved: the cli's `probePage` and `@opencraw/mcp`'s probe tool both
  * build on this, one printing the result, the other returning it as data.
  *
- * @param url - The page to probe.
+ * A PDF (by content type, or a local `.pdf` path) is read instead: its rows,
+ * and the rows that look like table headers.
+ *
+ * @param url - The page to probe, or a local file path.
  * @param options - Browser path, TLS and user agent, plus whether to render.
  * @returns What was found.
  * @throws Error when the fetch itself fails.

package/dist/src/probe/probe-report.mapper.d.ts

@@ -1,4 +1,5 @@
 import type { ProbeFindings } from './find-data.algorithm.js';
+import type { PdfFindings } from './pdf-findings.mapper.js';
 /**
  * Renders a probe's findings as text.
  *
@@ -9,4 +10,12 @@ import type { ProbeFindings } from './find-data.algorithm.js';
  * @returns The report.
  */
 export declare function probeReport(url: string, status: number, findings: ProbeFindings, observed?: string[]): string;
+/**
+ * Renders what a probe found in a PDF as text.
+ *
+ * @param url - The URL or file probed.
+ * @param pdf - What `describePdf` found.
+ * @returns The report.
+ */
+export declare function pdfReport(url: string, pdf: PdfFindings): string;
 //# sourceMappingURL=probe-report.mapper.d.ts.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opencraw/cli",
-  "version": "0.1.0",
+  "version": "0.1.1",
   "type": "module",
   "main": "./dist/index.esm.js",
   "module": "./dist/index.esm.js",
@@ -24,7 +24,7 @@
     "!**/*.js.map"
   ],
   "dependencies": {
-    "@opencraw/core": "^0.1.0",
+    "@opencraw/core": "^0.1.1",
     "playwright": "^1.63.0"
   },
   "publishConfig": {