@crawlee/utils 4.0.0-beta.159 → 4.0.0-beta.160

package/index.d.ts

@@ -4,6 +4,7 @@ export { EnqueueStrategy } from './internals/url.js';
 export type { DownloadListOfUrlsOptions, ExtractUrlsOptions } from './internals/extract-urls.js';
 export { sleep, expandShadowRoots } from './internals/general.js';
 export * as social from './internals/social.js';
+export * from './internals/extract-microdata.js';
 export * from './internals/open_graph_parser.js';
 export * from './internals/robots.js';
 export * from './internals/sitemap.js';

package/index.js

@@ -3,6 +3,7 @@ export { downloadListOfUrls, extractUrls } from './internals/extract-urls.js';
 export { EnqueueStrategy } from './internals/url.js';
 export { sleep, expandShadowRoots } from './internals/general.js';
 export * as social from './internals/social.js';
+export * from './internals/extract-microdata.js';
 export * from './internals/open_graph_parser.js';
 export * from './internals/robots.js';
 export * from './internals/sitemap.js';

package/internals/extract-microdata.d.ts

@@ -0,0 +1,24 @@
+import type { CheerioAPI } from 'cheerio';
+/** The value of a microdata property: either text or a nested item. */
+export type MicrodataValue = string | MicrodataItem;
+/** A single schema.org item extracted from a document. */
+export interface MicrodataItem {
+    /** Tokens of the item's `itemtype` attribute. */
+    type?: string[];
+    /** The item's `itemid` attribute. */
+    id?: string;
+    /** Values keyed by `itemprop` name, an array where the property repeats. */
+    properties: Record<string, MicrodataValue | MicrodataValue[]>;
+}
+/**
+ * Easily parse all schema.org microdata from a page with just a `CheerioAPI` object or raw HTML,
+ * following the [microdata processing model](https://html.spec.whatwg.org/multipage/microdata.html#microdata).
+ *
+ * Text values are trimmed and their inner whitespace collapsed. URL-valued attributes are returned
+ * verbatim rather than resolved against the document's base URL.
+ *
+ * @param htmlOrCheerioElement A `CheerioAPI` object, or a string of raw HTML.
+ * @returns The document's top-level items. Nested items are the property values of their parent.
+ */
+export declare function extractMicrodata(raw: string): Promise<MicrodataItem[]>;
+export declare function extractMicrodata($: CheerioAPI): Promise<MicrodataItem[]>;

package/internals/extract-microdata.js

@@ -0,0 +1,118 @@
+import { isTag } from 'domhandler';
+export async function extractMicrodata(htmlOrCheerioElement) {
+    // Dynamic so that importing `@crawlee/utils` does not pull in cheerio - see #3836.
+    const { load } = await import('cheerio');
+    const $ = typeof htmlOrCheerioElement === 'string' ? load(htmlOrCheerioElement) : htmlOrCheerioElement;
+    const context = { $ };
+    return $('[itemscope]')
+        .toArray()
+        .filter((element) => !('itemprop' in element.attribs))
+        .map((element) => parseItem(context, element, new Set()));
+}
+function parseItem(context, element, ancestors) {
+    const item = { properties: {} };
+    const type = uniqueTokens(element.attribs.itemtype);
+    const id = element.attribs.itemid;
+    if (type.length > 0) {
+        item.type = type;
+    }
+    if (id) {
+        item.id = id.trim();
+    }
+    ancestors.add(element);
+    for (const propertyElement of collectPropertyElements(context, element)) {
+        const value = getPropertyValue(context, propertyElement, ancestors);
+        for (const name of uniqueTokens(propertyElement.attribs.itemprop)) {
+            addProperty(item.properties, name, value);
+        }
+    }
+    ancestors.delete(element);
+    return item;
+}
+function collectPropertyElements(context, scope) {
+    const elements = [];
+    collectFromNodes(scope.children, elements);
+    for (const id of uniqueTokens(scope.attribs.itemref)) {
+        const referenced = (context.idIndex ??= indexIds(context.$)).get(id);
+        if (referenced) {
+            collectFromNodes([referenced], elements);
+        }
+    }
+    return elements;
+}
+function collectFromNodes(nodes, elements) {
+    for (const node of nodes) {
+        if (!isTag(node)) {
+            continue;
+        }
+        if ('itemprop' in node.attribs) {
+            elements.push(node);
+        }
+        // A nested item owns everything below it, so its subtree is not part of the enclosing item.
+        if (!('itemscope' in node.attribs)) {
+            collectFromNodes(node.children, elements);
+        }
+    }
+}
+function indexIds($) {
+    const index = new Map();
+    for (const element of $('[id]').toArray()) {
+        // Duplicate ids are invalid HTML; `getElementById` resolves them to the first element.
+        if (!index.has(element.attribs.id)) {
+            index.set(element.attribs.id, element);
+        }
+    }
+    return index;
+}
+function getPropertyValue(context, element, ancestors) {
+    if ('itemscope' in element.attribs) {
+        // `itemref` can point back at an enclosing item, which the spec treats as an error.
+        return ancestors.has(element) ? { properties: {} } : parseItem(context, element, ancestors);
+    }
+    const { attribs } = element;
+    switch (element.tagName.toLowerCase()) {
+        case 'meta':
+            return attribs.content ?? '';
+        case 'audio':
+        case 'embed':
+        case 'iframe':
+        case 'img':
+        case 'source':
+        case 'track':
+        case 'video':
+            return attribs.src ?? '';
+        case 'a':
+        case 'area':
+        case 'link':
+            return attribs.href ?? '';
+        case 'object':
+            return attribs.data ?? '';
+        case 'data':
+        case 'meter':
+            return attribs.value ?? '';
+        case 'time':
+            return attribs.datetime ?? context.$(element).text().replace(/\s+/g, ' ').trim();
+        default:
+            return context.$(element).text().replace(/\s+/g, ' ').trim();
+    }
+}
+function addProperty(properties, name, value) {
+    const existing = properties[name];
+    if (existing === undefined) {
+        properties[name] = value;
+    }
+    else if (Array.isArray(existing)) {
+        existing.push(value);
+    }
+    else {
+        properties[name] = [existing, value];
+    }
+}
+/** `itemtype`, `itemprop` and `itemref` are all unordered sets of unique space-separated tokens. */
+function uniqueTokens(value) {
+    if (!value) {
+        return [];
+    }
+    const tokens = value.split(/\s+/).filter(Boolean);
+    return tokens.length > 1 ? [...new Set(tokens)] : tokens;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/utils",
-    "version": "4.0.0-beta.159",
+    "version": "4.0.0-beta.160",
     "description": "A set of shared utilities that can be used by crawlers",
     "engines": {
         "node": ">=22.0.0"
@@ -43,8 +43,8 @@
     },
     "dependencies": {
         "@apify/ps-tree": "^1.2.0",
-        "@crawlee/http-client": "4.0.0-beta.159",
-        "@crawlee/types": "4.0.0-beta.159",
+        "@crawlee/http-client": "4.0.0-beta.160",
+        "@crawlee/types": "4.0.0-beta.160",
         "@types/sax": "^1.2.7",
         "cheerio": "^1.0.0",
         "domhandler": "^5.0.3",
@@ -63,5 +63,5 @@
             }
         }
     },
-    "gitHead": "9f91c03967915be4e9b077a717e0e3494ad0a101"
+    "gitHead": "09fbce5ade9b84a2b191c635463e172388e52678"
 }