npm - @xcrap/extractor - Versions diffs - 0.3.0 - Mend

@xcrap/extractor 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (45) hide show

package/LICENSE +21 -0
package/README.md +131 -0
package/dist/errors.d.ts +13 -0
package/dist/errors.js +29 -0
package/dist/html/extractor-model.d.ts +34 -0
package/dist/html/extractor-model.js +71 -0
package/dist/html/extractors.d.ts +61 -0
package/dist/html/extractors.js +145 -0
package/dist/html/index.d.ts +3 -0
package/dist/html/index.js +19 -0
package/dist/html/parser.d.ts +41 -0
package/dist/html/parser.js +72 -0
package/dist/index.d.ts +7 -0
package/dist/index.js +23 -0
package/dist/interfaces/extractor-model.d.ts +3 -0
package/dist/interfaces/extractor-model.js +2 -0
package/dist/json/extractor-model.d.ts +22 -0
package/dist/json/extractor-model.js +97 -0
package/dist/json/index.d.ts +2 -0
package/dist/json/index.js +18 -0
package/dist/json/parser.d.ts +6 -0
package/dist/json/parser.js +48 -0
package/dist/markdown/constants.d.ts +2 -0
package/dist/markdown/constants.js +11 -0
package/dist/markdown/extractor-model.d.ts +4 -0
package/dist/markdown/extractor-model.js +17 -0
package/dist/markdown/index.d.ts +2 -0
package/dist/markdown/index.js +18 -0
package/dist/markdown/parser.d.ts +5 -0
package/dist/markdown/parser.js +18 -0
package/dist/query-builders/html.d.ts +6 -0
package/dist/query-builders/html.js +16 -0
package/dist/query-builders/index.d.ts +1 -0
package/dist/query-builders/index.js +17 -0
package/dist/source-parser.d.ts +13 -0
package/dist/source-parser.js +22 -0
package/dist/utils/index.d.ts +3 -0
package/dist/utils/index.js +19 -0
package/dist/utils/select-first-element.d.ts +3 -0
package/dist/utils/select-first-element.js +60 -0
package/dist/utils/select-many-elements.d.ts +3 -0
package/dist/utils/select-many-elements.js +57 -0
package/dist/utils/slugify.d.ts +1 -0
package/dist/utils/slugify.js +10 -0
package/package.json +64 -0

package/LICENSE ADDED Viewed

@@ -0,0 +1,21 @@
+MIT License
+Copyright (c) 2025 Marcuth
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md ADDED Viewed

@@ -0,0 +1,131 @@
+# 🕷️ Xcrap Extractor: Parsing HTML and JSON using declarative models
+> Note: Xcrap Parser is now Xcrap Extractor
+Xcrap Extractor is a package of the Xcrap framework, it was developed to take care of the data extraction part of text files (currently supporting only HTML, JSON and Markdown) using declarative models.
+---
+## 📦 Installation
+Installing it is very simple, you can use NPM or any other package manager of your choice: such as PNPM, Yarn, etc.
+```cmd
+npm i @xcrap/extractor
+```
+---
+## 🛠️ How to Use
+Well, there are a few ways to use this parsing mechanism, from using the already created templates to expanding it by creating parsers for other file types and keeping the interleaving of these templates.
+### Providing HTML string
+```ts
+import { HtmlParser, extract } from "@xcrap/extractor"
+;(async () => {
+    const html = "<html><head><title>Page Title</title></head><body><><></body></html>"
+    const parser = new HtmlParser(html)
+})();
+```
+### Loading an HTML file
+```ts
+import { HtmlParser, extract } from "@xcrap/extractor"
+;(async () => {
+    const parser = await HtmlParser.loadFile("./path-to-html-file.html", { encoding: "utf-8" }) // Returns an instance of HtmlParser
+})();
+```
+### Data extraction without using models
+```ts
+import { HtmlParser, extract, css } from "@xcrap/extractor"
+;(async () => {
+    const html = `<html><head><title>Page Title</title></head><body><a href="https://example.com">Link</a></body></html>`
+    const parser = new HtmlParser(html)
+    // extractValue() searches and extracts something from the first element found
+    // extract(key: string, isAttribute?: boolean) is a generic extraction function, you can use some that are already created and ready to use by importing them from the same location :)
+    const title = await parser.extractValue({ query: css("title"), extractor: extract("innerText") })
+    // extractValues() fetches all the elements it finds with a query (you can limit the number of results) and uses the extractor to grab the data
+    const links = await parser.extractValues({ query: css("a"), extractor: extract("href", true) })
+    console.log(title) // "Page Title"
+    console.log(links) // ["https://example.com"]
+})();
+```
+### Data extraction with using models
+ExtractorModels are decoupled enough that you don't have to rely on using SourceParser instances, but we'll still use them:
+```ts
+import { HtmlParser, HtmlExtrctorModel, extract, css } from "@xcrap/extractor"
+;(async () => {
+    const html = `<html><body><h1>Heading</h1><div><p id="id">1</p><p id="name">Name</p><p class="age">23</p></div></body></html>`
+    const parser = new HtmlParser(html)
+    const rootModel = new HtmlExtrctorModel({
+        heading: {
+            query: css("h1"),
+            extractor: extract("innerText")
+        },
+        id: {
+            // You can also use xpath() query builder
+            // query: xpath("//*[@id='id']")
+            query: css("#id"),
+            extractor: extract("innerText")
+        },
+        name: {
+            query: css("#name"),
+            extractor: extract("innerText")
+        },
+        age: {
+            query: css(".age"),
+            extractor: extract("innerText")
+        }
+    })
+    const data = await parser.extractModel({ model: rootModel })
+    console.log(data) // { heading: "Heading", id: "1", name: "Name", age: "23" }
+})();
+```
+## 🧠 Create your own Parser: Concepts
+### What is a SourceParser?
+A SourceParser for this library is a class that deals in some way with a file type, loads that file, and may or may not have some methods to easily extract data.
+A parser has a default method called `extractWithModel` which is a wrapper that takes a `ExtractorModel` and calls the `extract()` method providing the internal `source` property.
+### What is a ExtractorModel?
+A Extractor Model is a class that receives a `shape` in its constructor and stores it as a property. It must have a method called `extract()` that will receive a `source`, which is the code/text that contains the information to be extracted.
+This `shape` is used to declare the form in which the information will be extracted from the `source`.
+## 🤝 Contributing
+**We are actively looking for contributors!** Whether you are a beginner or an experienced developer, your help is welcome. We have many tasks available, from simple documentation fixes to complex parser implementations.
+- Want to contribute? Follow these steps:
+- Fork the repository.
+- Create a new branch (git checkout -b feature-new).
+- Commit your changes (git commit -m 'Add new feature').
+- Push to the branch (git push origin feature-new).
+- Open a Pull Request.
+## 📝 License
+This project is licensed under the MIT License.

package/dist/errors.d.ts ADDED Viewed

@@ -0,0 +1,13 @@
+import { BuildedQuery } from "./query-builders";
+export declare class HTMLElementNotFoundError extends Error {
+    constructor(query?: BuildedQuery);
+}
+export declare class MultipleQueryError extends Error {
+    constructor();
+}
+export declare class FieldNotFoundError extends Error {
+    constructor(key: string);
+}
+export declare class ExtractorNotFoundError extends Error {
+    constructor(name: string);
+}

package/dist/errors.js ADDED Viewed

@@ -0,0 +1,29 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ExtractorNotFoundError = exports.FieldNotFoundError = exports.MultipleQueryError = exports.HTMLElementNotFoundError = void 0;
+class HTMLElementNotFoundError extends Error {
+    constructor(query) {
+        super(`Element with query "${JSON.stringify(query) || "no query provided"}" not found`);
+        this.name = "HTMLElementNotFoundError";
+    }
+}
+exports.HTMLElementNotFoundError = HTMLElementNotFoundError;
+class MultipleQueryError extends Error {
+    constructor() {
+        super("Multiple value must have a 'query'");
+        this.name = "MultipleQueryError";
+    }
+}
+exports.MultipleQueryError = MultipleQueryError;
+class FieldNotFoundError extends Error {
+    constructor(key) {
+        super(`Field with key "${key}" not found`);
+    }
+}
+exports.FieldNotFoundError = FieldNotFoundError;
+class ExtractorNotFoundError extends Error {
+    constructor(name) {
+        super(`Extractor with name "${name}" not found`);
+    }
+}
+exports.ExtractorNotFoundError = ExtractorNotFoundError;

package/dist/html/extractor-model.d.ts ADDED Viewed

@@ -0,0 +1,34 @@
+import { HTMLElement, Options as NodeHtmlOptions } from "node-html-parser";
+import { ExtractorModel } from "../interfaces/extractor-model";
+import { BuildedQuery } from "../query-builders";
+import { ExtractorFunction } from "./extractors";
+export type HtmlExtrctorModelShapeBaseValue = {
+    query?: BuildedQuery;
+    default?: string | string[] | null;
+    multiple?: boolean;
+    limit?: number;
+    extractor: ExtractorFunction;
+};
+export type HtmlExtrctorModelShapeNestedValue = {
+    query: BuildedQuery;
+    limit?: number;
+    multiple?: boolean;
+    model: ExtractorModel;
+    extractor?: ExtractorFunction;
+};
+export type HtmlExtrctorModelValue = HtmlExtrctorModelShapeBaseValue | HtmlExtrctorModelShapeNestedValue;
+export type HtmlExtrctorModelShape = {
+    [key: string]: HtmlExtrctorModelValue;
+};
+export type InferHtmlValue<V extends HtmlExtrctorModelValue> = V extends HtmlExtrctorModelShapeNestedValue ? V["multiple"] extends true ? V["model"] extends ExtractorModel<infer M> ? M[] : any : V["model"] extends ExtractorModel<infer M> ? M : any : V extends HtmlExtrctorModelShapeBaseValue ? V["multiple"] extends true ? Awaited<ReturnType<V["extractor"]>>[] : Awaited<ReturnType<V["extractor"]>> : never;
+export type InferHtmlShape<S extends HtmlExtrctorModelShape> = {
+    [K in keyof S]: InferHtmlValue<S[K]>;
+};
+export type ParseBaseValueReturnType = (undefined | string)[] | string | null | undefined;
+export declare class HtmlExtrctorModel<S extends HtmlExtrctorModelShape> implements ExtractorModel<InferHtmlShape<S>> {
+    readonly shape: S;
+    constructor(shape: S);
+    extract(source: string, options?: NodeHtmlOptions): Promise<InferHtmlShape<S>>;
+    protected extractBaseValue(value: HtmlExtrctorModelShapeBaseValue, root: HTMLElement): Promise<ParseBaseValueReturnType>;
+    protected extractNestedValue(value: HtmlExtrctorModelShapeNestedValue, root: HTMLElement): Promise<any>;
+}

package/dist/html/extractor-model.js ADDED Viewed

@@ -0,0 +1,71 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.HtmlExtrctorModel = void 0;
+const node_html_parser_1 = __importDefault(require("node-html-parser"));
+const errors_1 = require("../errors");
+const utils_1 = require("../utils");
+const parser_1 = require("./parser");
+class HtmlExtrctorModel {
+    constructor(shape) {
+        this.shape = shape;
+    }
+    async extract(source, options = parser_1.nodeHtmlParserOptions) {
+        const root = node_html_parser_1.default.parse(source, options);
+        const data = {};
+        for (const key in this.shape) {
+            const value = this.shape[key];
+            const isNestedValue = "model" in value;
+            if (isNestedValue) {
+                data[key] = await this.extractNestedValue(value, root);
+            }
+            else {
+                data[key] = await this.extractBaseValue(value, root);
+            }
+        }
+        return data;
+    }
+    async extractBaseValue(value, root) {
+        if (value.multiple) {
+            if (!value.query) {
+                throw new errors_1.MultipleQueryError();
+            }
+            const elements = (0, utils_1.selectManyElements)(value.query, root);
+            if (value.limit !== undefined) {
+                elements.splice(value.limit);
+            }
+            return await Promise.all(elements.map((element) => value.extractor(element)));
+        }
+        else {
+            const element = value.query ? (0, utils_1.selectFirstElement)(value.query, root) : root;
+            if (!element) {
+                if (value.default === undefined) {
+                    throw new errors_1.HTMLElementNotFoundError(value.query);
+                }
+                return value.default;
+            }
+            return await value.extractor(element);
+        }
+    }
+    async extractNestedValue(value, root) {
+        if (value.multiple) {
+            const elements = (0, utils_1.selectManyElements)(value.query, root);
+            if (value.limit !== undefined) {
+                elements.splice(value.limit);
+            }
+            return await Promise.all(elements.map((element) => value.model.extract(element.outerHTML)));
+        }
+        else {
+            const element = (0, utils_1.selectFirstElement)(value.query, root);
+            if (!element) {
+                throw new errors_1.HTMLElementNotFoundError(value.query);
+            }
+            const source = value.extractor ? (await value.extractor(element)) : element.outerHTML;
+            const data = await value.model.extract(source);
+            return data;
+        }
+    }
+}
+exports.HtmlExtrctorModel = HtmlExtrctorModel;

package/dist/html/extractors.d.ts ADDED Viewed

@@ -0,0 +1,61 @@
+import { Attributes, RawAttributes } from "node-html-parser/dist/nodes/html";
+import { HTMLElement, NodeType } from "node-html-parser";
+export type ExtractorFunctionReturnType = (string | undefined) | Promise<string | undefined>;
+export type ExtractorFunction<T = ExtractorFunctionReturnType> = (element: HTMLElement) => T;
+export type HtmlProperty = "innerText" | "textContent" | "text" | "innerHTML" | "outerHTML" | "tagName" | "classList" | "classNames" | "id" | "childElementCount" | "structure" | "structuredText" | "attributes" | "attrs" | "localName" | "nodeType" | "range" | "rawAttributes" | "rawAttrs" | "rawTagName" | "rawText";
+export type HtmlAttribute = "href" | "src" | "value" | "style" | "role" | "alt" | "title" | "placeholder" | "disabled" | "readonly" | "checked" | "selected" | "name" | "type" | "autocomplete" | "maxlength" | "minlength" | "pattern" | "required" | "aria-label" | "aria-hidden" | "aria-expanded" | "aria-checked" | "aria-disabled" | "data-*" | `data-${string}` | (string & {});
+export declare const propertyExtractors: Record<HtmlProperty, (element: HTMLElement) => unknown | undefined>;
+export declare function extract<T extends HtmlProperty | HtmlAttribute, R = string>(key: T, isAttribute?: boolean): ExtractorFunction<R | undefined>;
+export declare const extractInnerText: ExtractorFunction<string | undefined>;
+export declare const extractTextContent: ExtractorFunction<string | undefined>;
+export declare const extractText: ExtractorFunction<string | undefined>;
+export declare const extractInnerHtml: ExtractorFunction<string | undefined>;
+export declare const extractOuterHtml: ExtractorFunction<string | undefined>;
+export declare const extractTagName: ExtractorFunction<string | undefined>;
+export declare const extractClassList: ExtractorFunction<string[] | undefined>;
+export declare const extractId: ExtractorFunction<string | undefined>;
+export declare const extractHref: ExtractorFunction<string | undefined>;
+export declare const extractSrc: ExtractorFunction<string | undefined>;
+export declare const extractValue: ExtractorFunction<string | undefined>;
+export declare const extractStyle: ExtractorFunction<string | undefined>;
+export declare const extractRole: ExtractorFunction<string | undefined>;
+export declare const extractTitle: ExtractorFunction<string | undefined>;
+export declare const extractPlaceholder: ExtractorFunction<string | undefined>;
+export declare const extractDisabled: ExtractorFunction<string | undefined>;
+export declare const extractReadonly: ExtractorFunction<string | undefined>;
+export declare const extractChecked: ExtractorFunction<string | undefined>;
+export declare const extractSelected: ExtractorFunction<string | undefined>;
+export declare const extractName: ExtractorFunction<string | undefined>;
+export declare const extractType: ExtractorFunction<string | undefined>;
+export declare const extractAutocomplete: ExtractorFunction<string | undefined>;
+export declare const extractMaxLength: ExtractorFunction<string | undefined>;
+export declare const extractMinLength: ExtractorFunction<string | undefined>;
+export declare const extractPattern: ExtractorFunction<string | undefined>;
+export declare const extractRequired: ExtractorFunction<string | undefined>;
+export declare const extractAriaLabel: ExtractorFunction<string | undefined>;
+export declare const extractAriaHidden: ExtractorFunction<string | undefined>;
+export declare const extractAriaExpanded: ExtractorFunction<string | undefined>;
+export declare const extractAriaChecked: ExtractorFunction<string | undefined>;
+export declare const extractAriaDisabled: ExtractorFunction<string | undefined>;
+export declare const extractAllData: ExtractorFunction<string | undefined>;
+export declare const extractAttribute: <T extends string>(name: T) => ExtractorFunction<string | undefined>;
+export declare const extractRange: ExtractorFunction<[number, number] | undefined>;
+export declare const extarctAttributes: ExtractorFunction<Record<string, string> | undefined>;
+export declare const extractAttrs: ExtractorFunction<Attributes | undefined>;
+export declare const extractChildElementCount: ExtractorFunction<number | undefined>;
+export declare const extractClassNames: ExtractorFunction<string | undefined>;
+export declare const extractLocalName: ExtractorFunction<string | undefined>;
+export declare const extractNodeType: ExtractorFunction<NodeType | undefined>;
+export declare const extractRawTagName: ExtractorFunction<string | undefined>;
+export declare const extractRawText: ExtractorFunction<string | undefined>;
+export declare const extractStructure: ExtractorFunction<string | undefined>;
+export declare const extractStructuredText: ExtractorFunction<string | undefined>;
+export declare const extarctRawAttributes: ExtractorFunction<RawAttributes | undefined>;
+export declare const extractRawAttrs: ExtractorFunction<string | undefined>;
+export type FromNextOrPreviousElementSiblingOptions = {
+    shouldExists?: boolean;
+};
+export declare const fromNextElementSibling: (extractor: ExtractorFunction, { shouldExists }?: FromNextOrPreviousElementSiblingOptions) => ExtractorFunction;
+export declare const fromPreviousElementSibling: (extractor: ExtractorFunction, { shouldExists }?: FromNextOrPreviousElementSiblingOptions) => ExtractorFunction;
+export type RegexExtractor = (pattern: RegExp, index?: number) => ExtractorFunction<string | string[]>;
+export declare const matchRegexFrom: (extractor: ExtractorFunction<string | undefined>, pattern: RegExp, index?: number) => ExtractorFunction<string | string[] | undefined>;

package/dist/html/extractors.js ADDED Viewed

@@ -0,0 +1,145 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.matchRegexFrom = exports.fromPreviousElementSibling = exports.fromNextElementSibling = exports.extractRawAttrs = exports.extarctRawAttributes = exports.extractStructuredText = exports.extractStructure = exports.extractRawText = exports.extractRawTagName = exports.extractNodeType = exports.extractLocalName = exports.extractClassNames = exports.extractChildElementCount = exports.extractAttrs = exports.extarctAttributes = exports.extractRange = exports.extractAttribute = exports.extractAllData = exports.extractAriaDisabled = exports.extractAriaChecked = exports.extractAriaExpanded = exports.extractAriaHidden = exports.extractAriaLabel = exports.extractRequired = exports.extractPattern = exports.extractMinLength = exports.extractMaxLength = exports.extractAutocomplete = exports.extractType = exports.extractName = exports.extractSelected = exports.extractChecked = exports.extractReadonly = exports.extractDisabled = exports.extractPlaceholder = exports.extractTitle = exports.extractRole = exports.extractStyle = exports.extractValue = exports.extractSrc = exports.extractHref = exports.extractId = exports.extractClassList = exports.extractTagName = exports.extractOuterHtml = exports.extractInnerHtml = exports.extractText = exports.extractTextContent = exports.extractInnerText = exports.propertyExtractors = void 0;
+exports.extract = extract;
+const errors_1 = require("../errors");
+const htmlProperties = [
+    "innerText",
+    "textContent",
+    "text",
+    "innerHTML",
+    "outerHTML",
+    "tagName",
+    "classList",
+    "classNames",
+    "id",
+    "childElementCount",
+    "structure",
+    "structuredText",
+    "attributes",
+    "attrs",
+    "localName",
+    "nodeType",
+    "range",
+    "rawAttributes",
+    "rawAttrs",
+    "rawTagName",
+    "rawText",
+];
+exports.propertyExtractors = {
+    innerText: (element) => element.innerText,
+    textContent: (element) => element.textContent,
+    text: (element) => element.text,
+    innerHTML: (element) => element.innerHTML,
+    outerHTML: (element) => element.outerHTML,
+    tagName: (element) => element.tagName,
+    classList: (element) => Array.from(element.classList.values()),
+    id: (element) => element.id,
+    attributes: (element) => element.attributes,
+    attrs: (element) => element.attrs,
+    childElementCount: (element) => element.childElementCount,
+    classNames: (element) => element.classNames,
+    localName: (element) => element.localName,
+    nodeType: (element) => element.localName,
+    range: (element) => element.range,
+    rawAttributes: (element) => element.rawAttributes,
+    rawAttrs: (element) => element.rawAttrs,
+    rawTagName: (element) => element.rawTagName,
+    rawText: (element) => element.rawText,
+    structure: (element) => element.structure,
+    structuredText: (element) => element.structuredText,
+};
+function extract(key, isAttribute = false) {
+    return (element) => {
+        if (isAttribute || !htmlProperties.includes(key)) {
+            return element.getAttribute(key);
+        }
+        const extractor = exports.propertyExtractors[key];
+        if (!extractor) {
+            throw new errors_1.ExtractorNotFoundError(key);
+        }
+        return extractor(element);
+    };
+}
+exports.extractInnerText = extract("innerText");
+exports.extractTextContent = extract("textContent");
+exports.extractText = extract("text");
+exports.extractInnerHtml = extract("innerHTML");
+exports.extractOuterHtml = extract("outerHTML");
+exports.extractTagName = extract("tagName");
+exports.extractClassList = extract("classList");
+exports.extractId = extract("id");
+exports.extractHref = extract("href", true);
+exports.extractSrc = extract("src", true);
+exports.extractValue = extract("value", true);
+exports.extractStyle = extract("style", true);
+exports.extractRole = extract("role", true);
+exports.extractTitle = extract("title", true);
+exports.extractPlaceholder = extract("placeholder", true);
+exports.extractDisabled = extract("disabled", true);
+exports.extractReadonly = extract("readonly", true);
+exports.extractChecked = extract("checked", true);
+exports.extractSelected = extract("selected", true);
+exports.extractName = extract("name", true);
+exports.extractType = extract("type", true);
+exports.extractAutocomplete = extract("autocomplete", true);
+exports.extractMaxLength = extract("maxlength", true);
+exports.extractMinLength = extract("minlength", true);
+exports.extractPattern = extract("pattern", true);
+exports.extractRequired = extract("required", true);
+exports.extractAriaLabel = extract("aria-label", true);
+exports.extractAriaHidden = extract("aria-hidden", true);
+exports.extractAriaExpanded = extract("aria-expanded", true);
+exports.extractAriaChecked = extract("aria-checked", true);
+exports.extractAriaDisabled = extract("aria-disabled", true);
+exports.extractAllData = extract("data-*", true);
+const extractAttribute = (name) => extract(name, true);
+exports.extractAttribute = extractAttribute;
+exports.extractRange = extract("range");
+exports.extarctAttributes = extract("attributes");
+exports.extractAttrs = extract("attrs");
+exports.extractChildElementCount = extract("childElementCount");
+exports.extractClassNames = extract("classNames");
+exports.extractLocalName = extract("localName");
+exports.extractNodeType = extract("nodeType");
+exports.extractRawTagName = extract("rawTagName");
+exports.extractRawText = extract("rawText");
+exports.extractStructure = extract("structure");
+exports.extractStructuredText = extract("structuredText");
+exports.extarctRawAttributes = extract("rawAttributes");
+exports.extractRawAttrs = extract("rawAttrs");
+const fromNextElementSibling = (extractor, { shouldExists } = { shouldExists: true }) => {
+    return (element) => {
+        const nextElementSibling = element.nextElementSibling;
+        if (!nextElementSibling) {
+            if (shouldExists) {
+                throw new errors_1.HTMLElementNotFoundError();
+            }
+            return undefined;
+        }
+        return extractor(nextElementSibling);
+    };
+};
+exports.fromNextElementSibling = fromNextElementSibling;
+const fromPreviousElementSibling = (extractor, { shouldExists } = { shouldExists: true }) => {
+    return (element) => {
+        const previousElementSibling = element.previousElementSibling;
+        if (!previousElementSibling) {
+            if (shouldExists) {
+                throw new errors_1.HTMLElementNotFoundError();
+            }
+            return undefined;
+        }
+        return extractor(previousElementSibling);
+    };
+};
+exports.fromPreviousElementSibling = fromPreviousElementSibling;
+const matchRegexFrom = (extractor, pattern, index) => {
+    return (element) => {
+        var _a;
+        const value = (_a = extractor(element)) !== null && _a !== void 0 ? _a : "";
+        const matches = Array.from(value.matchAll(pattern), (match) => { var _a; return (_a = match[1]) !== null && _a !== void 0 ? _a : match[0]; });
+        return index !== undefined ? matches[index] : matches;
+    };
+};
+exports.matchRegexFrom = matchRegexFrom;

package/dist/html/index.d.ts ADDED Viewed

@@ -0,0 +1,3 @@
+export * from "./extractor-model";
+export * from "./parser";
+export * from "./extractors";

package/dist/html/index.js ADDED Viewed

@@ -0,0 +1,19 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./extractor-model"), exports);
+__exportStar(require("./parser"), exports);
+__exportStar(require("./extractors"), exports);

package/dist/html/parser.d.ts ADDED Viewed

@@ -0,0 +1,41 @@
+import { HTMLElement, Options as NodeHtmlOptions } from "node-html-parser";
+import { ExtractorModel } from "../interfaces/extractor-model";
+import { BuildedQuery } from "../query-builders";
+import { ExtractorFunction } from "./extractors";
+import { SourceParser } from "../source-parser";
+export type ExtractValuesOptions = {
+    query: BuildedQuery;
+    extractor: ExtractorFunction;
+    limit?: number;
+};
+export type ExtractValueOptions = {
+    query?: BuildedQuery;
+    extractor: ExtractorFunction;
+    default?: string | null;
+};
+export type ExtractModelOptions<T = any> = {
+    query?: BuildedQuery;
+    model: ExtractorModel<T>;
+};
+export type ExtractModelsOptions<T = any> = {
+    query: BuildedQuery;
+    model: ExtractorModel<T>;
+    limit?: number;
+};
+export declare const nodeHtmlParserOptions: {
+    blockTextElements: {
+        script: true;
+        noscript: true;
+        style: true;
+        code: true;
+    };
+};
+export declare class HtmlParser extends SourceParser {
+    readonly source: string;
+    readonly root: HTMLElement;
+    constructor(source: string, options?: NodeHtmlOptions);
+    extractValues({ query, extractor, limit }: ExtractValuesOptions): Promise<(string | undefined)[]>;
+    extractValue({ query, extractor, default: default_ }: ExtractValueOptions): Promise<any | undefined | null>;
+    extractModel<T>({ model, query }: ExtractModelOptions<T>): Promise<T>;
+    extractModels<T>({ model, query, limit }: ExtractModelsOptions<T>): Promise<T[]>;
+}

package/dist/html/parser.js ADDED Viewed

@@ -0,0 +1,72 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.HtmlParser = exports.nodeHtmlParserOptions = void 0;
+const node_html_parser_1 = __importDefault(require("node-html-parser"));
+const utils_1 = require("../utils");
+const errors_1 = require("../errors");
+const source_parser_1 = require("../source-parser");
+exports.nodeHtmlParserOptions = {
+    blockTextElements: {
+        script: true,
+        noscript: true,
+        style: true,
+        code: true,
+    },
+};
+class HtmlParser extends source_parser_1.SourceParser {
+    constructor(source, options = exports.nodeHtmlParserOptions) {
+        super(source);
+        this.source = source;
+        this.root = node_html_parser_1.default.parse(source, options);
+    }
+    async extractValues({ query, extractor, limit }) {
+        const elements = (0, utils_1.selectManyElements)(query, this.root);
+        let items = [];
+        for (const element of elements) {
+            if (limit != undefined && items.length >= limit)
+                break;
+            const data = await extractor(element);
+            items.push(data);
+        }
+        return items;
+    }
+    async extractValue({ query, extractor, default: default_ }) {
+        let data;
+        if (query) {
+            const element = (0, utils_1.selectFirstElement)(query, this.root);
+            if (!element) {
+                if (default_ !== undefined) {
+                    return default_;
+                }
+                throw new errors_1.HTMLElementNotFoundError(query);
+            }
+            data = await extractor(element);
+        }
+        else {
+            data = await extractor(this.root);
+        }
+        return data !== null && data !== void 0 ? data : default_;
+    }
+    async extractModel({ model, query }) {
+        const element = query ? (0, utils_1.selectFirstElement)(query, this.root) : this.root;
+        if (!element) {
+            throw new errors_1.HTMLElementNotFoundError(query);
+        }
+        return await model.extract(element.outerHTML);
+    }
+    async extractModels({ model, query, limit }) {
+        const elements = (0, utils_1.selectManyElements)(query, this.root);
+        let dataList = [];
+        for (const element of elements) {
+            if (limit != undefined && dataList.length >= limit)
+                break;
+            const data = await model.extract(element.outerHTML);
+            dataList.push(data);
+        }
+        return dataList;
+    }
+}
+exports.HtmlParser = HtmlParser;

package/dist/index.d.ts ADDED Viewed

@@ -0,0 +1,7 @@
+export * from "./errors";
+export * from "./html";
+export * from "./json";
+export * from "./source-parser";
+export * from "./interfaces/extractor-model";
+export * from "./markdown";
+export * from "./query-builders";