@peam-ai/parser 0.1.1

package/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

package/README.md

@@ -0,0 +1,11 @@
+# @peam-ai/parser
+
+Content parser for extracting page metadata.
+
+## Documentation
+
+For detailed documentation, visit [peam.ai/docs](https://peam.ai/docs)
+
+## License
+
+Apache-2.0

package/dist/index.d.mts

@@ -0,0 +1,105 @@
+import robotsParser from 'robots-parser';
+
+interface StructuredPage {
+    title: string;
+    description: string;
+    content: string;
+    textContent: string;
+    contentLength?: number;
+    author?: string;
+    direction?: string;
+    language?: string;
+    siteName?: string;
+    publishedTime?: string;
+    ogTitle?: string;
+    ogDescription?: string;
+    ogImage?: string;
+    keywords?: string[];
+    headings?: string[];
+    internalLinks?: string[];
+    externalLinks?: string[];
+    images?: string[];
+    imageAlts?: string[];
+}
+
+interface ParseOptions {
+    /**
+     * Keep CSS classes in the parsed HTML content
+     * @default false
+     */
+    keepClasses?: boolean;
+    /**
+     * Disable JSON-LD extraction
+     * @default true
+     */
+    disableJSONLD?: boolean;
+    /**
+     * Extract internal links
+     * @default true
+     */
+    extractLinks?: boolean;
+    /**
+     * Extract images
+     * @default true
+     */
+    extractImages?: boolean;
+}
+
+/**
+ * Checks if a path matches any of the given wildcard patterns
+ * Uses the matcher library which supports:
+ * - * (matches any characters except /)
+ * - ** (matches any characters including /)
+ * - ? (matches single character)
+ * - ! (negation)
+ * - [] (character ranges)
+ */
+declare function matchesExcludePattern(path: string, patterns: string[]): boolean;
+
+type RobotsParser = ReturnType<typeof robotsParser>;
+interface RobotsTxtResult {
+    parser: RobotsParser;
+    path: string;
+}
+declare function createRobotsParser(content: string): RobotsParser;
+/**
+ * Loads and parses robots.txt from custom path or standard locations
+ * Returns the parser and the path where robots.txt was found
+ */
+declare function loadRobotsTxt(projectDir: string, searchPaths: string[], robotsTxtPath?: string): RobotsTxtResult | null;
+declare function isPathAllowedByRobots(path: string, robots: RobotsParser | null): boolean;
+
+type PathFilterReason = 'included' | 'dynamic-route' | 'internal-route' | 'rsc-file' | 'segment-file' | 'static-asset' | 'robots-txt' | 'exclude-pattern';
+interface PathFilterResult {
+    included: boolean;
+    reason: PathFilterReason;
+}
+/**
+ * Determines if a pathname should be included in the index
+ * Returns both the decision and the reason for exclusion
+ */
+declare function shouldIncludePath(pathname: string, robots: RobotsParser | null, excludePatterns: string[], respectRobotsTxt: boolean): PathFilterResult;
+
+/**
+ * Convert file path to URL pathname
+ *
+ * Examples:
+ *   index.html -> /
+ *   about.html -> /about
+ *   about/index.html -> /about/
+ *   blog/post-1.html -> /blog/post-1
+ *   blog/post-1/index.html -> /blog/post-1/
+ *   server/pages/contact.html -> /contact
+ *   server/app/about.html -> /about
+ */
+declare function filePathToPathname(filePath: string): string;
+
+/**
+ * Parse HTML content and convert it to a StructuredPage
+ * @param html - HTML string to parse
+ * @param options - Parsing options
+ * @returns StructuredPage or undefined if parsing fails
+ */
+declare function parseHTML(html: string, options?: ParseOptions): StructuredPage | undefined;
+
+export { type ParseOptions, type PathFilterReason, type PathFilterResult, type RobotsParser, type RobotsTxtResult, type StructuredPage, createRobotsParser, filePathToPathname, isPathAllowedByRobots, loadRobotsTxt, matchesExcludePattern, parseHTML, shouldIncludePath };

package/dist/index.d.ts

@@ -0,0 +1,105 @@
+import robotsParser from 'robots-parser';
+
+interface StructuredPage {
+    title: string;
+    description: string;
+    content: string;
+    textContent: string;
+    contentLength?: number;
+    author?: string;
+    direction?: string;
+    language?: string;
+    siteName?: string;
+    publishedTime?: string;
+    ogTitle?: string;
+    ogDescription?: string;
+    ogImage?: string;
+    keywords?: string[];
+    headings?: string[];
+    internalLinks?: string[];
+    externalLinks?: string[];
+    images?: string[];
+    imageAlts?: string[];
+}
+
+interface ParseOptions {
+    /**
+     * Keep CSS classes in the parsed HTML content
+     * @default false
+     */
+    keepClasses?: boolean;
+    /**
+     * Disable JSON-LD extraction
+     * @default true
+     */
+    disableJSONLD?: boolean;
+    /**
+     * Extract internal links
+     * @default true
+     */
+    extractLinks?: boolean;
+    /**
+     * Extract images
+     * @default true
+     */
+    extractImages?: boolean;
+}
+
+/**
+ * Checks if a path matches any of the given wildcard patterns
+ * Uses the matcher library which supports:
+ * - * (matches any characters except /)
+ * - ** (matches any characters including /)
+ * - ? (matches single character)
+ * - ! (negation)
+ * - [] (character ranges)
+ */
+declare function matchesExcludePattern(path: string, patterns: string[]): boolean;
+
+type RobotsParser = ReturnType<typeof robotsParser>;
+interface RobotsTxtResult {
+    parser: RobotsParser;
+    path: string;
+}
+declare function createRobotsParser(content: string): RobotsParser;
+/**
+ * Loads and parses robots.txt from custom path or standard locations
+ * Returns the parser and the path where robots.txt was found
+ */
+declare function loadRobotsTxt(projectDir: string, searchPaths: string[], robotsTxtPath?: string): RobotsTxtResult | null;
+declare function isPathAllowedByRobots(path: string, robots: RobotsParser | null): boolean;
+
+type PathFilterReason = 'included' | 'dynamic-route' | 'internal-route' | 'rsc-file' | 'segment-file' | 'static-asset' | 'robots-txt' | 'exclude-pattern';
+interface PathFilterResult {
+    included: boolean;
+    reason: PathFilterReason;
+}
+/**
+ * Determines if a pathname should be included in the index
+ * Returns both the decision and the reason for exclusion
+ */
+declare function shouldIncludePath(pathname: string, robots: RobotsParser | null, excludePatterns: string[], respectRobotsTxt: boolean): PathFilterResult;
+
+/**
+ * Convert file path to URL pathname
+ *
+ * Examples:
+ *   index.html -> /
+ *   about.html -> /about
+ *   about/index.html -> /about/
+ *   blog/post-1.html -> /blog/post-1
+ *   blog/post-1/index.html -> /blog/post-1/
+ *   server/pages/contact.html -> /contact
+ *   server/app/about.html -> /about
+ */
+declare function filePathToPathname(filePath: string): string;
+
+/**
+ * Parse HTML content and convert it to a StructuredPage
+ * @param html - HTML string to parse
+ * @param options - Parsing options
+ * @returns StructuredPage or undefined if parsing fails
+ */
+declare function parseHTML(html: string, options?: ParseOptions): StructuredPage | undefined;
+
+export { type ParseOptions, type PathFilterReason, type PathFilterResult, type RobotsParser, type RobotsTxtResult, type StructuredPage, createRobotsParser, filePathToPathname, isPathAllowedByRobots, loadRobotsTxt, matchesExcludePattern, parseHTML, shouldIncludePath };

package/dist/index.js

@@ -0,0 +1,333 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getOwnPropSymbols = Object.getOwnPropertySymbols;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __propIsEnum = Object.prototype.propertyIsEnumerable;
+var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
+var __spreadValues = (a, b) => {
+  for (var prop in b || (b = {}))
+    if (__hasOwnProp.call(b, prop))
+      __defNormalProp(a, prop, b[prop]);
+  if (__getOwnPropSymbols)
+    for (var prop of __getOwnPropSymbols(b)) {
+      if (__propIsEnum.call(b, prop))
+        __defNormalProp(a, prop, b[prop]);
+    }
+  return a;
+};
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/index.ts
+var index_exports = {};
+__export(index_exports, {
+  createRobotsParser: () => createRobotsParser,
+  filePathToPathname: () => filePathToPathname,
+  isPathAllowedByRobots: () => isPathAllowedByRobots,
+  loadRobotsTxt: () => loadRobotsTxt,
+  matchesExcludePattern: () => matchesExcludePattern,
+  parseHTML: () => parseHTML,
+  shouldIncludePath: () => shouldIncludePath
+});
+module.exports = __toCommonJS(index_exports);
+
+// src/utils/excludePatterns.ts
+var import_matcher = require("matcher");
+function matchesExcludePattern(path, patterns) {
+  if (!patterns || patterns.length === 0) {
+    return false;
+  }
+  const normalize = (p) => p.startsWith("/") ? p : `/${p}`;
+  const normalizedPath = normalize(path);
+  const normalizedPatterns = patterns.map(normalize);
+  return (0, import_matcher.isMatch)(normalizedPath, normalizedPatterns);
+}
+
+// src/utils/robotsParser.ts
+var import_fs = require("fs");
+var import_path = require("path");
+var import_robots_parser = __toESM(require("robots-parser"));
+function createRobotsParser(content) {
+  return (0, import_robots_parser.default)("https://robots.invalid/robots.txt", content);
+}
+function loadRobotsTxt(projectDir, searchPaths, robotsTxtPath) {
+  let robotsContent = null;
+  let foundPath = null;
+  if (robotsTxtPath) {
+    const customPath = (0, import_path.join)(projectDir, robotsTxtPath);
+    if ((0, import_fs.existsSync)(customPath)) {
+      robotsContent = (0, import_fs.readFileSync)(customPath, "utf-8");
+      foundPath = customPath;
+    }
+  }
+  if (!robotsContent) {
+    for (const searchPath of searchPaths) {
+      const fullPath = (0, import_path.join)(projectDir, searchPath);
+      if ((0, import_fs.existsSync)(fullPath)) {
+        robotsContent = (0, import_fs.readFileSync)(fullPath, "utf-8");
+        foundPath = fullPath;
+        break;
+      }
+    }
+  }
+  if (!robotsContent) {
+    return null;
+  }
+  return {
+    parser: createRobotsParser(robotsContent),
+    path: foundPath || ""
+  };
+}
+function isPathAllowedByRobots(path, robots) {
+  if (!robots) {
+    return true;
+  }
+  const normalizedPath = path.startsWith("/") ? path : `/${path}`;
+  const testUrl = `https://robots.invalid${normalizedPath}`;
+  const isAllowed = robots.isAllowed(testUrl, "*");
+  return isAllowed !== false;
+}
+
+// src/utils/pathFilter.ts
+function shouldIncludePath(pathname, robots, excludePatterns, respectRobotsTxt) {
+  if (pathname.includes("[") && pathname.includes("]")) {
+    return { included: false, reason: "dynamic-route" };
+  }
+  if (pathname.startsWith("/_not-found") || pathname.startsWith("/_global-error")) {
+    return { included: false, reason: "internal-route" };
+  }
+  if (pathname.endsWith(".rsc")) {
+    return { included: false, reason: "rsc-file" };
+  }
+  if (pathname.includes(".segments/")) {
+    return { included: false, reason: "segment-file" };
+  }
+  if (pathname.match(/\.(ico|png|jpg|jpeg|svg|gif|webp|txt|xml|json|css|js|woff|woff2|ttf|eot)$/i)) {
+    return { included: false, reason: "static-asset" };
+  }
+  if (respectRobotsTxt && !isPathAllowedByRobots(pathname, robots)) {
+    return { included: false, reason: "robots-txt" };
+  }
+  if (matchesExcludePattern(pathname, excludePatterns)) {
+    return { included: false, reason: "exclude-pattern" };
+  }
+  return { included: true, reason: "included" };
+}
+
+// src/utils/pathUtils.ts
+var import_path2 = require("path");
+var artifactPrefixes = [
+  "server/pages/",
+  "server/app/",
+  "static/chunks/app/",
+  "static/chunks/pages/",
+  "static/",
+  "server/"
+];
+function filePathToPathname(filePath) {
+  let pathname = filePath.split(import_path2.sep).join("/");
+  for (const prefix of artifactPrefixes) {
+    if (pathname.startsWith(prefix)) {
+      pathname = pathname.substring(prefix.length);
+      break;
+    }
+  }
+  pathname = pathname.replace(/\.html?$/, "");
+  if (pathname === "index" || pathname === "") {
+    return "/";
+  }
+  if (pathname.endsWith("/index")) {
+    pathname = pathname.slice(0, -5);
+  }
+  if (!pathname.startsWith("/")) {
+    pathname = "/" + pathname;
+  }
+  return pathname;
+}
+
+// src/parseHtml.ts
+var import_logger = require("@peam-ai/logger");
+var import_jsdom = require("jsdom");
+
+// src/parsers/cssSelectorParser.ts
+var CssSelectorParser = class {
+  parse(document, options) {
+    var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j;
+    const title = (_a = document.querySelector("title")) == null ? void 0 : _a.textContent;
+    const description = (_b = document.querySelector('meta[name="description"]')) == null ? void 0 : _b.getAttribute("content");
+    const content = document.body.innerHTML;
+    const bodyClone = document.body.cloneNode(true);
+    bodyClone.querySelectorAll("script, style, noscript, iframe, [hidden], nav, header, footer, aside").forEach((el) => el.remove());
+    const textContent = (_c = bodyClone.textContent) == null ? void 0 : _c.replace(/\s+/g, " ").trim();
+    if (!title || title.trim().length === 0 || !textContent || textContent.trim().length === 0 || !content || content.trim().length === 0 || !description || description.trim().length === 0) {
+      return void 0;
+    }
+    const page = {
+      title,
+      description,
+      content,
+      textContent,
+      contentLength: textContent.length
+    };
+    const htmlLang = document.documentElement.getAttribute("lang");
+    if (htmlLang) {
+      page.language = htmlLang;
+    }
+    const dir = document.documentElement.getAttribute("dir");
+    if (dir) {
+      page.direction = dir;
+    }
+    const ogTitle = (_d = document.querySelector('meta[property="og:title"]')) == null ? void 0 : _d.getAttribute("content");
+    if (ogTitle && ogTitle !== page.title) {
+      page.ogTitle = ogTitle;
+    }
+    const ogDescription = (_e = document.querySelector('meta[property="og:description"]')) == null ? void 0 : _e.getAttribute("content");
+    if (ogDescription) {
+      page.ogDescription = ogDescription;
+    }
+    const ogImage = (_f = document.querySelector('meta[property="og:image"]')) == null ? void 0 : _f.getAttribute("content");
+    if (ogImage) {
+      page.ogImage = ogImage;
+    }
+    const ogSiteName = (_g = document.querySelector('meta[property="og:site_name"]')) == null ? void 0 : _g.getAttribute("content");
+    if (ogSiteName) {
+      page.siteName = ogSiteName;
+    }
+    const author = (_h = document.querySelector('meta[name="author"]')) == null ? void 0 : _h.getAttribute("content");
+    if (author) {
+      page.author = author;
+    }
+    const keywords = (_i = document.querySelector('meta[name="keywords"]')) == null ? void 0 : _i.getAttribute("content");
+    if (keywords) {
+      page.keywords = keywords.split(",").map((k) => k.trim()).filter((k) => k);
+    }
+    const publishedTime = (_j = document.querySelector('meta[property="article:published_time"]')) == null ? void 0 : _j.getAttribute("content");
+    if (publishedTime) {
+      page.publishedTime = publishedTime;
+    }
+    const headings = Array.from(document.querySelectorAll("h1, h2, h3, h4, h5, h6")).map((h) => {
+      var _a2;
+      return (_a2 = h.textContent) == null ? void 0 : _a2.trim();
+    }).filter((h) => !!h && h.length > 0);
+    if (headings.length > 0) {
+      page.headings = headings;
+    }
+    if ((options == null ? void 0 : options.extractLinks) !== false) {
+      const links = Array.from(document.querySelectorAll("a[href]")).map((a) => a.getAttribute("href")).filter((href) => {
+        if (!href) return false;
+        return href.startsWith("/") && !href.startsWith("/_next") && !href.startsWith("/_");
+      });
+      if (links.length > 0) {
+        page.internalLinks = Array.from(new Set(links));
+      }
+      const externalLinks = Array.from(document.querySelectorAll("a[href]")).map((a) => a.getAttribute("href")).filter((href) => {
+        if (!href) return false;
+        return href.startsWith("http://") || href.startsWith("https://");
+      });
+      if (externalLinks.length > 0) {
+        page.externalLinks = Array.from(new Set(externalLinks));
+      }
+    }
+    if ((options == null ? void 0 : options.extractImages) !== false) {
+      const images = Array.from(document.querySelectorAll("img[src], img[data-nimg], img[data-src]")).map((img) => {
+        const src = img.getAttribute("src") || img.getAttribute("data-src") || img.getAttribute("data-nimg");
+        const alt = img.getAttribute("alt");
+        return { src, alt };
+      }).filter((img) => img.src && !img.src.startsWith("data:"));
+      if (images.length > 0) {
+        page.images = images.map((img) => img.src || "").filter((src) => src);
+        page.imageAlts = images.map((img) => img.alt || "");
+      }
+    }
+    return page;
+  }
+};
+
+// src/parsers/readabilityParser.ts
+var import_readability = require("@mozilla/readability");
+var ReadabilityParser = class {
+  parse(document, options) {
+    const { keepClasses, disableJSONLD } = options || {};
+    const reader = new import_readability.Readability(document, {
+      keepClasses,
+      disableJSONLD
+    });
+    const parsedPage = reader.parse();
+    if (parsedPage) {
+      const page = {
+        title: parsedPage.title,
+        description: parsedPage.excerpt,
+        content: parsedPage.content,
+        textContent: parsedPage.textContent,
+        contentLength: parsedPage.length,
+        author: parsedPage.byline,
+        direction: parsedPage.dir,
+        language: parsedPage.lang,
+        siteName: parsedPage.siteName,
+        publishedTime: parsedPage.publishedTime
+      };
+      return page;
+    }
+    return void 0;
+  }
+};
+
+// src/parseHtml.ts
+var log = import_logger.loggers.parser;
+function parseHTML(html, options = {}) {
+  if (!html || html.trim().length === 0) {
+    log.warn("Empty or invalid HTML input");
+    return void 0;
+  }
+  log.debug("Starting parse with options", options);
+  const dom = new import_jsdom.JSDOM(html);
+  const document = dom.window.document;
+  const cssSelectorParser = new CssSelectorParser();
+  const cssSelectorStructuredPage = cssSelectorParser.parse(document, options);
+  const readabilityParser = new ReadabilityParser();
+  const readabilityStructuredPage = readabilityParser.parse(document, options);
+  if (!cssSelectorStructuredPage && !readabilityStructuredPage) {
+    log.warn("Failed to extract content");
+    return void 0;
+  }
+  return __spreadValues(__spreadValues(__spreadValues({}, {
+    title: "",
+    description: "",
+    content: "",
+    textContent: ""
+  }), cssSelectorStructuredPage), readabilityStructuredPage);
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  createRobotsParser,
+  filePathToPathname,
+  isPathAllowedByRobots,
+  loadRobotsTxt,
+  matchesExcludePattern,
+  parseHTML,
+  shouldIncludePath
+});
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/index.ts","../src/utils/excludePatterns.ts","../src/utils/robotsParser.ts","../src/utils/pathFilter.ts","../src/utils/pathUtils.ts","../src/parseHtml.ts","../src/parsers/cssSelectorParser.ts","../src/parsers/readabilityParser.ts"],"sourcesContent":["export { ParseOptions } from './parsers/parser';\nexport { StructuredPage } from './structuredPage';\n\nexport { matchesExcludePattern } from './utils/excludePatterns';\nexport { shouldIncludePath, type PathFilterReason, type PathFilterResult } from './utils/pathFilter';\nexport { filePathToPathname } from './utils/pathUtils';\nexport {\n  createRobotsParser,\n  isPathAllowedByRobots,\n  loadRobotsTxt,\n  type RobotsParser,\n  type RobotsTxtResult,\n} from './utils/robotsParser';\n\nexport { parseHTML } from './parseHtml';\n","import { isMatch } from 'matcher';\n\n/**\n * Checks if a path matches any of the given wildcard patterns\n * Uses the matcher library which supports:\n * - * (matches any characters except /)\n * - ** (matches any characters including /)\n * - ? (matches single character)\n * - ! (negation)\n * - [] (character ranges)\n */\nexport function matchesExcludePattern(path: string, patterns: string[]): boolean {\n  if (!patterns || patterns.length === 0) {\n    return false;\n  }\n\n  const normalize = (p: string) => (p.startsWith('/') ? p : `/${p}`);\n  const normalizedPath = normalize(path);\n  const normalizedPatterns = patterns.map(normalize);\n\n  return isMatch(normalizedPath, normalizedPatterns);\n}\n","import { existsSync, readFileSync } from 'fs';\nimport { join } from 'path';\nimport robotsParser from 'robots-parser';\n\nexport type RobotsParser = ReturnType<typeof robotsParser>;\n\nexport interface RobotsTxtResult {\n  parser: RobotsParser;\n  path: string;\n}\n\nexport function createRobotsParser(content: string): RobotsParser {\n  return robotsParser('https://robots.invalid/robots.txt', content);\n}\n\n/**\n * Loads and parses robots.txt from custom path or standard locations\n * Returns the parser and the path where robots.txt was found\n */\nexport function loadRobotsTxt(\n  projectDir: string,\n  searchPaths: string[],\n  robotsTxtPath?: string\n): RobotsTxtResult | null {\n  let robotsContent: string | null = null;\n  let foundPath: string | null = null;\n\n  if (robotsTxtPath) {\n    const customPath = join(projectDir, robotsTxtPath);\n    if (existsSync(customPath)) {\n      robotsContent = readFileSync(customPath, 'utf-8');\n      foundPath = customPath;\n    }\n  }\n\n  if (!robotsContent) {\n    for (const searchPath of searchPaths) {\n      const fullPath = join(projectDir, searchPath);\n      if (existsSync(fullPath)) {\n        robotsContent = readFileSync(fullPath, 'utf-8');\n        foundPath = fullPath;\n        break;\n      }\n    }\n  }\n\n  if (!robotsContent) {\n    return null;\n  }\n\n  return {\n    parser: createRobotsParser(robotsContent),\n    path: foundPath || '',\n  };\n}\n\nexport function isPathAllowedByRobots(path: string, robots: RobotsParser | null): boolean {\n  if (!robots) {\n    return true;\n  }\n\n  const normalizedPath = path.startsWith('/') ? path : `/${path}`;\n  const testUrl = `https://robots.invalid${normalizedPath}`;\n  const isAllowed = robots.isAllowed(testUrl, '*');\n\n  return isAllowed !== false;\n}\n","import { matchesExcludePattern } from './excludePatterns';\nimport type { RobotsParser } from './robotsParser';\nimport { isPathAllowedByRobots } from './robotsParser';\n\nexport type PathFilterReason =\n  | 'included'\n  | 'dynamic-route'\n  | 'internal-route'\n  | 'rsc-file'\n  | 'segment-file'\n  | 'static-asset'\n  | 'robots-txt'\n  | 'exclude-pattern';\n\nexport interface PathFilterResult {\n  included: boolean;\n  reason: PathFilterReason;\n}\n\n/**\n * Determines if a pathname should be included in the index\n * Returns both the decision and the reason for exclusion\n */\nexport function shouldIncludePath(\n  pathname: string,\n  robots: RobotsParser | null,\n  excludePatterns: string[],\n  respectRobotsTxt: boolean\n): PathFilterResult {\n  // Exclude routes with dynamic parameters (e.g., /session/[session_id])\n  if (pathname.includes('[') && pathname.includes(']')) {\n    return { included: false, reason: 'dynamic-route' };\n  }\n\n  // Exclude Next.js internal routes\n  if (pathname.startsWith('/_not-found') || pathname.startsWith('/_global-error')) {\n    return { included: false, reason: 'internal-route' };\n  }\n\n  // Exclude RSC files\n  if (pathname.endsWith('.rsc')) {\n    return { included: false, reason: 'rsc-file' };\n  }\n\n  // Exclude segment files\n  if (pathname.includes('.segments/')) {\n    return { included: false, reason: 'segment-file' };\n  }\n\n  // Exclude static assets\n  if (pathname.match(/\\.(ico|png|jpg|jpeg|svg|gif|webp|txt|xml|json|css|js|woff|woff2|ttf|eot)$/i)) {\n    return { included: false, reason: 'static-asset' };\n  }\n\n  // Check robots.txt rules\n  if (respectRobotsTxt && !isPathAllowedByRobots(pathname, robots)) {\n    return { included: false, reason: 'robots-txt' };\n  }\n\n  // Check user-defined exclude patterns\n  if (matchesExcludePattern(pathname, excludePatterns)) {\n    return { included: false, reason: 'exclude-pattern' };\n  }\n\n  return { included: true, reason: 'included' };\n}\n","import { sep } from 'path';\n\nconst artifactPrefixes = [\n  'server/pages/',\n  'server/app/',\n  'static/chunks/app/',\n  'static/chunks/pages/',\n  'static/',\n  'server/',\n];\n\n/**\n * Convert file path to URL pathname\n *\n * Examples:\n *   index.html -> /\n *   about.html -> /about\n *   about/index.html -> /about/\n *   blog/post-1.html -> /blog/post-1\n *   blog/post-1/index.html -> /blog/post-1/\n *   server/pages/contact.html -> /contact\n *   server/app/about.html -> /about\n */\nexport function filePathToPathname(filePath: string): string {\n  let pathname = filePath.split(sep).join('/');\n\n  for (const prefix of artifactPrefixes) {\n    if (pathname.startsWith(prefix)) {\n      pathname = pathname.substring(prefix.length);\n      break;\n    }\n  }\n\n  pathname = pathname.replace(/\\.html?$/, '');\n\n  if (pathname === 'index' || pathname === '') {\n    return '/';\n  }\n\n  if (pathname.endsWith('/index')) {\n    pathname = pathname.slice(0, -5);\n  }\n\n  if (!pathname.startsWith('/')) {\n    pathname = '/' + pathname;\n  }\n\n  return pathname;\n}\n","import { loggers } from '@peam-ai/logger';\nimport { JSDOM } from 'jsdom';\nimport { CssSelectorParser } from './parsers/cssSelectorParser';\nimport { ParseOptions } from './parsers/parser';\nimport { ReadabilityParser } from './parsers/readabilityParser';\nimport { StructuredPage } from './structuredPage';\n\nconst log = loggers.parser;\n\n/**\n * Parse HTML content and convert it to a StructuredPage\n * @param html - HTML string to parse\n * @param options - Parsing options\n * @returns StructuredPage or undefined if parsing fails\n */\nexport function parseHTML(html: string, options: ParseOptions = {}): StructuredPage | undefined {\n  if (!html || html.trim().length === 0) {\n    log.warn('Empty or invalid HTML input');\n    return undefined;\n  }\n\n  log.debug('Starting parse with options', options);\n  const dom = new JSDOM(html);\n  const document = dom.window.document;\n\n  const cssSelectorParser = new CssSelectorParser();\n  const cssSelectorStructuredPage = cssSelectorParser.parse(document, options);\n  const readabilityParser = new ReadabilityParser();\n  const readabilityStructuredPage = readabilityParser.parse(document, options);\n\n  if (!cssSelectorStructuredPage && !readabilityStructuredPage) {\n    log.warn('Failed to extract content');\n    return undefined;\n  }\n\n  // Merge results, prioritizing Readability data\n  return {\n    ...{\n      title: '',\n      description: '',\n      content: '',\n      textContent: '',\n    },\n    ...cssSelectorStructuredPage,\n    ...readabilityStructuredPage,\n  };\n}\n","import { StructuredPage } from '../structuredPage';\nimport { ParseOptions, Parser } from './parser';\n\nexport class CssSelectorParser implements Parser {\n  parse(document: Document, options?: ParseOptions): StructuredPage | undefined {\n    const title = document.querySelector('title')?.textContent;\n    const description = document.querySelector('meta[name=\"description\"]')?.getAttribute('content');\n    const content = document.body.innerHTML;\n\n    const bodyClone = document.body.cloneNode(true) as HTMLElement;\n    bodyClone\n      .querySelectorAll('script, style, noscript, iframe, [hidden], nav, header, footer, aside')\n      .forEach((el) => el.remove());\n\n    const textContent = bodyClone.textContent?.replace(/\\s+/g, ' ').trim();\n\n    if (\n      !title ||\n      title.trim().length === 0 ||\n      !textContent ||\n      textContent.trim().length === 0 ||\n      !content ||\n      content.trim().length === 0 ||\n      !description ||\n      description.trim().length === 0\n    ) {\n      return undefined;\n    }\n\n    const page: StructuredPage = {\n      title,\n      description,\n      content,\n      textContent,\n      contentLength: textContent.length,\n    };\n\n    const htmlLang = document.documentElement.getAttribute('lang');\n    if (htmlLang) {\n      page.language = htmlLang;\n    }\n\n    const dir = document.documentElement.getAttribute('dir');\n    if (dir) {\n      page.direction = dir;\n    }\n\n    const ogTitle = document.querySelector('meta[property=\"og:title\"]')?.getAttribute('content');\n    if (ogTitle && ogTitle !== page.title) {\n      page.ogTitle = ogTitle;\n    }\n\n    const ogDescription = document.querySelector('meta[property=\"og:description\"]')?.getAttribute('content');\n    if (ogDescription) {\n      page.ogDescription = ogDescription;\n    }\n\n    const ogImage = document.querySelector('meta[property=\"og:image\"]')?.getAttribute('content');\n    if (ogImage) {\n      page.ogImage = ogImage;\n    }\n\n    const ogSiteName = document.querySelector('meta[property=\"og:site_name\"]')?.getAttribute('content');\n    if (ogSiteName) {\n      page.siteName = ogSiteName;\n    }\n\n    const author = document.querySelector('meta[name=\"author\"]')?.getAttribute('content');\n    if (author) {\n      page.author = author;\n    }\n\n    const keywords = document.querySelector('meta[name=\"keywords\"]')?.getAttribute('content');\n    if (keywords) {\n      page.keywords = keywords\n        .split(',')\n        .map((k) => k.trim())\n        .filter((k) => k);\n    }\n\n    const publishedTime = document.querySelector('meta[property=\"article:published_time\"]')?.getAttribute('content');\n    if (publishedTime) {\n      page.publishedTime = publishedTime;\n    }\n\n    const headings = Array.from(document.querySelectorAll('h1, h2, h3, h4, h5, h6'))\n      .map((h) => h.textContent?.trim())\n      .filter((h): h is string => !!h && h.length > 0);\n\n    if (headings.length > 0) {\n      page.headings = headings;\n    }\n\n    // Extract links if enabled\n    if (options?.extractLinks !== false) {\n      // Extract internal links\n      const links = Array.from(document.querySelectorAll('a[href]'))\n        .map((a) => a.getAttribute('href'))\n        .filter((href): href is string => {\n          if (!href) return false;\n          return href.startsWith('/') && !href.startsWith('/_next') && !href.startsWith('/_');\n        });\n\n      if (links.length > 0) {\n        page.internalLinks = Array.from(new Set(links));\n      }\n\n      // Extract external links\n      const externalLinks = Array.from(document.querySelectorAll('a[href]'))\n        .map((a) => a.getAttribute('href'))\n        .filter((href): href is string => {\n          if (!href) return false;\n          return href.startsWith('http://') || href.startsWith('https://');\n        });\n\n      if (externalLinks.length > 0) {\n        page.externalLinks = Array.from(new Set(externalLinks));\n      }\n    }\n\n    // Extract images if enabled\n    if (options?.extractImages !== false) {\n      const images = Array.from(document.querySelectorAll('img[src], img[data-nimg], img[data-src]'))\n        .map((img) => {\n          const src = img.getAttribute('src') || img.getAttribute('data-src') || img.getAttribute('data-nimg');\n          const alt = img.getAttribute('alt');\n          return { src, alt };\n        })\n        .filter((img) => img.src && !img.src.startsWith('data:'));\n\n      if (images.length > 0) {\n        page.images = images.map((img) => img.src || '').filter((src) => src);\n        page.imageAlts = images.map((img) => img.alt || '');\n      }\n    }\n\n    return page;\n  }\n}\n","import { Readability } from '@mozilla/readability';\nimport { StructuredPage } from '../structuredPage';\nimport { ParseOptions, Parser } from './parser';\n\nexport class ReadabilityParser implements Parser {\n  parse(document: Document, options?: ParseOptions): StructuredPage | undefined {\n    const { keepClasses, disableJSONLD } = options || {};\n\n    const reader = new Readability(document, {\n      keepClasses,\n      disableJSONLD,\n    });\n\n    const parsedPage = reader.parse();\n\n    if (parsedPage) {\n      const page: StructuredPage = {\n        title: parsedPage.title,\n        description: parsedPage.excerpt,\n        content: parsedPage.content,\n        textContent: parsedPage.textContent,\n        contentLength: parsedPage.length,\n        author: parsedPage.byline,\n        direction: parsedPage.dir,\n        language: parsedPage.lang,\n        siteName: parsedPage.siteName,\n        publishedTime: parsedPage.publishedTime,\n      };\n\n      return page;\n    }\n\n    return undefined;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACAA,qBAAwB;AAWjB,SAAS,sBAAsB,MAAc,UAA6B;AAC/E,MAAI,CAAC,YAAY,SAAS,WAAW,GAAG;AACtC,WAAO;AAAA,EACT;AAEA,QAAM,YAAY,CAAC,MAAe,EAAE,WAAW,GAAG,IAAI,IAAI,IAAI,CAAC;AAC/D,QAAM,iBAAiB,UAAU,IAAI;AACrC,QAAM,qBAAqB,SAAS,IAAI,SAAS;AAEjD,aAAO,wBAAQ,gBAAgB,kBAAkB;AACnD;;;ACrBA,gBAAyC;AACzC,kBAAqB;AACrB,2BAAyB;AASlB,SAAS,mBAAmB,SAA+B;AAChE,aAAO,qBAAAA,SAAa,qCAAqC,OAAO;AAClE;AAMO,SAAS,cACd,YACA,aACA,eACwB;AACxB,MAAI,gBAA+B;AACnC,MAAI,YAA2B;AAE/B,MAAI,eAAe;AACjB,UAAM,iBAAa,kBAAK,YAAY,aAAa;AACjD,YAAI,sBAAW,UAAU,GAAG;AAC1B,0BAAgB,wBAAa,YAAY,OAAO;AAChD,kBAAY;AAAA,IACd;AAAA,EACF;AAEA,MAAI,CAAC,eAAe;AAClB,eAAW,cAAc,aAAa;AACpC,YAAM,eAAW,kBAAK,YAAY,UAAU;AAC5C,cAAI,sBAAW,QAAQ,GAAG;AACxB,4BAAgB,wBAAa,UAAU,OAAO;AAC9C,oBAAY;AACZ;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,MAAI,CAAC,eAAe;AAClB,WAAO;AAAA,EACT;AAEA,SAAO;AAAA,IACL,QAAQ,mBAAmB,aAAa;AAAA,IACxC,MAAM,aAAa;AAAA,EACrB;AACF;AAEO,SAAS,sBAAsB,MAAc,QAAsC;AACxF,MAAI,CAAC,QAAQ;AACX,WAAO;AAAA,EACT;AAEA,QAAM,iBAAiB,KAAK,WAAW,GAAG,IAAI,OAAO,IAAI,IAAI;AAC7D,QAAM,UAAU,yBAAyB,cAAc;AACvD,QAAM,YAAY,OAAO,UAAU,SAAS,GAAG;AAE/C,SAAO,cAAc;AACvB;;;AC3CO,SAAS,kBACd,UACA,QACA,iBACA,kBACkB;AAElB,MAAI,SAAS,SAAS,GAAG,KAAK,SAAS,SAAS,GAAG,GAAG;AACpD,WAAO,EAAE,UAAU,OAAO,QAAQ,gBAAgB;AAAA,EACpD;AAGA,MAAI,SAAS,WAAW,aAAa,KAAK,SAAS,WAAW,gBAAgB,GAAG;AAC/E,WAAO,EAAE,UAAU,OAAO,QAAQ,iBAAiB;AAAA,EACrD;AAGA,MAAI,SAAS,SAAS,MAAM,GAAG;AAC7B,WAAO,EAAE,UAAU,OAAO,QAAQ,WAAW;AAAA,EAC/C;AAGA,MAAI,SAAS,SAAS,YAAY,GAAG;AACnC,WAAO,EAAE,UAAU,OAAO,QAAQ,eAAe;AAAA,EACnD;AAGA,MAAI,SAAS,MAAM,4EAA4E,GAAG;AAChG,WAAO,EAAE,UAAU,OAAO,QAAQ,eAAe;AAAA,EACnD;AAGA,MAAI,oBAAoB,CAAC,sBAAsB,UAAU,MAAM,GAAG;AAChE,WAAO,EAAE,UAAU,OAAO,QAAQ,aAAa;AAAA,EACjD;AAGA,MAAI,sBAAsB,UAAU,eAAe,GAAG;AACpD,WAAO,EAAE,UAAU,OAAO,QAAQ,kBAAkB;AAAA,EACtD;AAEA,SAAO,EAAE,UAAU,MAAM,QAAQ,WAAW;AAC9C;;;ACjEA,IAAAC,eAAoB;AAEpB,IAAM,mBAAmB;AAAA,EACvB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAcO,SAAS,mBAAmB,UAA0B;AAC3D,MAAI,WAAW,SAAS,MAAM,gBAAG,EAAE,KAAK,GAAG;AAE3C,aAAW,UAAU,kBAAkB;AACrC,QAAI,SAAS,WAAW,MAAM,GAAG;AAC/B,iBAAW,SAAS,UAAU,OAAO,MAAM;AAC3C;AAAA,IACF;AAAA,EACF;AAEA,aAAW,SAAS,QAAQ,YAAY,EAAE;AAE1C,MAAI,aAAa,WAAW,aAAa,IAAI;AAC3C,WAAO;AAAA,EACT;AAEA,MAAI,SAAS,SAAS,QAAQ,GAAG;AAC/B,eAAW,SAAS,MAAM,GAAG,EAAE;AAAA,EACjC;AAEA,MAAI,CAAC,SAAS,WAAW,GAAG,GAAG;AAC7B,eAAW,MAAM;AAAA,EACnB;AAEA,SAAO;AACT;;;AChDA,oBAAwB;AACxB,mBAAsB;;;ACEf,IAAM,oBAAN,MAA0C;AAAA,EAC/C,MAAM,UAAoB,SAAoD;AAJhF;AAKI,UAAM,SAAQ,cAAS,cAAc,OAAO,MAA9B,mBAAiC;AAC/C,UAAM,eAAc,cAAS,cAAc,0BAA0B,MAAjD,mBAAoD,aAAa;AACrF,UAAM,UAAU,SAAS,KAAK;AAE9B,UAAM,YAAY,SAAS,KAAK,UAAU,IAAI;AAC9C,cACG,iBAAiB,uEAAuE,EACxF,QAAQ,CAAC,OAAO,GAAG,OAAO,CAAC;AAE9B,UAAM,eAAc,eAAU,gBAAV,mBAAuB,QAAQ,QAAQ,KAAK;AAEhE,QACE,CAAC,SACD,MAAM,KAAK,EAAE,WAAW,KACxB,CAAC,eACD,YAAY,KAAK,EAAE,WAAW,KAC9B,CAAC,WACD,QAAQ,KAAK,EAAE,WAAW,KAC1B,CAAC,eACD,YAAY,KAAK,EAAE,WAAW,GAC9B;AACA,aAAO;AAAA,IACT;AAEA,UAAM,OAAuB;AAAA,MAC3B;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,eAAe,YAAY;AAAA,IAC7B;AAEA,UAAM,WAAW,SAAS,gBAAgB,aAAa,MAAM;AAC7D,QAAI,UAAU;AACZ,WAAK,WAAW;AAAA,IAClB;AAEA,UAAM,MAAM,SAAS,gBAAgB,aAAa,KAAK;AACvD,QAAI,KAAK;AACP,WAAK,YAAY;AAAA,IACnB;AAEA,UAAM,WAAU,cAAS,cAAc,2BAA2B,MAAlD,mBAAqD,aAAa;AAClF,QAAI,WAAW,YAAY,KAAK,OAAO;AACrC,WAAK,UAAU;AAAA,IACjB;AAEA,UAAM,iBAAgB,cAAS,cAAc,iCAAiC,MAAxD,mBAA2D,aAAa;AAC9F,QAAI,eAAe;AACjB,WAAK,gBAAgB;AAAA,IACvB;AAEA,UAAM,WAAU,cAAS,cAAc,2BAA2B,MAAlD,mBAAqD,aAAa;AAClF,QAAI,SAAS;AACX,WAAK,UAAU;AAAA,IACjB;AAEA,UAAM,cAAa,cAAS,cAAc,+BAA+B,MAAtD,mBAAyD,aAAa;AACzF,QAAI,YAAY;AACd,WAAK,WAAW;AAAA,IAClB;AAEA,UAAM,UAAS,cAAS,cAAc,qBAAqB,MAA5C,mBAA+C,aAAa;AAC3E,QAAI,QAAQ;AACV,WAAK,SAAS;AAAA,IAChB;AAEA,UAAM,YAAW,cAAS,cAAc,uBAAuB,MAA9C,mBAAiD,aAAa;AAC/E,QAAI,UAAU;AACZ,WAAK,WAAW,SACb,MAAM,GAAG,EACT,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,EACnB,OAAO,CAAC,MAAM,CAAC;AAAA,IACpB;AAEA,UAAM,iBAAgB,cAAS,cAAc,yCAAyC,MAAhE,mBAAmE,aAAa;AACtG,QAAI,eAAe;AACjB,WAAK,gBAAgB;AAAA,IACvB;AAEA,UAAM,WAAW,MAAM,KAAK,SAAS,iBAAiB,wBAAwB,CAAC,EAC5E,IAAI,CAAC,MAAG;AAtFf,UAAAC;AAsFkB,cAAAA,MAAA,EAAE,gBAAF,gBAAAA,IAAe;AAAA,KAAM,EAChC,OAAO,CAAC,MAAmB,CAAC,CAAC,KAAK,EAAE,SAAS,CAAC;AAEjD,QAAI,SAAS,SAAS,GAAG;AACvB,WAAK,WAAW;AAAA,IAClB;AAGA,SAAI,mCAAS,kBAAiB,OAAO;AAEnC,YAAM,QAAQ,MAAM,KAAK,SAAS,iBAAiB,SAAS,CAAC,EAC1D,IAAI,CAAC,MAAM,EAAE,aAAa,MAAM,CAAC,EACjC,OAAO,CAAC,SAAyB;AAChC,YAAI,CAAC,KAAM,QAAO;AAClB,eAAO,KAAK,WAAW,GAAG,KAAK,CAAC,KAAK,WAAW,QAAQ,KAAK,CAAC,KAAK,WAAW,IAAI;AAAA,MACpF,CAAC;AAEH,UAAI,MAAM,SAAS,GAAG;AACpB,aAAK,gBAAgB,MAAM,KAAK,IAAI,IAAI,KAAK,CAAC;AAAA,MAChD;AAGA,YAAM,gBAAgB,MAAM,KAAK,SAAS,iBAAiB,SAAS,CAAC,EAClE,IAAI,CAAC,MAAM,EAAE,aAAa,MAAM,CAAC,EACjC,OAAO,CAAC,SAAyB;AAChC,YAAI,CAAC,KAAM,QAAO;AAClB,eAAO,KAAK,WAAW,SAAS,KAAK,KAAK,WAAW,UAAU;AAAA,MACjE,CAAC;AAEH,UAAI,cAAc,SAAS,GAAG;AAC5B,aAAK,gBAAgB,MAAM,KAAK,IAAI,IAAI,aAAa,CAAC;AAAA,MACxD;AAAA,IACF;AAGA,SAAI,mCAAS,mBAAkB,OAAO;AACpC,YAAM,SAAS,MAAM,KAAK,SAAS,iBAAiB,yCAAyC,CAAC,EAC3F,IAAI,CAAC,QAAQ;AACZ,cAAM,MAAM,IAAI,aAAa,KAAK,KAAK,IAAI,aAAa,UAAU,KAAK,IAAI,aAAa,WAAW;AACnG,cAAM,MAAM,IAAI,aAAa,KAAK;AAClC,eAAO,EAAE,KAAK,IAAI;AAAA,MACpB,CAAC,EACA,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,IAAI,IAAI,WAAW,OAAO,CAAC;AAE1D,UAAI,OAAO,SAAS,GAAG;AACrB,aAAK,SAAS,OAAO,IAAI,CAAC,QAAQ,IAAI,OAAO,EAAE,EAAE,OAAO,CAAC,QAAQ,GAAG;AACpE,aAAK,YAAY,OAAO,IAAI,CAAC,QAAQ,IAAI,OAAO,EAAE;AAAA,MACpD;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AACF;;;AC1IA,yBAA4B;AAIrB,IAAM,oBAAN,MAA0C;AAAA,EAC/C,MAAM,UAAoB,SAAoD;AAC5E,UAAM,EAAE,aAAa,cAAc,IAAI,WAAW,CAAC;AAEnD,UAAM,SAAS,IAAI,+BAAY,UAAU;AAAA,MACvC;AAAA,MACA;AAAA,IACF,CAAC;AAED,UAAM,aAAa,OAAO,MAAM;AAEhC,QAAI,YAAY;AACd,YAAM,OAAuB;AAAA,QAC3B,OAAO,WAAW;AAAA,QAClB,aAAa,WAAW;AAAA,QACxB,SAAS,WAAW;AAAA,QACpB,aAAa,WAAW;AAAA,QACxB,eAAe,WAAW;AAAA,QAC1B,QAAQ,WAAW;AAAA,QACnB,WAAW,WAAW;AAAA,QACtB,UAAU,WAAW;AAAA,QACrB,UAAU,WAAW;AAAA,QACrB,eAAe,WAAW;AAAA,MAC5B;AAEA,aAAO;AAAA,IACT;AAEA,WAAO;AAAA,EACT;AACF;;;AF3BA,IAAM,MAAM,sBAAQ;AAQb,SAAS,UAAU,MAAc,UAAwB,CAAC,GAA+B;AAC9F,MAAI,CAAC,QAAQ,KAAK,KAAK,EAAE,WAAW,GAAG;AACrC,QAAI,KAAK,6BAA6B;AACtC,WAAO;AAAA,EACT;AAEA,MAAI,MAAM,+BAA+B,OAAO;AAChD,QAAM,MAAM,IAAI,mBAAM,IAAI;AAC1B,QAAM,WAAW,IAAI,OAAO;AAE5B,QAAM,oBAAoB,IAAI,kBAAkB;AAChD,QAAM,4BAA4B,kBAAkB,MAAM,UAAU,OAAO;AAC3E,QAAM,oBAAoB,IAAI,kBAAkB;AAChD,QAAM,4BAA4B,kBAAkB,MAAM,UAAU,OAAO;AAE3E,MAAI,CAAC,6BAA6B,CAAC,2BAA2B;AAC5D,QAAI,KAAK,2BAA2B;AACpC,WAAO;AAAA,EACT;AAGA,SAAO,iDACF;AAAA,IACD,OAAO;AAAA,IACP,aAAa;AAAA,IACb,SAAS;AAAA,IACT,aAAa;AAAA,EACf,IACG,4BACA;AAEP;","names":["robotsParser","import_path","_a"]}

package/dist/index.mjs

@@ -0,0 +1,293 @@
+var __defProp = Object.defineProperty;
+var __getOwnPropSymbols = Object.getOwnPropertySymbols;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __propIsEnum = Object.prototype.propertyIsEnumerable;
+var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
+var __spreadValues = (a, b) => {
+  for (var prop in b || (b = {}))
+    if (__hasOwnProp.call(b, prop))
+      __defNormalProp(a, prop, b[prop]);
+  if (__getOwnPropSymbols)
+    for (var prop of __getOwnPropSymbols(b)) {
+      if (__propIsEnum.call(b, prop))
+        __defNormalProp(a, prop, b[prop]);
+    }
+  return a;
+};
+
+// src/utils/excludePatterns.ts
+import { isMatch } from "matcher";
+function matchesExcludePattern(path, patterns) {
+  if (!patterns || patterns.length === 0) {
+    return false;
+  }
+  const normalize = (p) => p.startsWith("/") ? p : `/${p}`;
+  const normalizedPath = normalize(path);
+  const normalizedPatterns = patterns.map(normalize);
+  return isMatch(normalizedPath, normalizedPatterns);
+}
+
+// src/utils/robotsParser.ts
+import { existsSync, readFileSync } from "fs";
+import { join } from "path";
+import robotsParser from "robots-parser";
+function createRobotsParser(content) {
+  return robotsParser("https://robots.invalid/robots.txt", content);
+}
+function loadRobotsTxt(projectDir, searchPaths, robotsTxtPath) {
+  let robotsContent = null;
+  let foundPath = null;
+  if (robotsTxtPath) {
+    const customPath = join(projectDir, robotsTxtPath);
+    if (existsSync(customPath)) {
+      robotsContent = readFileSync(customPath, "utf-8");
+      foundPath = customPath;
+    }
+  }
+  if (!robotsContent) {
+    for (const searchPath of searchPaths) {
+      const fullPath = join(projectDir, searchPath);
+      if (existsSync(fullPath)) {
+        robotsContent = readFileSync(fullPath, "utf-8");
+        foundPath = fullPath;
+        break;
+      }
+    }
+  }
+  if (!robotsContent) {
+    return null;
+  }
+  return {
+    parser: createRobotsParser(robotsContent),
+    path: foundPath || ""
+  };
+}
+function isPathAllowedByRobots(path, robots) {
+  if (!robots) {
+    return true;
+  }
+  const normalizedPath = path.startsWith("/") ? path : `/${path}`;
+  const testUrl = `https://robots.invalid${normalizedPath}`;
+  const isAllowed = robots.isAllowed(testUrl, "*");
+  return isAllowed !== false;
+}
+
+// src/utils/pathFilter.ts
+function shouldIncludePath(pathname, robots, excludePatterns, respectRobotsTxt) {
+  if (pathname.includes("[") && pathname.includes("]")) {
+    return { included: false, reason: "dynamic-route" };
+  }
+  if (pathname.startsWith("/_not-found") || pathname.startsWith("/_global-error")) {
+    return { included: false, reason: "internal-route" };
+  }
+  if (pathname.endsWith(".rsc")) {
+    return { included: false, reason: "rsc-file" };
+  }
+  if (pathname.includes(".segments/")) {
+    return { included: false, reason: "segment-file" };
+  }
+  if (pathname.match(/\.(ico|png|jpg|jpeg|svg|gif|webp|txt|xml|json|css|js|woff|woff2|ttf|eot)$/i)) {
+    return { included: false, reason: "static-asset" };
+  }
+  if (respectRobotsTxt && !isPathAllowedByRobots(pathname, robots)) {
+    return { included: false, reason: "robots-txt" };
+  }
+  if (matchesExcludePattern(pathname, excludePatterns)) {
+    return { included: false, reason: "exclude-pattern" };
+  }
+  return { included: true, reason: "included" };
+}
+
+// src/utils/pathUtils.ts
+import { sep } from "path";
+var artifactPrefixes = [
+  "server/pages/",
+  "server/app/",
+  "static/chunks/app/",
+  "static/chunks/pages/",
+  "static/",
+  "server/"
+];
+function filePathToPathname(filePath) {
+  let pathname = filePath.split(sep).join("/");
+  for (const prefix of artifactPrefixes) {
+    if (pathname.startsWith(prefix)) {
+      pathname = pathname.substring(prefix.length);
+      break;
+    }
+  }
+  pathname = pathname.replace(/\.html?$/, "");
+  if (pathname === "index" || pathname === "") {
+    return "/";
+  }
+  if (pathname.endsWith("/index")) {
+    pathname = pathname.slice(0, -5);
+  }
+  if (!pathname.startsWith("/")) {
+    pathname = "/" + pathname;
+  }
+  return pathname;
+}
+
+// src/parseHtml.ts
+import { loggers } from "@peam-ai/logger";
+import { JSDOM } from "jsdom";
+
+// src/parsers/cssSelectorParser.ts
+var CssSelectorParser = class {
+  parse(document, options) {
+    var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j;
+    const title = (_a = document.querySelector("title")) == null ? void 0 : _a.textContent;
+    const description = (_b = document.querySelector('meta[name="description"]')) == null ? void 0 : _b.getAttribute("content");
+    const content = document.body.innerHTML;
+    const bodyClone = document.body.cloneNode(true);
+    bodyClone.querySelectorAll("script, style, noscript, iframe, [hidden], nav, header, footer, aside").forEach((el) => el.remove());
+    const textContent = (_c = bodyClone.textContent) == null ? void 0 : _c.replace(/\s+/g, " ").trim();
+    if (!title || title.trim().length === 0 || !textContent || textContent.trim().length === 0 || !content || content.trim().length === 0 || !description || description.trim().length === 0) {
+      return void 0;
+    }
+    const page = {
+      title,
+      description,
+      content,
+      textContent,
+      contentLength: textContent.length
+    };
+    const htmlLang = document.documentElement.getAttribute("lang");
+    if (htmlLang) {
+      page.language = htmlLang;
+    }
+    const dir = document.documentElement.getAttribute("dir");
+    if (dir) {
+      page.direction = dir;
+    }
+    const ogTitle = (_d = document.querySelector('meta[property="og:title"]')) == null ? void 0 : _d.getAttribute("content");
+    if (ogTitle && ogTitle !== page.title) {
+      page.ogTitle = ogTitle;
+    }
+    const ogDescription = (_e = document.querySelector('meta[property="og:description"]')) == null ? void 0 : _e.getAttribute("content");
+    if (ogDescription) {
+      page.ogDescription = ogDescription;
+    }
+    const ogImage = (_f = document.querySelector('meta[property="og:image"]')) == null ? void 0 : _f.getAttribute("content");
+    if (ogImage) {
+      page.ogImage = ogImage;
+    }
+    const ogSiteName = (_g = document.querySelector('meta[property="og:site_name"]')) == null ? void 0 : _g.getAttribute("content");
+    if (ogSiteName) {
+      page.siteName = ogSiteName;
+    }
+    const author = (_h = document.querySelector('meta[name="author"]')) == null ? void 0 : _h.getAttribute("content");
+    if (author) {
+      page.author = author;
+    }
+    const keywords = (_i = document.querySelector('meta[name="keywords"]')) == null ? void 0 : _i.getAttribute("content");
+    if (keywords) {
+      page.keywords = keywords.split(",").map((k) => k.trim()).filter((k) => k);
+    }
+    const publishedTime = (_j = document.querySelector('meta[property="article:published_time"]')) == null ? void 0 : _j.getAttribute("content");
+    if (publishedTime) {
+      page.publishedTime = publishedTime;
+    }
+    const headings = Array.from(document.querySelectorAll("h1, h2, h3, h4, h5, h6")).map((h) => {
+      var _a2;
+      return (_a2 = h.textContent) == null ? void 0 : _a2.trim();
+    }).filter((h) => !!h && h.length > 0);
+    if (headings.length > 0) {
+      page.headings = headings;
+    }
+    if ((options == null ? void 0 : options.extractLinks) !== false) {
+      const links = Array.from(document.querySelectorAll("a[href]")).map((a) => a.getAttribute("href")).filter((href) => {
+        if (!href) return false;
+        return href.startsWith("/") && !href.startsWith("/_next") && !href.startsWith("/_");
+      });
+      if (links.length > 0) {
+        page.internalLinks = Array.from(new Set(links));
+      }
+      const externalLinks = Array.from(document.querySelectorAll("a[href]")).map((a) => a.getAttribute("href")).filter((href) => {
+        if (!href) return false;
+        return href.startsWith("http://") || href.startsWith("https://");
+      });
+      if (externalLinks.length > 0) {
+        page.externalLinks = Array.from(new Set(externalLinks));
+      }
+    }
+    if ((options == null ? void 0 : options.extractImages) !== false) {
+      const images = Array.from(document.querySelectorAll("img[src], img[data-nimg], img[data-src]")).map((img) => {
+        const src = img.getAttribute("src") || img.getAttribute("data-src") || img.getAttribute("data-nimg");
+        const alt = img.getAttribute("alt");
+        return { src, alt };
+      }).filter((img) => img.src && !img.src.startsWith("data:"));
+      if (images.length > 0) {
+        page.images = images.map((img) => img.src || "").filter((src) => src);
+        page.imageAlts = images.map((img) => img.alt || "");
+      }
+    }
+    return page;
+  }
+};
+
+// src/parsers/readabilityParser.ts
+import { Readability } from "@mozilla/readability";
+var ReadabilityParser = class {
+  parse(document, options) {
+    const { keepClasses, disableJSONLD } = options || {};
+    const reader = new Readability(document, {
+      keepClasses,
+      disableJSONLD
+    });
+    const parsedPage = reader.parse();
+    if (parsedPage) {
+      const page = {
+        title: parsedPage.title,
+        description: parsedPage.excerpt,
+        content: parsedPage.content,
+        textContent: parsedPage.textContent,
+        contentLength: parsedPage.length,
+        author: parsedPage.byline,
+        direction: parsedPage.dir,
+        language: parsedPage.lang,
+        siteName: parsedPage.siteName,
+        publishedTime: parsedPage.publishedTime
+      };
+      return page;
+    }
+    return void 0;
+  }
+};
+
+// src/parseHtml.ts
+var log = loggers.parser;
+function parseHTML(html, options = {}) {
+  if (!html || html.trim().length === 0) {
+    log.warn("Empty or invalid HTML input");
+    return void 0;
+  }
+  log.debug("Starting parse with options", options);
+  const dom = new JSDOM(html);
+  const document = dom.window.document;
+  const cssSelectorParser = new CssSelectorParser();
+  const cssSelectorStructuredPage = cssSelectorParser.parse(document, options);
+  const readabilityParser = new ReadabilityParser();
+  const readabilityStructuredPage = readabilityParser.parse(document, options);
+  if (!cssSelectorStructuredPage && !readabilityStructuredPage) {
+    log.warn("Failed to extract content");
+    return void 0;
+  }
+  return __spreadValues(__spreadValues(__spreadValues({}, {
+    title: "",
+    description: "",
+    content: "",
+    textContent: ""
+  }), cssSelectorStructuredPage), readabilityStructuredPage);
+}
+export {
+  createRobotsParser,
+  filePathToPathname,
+  isPathAllowedByRobots,
+  loadRobotsTxt,
+  matchesExcludePattern,
+  parseHTML,
+  shouldIncludePath
+};
+//# sourceMappingURL=index.mjs.map

package/dist/index.mjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/utils/excludePatterns.ts","../src/utils/robotsParser.ts","../src/utils/pathFilter.ts","../src/utils/pathUtils.ts","../src/parseHtml.ts","../src/parsers/cssSelectorParser.ts","../src/parsers/readabilityParser.ts"],"sourcesContent":["import { isMatch } from 'matcher';\n\n/**\n * Checks if a path matches any of the given wildcard patterns\n * Uses the matcher library which supports:\n * - * (matches any characters except /)\n * - ** (matches any characters including /)\n * - ? (matches single character)\n * - ! (negation)\n * - [] (character ranges)\n */\nexport function matchesExcludePattern(path: string, patterns: string[]): boolean {\n  if (!patterns || patterns.length === 0) {\n    return false;\n  }\n\n  const normalize = (p: string) => (p.startsWith('/') ? p : `/${p}`);\n  const normalizedPath = normalize(path);\n  const normalizedPatterns = patterns.map(normalize);\n\n  return isMatch(normalizedPath, normalizedPatterns);\n}\n","import { existsSync, readFileSync } from 'fs';\nimport { join } from 'path';\nimport robotsParser from 'robots-parser';\n\nexport type RobotsParser = ReturnType<typeof robotsParser>;\n\nexport interface RobotsTxtResult {\n  parser: RobotsParser;\n  path: string;\n}\n\nexport function createRobotsParser(content: string): RobotsParser {\n  return robotsParser('https://robots.invalid/robots.txt', content);\n}\n\n/**\n * Loads and parses robots.txt from custom path or standard locations\n * Returns the parser and the path where robots.txt was found\n */\nexport function loadRobotsTxt(\n  projectDir: string,\n  searchPaths: string[],\n  robotsTxtPath?: string\n): RobotsTxtResult | null {\n  let robotsContent: string | null = null;\n  let foundPath: string | null = null;\n\n  if (robotsTxtPath) {\n    const customPath = join(projectDir, robotsTxtPath);\n    if (existsSync(customPath)) {\n      robotsContent = readFileSync(customPath, 'utf-8');\n      foundPath = customPath;\n    }\n  }\n\n  if (!robotsContent) {\n    for (const searchPath of searchPaths) {\n      const fullPath = join(projectDir, searchPath);\n      if (existsSync(fullPath)) {\n        robotsContent = readFileSync(fullPath, 'utf-8');\n        foundPath = fullPath;\n        break;\n      }\n    }\n  }\n\n  if (!robotsContent) {\n    return null;\n  }\n\n  return {\n    parser: createRobotsParser(robotsContent),\n    path: foundPath || '',\n  };\n}\n\nexport function isPathAllowedByRobots(path: string, robots: RobotsParser | null): boolean {\n  if (!robots) {\n    return true;\n  }\n\n  const normalizedPath = path.startsWith('/') ? path : `/${path}`;\n  const testUrl = `https://robots.invalid${normalizedPath}`;\n  const isAllowed = robots.isAllowed(testUrl, '*');\n\n  return isAllowed !== false;\n}\n","import { matchesExcludePattern } from './excludePatterns';\nimport type { RobotsParser } from './robotsParser';\nimport { isPathAllowedByRobots } from './robotsParser';\n\nexport type PathFilterReason =\n  | 'included'\n  | 'dynamic-route'\n  | 'internal-route'\n  | 'rsc-file'\n  | 'segment-file'\n  | 'static-asset'\n  | 'robots-txt'\n  | 'exclude-pattern';\n\nexport interface PathFilterResult {\n  included: boolean;\n  reason: PathFilterReason;\n}\n\n/**\n * Determines if a pathname should be included in the index\n * Returns both the decision and the reason for exclusion\n */\nexport function shouldIncludePath(\n  pathname: string,\n  robots: RobotsParser | null,\n  excludePatterns: string[],\n  respectRobotsTxt: boolean\n): PathFilterResult {\n  // Exclude routes with dynamic parameters (e.g., /session/[session_id])\n  if (pathname.includes('[') && pathname.includes(']')) {\n    return { included: false, reason: 'dynamic-route' };\n  }\n\n  // Exclude Next.js internal routes\n  if (pathname.startsWith('/_not-found') || pathname.startsWith('/_global-error')) {\n    return { included: false, reason: 'internal-route' };\n  }\n\n  // Exclude RSC files\n  if (pathname.endsWith('.rsc')) {\n    return { included: false, reason: 'rsc-file' };\n  }\n\n  // Exclude segment files\n  if (pathname.includes('.segments/')) {\n    return { included: false, reason: 'segment-file' };\n  }\n\n  // Exclude static assets\n  if (pathname.match(/\\.(ico|png|jpg|jpeg|svg|gif|webp|txt|xml|json|css|js|woff|woff2|ttf|eot)$/i)) {\n    return { included: false, reason: 'static-asset' };\n  }\n\n  // Check robots.txt rules\n  if (respectRobotsTxt && !isPathAllowedByRobots(pathname, robots)) {\n    return { included: false, reason: 'robots-txt' };\n  }\n\n  // Check user-defined exclude patterns\n  if (matchesExcludePattern(pathname, excludePatterns)) {\n    return { included: false, reason: 'exclude-pattern' };\n  }\n\n  return { included: true, reason: 'included' };\n}\n","import { sep } from 'path';\n\nconst artifactPrefixes = [\n  'server/pages/',\n  'server/app/',\n  'static/chunks/app/',\n  'static/chunks/pages/',\n  'static/',\n  'server/',\n];\n\n/**\n * Convert file path to URL pathname\n *\n * Examples:\n *   index.html -> /\n *   about.html -> /about\n *   about/index.html -> /about/\n *   blog/post-1.html -> /blog/post-1\n *   blog/post-1/index.html -> /blog/post-1/\n *   server/pages/contact.html -> /contact\n *   server/app/about.html -> /about\n */\nexport function filePathToPathname(filePath: string): string {\n  let pathname = filePath.split(sep).join('/');\n\n  for (const prefix of artifactPrefixes) {\n    if (pathname.startsWith(prefix)) {\n      pathname = pathname.substring(prefix.length);\n      break;\n    }\n  }\n\n  pathname = pathname.replace(/\\.html?$/, '');\n\n  if (pathname === 'index' || pathname === '') {\n    return '/';\n  }\n\n  if (pathname.endsWith('/index')) {\n    pathname = pathname.slice(0, -5);\n  }\n\n  if (!pathname.startsWith('/')) {\n    pathname = '/' + pathname;\n  }\n\n  return pathname;\n}\n","import { loggers } from '@peam-ai/logger';\nimport { JSDOM } from 'jsdom';\nimport { CssSelectorParser } from './parsers/cssSelectorParser';\nimport { ParseOptions } from './parsers/parser';\nimport { ReadabilityParser } from './parsers/readabilityParser';\nimport { StructuredPage } from './structuredPage';\n\nconst log = loggers.parser;\n\n/**\n * Parse HTML content and convert it to a StructuredPage\n * @param html - HTML string to parse\n * @param options - Parsing options\n * @returns StructuredPage or undefined if parsing fails\n */\nexport function parseHTML(html: string, options: ParseOptions = {}): StructuredPage | undefined {\n  if (!html || html.trim().length === 0) {\n    log.warn('Empty or invalid HTML input');\n    return undefined;\n  }\n\n  log.debug('Starting parse with options', options);\n  const dom = new JSDOM(html);\n  const document = dom.window.document;\n\n  const cssSelectorParser = new CssSelectorParser();\n  const cssSelectorStructuredPage = cssSelectorParser.parse(document, options);\n  const readabilityParser = new ReadabilityParser();\n  const readabilityStructuredPage = readabilityParser.parse(document, options);\n\n  if (!cssSelectorStructuredPage && !readabilityStructuredPage) {\n    log.warn('Failed to extract content');\n    return undefined;\n  }\n\n  // Merge results, prioritizing Readability data\n  return {\n    ...{\n      title: '',\n      description: '',\n      content: '',\n      textContent: '',\n    },\n    ...cssSelectorStructuredPage,\n    ...readabilityStructuredPage,\n  };\n}\n","import { StructuredPage } from '../structuredPage';\nimport { ParseOptions, Parser } from './parser';\n\nexport class CssSelectorParser implements Parser {\n  parse(document: Document, options?: ParseOptions): StructuredPage | undefined {\n    const title = document.querySelector('title')?.textContent;\n    const description = document.querySelector('meta[name=\"description\"]')?.getAttribute('content');\n    const content = document.body.innerHTML;\n\n    const bodyClone = document.body.cloneNode(true) as HTMLElement;\n    bodyClone\n      .querySelectorAll('script, style, noscript, iframe, [hidden], nav, header, footer, aside')\n      .forEach((el) => el.remove());\n\n    const textContent = bodyClone.textContent?.replace(/\\s+/g, ' ').trim();\n\n    if (\n      !title ||\n      title.trim().length === 0 ||\n      !textContent ||\n      textContent.trim().length === 0 ||\n      !content ||\n      content.trim().length === 0 ||\n      !description ||\n      description.trim().length === 0\n    ) {\n      return undefined;\n    }\n\n    const page: StructuredPage = {\n      title,\n      description,\n      content,\n      textContent,\n      contentLength: textContent.length,\n    };\n\n    const htmlLang = document.documentElement.getAttribute('lang');\n    if (htmlLang) {\n      page.language = htmlLang;\n    }\n\n    const dir = document.documentElement.getAttribute('dir');\n    if (dir) {\n      page.direction = dir;\n    }\n\n    const ogTitle = document.querySelector('meta[property=\"og:title\"]')?.getAttribute('content');\n    if (ogTitle && ogTitle !== page.title) {\n      page.ogTitle = ogTitle;\n    }\n\n    const ogDescription = document.querySelector('meta[property=\"og:description\"]')?.getAttribute('content');\n    if (ogDescription) {\n      page.ogDescription = ogDescription;\n    }\n\n    const ogImage = document.querySelector('meta[property=\"og:image\"]')?.getAttribute('content');\n    if (ogImage) {\n      page.ogImage = ogImage;\n    }\n\n    const ogSiteName = document.querySelector('meta[property=\"og:site_name\"]')?.getAttribute('content');\n    if (ogSiteName) {\n      page.siteName = ogSiteName;\n    }\n\n    const author = document.querySelector('meta[name=\"author\"]')?.getAttribute('content');\n    if (author) {\n      page.author = author;\n    }\n\n    const keywords = document.querySelector('meta[name=\"keywords\"]')?.getAttribute('content');\n    if (keywords) {\n      page.keywords = keywords\n        .split(',')\n        .map((k) => k.trim())\n        .filter((k) => k);\n    }\n\n    const publishedTime = document.querySelector('meta[property=\"article:published_time\"]')?.getAttribute('content');\n    if (publishedTime) {\n      page.publishedTime = publishedTime;\n    }\n\n    const headings = Array.from(document.querySelectorAll('h1, h2, h3, h4, h5, h6'))\n      .map((h) => h.textContent?.trim())\n      .filter((h): h is string => !!h && h.length > 0);\n\n    if (headings.length > 0) {\n      page.headings = headings;\n    }\n\n    // Extract links if enabled\n    if (options?.extractLinks !== false) {\n      // Extract internal links\n      const links = Array.from(document.querySelectorAll('a[href]'))\n        .map((a) => a.getAttribute('href'))\n        .filter((href): href is string => {\n          if (!href) return false;\n          return href.startsWith('/') && !href.startsWith('/_next') && !href.startsWith('/_');\n        });\n\n      if (links.length > 0) {\n        page.internalLinks = Array.from(new Set(links));\n      }\n\n      // Extract external links\n      const externalLinks = Array.from(document.querySelectorAll('a[href]'))\n        .map((a) => a.getAttribute('href'))\n        .filter((href): href is string => {\n          if (!href) return false;\n          return href.startsWith('http://') || href.startsWith('https://');\n        });\n\n      if (externalLinks.length > 0) {\n        page.externalLinks = Array.from(new Set(externalLinks));\n      }\n    }\n\n    // Extract images if enabled\n    if (options?.extractImages !== false) {\n      const images = Array.from(document.querySelectorAll('img[src], img[data-nimg], img[data-src]'))\n        .map((img) => {\n          const src = img.getAttribute('src') || img.getAttribute('data-src') || img.getAttribute('data-nimg');\n          const alt = img.getAttribute('alt');\n          return { src, alt };\n        })\n        .filter((img) => img.src && !img.src.startsWith('data:'));\n\n      if (images.length > 0) {\n        page.images = images.map((img) => img.src || '').filter((src) => src);\n        page.imageAlts = images.map((img) => img.alt || '');\n      }\n    }\n\n    return page;\n  }\n}\n","import { Readability } from '@mozilla/readability';\nimport { StructuredPage } from '../structuredPage';\nimport { ParseOptions, Parser } from './parser';\n\nexport class ReadabilityParser implements Parser {\n  parse(document: Document, options?: ParseOptions): StructuredPage | undefined {\n    const { keepClasses, disableJSONLD } = options || {};\n\n    const reader = new Readability(document, {\n      keepClasses,\n      disableJSONLD,\n    });\n\n    const parsedPage = reader.parse();\n\n    if (parsedPage) {\n      const page: StructuredPage = {\n        title: parsedPage.title,\n        description: parsedPage.excerpt,\n        content: parsedPage.content,\n        textContent: parsedPage.textContent,\n        contentLength: parsedPage.length,\n        author: parsedPage.byline,\n        direction: parsedPage.dir,\n        language: parsedPage.lang,\n        siteName: parsedPage.siteName,\n        publishedTime: parsedPage.publishedTime,\n      };\n\n      return page;\n    }\n\n    return undefined;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA,SAAS,eAAe;AAWjB,SAAS,sBAAsB,MAAc,UAA6B;AAC/E,MAAI,CAAC,YAAY,SAAS,WAAW,GAAG;AACtC,WAAO;AAAA,EACT;AAEA,QAAM,YAAY,CAAC,MAAe,EAAE,WAAW,GAAG,IAAI,IAAI,IAAI,CAAC;AAC/D,QAAM,iBAAiB,UAAU,IAAI;AACrC,QAAM,qBAAqB,SAAS,IAAI,SAAS;AAEjD,SAAO,QAAQ,gBAAgB,kBAAkB;AACnD;;;ACrBA,SAAS,YAAY,oBAAoB;AACzC,SAAS,YAAY;AACrB,OAAO,kBAAkB;AASlB,SAAS,mBAAmB,SAA+B;AAChE,SAAO,aAAa,qCAAqC,OAAO;AAClE;AAMO,SAAS,cACd,YACA,aACA,eACwB;AACxB,MAAI,gBAA+B;AACnC,MAAI,YAA2B;AAE/B,MAAI,eAAe;AACjB,UAAM,aAAa,KAAK,YAAY,aAAa;AACjD,QAAI,WAAW,UAAU,GAAG;AAC1B,sBAAgB,aAAa,YAAY,OAAO;AAChD,kBAAY;AAAA,IACd;AAAA,EACF;AAEA,MAAI,CAAC,eAAe;AAClB,eAAW,cAAc,aAAa;AACpC,YAAM,WAAW,KAAK,YAAY,UAAU;AAC5C,UAAI,WAAW,QAAQ,GAAG;AACxB,wBAAgB,aAAa,UAAU,OAAO;AAC9C,oBAAY;AACZ;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,MAAI,CAAC,eAAe;AAClB,WAAO;AAAA,EACT;AAEA,SAAO;AAAA,IACL,QAAQ,mBAAmB,aAAa;AAAA,IACxC,MAAM,aAAa;AAAA,EACrB;AACF;AAEO,SAAS,sBAAsB,MAAc,QAAsC;AACxF,MAAI,CAAC,QAAQ;AACX,WAAO;AAAA,EACT;AAEA,QAAM,iBAAiB,KAAK,WAAW,GAAG,IAAI,OAAO,IAAI,IAAI;AAC7D,QAAM,UAAU,yBAAyB,cAAc;AACvD,QAAM,YAAY,OAAO,UAAU,SAAS,GAAG;AAE/C,SAAO,cAAc;AACvB;;;AC3CO,SAAS,kBACd,UACA,QACA,iBACA,kBACkB;AAElB,MAAI,SAAS,SAAS,GAAG,KAAK,SAAS,SAAS,GAAG,GAAG;AACpD,WAAO,EAAE,UAAU,OAAO,QAAQ,gBAAgB;AAAA,EACpD;AAGA,MAAI,SAAS,WAAW,aAAa,KAAK,SAAS,WAAW,gBAAgB,GAAG;AAC/E,WAAO,EAAE,UAAU,OAAO,QAAQ,iBAAiB;AAAA,EACrD;AAGA,MAAI,SAAS,SAAS,MAAM,GAAG;AAC7B,WAAO,EAAE,UAAU,OAAO,QAAQ,WAAW;AAAA,EAC/C;AAGA,MAAI,SAAS,SAAS,YAAY,GAAG;AACnC,WAAO,EAAE,UAAU,OAAO,QAAQ,eAAe;AAAA,EACnD;AAGA,MAAI,SAAS,MAAM,4EAA4E,GAAG;AAChG,WAAO,EAAE,UAAU,OAAO,QAAQ,eAAe;AAAA,EACnD;AAGA,MAAI,oBAAoB,CAAC,sBAAsB,UAAU,MAAM,GAAG;AAChE,WAAO,EAAE,UAAU,OAAO,QAAQ,aAAa;AAAA,EACjD;AAGA,MAAI,sBAAsB,UAAU,eAAe,GAAG;AACpD,WAAO,EAAE,UAAU,OAAO,QAAQ,kBAAkB;AAAA,EACtD;AAEA,SAAO,EAAE,UAAU,MAAM,QAAQ,WAAW;AAC9C;;;ACjEA,SAAS,WAAW;AAEpB,IAAM,mBAAmB;AAAA,EACvB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAcO,SAAS,mBAAmB,UAA0B;AAC3D,MAAI,WAAW,SAAS,MAAM,GAAG,EAAE,KAAK,GAAG;AAE3C,aAAW,UAAU,kBAAkB;AACrC,QAAI,SAAS,WAAW,MAAM,GAAG;AAC/B,iBAAW,SAAS,UAAU,OAAO,MAAM;AAC3C;AAAA,IACF;AAAA,EACF;AAEA,aAAW,SAAS,QAAQ,YAAY,EAAE;AAE1C,MAAI,aAAa,WAAW,aAAa,IAAI;AAC3C,WAAO;AAAA,EACT;AAEA,MAAI,SAAS,SAAS,QAAQ,GAAG;AAC/B,eAAW,SAAS,MAAM,GAAG,EAAE;AAAA,EACjC;AAEA,MAAI,CAAC,SAAS,WAAW,GAAG,GAAG;AAC7B,eAAW,MAAM;AAAA,EACnB;AAEA,SAAO;AACT;;;AChDA,SAAS,eAAe;AACxB,SAAS,aAAa;;;ACEf,IAAM,oBAAN,MAA0C;AAAA,EAC/C,MAAM,UAAoB,SAAoD;AAJhF;AAKI,UAAM,SAAQ,cAAS,cAAc,OAAO,MAA9B,mBAAiC;AAC/C,UAAM,eAAc,cAAS,cAAc,0BAA0B,MAAjD,mBAAoD,aAAa;AACrF,UAAM,UAAU,SAAS,KAAK;AAE9B,UAAM,YAAY,SAAS,KAAK,UAAU,IAAI;AAC9C,cACG,iBAAiB,uEAAuE,EACxF,QAAQ,CAAC,OAAO,GAAG,OAAO,CAAC;AAE9B,UAAM,eAAc,eAAU,gBAAV,mBAAuB,QAAQ,QAAQ,KAAK;AAEhE,QACE,CAAC,SACD,MAAM,KAAK,EAAE,WAAW,KACxB,CAAC,eACD,YAAY,KAAK,EAAE,WAAW,KAC9B,CAAC,WACD,QAAQ,KAAK,EAAE,WAAW,KAC1B,CAAC,eACD,YAAY,KAAK,EAAE,WAAW,GAC9B;AACA,aAAO;AAAA,IACT;AAEA,UAAM,OAAuB;AAAA,MAC3B;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,eAAe,YAAY;AAAA,IAC7B;AAEA,UAAM,WAAW,SAAS,gBAAgB,aAAa,MAAM;AAC7D,QAAI,UAAU;AACZ,WAAK,WAAW;AAAA,IAClB;AAEA,UAAM,MAAM,SAAS,gBAAgB,aAAa,KAAK;AACvD,QAAI,KAAK;AACP,WAAK,YAAY;AAAA,IACnB;AAEA,UAAM,WAAU,cAAS,cAAc,2BAA2B,MAAlD,mBAAqD,aAAa;AAClF,QAAI,WAAW,YAAY,KAAK,OAAO;AACrC,WAAK,UAAU;AAAA,IACjB;AAEA,UAAM,iBAAgB,cAAS,cAAc,iCAAiC,MAAxD,mBAA2D,aAAa;AAC9F,QAAI,eAAe;AACjB,WAAK,gBAAgB;AAAA,IACvB;AAEA,UAAM,WAAU,cAAS,cAAc,2BAA2B,MAAlD,mBAAqD,aAAa;AAClF,QAAI,SAAS;AACX,WAAK,UAAU;AAAA,IACjB;AAEA,UAAM,cAAa,cAAS,cAAc,+BAA+B,MAAtD,mBAAyD,aAAa;AACzF,QAAI,YAAY;AACd,WAAK,WAAW;AAAA,IAClB;AAEA,UAAM,UAAS,cAAS,cAAc,qBAAqB,MAA5C,mBAA+C,aAAa;AAC3E,QAAI,QAAQ;AACV,WAAK,SAAS;AAAA,IAChB;AAEA,UAAM,YAAW,cAAS,cAAc,uBAAuB,MAA9C,mBAAiD,aAAa;AAC/E,QAAI,UAAU;AACZ,WAAK,WAAW,SACb,MAAM,GAAG,EACT,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,EACnB,OAAO,CAAC,MAAM,CAAC;AAAA,IACpB;AAEA,UAAM,iBAAgB,cAAS,cAAc,yCAAyC,MAAhE,mBAAmE,aAAa;AACtG,QAAI,eAAe;AACjB,WAAK,gBAAgB;AAAA,IACvB;AAEA,UAAM,WAAW,MAAM,KAAK,SAAS,iBAAiB,wBAAwB,CAAC,EAC5E,IAAI,CAAC,MAAG;AAtFf,UAAAA;AAsFkB,cAAAA,MAAA,EAAE,gBAAF,gBAAAA,IAAe;AAAA,KAAM,EAChC,OAAO,CAAC,MAAmB,CAAC,CAAC,KAAK,EAAE,SAAS,CAAC;AAEjD,QAAI,SAAS,SAAS,GAAG;AACvB,WAAK,WAAW;AAAA,IAClB;AAGA,SAAI,mCAAS,kBAAiB,OAAO;AAEnC,YAAM,QAAQ,MAAM,KAAK,SAAS,iBAAiB,SAAS,CAAC,EAC1D,IAAI,CAAC,MAAM,EAAE,aAAa,MAAM,CAAC,EACjC,OAAO,CAAC,SAAyB;AAChC,YAAI,CAAC,KAAM,QAAO;AAClB,eAAO,KAAK,WAAW,GAAG,KAAK,CAAC,KAAK,WAAW,QAAQ,KAAK,CAAC,KAAK,WAAW,IAAI;AAAA,MACpF,CAAC;AAEH,UAAI,MAAM,SAAS,GAAG;AACpB,aAAK,gBAAgB,MAAM,KAAK,IAAI,IAAI,KAAK,CAAC;AAAA,MAChD;AAGA,YAAM,gBAAgB,MAAM,KAAK,SAAS,iBAAiB,SAAS,CAAC,EAClE,IAAI,CAAC,MAAM,EAAE,aAAa,MAAM,CAAC,EACjC,OAAO,CAAC,SAAyB;AAChC,YAAI,CAAC,KAAM,QAAO;AAClB,eAAO,KAAK,WAAW,SAAS,KAAK,KAAK,WAAW,UAAU;AAAA,MACjE,CAAC;AAEH,UAAI,cAAc,SAAS,GAAG;AAC5B,aAAK,gBAAgB,MAAM,KAAK,IAAI,IAAI,aAAa,CAAC;AAAA,MACxD;AAAA,IACF;AAGA,SAAI,mCAAS,mBAAkB,OAAO;AACpC,YAAM,SAAS,MAAM,KAAK,SAAS,iBAAiB,yCAAyC,CAAC,EAC3F,IAAI,CAAC,QAAQ;AACZ,cAAM,MAAM,IAAI,aAAa,KAAK,KAAK,IAAI,aAAa,UAAU,KAAK,IAAI,aAAa,WAAW;AACnG,cAAM,MAAM,IAAI,aAAa,KAAK;AAClC,eAAO,EAAE,KAAK,IAAI;AAAA,MACpB,CAAC,EACA,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,IAAI,IAAI,WAAW,OAAO,CAAC;AAE1D,UAAI,OAAO,SAAS,GAAG;AACrB,aAAK,SAAS,OAAO,IAAI,CAAC,QAAQ,IAAI,OAAO,EAAE,EAAE,OAAO,CAAC,QAAQ,GAAG;AACpE,aAAK,YAAY,OAAO,IAAI,CAAC,QAAQ,IAAI,OAAO,EAAE;AAAA,MACpD;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AACF;;;AC1IA,SAAS,mBAAmB;AAIrB,IAAM,oBAAN,MAA0C;AAAA,EAC/C,MAAM,UAAoB,SAAoD;AAC5E,UAAM,EAAE,aAAa,cAAc,IAAI,WAAW,CAAC;AAEnD,UAAM,SAAS,IAAI,YAAY,UAAU;AAAA,MACvC;AAAA,MACA;AAAA,IACF,CAAC;AAED,UAAM,aAAa,OAAO,MAAM;AAEhC,QAAI,YAAY;AACd,YAAM,OAAuB;AAAA,QAC3B,OAAO,WAAW;AAAA,QAClB,aAAa,WAAW;AAAA,QACxB,SAAS,WAAW;AAAA,QACpB,aAAa,WAAW;AAAA,QACxB,eAAe,WAAW;AAAA,QAC1B,QAAQ,WAAW;AAAA,QACnB,WAAW,WAAW;AAAA,QACtB,UAAU,WAAW;AAAA,QACrB,UAAU,WAAW;AAAA,QACrB,eAAe,WAAW;AAAA,MAC5B;AAEA,aAAO;AAAA,IACT;AAEA,WAAO;AAAA,EACT;AACF;;;AF3BA,IAAM,MAAM,QAAQ;AAQb,SAAS,UAAU,MAAc,UAAwB,CAAC,GAA+B;AAC9F,MAAI,CAAC,QAAQ,KAAK,KAAK,EAAE,WAAW,GAAG;AACrC,QAAI,KAAK,6BAA6B;AACtC,WAAO;AAAA,EACT;AAEA,MAAI,MAAM,+BAA+B,OAAO;AAChD,QAAM,MAAM,IAAI,MAAM,IAAI;AAC1B,QAAM,WAAW,IAAI,OAAO;AAE5B,QAAM,oBAAoB,IAAI,kBAAkB;AAChD,QAAM,4BAA4B,kBAAkB,MAAM,UAAU,OAAO;AAC3E,QAAM,oBAAoB,IAAI,kBAAkB;AAChD,QAAM,4BAA4B,kBAAkB,MAAM,UAAU,OAAO;AAE3E,MAAI,CAAC,6BAA6B,CAAC,2BAA2B;AAC5D,QAAI,KAAK,2BAA2B;AACpC,WAAO;AAAA,EACT;AAGA,SAAO,iDACF;AAAA,IACD,OAAO;AAAA,IACP,aAAa;AAAA,IACb,SAAS;AAAA,IACT,aAAa;AAAA,EACf,IACG,4BACA;AAEP;","names":["_a"]}

package/package.json

@@ -0,0 +1,49 @@
+{
+  "name": "@peam-ai/parser",
+  "description": "Content parser for extracting page metadata",
+  "version": "0.1.1",
+  "main": "./dist/index.js",
+  "module": "./dist/index.mjs",
+  "types": "./dist/index.d.ts",
+  "homepage": "https://peam.ai",
+  "author": "Afshin Mehrabani",
+  "license": "Apache-2.0",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/peam-ai/peam.git",
+    "directory": "packages/parser"
+  },
+  "bugs": {
+    "url": "https://github.com/peam-ai/peam/issues"
+  },
+  "files": [
+    "dist/**/*"
+  ],
+  "exports": {
+    "./package.json": "./package.json",
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.mjs",
+      "require": "./dist/index.js"
+    }
+  },
+  "devDependencies": {
+    "@types/jsdom": "^27.0.0",
+    "tsup": "^8.2.4",
+    "typescript": "^5.5.4"
+  },
+  "dependencies": {
+    "@mozilla/readability": "^0.5.0",
+    "jsdom": "^27.3.0",
+    "matcher": "^6.0.0",
+    "robots-parser": "^3.0.1",
+    "@peam-ai/logger": "0.1.1"
+  },
+  "scripts": {
+    "build": "tsup",
+    "clean": "rm -rf dist",
+    "test:eslint": "eslint \"src/**/*.ts*\"",
+    "test:prettier": "prettier --check \"src/**/*.ts*\"",
+    "test:jest": "jest"
+  }
+}