npm - minipug - Versions diffs - 1.0.1 - Mend

minipug 1.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (5) hide show

package/LICENSE ADDED Viewed

@@ -0,0 +1,21 @@
+MIT License
+Copyright (c) 2025 ChromeAutopilot
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md ADDED Viewed

@@ -0,0 +1,54 @@
+# minipug
+Extract LLM-friendly representation of the DOM in [Pug](https://pugjs.org/) format.
+## Install
+```sh
+npm i -s minipug
+```
+## Usage
+```js
+import MiniPug from 'minipug'
+const minipug = new MiniPug(document)
+const pug = minipug.convert()
+```
+## Example
+Input:
+```html
+<html>
+  <head>
+    <title>Example Domain</title>
+    <meta charset="utf-8">
+    <meta http-equiv="Content-type" content="text/html; charset=utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <style type="text/css">
+      body {
+        background-color: #f0f0f2;
+        margin: 0;
+        padding: 0;
+      }
+    </style>
+  </head>
+  <body>
+    <h1>Hello World</h1>
+    <section class="example-section">
+      <p>This is an example.</p>
+      <button aria-label="Next page" class="z2iX5wAef9nHv">Next page</button>
+    </section>
+  </body>
+</html>
+```
+Output:
+```pug
+h1 Hello World
+section
+  p This is an example.
+  button(aria-label="Next page") Next page
+```

package/index.mjs ADDED Viewed

@@ -0,0 +1,332 @@
+export default class MiniPug {
+  constructor(document) {
+      this.document = document;
+      this.ignoreTags = new Set(['html', 'body']);
+      this.skipDescendantsTags = new Set(['noscript']);
+      this.whitelistedTags = new Set([
+          'nav', 'main', 'header', 'footer', 'aside',
+          'article', 'section',
+          'a', 'form', 'input', 'textarea', 'button', 'select', 'option',
+          'h1', 'h2', 'h3', 'h4', 'h5',
+          'ul', 'ol', 'li', 'dl', 'dt', 'dd',
+          'caption',
+          'pre', 'code',
+          'fieldset', 'legend',
+          'dialog', 'details', 'summary',
+          'iframe', 'br', 'hr'
+      ]);
+      this.tableTags = new Set(['table', 'tr', 'th', 'td']);
+      this.whitelistedClassSubstrings = new Set([
+          'up', 'down', 'left', 'right', 'arrow', 'caret', 'chevron', 'star',
+          'increase', 'decrease', 'plus', 'minus', 'expand', 'collapse', 'open', 'close',
+          'success', 'error', 'warning', 'valid', 'invalid',
+          'active', 'inactive', 'enabled', 'disabled', 'next', 'prev', 'previous', 'first', 'last'
+      ]);
+      this.whitelistedAttributes = new Set([
+          'href', 'target', 'download',
+          'action', 'method', 'type', 'name', 'value', 'placeholder', 'required', 'checked', 'selected',
+          'aria-label', 'aria-expanded', 'aria-hidden', 'aria-controls', 'aria-current',
+          'aria-describedby', 'aria-disabled', 'aria-haspopup', 'aria-invalid',
+          'aria-labelledby', 'aria-live', 'aria-pressed', 'aria-required', 'aria-selected',
+          'aria-checked', 'aria-valuenow', 'aria-valuemin', 'aria-valuemax', 'role', 'title', 'alt',
+          'disabled', 'readonly'
+      ]);
+      this.booleanAttributes = new Set([
+          'checked', 'selected', 'disabled', 'readonly', 'required'
+      ]);
+  }
+  isVisible(element) {
+      if (!(element instanceof Element)) return true;
+      if (element.tagName.toLowerCase() === 'body') return true; // Always consider body visible
+      const style = window.getComputedStyle(element);
+      const rect = element.getBoundingClientRect();
+      // If element has zero dimensions, check if it has visible absolutely positioned children
+      if (rect.width === 0 && rect.height === 0) {
+          for (const child of element.children) {
+              const childStyle = window.getComputedStyle(child);
+              if ((childStyle.position === 'absolute' || childStyle.position === 'fixed') &&
+                  this.isVisible(child)) {
+                  return true;
+              }
+          }
+      }
+      return !(
+          style.display === 'none' ||
+          style.visibility === 'hidden' ||
+          element.getAttribute('aria-hidden') === 'true' ||
+          (rect.width === 0 && rect.height === 0) ||
+          style.opacity === '0' ||
+          (
+              (style.position === 'absolute' || style.position === 'fixed') &&
+              (
+                  rect.left + rect.width < 0 ||
+                  rect.top + rect.height < 0 ||
+                  rect.left > window.innerWidth ||
+                  rect.top > window.innerHeight
+              )
+          )
+      );
+  }
+  // hasWhitelistedClass(element) {
+  //     const classAttr = element.getAttribute('class');
+  //     if (!classAttr) return false;
+  //     const normalizedClasses = classAttr.toLowerCase();
+  //     for (const substring of this.whitelistedClassSubstrings) {
+  //         if (normalizedClasses.includes(substring)) {
+  //             return true;
+  //         }
+  //     }
+  //     return false;
+  // }
+  findWhitelistedClasses(element, firstMatchOnly = false) {
+    const classAttr = element.getAttribute('class');
+    if (!classAttr) return [];
+    const matches = [];
+    const classes = classAttr.split(/\s+/);
+    const normalizedClasses = classes.map(c => c.toLowerCase());
+    for (const className of classes) {
+      const normalizedClass = className.toLowerCase();
+      for (const substring of this.whitelistedClassSubstrings) {
+        if (normalizedClass.includes(substring)) {
+          if (firstMatchOnly) return [className];
+          matches.push(className);
+          break; // Move to next className after finding a match
+        }
+      }
+    }
+    return matches;
+  }
+  hasWhitelistedClass(element) {
+      return this.findWhitelistedClasses(element, true).length > 0;
+  }
+  hasWhitelistedAttribute(element) {
+      for (const attr of element.attributes) {
+          if (this.whitelistedAttributes.has(attr.name)) {
+              if (this.booleanAttributes.has(attr.name)) {
+                  return true; // Boolean attributes are valid just by existing
+              }
+              if (attr.value.trim() !== '') {
+                  return true; // Non-boolean attributes need non-empty values
+              }
+          }
+      }
+      return false;
+  }
+  hasDirectTextNode(element) {
+      for (const child of element.childNodes) {
+          if (child.nodeType === Node.TEXT_NODE && child.textContent.trim()) {
+              return true;
+          }
+      }
+      return false;
+  }
+  hasIncludedDescendant(element) {
+      for (const child of element.children) {
+          if (this.shouldIncludeElement(child)) {
+              return true;
+          }
+          if (this.hasIncludedDescendant(child)) {
+              return true;
+          }
+      }
+      return false;
+  }
+  shouldIncludeElement(element) {
+      if (this.ignoreTags.has(element.tagName.toLowerCase())) {
+          return false;
+      }
+      // TODO: revisit tables
+      // const tagName = element.tagName.toLowerCase();
+      // if (this.tableTags.has(tagName)) {
+      //     // For table elements, check if they have any included descendants
+      //     return this.hasIncludedDescendant(element);
+      // }
+      const isWhitelistedTag = this.whitelistedTags.has(element.tagName.toLowerCase());
+      if (isWhitelistedTag) return true;
+      // For non-whitelisted tags, check if element has any meaningful content
+      const hasText = this.hasDirectTextNode(element);
+      if (hasText) return true;
+      const hasWhitelistedClass = this.hasWhitelistedClass(element);
+      if (hasWhitelistedClass) return true;
+      const hasWhitelistedAttribute = this.hasWhitelistedAttribute(element);
+      if (hasWhitelistedAttribute) return true;
+      // const hasIncludedChildren = this.hasIncludedDescendant(element);
+      // if (hasIncludedChildren) return true;
+      return false;
+  }
+  getCapturedValue(element) {
+      const tagName = element.tagName.toLowerCase();
+      const type = element.type?.toLowerCase();
+      if (tagName === 'input') {
+          if (type === 'checkbox' || type === 'radio') {
+              return element.checked;
+          }
+          return element.value;
+      }
+      if (tagName === 'textarea' || tagName === 'select') {
+          return element.value;
+      }
+      return null;
+  }
+  getAttributes(element) {
+      const attributes = {};
+      const tagName = element.tagName.toLowerCase();
+      for (const attr of element.attributes) {
+          if (!this.whitelistedAttributes.has(attr.name)) continue;
+          // Skip href attribute for img tags
+          if (['img', 'image'].includes(tagName) && attr.name === 'href') continue;
+          if (this.booleanAttributes.has(attr.name)) {
+              // For boolean attributes, include them only if they exist
+              attributes[attr.name] = true;
+          } else if (attr.value.trim() !== '') {
+              // For non-boolean attributes, include only if they have non-empty values
+              attributes[attr.name] = attr.value;
+          }
+      }
+      // Capture live DOM values
+      const capturedValue = this.getCapturedValue(element);
+      if (capturedValue !== null) {
+          if (typeof capturedValue === 'boolean') {
+              if (capturedValue) {
+                  attributes['checked'] = true;
+              }
+          } else if (capturedValue.trim() !== '') {
+              attributes['value'] = capturedValue;
+          }
+      }
+      return attributes;
+  }
+  onlyHasTextNodes(node) {
+      for (const child of node.childNodes) {
+          if (child.nodeType !== Node.TEXT_NODE) {
+              return false;
+          }
+      }
+      return true;
+  }
+  convertNode(node, depth = 0) {
+      if (node.nodeType !== Node.ELEMENT_NODE) {
+        return '';
+      }
+      if (!this.isVisible(node)) {
+        return '';
+      }
+      const tagName = node.tagName.toLowerCase();
+      if (this.skipDescendantsTags.has(tagName)) {
+        return '';
+      }
+      let output = '';
+      const shouldInclude = this.shouldIncludeElement(node);
+      if (shouldInclude) {
+          output += `${' '.repeat(depth * 2)}${tagName}`;
+          const attributes = this.getAttributes(node);
+          if (Object.keys(attributes).length > 0) {
+              const attrStrings = [];
+              for (const [key, value] of Object.entries(attributes)) {
+                  if (value === true) {
+                      attrStrings.push(key);
+                  } else {
+                      attrStrings.push(`${key}="${value}"`);
+                  }
+              }
+              output += `(${attrStrings.join(' ')})`;
+          }
+          // Only output whitelisted classes
+          const relevantClasses = this.findWhitelistedClasses(node);
+          if (relevantClasses.length > 0) {
+              output += `.${relevantClasses.join('.')}`;
+          }
+          // Handle text-only nodes differently
+          if (this.onlyHasTextNodes(node)) {
+              const text = Array.from(node.childNodes)
+                  .map(child => child.textContent.trim())
+                  .filter(Boolean)
+                  .join(' ');
+              if (text) {
+                  output += ` ${text}`;
+              }
+              output += '\n';
+          } else {
+              output += '\n';
+              // Process mixed content nodes in order
+              let pendingText = '';
+              for (const child of node.childNodes) {
+                  if (child.nodeType === Node.TEXT_NODE) {
+                      const text = child.textContent.trim();
+                      if (text) {
+                          pendingText += (pendingText ? ' ' : '') + text;
+                      }
+                  } else if (child.nodeType === Node.ELEMENT_NODE) {
+                      // Output any pending text before the element
+                      if (pendingText) {
+                          output += `${' '.repeat((depth + 1) * 2)}| ${pendingText}\n`;
+                          pendingText = '';
+                      }
+                      output += this.convertNode(child, depth + 1);
+                  }
+              }
+              // Output any remaining text
+              if (pendingText) {
+                  output += `${' '.repeat((depth + 1) * 2)}| ${pendingText}\n`;
+              }
+          }
+      } else {
+          // Even if we don't include this element, we still need to process its children
+          for (const child of node.childNodes) {
+              if (child.nodeType === Node.ELEMENT_NODE) {
+                  output += this.convertNode(child, depth);
+              }
+          }
+      }
+      return output;
+  }
+  convert() {
+      return this.convertNode(this.document.body).trim();
+  }
+}

package/package.json ADDED Viewed

@@ -0,0 +1,23 @@
+{
+  "name": "minipug",
+  "version": "1.0.1",
+  "description": "Extract LLM-friendly representation of a webpage",
+  "main": "index.mjs",
+  "type": "module",
+  "scripts": {
+    "test": "node test.js"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/will123195/minipug.git"
+  },
+  "author": "will123195",
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/will123195/minipug/issues"
+  },
+  "homepage": "https://github.com/will123195/minipug#readme",
+  "devDependencies": {
+    "puppeteer": "^21.5.0"
+  }
+}

package/test.js ADDED Viewed

@@ -0,0 +1,86 @@
+import puppeteer from 'puppeteer';
+import fs from 'fs/promises';
+import path from 'path';
+import { fileURLToPath } from 'url';
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const createTestHtml = async () => {
+  const htmlContent = `
+<html>
+  <head>
+    <title>Example Domain</title>
+    <meta charset="utf-8">
+    <meta http-equiv="Content-type" content="text/html; charset=utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <style type="text/css">
+      body {
+        background-color: #f0f0f2;
+        margin: 0;
+        padding: 0;
+      }
+    </style>
+  </head>
+  <body>
+    <h1>Hello World</h1>
+    <section class="example-section">
+      <p>This is an example.</p>
+      <button aria-label="Next page" class="z2iX5wAef9nHv">Next page</button>
+    </section>
+  </body>
+</html>
+  `;
+  const testFilePath = path.join(__dirname, 'test.html');
+  await fs.writeFile(testFilePath, htmlContent);
+  return testFilePath;
+};
+const runTest = async () => {
+  console.log('Running MiniPug test with Puppeteer...');
+  const testFilePath = await createTestHtml();
+  const testFileUrl = `file://${testFilePath}`;
+  const browser = await puppeteer.launch({ headless: 'new' });
+  const page = await browser.newPage();
+  try {
+    await page.goto(testFileUrl);
+    const miniPugCode = await fs.readFile('./index.mjs', 'utf8');
+    const result = await page.evaluate(async (code) => {
+      const blob = new Blob([code], { type: 'application/javascript' });
+      const url = URL.createObjectURL(blob);
+      const MiniPugModule = await import(url);
+      const MiniPug = MiniPugModule.default;
+      const minipug = new MiniPug(document);
+      return minipug.convert();
+    }, miniPugCode);
+    const expectedOutput = `h1 Hello World\nsection\n  p This is an example.\n  button(aria-label="Next page") Next page`;
+    console.log('\nValidation:');
+    if (result.trim() === expectedOutput.trim()) {
+      console.log('✅ Test passed! Output matches expected result.');
+    } else {
+      console.log('❌ Test failed! Output does not match expected result.');
+      console.log('\nExpected:');
+      console.log(expectedOutput);
+      console.log(expectedOutput.length);
+      console.log('\nActual:');
+      console.log(result);
+      console.log(result.length);
+      process.exitCode = 1;
+    }
+  } finally {
+    // Always close the browser and clean up, regardless of success or failure
+    await browser.close();
+    await fs.unlink(testFilePath).catch(() => {});
+  }
+};
+// This will catch any errors that occur during the test execution
+runTest().catch(error => {
+  console.error('Test error:', error);
+  process.exitCode = 1;
+});