rubycrawl 0.1.0

data/node/package-lock.json

@@ -0,0 +1,72 @@
+{
+  "name": "rubycrawl-node",
+  "version": "0.1.0",
+  "lockfileVersion": 3,
+  "requires": true,
+  "packages": {
+    "": {
+      "name": "rubycrawl-node",
+      "version": "0.1.0",
+      "dependencies": {
+        "dotenv": "^16.4.5",
+        "playwright": "^1.41.0"
+      }
+    },
+    "node_modules/dotenv": {
+      "version": "16.6.1",
+      "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.6.1.tgz",
+      "integrity": "sha512-uBq4egWHTcTt33a72vpSG0z3HnPuIl6NqYcTrKEg2azoEyl2hpW0zqlxysq2pK9HlDIHyHyakeYaYnSAwd8bow==",
+      "license": "BSD-2-Clause",
+      "engines": {
+        "node": ">=12"
+      },
+      "funding": {
+        "url": "https://dotenvx.com"
+      }
+    },
+    "node_modules/fsevents": {
+      "version": "2.3.2",
+      "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
+      "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
+      "hasInstallScript": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": "^8.16.0 || ^10.6.0 || >=11.0.0"
+      }
+    },
+    "node_modules/playwright": {
+      "version": "1.58.0",
+      "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.58.0.tgz",
+      "integrity": "sha512-2SVA0sbPktiIY/MCOPX8e86ehA/e+tDNq+e5Y8qjKYti2Z/JG7xnronT/TXTIkKbYGWlCbuucZ6dziEgkoEjQQ==",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "playwright-core": "1.58.0"
+      },
+      "bin": {
+        "playwright": "cli.js"
+      },
+      "engines": {
+        "node": ">=18"
+      },
+      "optionalDependencies": {
+        "fsevents": "2.3.2"
+      }
+    },
+    "node_modules/playwright-core": {
+      "version": "1.58.0",
+      "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.58.0.tgz",
+      "integrity": "sha512-aaoB1RWrdNi3//rOeKuMiS65UCcgOVljU46At6eFcOFPFHWtd2weHRRow6z/n+Lec0Lvu0k9ZPKJSjPugikirw==",
+      "license": "Apache-2.0",
+      "bin": {
+        "playwright-core": "cli.js"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    }
+  }
+}

data/node/package.json

@@ -0,0 +1,14 @@
+{
+  "name": "rubycrawl-node",
+  "version": "0.1.0",
+  "private": true,
+  "type": "module",
+  "main": "src/index.js",
+  "scripts": {
+    "start": "node src/index.js"
+  },
+  "dependencies": {
+    "dotenv": "^16.4.5",
+    "playwright": "^1.41.0"
+  }
+}

data/node/src/index.js

@@ -0,0 +1,215 @@
+import "dotenv/config";
+import http from "node:http";
+import { chromium } from "playwright";
+
+const HOST = "127.0.0.1";
+const PORT = process.env.RUBYCRAWL_NODE_PORT || 3344;
+const DEFAULT_BLOCK_RESOURCES = true;
+const BLOCKED_RESOURCE_TYPES = new Set([
+  "image",
+  "media",
+  "font",
+  "stylesheet",
+]);
+
+function json(res, statusCode, body) {
+  const payload = JSON.stringify(body);
+  res.writeHead(statusCode, {
+    "content-type": "application/json",
+    "content-length": Buffer.byteLength(payload),
+  });
+  res.end(payload);
+}
+
+function readJson(req) {
+  return new Promise((resolve, reject) => {
+    let data = "";
+    req.on("data", (chunk) => {
+      data += chunk;
+    });
+    req.on("end", () => {
+      if (!data) return resolve({});
+      try {
+        resolve(JSON.parse(data));
+      } catch (error) {
+        reject(error);
+      }
+    });
+    req.on("error", reject);
+  });
+}
+
+function validateRequest(body) {
+  if (!body || typeof body.url !== "string" || body.url.trim() === "") {
+    return { ok: false, error: "url is required" };
+  }
+  return { ok: true };
+}
+
+let browserPromise;
+let contextPromise;
+
+function getBrowser() {
+  if (!browserPromise) {
+    browserPromise = chromium.launch({ headless: true });
+  }
+  return browserPromise;
+}
+
+async function getContext() {
+  if (!contextPromise) {
+    const browser = await getBrowser();
+    contextPromise = browser.newContext();
+  }
+
+  return contextPromise;
+}
+
+/**
+ * Extract HTML metadata from the page
+ * @param {import('playwright').Page} page - The Playwright page object
+ * @returns {Promise<Object>} Metadata object with title, description, OG tags, etc.
+ */
+async function extractMetadata(page) {
+  return page.evaluate(() => {
+    const getMeta = (name) => {
+      const meta = document.querySelector(
+        `meta[name="${name}"], meta[property="${name}"]`,
+      );
+      return meta?.getAttribute("content") || null;
+    };
+
+    const getLink = (rel) => {
+      const link = document.querySelector(`link[rel="${rel}"]`);
+      return link?.getAttribute("href") || null;
+    };
+
+    return {
+      title: document.title || null,
+      description: getMeta("description") || getMeta("og:description") || null,
+      keywords: getMeta("keywords"),
+      author: getMeta("author"),
+      og_title: getMeta("og:title"),
+      og_description: getMeta("og:description"),
+      og_image: getMeta("og:image"),
+      og_url: getMeta("og:url"),
+      og_type: getMeta("og:type"),
+      twitter_card: getMeta("twitter:card"),
+      twitter_title: getMeta("twitter:title"),
+      twitter_description: getMeta("twitter:description"),
+      twitter_image: getMeta("twitter:image"),
+      canonical: getLink("canonical"),
+      lang: document.documentElement.lang || null,
+      charset: document.characterSet || null,
+    };
+  });
+}
+
+/**
+ * Extract links from the page.
+ * @param {import('playwright').Page} page - The Playwright page object
+ * @returns {Promise<Array>} Array of link objects
+ */
+async function extractLinks(page) {
+  return page.evaluate(() => {
+    const links = Array.from(document.querySelectorAll("a[href]"));
+    return links.map((link) => ({
+      url: link.href,
+      text: (link.textContent || "").trim(),
+      title: link.getAttribute("title") || null,
+      rel: link.getAttribute("rel") || null,
+    }));
+  });
+}
+
+async function handleCrawl(req, res) {
+  try {
+    const body = await readJson(req);
+    const validation = validateRequest(body);
+    if (!validation.ok) {
+      return json(res, 422, { error: validation.error });
+    }
+
+    const waitUntil = body.wait_until || "load";
+    const blockResources =
+      typeof body.block_resources === "boolean"
+        ? body.block_resources
+        : DEFAULT_BLOCK_RESOURCES;
+
+    const start = Date.now();
+    // eslint-disable-next-line no-console
+    console.log(`[rubycrawl] crawl start ${body.url}`);
+
+    const context = await getContext();
+    const page = await context.newPage();
+
+    try {
+      if (blockResources) {
+        await page.route("**/*", (route) => {
+          const type = route.request().resourceType();
+          if (BLOCKED_RESOURCE_TYPES.has(type)) {
+            return route.abort();
+          }
+          return route.continue();
+        });
+      }
+
+      const response = await page.goto(body.url, {
+        waitUntil,
+        timeout: 30_000,
+      });
+
+      const html = await page.content();
+      const finalUrl = page.url();
+      const status = response ? response.status() : null;
+      const htmlMetadata = await extractMetadata(page);
+      const links = await extractLinks(page);
+
+      // eslint-disable-next-line no-console
+      console.log(
+        `[rubycrawl] crawl done ${body.url} status=${status} ms=${Date.now() - start}`,
+      );
+
+      return json(res, 200, {
+        ok: true,
+        url: body.url,
+        html,
+        text: "",
+        markdown: "",
+        links,
+        metadata: {
+          status,
+          final_url: finalUrl,
+          ...htmlMetadata,
+        },
+      });
+    } finally {
+      await page.close();
+    }
+  } catch (error) {
+    const code =
+      error?.name === "SyntaxError" ? "invalid_json" : "crawl_failed";
+    // eslint-disable-next-line no-console
+    console.log(`[rubycrawl] crawl error ${code} ${error?.message || ""}`);
+    return json(res, 400, { error: code, message: error?.message });
+  }
+}
+
+const server = http.createServer((req, res) => {
+  // eslint-disable-next-line no-console
+  console.log(`[rubycrawl] request ${req.method} ${req.url}`);
+  if (req.method === "POST" && req.url === "/crawl") {
+    return handleCrawl(req, res);
+  }
+
+  if (req.method === "GET" && req.url === "/health") {
+    return json(res, 200, { ok: true });
+  }
+
+  return json(res, 404, { error: "not_found" });
+});
+
+server.listen(PORT, HOST, () => {
+  // eslint-disable-next-line no-console
+  console.log(`rubycrawl node service listening on http://${HOST}:${PORT}`);
+});

data/rubycrawl.gemspec

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require_relative 'lib/rubycrawl/version'
+
+Gem::Specification.new do |spec|
+  spec.name = 'rubycrawl'
+  spec.version = RubyCrawl::VERSION
+  spec.authors = ['RubyCrawl contributors']
+  spec.email = ['ganesh.navale@zohomail.in']
+
+  spec.summary = 'Playwright-based web crawler for Ruby'
+  spec.description = 'A Ruby-first web crawler that orchestrates a local Playwright service.'
+  spec.homepage = 'https://github.com/craft-wise/rubycrawl'
+  spec.license = 'MIT'
+
+  spec.required_ruby_version = '>= 3.0'
+
+  spec.files = Dir.glob('{bin,lib,spec,node}/**/*', File::FNM_DOTMATCH).reject { |f| File.directory?(f) }
+  spec.files += %w[README.md LICENSE Gemfile Rakefile rubycrawl.gemspec .rspec]
+
+  spec.bindir = 'bin'
+  spec.executables = []
+  spec.require_paths = ['lib']
+
+  # For HTML to Markdown conversion
+  spec.add_dependency 'reverse_markdown', '~> 2.1'
+
+  spec.metadata['rubygems_mfa_required'] = 'true'
+end

data/spec/rubycrawl_spec.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+# rubocop:disable Metrics/BlockLength
+
+require 'spec_helper'
+
+RSpec.describe RubyCrawl do
+  describe '.crawl' do
+    it 'returns a result object with HTML and metadata' do
+      result = described_class.crawl('https://example.com')
+
+      expect(result).to be_a(RubyCrawl::Result)
+      expect(result.html).to be_a(String)
+      expect(result.links).to be_a(Array)
+      expect(result.metadata).to be_a(Hash)
+      expect(result.metadata).to include('status', 'final_url')
+    end
+
+    it 'includes HTML metadata when available' do
+      result = described_class.crawl('https://example.com')
+
+      # Metadata keys that should be present (may be nil)
+      expect(result.metadata.keys).to include('title', 'description', 'canonical')
+    end
+  end
+
+  describe 'error handling' do
+    it 'raises ConfigurationError for invalid URLs' do
+      expect do
+        described_class.crawl('not-a-url')
+      end.to raise_error(RubyCrawl::ConfigurationError, /Invalid URL/)
+    end
+
+    it 'raises ConfigurationError for non-HTTP(S) URLs' do
+      expect do
+        described_class.crawl('ftp://example.com')
+      end.to raise_error(RubyCrawl::ConfigurationError, /Only HTTP\(S\) URLs are supported/)
+    end
+  end
+
+  describe 'custom exceptions' do
+    it 'defines error hierarchy' do
+      expect(RubyCrawl::Error).to be < StandardError
+      expect(RubyCrawl::ServiceError).to be < RubyCrawl::Error
+      expect(RubyCrawl::NavigationError).to be < RubyCrawl::Error
+      expect(RubyCrawl::TimeoutError).to be < RubyCrawl::Error
+      expect(RubyCrawl::ConfigurationError).to be < RubyCrawl::Error
+    end
+  end
+end
+# rubocop:enable Metrics/BlockLength

data/spec/spec_helper.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'rubycrawl'
+
+RSpec.configure do |config|
+  config.disable_monkey_patching!
+  config.expect_with :rspec do |c|
+    c.syntax = :expect
+  end
+end