rubycrawl 0.1.1 → 0.1.3

data/node/package-lock.json

@@ -1,72 +0,0 @@
-{
-  "name": "rubycrawl-node",
-  "version": "0.1.0",
-  "lockfileVersion": 3,
-  "requires": true,
-  "packages": {
-    "": {
-      "name": "rubycrawl-node",
-      "version": "0.1.0",
-      "dependencies": {
-        "dotenv": "^16.4.5",
-        "playwright": "^1.41.0"
-      }
-    },
-    "node_modules/dotenv": {
-      "version": "16.6.1",
-      "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.6.1.tgz",
-      "integrity": "sha512-uBq4egWHTcTt33a72vpSG0z3HnPuIl6NqYcTrKEg2azoEyl2hpW0zqlxysq2pK9HlDIHyHyakeYaYnSAwd8bow==",
-      "license": "BSD-2-Clause",
-      "engines": {
-        "node": ">=12"
-      },
-      "funding": {
-        "url": "https://dotenvx.com"
-      }
-    },
-    "node_modules/fsevents": {
-      "version": "2.3.2",
-      "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
-      "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
-      "hasInstallScript": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "darwin"
-      ],
-      "engines": {
-        "node": "^8.16.0 || ^10.6.0 || >=11.0.0"
-      }
-    },
-    "node_modules/playwright": {
-      "version": "1.58.0",
-      "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.58.0.tgz",
-      "integrity": "sha512-2SVA0sbPktiIY/MCOPX8e86ehA/e+tDNq+e5Y8qjKYti2Z/JG7xnronT/TXTIkKbYGWlCbuucZ6dziEgkoEjQQ==",
-      "license": "Apache-2.0",
-      "dependencies": {
-        "playwright-core": "1.58.0"
-      },
-      "bin": {
-        "playwright": "cli.js"
-      },
-      "engines": {
-        "node": ">=18"
-      },
-      "optionalDependencies": {
-        "fsevents": "2.3.2"
-      }
-    },
-    "node_modules/playwright-core": {
-      "version": "1.58.0",
-      "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.58.0.tgz",
-      "integrity": "sha512-aaoB1RWrdNi3//rOeKuMiS65UCcgOVljU46At6eFcOFPFHWtd2weHRRow6z/n+Lec0Lvu0k9ZPKJSjPugikirw==",
-      "license": "Apache-2.0",
-      "bin": {
-        "playwright-core": "cli.js"
-      },
-      "engines": {
-        "node": ">=18"
-      }
-    }
-  }
-}

data/node/package.json

@@ -1,14 +0,0 @@
-{
-  "name": "rubycrawl-node",
-  "version": "0.1.0",
-  "private": true,
-  "type": "module",
-  "main": "src/index.js",
-  "scripts": {
-    "start": "node src/index.js"
-  },
-  "dependencies": {
-    "dotenv": "^16.4.5",
-    "playwright": "^1.41.0"
-  }
-}

data/node/src/index.js

@@ -1,215 +0,0 @@
-import "dotenv/config";
-import http from "node:http";
-import { chromium } from "playwright";
-
-const HOST = "127.0.0.1";
-const PORT = process.env.RUBYCRAWL_NODE_PORT || 3344;
-const DEFAULT_BLOCK_RESOURCES = true;
-const BLOCKED_RESOURCE_TYPES = new Set([
-  "image",
-  "media",
-  "font",
-  "stylesheet",
-]);
-
-function json(res, statusCode, body) {
-  const payload = JSON.stringify(body);
-  res.writeHead(statusCode, {
-    "content-type": "application/json",
-    "content-length": Buffer.byteLength(payload),
-  });
-  res.end(payload);
-}
-
-function readJson(req) {
-  return new Promise((resolve, reject) => {
-    let data = "";
-    req.on("data", (chunk) => {
-      data += chunk;
-    });
-    req.on("end", () => {
-      if (!data) return resolve({});
-      try {
-        resolve(JSON.parse(data));
-      } catch (error) {
-        reject(error);
-      }
-    });
-    req.on("error", reject);
-  });
-}
-
-function validateRequest(body) {
-  if (!body || typeof body.url !== "string" || body.url.trim() === "") {
-    return { ok: false, error: "url is required" };
-  }
-  return { ok: true };
-}
-
-let browserPromise;
-let contextPromise;
-
-function getBrowser() {
-  if (!browserPromise) {
-    browserPromise = chromium.launch({ headless: true });
-  }
-  return browserPromise;
-}
-
-async function getContext() {
-  if (!contextPromise) {
-    const browser = await getBrowser();
-    contextPromise = browser.newContext();
-  }
-
-  return contextPromise;
-}
-
-/**
- * Extract HTML metadata from the page
- * @param {import('playwright').Page} page - The Playwright page object
- * @returns {Promise<Object>} Metadata object with title, description, OG tags, etc.
- */
-async function extractMetadata(page) {
-  return page.evaluate(() => {
-    const getMeta = (name) => {
-      const meta = document.querySelector(
-        `meta[name="${name}"], meta[property="${name}"]`,
-      );
-      return meta?.getAttribute("content") || null;
-    };
-
-    const getLink = (rel) => {
-      const link = document.querySelector(`link[rel="${rel}"]`);
-      return link?.getAttribute("href") || null;
-    };
-
-    return {
-      title: document.title || null,
-      description: getMeta("description") || getMeta("og:description") || null,
-      keywords: getMeta("keywords"),
-      author: getMeta("author"),
-      og_title: getMeta("og:title"),
-      og_description: getMeta("og:description"),
-      og_image: getMeta("og:image"),
-      og_url: getMeta("og:url"),
-      og_type: getMeta("og:type"),
-      twitter_card: getMeta("twitter:card"),
-      twitter_title: getMeta("twitter:title"),
-      twitter_description: getMeta("twitter:description"),
-      twitter_image: getMeta("twitter:image"),
-      canonical: getLink("canonical"),
-      lang: document.documentElement.lang || null,
-      charset: document.characterSet || null,
-    };
-  });
-}
-
-/**
- * Extract links from the page.
- * @param {import('playwright').Page} page - The Playwright page object
- * @returns {Promise<Array>} Array of link objects
- */
-async function extractLinks(page) {
-  return page.evaluate(() => {
-    const links = Array.from(document.querySelectorAll("a[href]"));
-    return links.map((link) => ({
-      url: link.href,
-      text: (link.textContent || "").trim(),
-      title: link.getAttribute("title") || null,
-      rel: link.getAttribute("rel") || null,
-    }));
-  });
-}
-
-async function handleCrawl(req, res) {
-  try {
-    const body = await readJson(req);
-    const validation = validateRequest(body);
-    if (!validation.ok) {
-      return json(res, 422, { error: validation.error });
-    }
-
-    const waitUntil = body.wait_until || "load";
-    const blockResources =
-      typeof body.block_resources === "boolean"
-        ? body.block_resources
-        : DEFAULT_BLOCK_RESOURCES;
-
-    const start = Date.now();
-    // eslint-disable-next-line no-console
-    console.log(`[rubycrawl] crawl start ${body.url}`);
-
-    const context = await getContext();
-    const page = await context.newPage();
-
-    try {
-      if (blockResources) {
-        await page.route("**/*", (route) => {
-          const type = route.request().resourceType();
-          if (BLOCKED_RESOURCE_TYPES.has(type)) {
-            return route.abort();
-          }
-          return route.continue();
-        });
-      }
-
-      const response = await page.goto(body.url, {
-        waitUntil,
-        timeout: 30_000,
-      });
-
-      const html = await page.content();
-      const finalUrl = page.url();
-      const status = response ? response.status() : null;
-      const htmlMetadata = await extractMetadata(page);
-      const links = await extractLinks(page);
-
-      // eslint-disable-next-line no-console
-      console.log(
-        `[rubycrawl] crawl done ${body.url} status=${status} ms=${Date.now() - start}`,
-      );
-
-      return json(res, 200, {
-        ok: true,
-        url: body.url,
-        html,
-        text: "",
-        markdown: "",
-        links,
-        metadata: {
-          status,
-          final_url: finalUrl,
-          ...htmlMetadata,
-        },
-      });
-    } finally {
-      await page.close();
-    }
-  } catch (error) {
-    const code =
-      error?.name === "SyntaxError" ? "invalid_json" : "crawl_failed";
-    // eslint-disable-next-line no-console
-    console.log(`[rubycrawl] crawl error ${code} ${error?.message || ""}`);
-    return json(res, 400, { error: code, message: error?.message });
-  }
-}
-
-const server = http.createServer((req, res) => {
-  // eslint-disable-next-line no-console
-  console.log(`[rubycrawl] request ${req.method} ${req.url}`);
-  if (req.method === "POST" && req.url === "/crawl") {
-    return handleCrawl(req, res);
-  }
-
-  if (req.method === "GET" && req.url === "/health") {
-    return json(res, 200, { ok: true });
-  }
-
-  return json(res, 404, { error: "not_found" });
-});
-
-server.listen(PORT, HOST, () => {
-  // eslint-disable-next-line no-console
-  console.log(`rubycrawl node service listening on http://${HOST}:${PORT}`);
-});

data/spec/rubycrawl_spec.rb

@@ -1,51 +0,0 @@
-# frozen_string_literal: true
-
-# rubocop:disable Metrics/BlockLength
-
-require 'spec_helper'
-
-RSpec.describe RubyCrawl do
-  describe '.crawl' do
-    it 'returns a result object with HTML and metadata' do
-      result = described_class.crawl('https://example.com')
-
-      expect(result).to be_a(RubyCrawl::Result)
-      expect(result.html).to be_a(String)
-      expect(result.links).to be_a(Array)
-      expect(result.metadata).to be_a(Hash)
-      expect(result.metadata).to include('status', 'final_url')
-    end
-
-    it 'includes HTML metadata when available' do
-      result = described_class.crawl('https://example.com')
-
-      # Metadata keys that should be present (may be nil)
-      expect(result.metadata.keys).to include('title', 'description', 'canonical')
-    end
-  end
-
-  describe 'error handling' do
-    it 'raises ConfigurationError for invalid URLs' do
-      expect do
-        described_class.crawl('not-a-url')
-      end.to raise_error(RubyCrawl::ConfigurationError, /Invalid URL/)
-    end
-
-    it 'raises ConfigurationError for non-HTTP(S) URLs' do
-      expect do
-        described_class.crawl('ftp://example.com')
-      end.to raise_error(RubyCrawl::ConfigurationError, /Only HTTP\(S\) URLs are supported/)
-    end
-  end
-
-  describe 'custom exceptions' do
-    it 'defines error hierarchy' do
-      expect(RubyCrawl::Error).to be < StandardError
-      expect(RubyCrawl::ServiceError).to be < RubyCrawl::Error
-      expect(RubyCrawl::NavigationError).to be < RubyCrawl::Error
-      expect(RubyCrawl::TimeoutError).to be < RubyCrawl::Error
-      expect(RubyCrawl::ConfigurationError).to be < RubyCrawl::Error
-    end
-  end
-end
-# rubocop:enable Metrics/BlockLength

data/spec/spec_helper.rb

@@ -1,11 +0,0 @@
-# frozen_string_literal: true
-
-require 'bundler/setup'
-require 'rubycrawl'
-
-RSpec.configure do |config|
-  config.disable_monkey_patching!
-  config.expect_with :rspec do |c|
-    c.syntax = :expect
-  end
-end