rubycrawl 0.1.2 → 0.1.4

data/lib/rubycrawl/helpers.rb

@@ -17,14 +17,22 @@ class RubyCrawl
       if uri.host&.match?(/^(localhost|127\.|192\.168\.|10\.|172\.(1[6-9]|2[0-9]|3[01]))/)
         warn '[rubycrawl] Warning: Crawling internal/private IP addresses'
       end
-    rescue URI::InvalidURIError => e
+    rescue URI::InvalidURIError, TypeError => e
       raise ConfigurationError, "Invalid URL: #{e.message}"
     end
 
-    def build_payload(url, wait_until, block_resources)
+    VALID_WAIT_UNTIL = %w[load domcontentloaded networkidle commit].freeze
+
+    def build_payload(url, wait_until, block_resources, session_id = nil)
+      if wait_until && !VALID_WAIT_UNTIL.include?(wait_until.to_s)
+        raise ConfigurationError,
+              "Invalid wait_until: #{wait_until.inspect}. Must be one of: #{VALID_WAIT_UNTIL.join(', ')}"
+      end
+
       payload = { url: url }
       payload[:wait_until] = wait_until if wait_until
       payload[:block_resources] = block_resources unless block_resources.nil?
+      payload[:session_id] = session_id if session_id
       payload
     end
 
@@ -39,11 +47,9 @@ class RubyCrawl
 
     def error_class_for(error_code)
       case error_code
-      when 'navigation_timeout', 'crawl_timeout'
-        TimeoutError
-      when 'navigation_failed', 'crawl_failed'
+      when 'crawl_failed'
         NavigationError
-      when 'invalid_json', 'invalid_json_response'
+      when 'invalid_json', 'session_create_failed', 'session_destroy_failed'
         ServiceError
       else
         Error
@@ -52,12 +58,10 @@ class RubyCrawl
 
     def error_message_for(error_code, error_message)
       case error_code
-      when 'navigation_timeout', 'crawl_timeout'
-        "Crawl timeout: #{error_message}"
-      when 'navigation_failed', 'crawl_failed'
+      when 'crawl_failed'
         "Navigation failed: #{error_message}"
-      when 'invalid_json', 'invalid_json_response'
-        "Node service returned invalid JSON: #{error_message}"
+      when 'invalid_json', 'session_create_failed', 'session_destroy_failed'
+        "Service error [#{error_code}]: #{error_message}"
       else
         "Crawl error [#{error_code}]: #{error_message}"
       end

data/lib/rubycrawl/markdown_converter.rb

@@ -15,10 +15,10 @@ class RubyCrawl
 
     # Convert HTML to Markdown with resolved URLs.
     #
-    # @param html [String] The HTML content to convert
+    # @param html [String] The page HTML to convert
     # @param base_url [String, nil] Base URL to resolve relative URLs
-    # @param options [Hash] Options for conversion
-    # @return [String] The Markdown content with absolute URLs
+    # @param options [Hash] Options passed to ReverseMarkdown
+    # @return [String] Markdown content with absolute URLs
     def convert(html, base_url: nil, **options)
       return '' if html.nil? || html.empty?
 

data/lib/rubycrawl/result.rb

@@ -1,38 +1,37 @@
 # frozen_string_literal: true
 
 class RubyCrawl
-  # Result object with lazy markdown conversion.
+  # Result object with lazy clean_markdown conversion.
   class Result
     attr_reader :text, :html, :links, :metadata
 
-    def initialize(text:, html:, links:, metadata:, markdown: nil)
+    def initialize(text:, html:, links:, metadata:)
       @text = text
       @html = html
       @links = links
       @metadata = metadata
-      @markdown = markdown unless markdown.to_s.empty?
     end
 
-    # Returns markdown, converting from HTML lazily if needed.
+    # Returns clean markdown converted from the page HTML.
     # Relative URLs are resolved using the page's final_url.
     #
     # @return [String] Markdown content with absolute URLs
-    def markdown
-      @markdown ||= MarkdownConverter.convert(html, base_url: final_url)
+    def clean_markdown
+      @clean_markdown ||= MarkdownConverter.convert(html, base_url: final_url)
     end
 
     # The final URL after redirects.
     #
     # @return [String, nil]
     def final_url
-      metadata['final_url'] || metadata[:final_url]
+      metadata['final_url']
     end
 
-    # Check if markdown has been computed.
+    # Check if clean_markdown has been computed.
     #
     # @return [Boolean]
-    def markdown?
-      !@markdown.nil?
+    def clean_markdown?
+      !@clean_markdown.nil?
     end
 
     def to_h
@@ -41,7 +40,7 @@ class RubyCrawl
         html: html,
         links: links,
         metadata: metadata,
-        markdown: markdown
+        clean_markdown: @clean_markdown
       }
     end
   end

data/lib/rubycrawl/service_client.rb

@@ -36,6 +36,24 @@ class RubyCrawl
       raise TimeoutError, "Request to node service timed out: #{e.message}"
     end
 
+    # Create a session for reusing browser context across multiple crawls.
+    # @return [String] session_id
+    def create_session
+      response = post_json('/session/create', {})
+      raise ServiceError, "Failed to create session: #{response['error']}" if response['error']
+
+      response['session_id']
+    end
+
+    # Destroy a session and close its browser context.
+    # @param session_id [String]
+    def destroy_session(session_id)
+      post_json('/session/destroy', { session_id: session_id })
+    rescue StandardError
+      # Ignore errors on destroy - context may already be closed
+      nil
+    end
+
     private
 
     def build_request(uri, body)
@@ -55,9 +73,13 @@ class RubyCrawl
       raise ServiceError, "rubycrawl node service directory not found: #{@node_dir}" unless Dir.exist?(@node_dir)
 
       env = { 'RUBYCRAWL_NODE_PORT' => @port.to_s }
-      out = @node_log ? File.open(@node_log, 'a') : File::NULL
-      err = @node_log ? out : File::NULL
-      @node_pid = Process.spawn(env, @node_bin, 'src/index.js', chdir: @node_dir, out: out, err: err)
+      if @node_log
+        out = File.open(@node_log, 'a')
+        @node_pid = Process.spawn(env, @node_bin, 'src/index.js', chdir: @node_dir, out: out, err: out)
+        out.close
+      else
+        @node_pid = Process.spawn(env, @node_bin, 'src/index.js', chdir: @node_dir, out: File::NULL, err: File::NULL)
+      end
       Process.detach(@node_pid)
     end
 

data/lib/rubycrawl/site_crawler.rb

@@ -5,7 +5,7 @@ require 'set'
 class RubyCrawl
   # BFS crawler that follows links with deduplication.
   class SiteCrawler
-    # Page result yielded to the block with lazy markdown.
+    # Page result yielded to the block with lazy clean_markdown.
     class PageResult
       attr_reader :url, :html, :links, :metadata, :depth
 
@@ -17,14 +17,15 @@ class RubyCrawl
         @depth = depth
       end
 
-      # Lazy markdown conversion with resolved URLs.
-      def markdown
-        @markdown ||= MarkdownConverter.convert(html, base_url: final_url)
+      # Returns clean markdown converted from the page HTML.
+      # Relative URLs are resolved using the page's final_url.
+      def clean_markdown
+        @clean_markdown ||= MarkdownConverter.convert(html, base_url: final_url)
       end
 
       # The final URL after redirects.
       def final_url
-        metadata['final_url'] || metadata[:final_url] || url
+        metadata['final_url'] || url
       end
     end
 
@@ -35,8 +36,10 @@ class RubyCrawl
       @same_host_only = options.fetch(:same_host_only, true)
       @wait_until = options.fetch(:wait_until, nil)
       @block_resources = options.fetch(:block_resources, nil)
+      @max_attempts = options.fetch(:max_attempts, nil)
       @visited = Set.new
       @queue = []
+      @session_id = nil
     end
 
     def crawl(start_url, &block)
@@ -46,8 +49,11 @@ class RubyCrawl
       raise ConfigurationError, "Invalid start URL: #{start_url}" unless normalized
 
       @base_url = normalized
+      @session_id = @client.create_session
       enqueue(normalized, 0)
       process_queue(&block)
+    ensure
+      @client.destroy_session(@session_id) if @session_id
     end
 
     private
@@ -77,7 +83,9 @@ class RubyCrawl
     end
 
     def crawl_page(url, depth)
-      result = @client.crawl(url, wait_until: @wait_until, block_resources: @block_resources)
+      opts = { wait_until: @wait_until, block_resources: @block_resources, session_id: @session_id }
+      opts[:max_attempts] = @max_attempts if @max_attempts
+      result = @client.crawl(url, **opts)
       build_page_result(url, depth, result)
     rescue Error => e
       warn "[rubycrawl] Failed to crawl #{url}: #{e.message}"

data/lib/rubycrawl/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class RubyCrawl
-  VERSION = '0.1.2'
+  VERSION = '0.1.4'
 end

data/lib/rubycrawl.rb

@@ -45,6 +45,18 @@ class RubyCrawl
       client.crawl_site(url, ...)
     end
 
+    # Create a session for reusing browser context across multiple crawls.
+    # @return [String] session_id
+    def create_session
+      client.create_session
+    end
+
+    # Destroy a session and close its browser context.
+    # @param session_id [String]
+    def destroy_session(session_id)
+      client.destroy_session(session_id)
+    end
+
     def configure(**options)
       @client = new(**options)
     end
@@ -55,17 +67,30 @@ class RubyCrawl
     build_service_client
   end
 
-  def crawl(url, wait_until: @wait_until, block_resources: @block_resources, retries: @max_retries)
+  def crawl(url, wait_until: @wait_until, block_resources: @block_resources, max_attempts: @max_attempts, session_id: nil)
     validate_url!(url)
     @service_client.ensure_running
-    with_retries(retries) do
-      payload = build_payload(url, wait_until, block_resources)
+    with_retries(max_attempts) do
+      payload = build_payload(url, wait_until, block_resources, session_id)
       response = @service_client.post_json('/crawl', payload)
       raise_node_error!(response)
       build_result(response)
     end
   end
 
+  # Create a session for reusing browser context.
+  # @return [String] session_id
+  def create_session
+    @service_client.ensure_running
+    @service_client.create_session
+  end
+
+  # Destroy a session.
+  # @param session_id [String]
+  def destroy_session(session_id)
+    @service_client.destroy_session(session_id)
+  end
+
   # Crawl multiple pages starting from a URL, following links.
   # @see RubyCrawl.crawl_site
   def crawl_site(url, **options, &block)
@@ -106,7 +131,7 @@ class RubyCrawl
     @node_log = options.fetch(:node_log, ENV.fetch('RUBYCRAWL_NODE_LOG', nil))
     @wait_until = options.fetch(:wait_until, nil)
     @block_resources = options.fetch(:block_resources, nil)
-    @max_retries = options.fetch(:max_retries, 3)
+    @max_attempts = options.fetch(:max_attempts, 3)
   end
 
   def build_service_client
@@ -119,9 +144,9 @@ class RubyCrawl
     )
   end
 
-  def retry_with_backoff(attempt, retries, error)
+  def retry_with_backoff(attempt, max_attempts, error)
     backoff_seconds = 2**attempt
-    warn "[rubycrawl] Retry #{attempt}/#{retries - 1} after #{backoff_seconds}s: #{error.message}"
+    warn "[rubycrawl] Attempt #{attempt + 1}/#{max_attempts} failed, retrying in #{backoff_seconds}s: #{error.message}"
     sleep(backoff_seconds)
   end
 
@@ -131,7 +156,8 @@ class RubyCrawl
       max_depth: options.fetch(:max_depth, 3),
       same_host_only: options.fetch(:same_host_only, true),
       wait_until: options.fetch(:wait_until, @wait_until),
-      block_resources: options.fetch(:block_resources, @block_resources)
+      block_resources: options.fetch(:block_resources, @block_resources),
+      max_attempts: options.fetch(:max_attempts, @max_attempts)
     }
   end
 

data/node/src/index.js

@@ -1,5 +1,6 @@
 import "dotenv/config";
 import http from "node:http";
+import crypto from "node:crypto";
 import { chromium } from "playwright";
 
 const HOST = "127.0.0.1";
@@ -21,10 +22,19 @@ function json(res, statusCode, body) {
   res.end(payload);
 }
 
+const MAX_BODY_SIZE = 1 * 1024 * 1024; // 1 MB
+
 function readJson(req) {
   return new Promise((resolve, reject) => {
     let data = "";
+    let size = 0;
     req.on("data", (chunk) => {
+      size += chunk.length;
+      if (size > MAX_BODY_SIZE) {
+        reject(new Error("Request body too large"));
+        req.destroy();
+        return;
+      }
       data += chunk;
     });
     req.on("end", () => {
@@ -46,29 +56,100 @@ function validateRequest(body) {
   return { ok: true };
 }
 
-let browserPromise;
-let contextPromise;
+let browser = null;
+
+// Session storage: session_id -> { context, createdAt, lastUsedAt }
+const sessions = new Map();
+
+// Session TTL: 30 minutes of inactivity
+const SESSION_TTL_MS = 30 * 60 * 1000;
+// Cleanup interval: every 5 minutes
+const CLEANUP_INTERVAL_MS = 5 * 60 * 1000;
+
+function generateSessionId() {
+  return `sess_${crypto.randomBytes(16).toString("hex")}`;
+}
+
+async function getBrowser() {
+  if (browser && browser.isConnected()) return browser;
+  browser = await chromium.launch({ headless: true });
+  return browser;
+}
 
-function getBrowser() {
-  if (!browserPromise) {
-    browserPromise = chromium.launch({ headless: true });
+/**
+ * Create a fresh browser context.
+ */
+async function createContext() {
+  const browser = await getBrowser();
+  return browser.newContext();
+}
+
+/**
+ * Get or create context based on session_id.
+ * If session_id provided and exists, reuse existing context.
+ * If session_id provided but expired/destroyed, create new context (handles retries).
+ * Otherwise create a fresh one-off context.
+ */
+async function getContext(sessionId) {
+  if (sessionId && sessions.has(sessionId)) {
+    // Update last used time
+    const session = sessions.get(sessionId);
+    session.lastUsedAt = Date.now();
+    return { context: session.context, isSession: true };
   }
-  return browserPromise;
+
+  // If session_id provided but doesn't exist (expired/destroyed), recreate it
+  // This handles job retries gracefully
+  if (sessionId) {
+    const context = await createContext();
+    const now = Date.now();
+    sessions.set(sessionId, { context, createdAt: now, lastUsedAt: now });
+    // eslint-disable-next-line no-console
+    console.log(
+      `[rubycrawl] session recreated ${sessionId} (was expired or destroyed)`,
+    );
+    return { context, isSession: true };
+  }
+
+  return { context: await createContext(), isSession: false };
 }
 
-async function getContext() {
-  if (!contextPromise) {
-    const browser = await getBrowser();
-    contextPromise = browser.newContext();
+/**
+ * Cleanup expired sessions (no activity for SESSION_TTL_MS).
+ */
+async function cleanupExpiredSessions() {
+  const now = Date.now();
+  const expiredIds = [];
+
+  for (const [sessionId, session] of sessions) {
+    if (now - session.lastUsedAt > SESSION_TTL_MS) {
+      expiredIds.push(sessionId);
+    }
   }
 
-  return contextPromise;
+  for (const sessionId of expiredIds) {
+    const session = sessions.get(sessionId);
+    await session.context.close().catch(() => {});
+    sessions.delete(sessionId);
+    // eslint-disable-next-line no-console
+    console.log(
+      `[rubycrawl] session expired ${sessionId} (inactive for ${SESSION_TTL_MS / 60000} min)`,
+    );
+  }
+
+  if (expiredIds.length > 0) {
+    // eslint-disable-next-line no-console
+    console.log(
+      `[rubycrawl] cleanup: ${expiredIds.length} expired, ${sessions.size} active`,
+    );
+  }
 }
 
+// Start cleanup interval
+setInterval(cleanupExpiredSessions, CLEANUP_INTERVAL_MS);
+
 /**
  * Extract HTML metadata from the page
- * @param {import('playwright').Page} page - The Playwright page object
- * @returns {Promise<Object>} Metadata object with title, description, OG tags, etc.
  */
 async function extractMetadata(page) {
   return page.evaluate(() => {
@@ -107,8 +188,6 @@ async function extractMetadata(page) {
 
 /**
  * Extract links from the page.
- * @param {import('playwright').Page} page - The Playwright page object
- * @returns {Promise<Array>} Array of link objects
  */
 async function extractLinks(page) {
   return page.evaluate(() => {
@@ -122,7 +201,17 @@ async function extractLinks(page) {
   });
 }
 
+/**
+ * Extract plain text content from the page using innerText.
+ */
+async function extractText(page) {
+  return page.evaluate(() => (document.body?.innerText || "").trim());
+}
+
 async function handleCrawl(req, res) {
+  let context = null;
+  let isSession = false;
+
   try {
     const body = await readJson(req);
     const validation = validateRequest(body);
@@ -138,9 +227,15 @@ async function handleCrawl(req, res) {
 
     const start = Date.now();
     // eslint-disable-next-line no-console
-    console.log(`[rubycrawl] crawl start ${body.url}`);
+    console.log(
+      `[rubycrawl] crawl start ${body.url}${body.session_id ? ` (session=${body.session_id})` : ""}`,
+    );
+
+    // Get context (reuse if session_id provided)
+    const ctxResult = await getContext(body.session_id);
+    context = ctxResult.context;
+    isSession = ctxResult.isSession;
 
-    const context = await getContext();
     const page = await context.newPage();
 
     try {
@@ -164,6 +259,7 @@ async function handleCrawl(req, res) {
       const status = response ? response.status() : null;
       const htmlMetadata = await extractMetadata(page);
       const links = await extractLinks(page);
+      const text = await extractText(page);
 
       // eslint-disable-next-line no-console
       console.log(
@@ -174,8 +270,7 @@ async function handleCrawl(req, res) {
         ok: true,
         url: body.url,
         html,
-        text: "",
-        markdown: "",
+        text,
         links,
         metadata: {
           status,
@@ -192,16 +287,95 @@ async function handleCrawl(req, res) {
     // eslint-disable-next-line no-console
     console.log(`[rubycrawl] crawl error ${code} ${error?.message || ""}`);
     return json(res, 400, { error: code, message: error?.message });
+  } finally {
+    // Only close context if not a session (sessions are managed separately)
+    if (context && !isSession) {
+      await context.close().catch(() => {});
+    }
+  }
+}
+
+/**
+ * Create a new session with a reusable browser context.
+ */
+async function handleSessionCreate(req, res) {
+  try {
+    const sessionId = generateSessionId();
+    const context = await createContext();
+    const now = Date.now();
+    sessions.set(sessionId, { context, createdAt: now, lastUsedAt: now });
+
+    // eslint-disable-next-line no-console
+    console.log(
+      `[rubycrawl] session created ${sessionId} (active=${sessions.size})`,
+    );
+
+    return json(res, 200, { ok: true, session_id: sessionId });
+  } catch (error) {
+    // eslint-disable-next-line no-console
+    console.log(`[rubycrawl] session create error ${error?.message || ""}`);
+    return json(res, 400, {
+      error: "session_create_failed",
+      message: error?.message,
+    });
+  }
+}
+
+/**
+ * Destroy a session and close its browser context.
+ * Returns success even if session doesn't exist (idempotent for retries).
+ */
+async function handleSessionDestroy(req, res) {
+  try {
+    const body = await readJson(req);
+    const sessionId = body.session_id;
+
+    if (!sessionId) {
+      return json(res, 422, { error: "session_id required" });
+    }
+
+    // Idempotent: if session doesn't exist, still return success
+    if (!sessions.has(sessionId)) {
+      return json(res, 200, {
+        ok: true,
+        message: "session already destroyed or expired",
+      });
+    }
+
+    const session = sessions.get(sessionId);
+    await session.context.close().catch(() => {});
+    sessions.delete(sessionId);
+
+    // eslint-disable-next-line no-console
+    console.log(`[rubycrawl] session destroyed ${sessionId}`);
+
+    return json(res, 200, { ok: true });
+  } catch (error) {
+    // eslint-disable-next-line no-console
+    console.log(`[rubycrawl] session destroy error ${error?.message || ""}`);
+    return json(res, 400, {
+      error: "session_destroy_failed",
+      message: error?.message,
+    });
   }
 }
 
 const server = http.createServer((req, res) => {
   // eslint-disable-next-line no-console
   console.log(`[rubycrawl] request ${req.method} ${req.url}`);
+
   if (req.method === "POST" && req.url === "/crawl") {
     return handleCrawl(req, res);
   }
 
+  if (req.method === "POST" && req.url === "/session/create") {
+    return handleSessionCreate(req, res);
+  }
+
+  if (req.method === "POST" && req.url === "/session/destroy") {
+    return handleSessionDestroy(req, res);
+  }
+
   if (req.method === "GET" && req.url === "/health") {
     return json(res, 200, { ok: true });
   }

data/rubycrawl.gemspec

@@ -15,8 +15,9 @@ Gem::Specification.new do |spec|
 
   spec.required_ruby_version = '>= 3.0'
 
-  spec.files = Dir.glob('{bin,lib,spec,node}/**/*', File::FNM_DOTMATCH).reject { |f| File.directory?(f) }
-  spec.files += %w[README.md LICENSE Gemfile Rakefile rubycrawl.gemspec .rspec]
+  spec.files  = Dir.glob('{lib}/**/*', File::FNM_DOTMATCH).reject { |f| File.directory?(f) }
+  spec.files += Dir.glob('node/**/*', File::FNM_DOTMATCH).reject { |f| File.directory?(f) || f.include?('node_modules') }
+  spec.files += %w[README.md LICENSE Rakefile rubycrawl.gemspec .rspec]
 
   spec.bindir = 'bin'
   spec.executables = []