opencraw 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 7e0794fc1cda09d04c2effc3b94f4ca71b5fb782e337550ae186f42761b84f74
+  data.tar.gz: 25588199f1e6d378952986bc973945dca5da39861d4c97aff2821b10ef90e1ad
+SHA512:
+  metadata.gz: ae93dc575f790bf105aa5c1c9492a125be4d74db9badd2371658ce5b328d185fbd08902ca5cc6ecbabe9af1c65fb7e73a9f63c4e00231d9b40d58b2ccfb1aed2
+  data.tar.gz: 3b2a0ef4e130d0fcb02e6c41cfb152ba181a31c0038a01b18ab91ebf8acecd501e0cb3c28882c7fbb61b09764654c1bebfb5070d2c60ef273431cb52880680a2

data/README.md

@@ -0,0 +1,38 @@
+# opencraw (Ruby)
+
+Crawl with [OpenCraw](https://github.com/Open-Craw/OpenCraw) from Ruby, with no Node to install. `OpenCraw::Crawler`
+downloads the driver's runtime for this platform once (a Node binary and the engine in one archive, checked against
+its SHA-256 and cached in `~/.cache/opencraw/driver/`), starts it, and drives it over its pipes. Standard library
+only; Ruby 3.2 or later.
+
+```sh
+gem install opencraw
+```
+
+```ruby
+require 'opencraw'
+
+OpenCraw::Crawler.open(version: '0.7.1') do |crawler|
+  crawler.hook('tidy') { |value, _args, _request| value.to_s.strip.downcase }
+
+  run = crawler.run(paths: ['recipes/'])
+  run.records.each { |record| puts record } # as they are emitted
+  puts run.report['records']
+end
+```
+
+- **`run`** takes the driver's run options (`out:`, `only:`, `dryRun:`, `parallel:`, `access:`, ...); `records` and
+  `events` are enumerators that yield as it crawls, `report` waits for the end, `cancel` stops it.
+- **`work(next_item, ...)`** is worker mode: `next_item.call` returns each window's item (`{ id:, vars: }`), or `nil`
+  to end.
+- **`hook`, `captcha_solver`, `access_plugin`** register blocks by name; raising fails that one call
+  (`OpenCraw::CalloutError` for a plain message), never the run.
+- **`validate`, `probe`, `diff`, `export`** answer as the driver does.
+- `OPENCRAW_DRIVER=<folder>` uses an unpacked runtime; `OPENCRAW_DRIVER_OFFLINE=1` (or `offline: true`) never
+  downloads; `command:` starts any other driver.
+
+## Layout and development
+
+One slice per outcome under `lib/opencraw/`: `driver_runtime` (the runtime's download and cache), `driver_channel`
+(JSON-RPC over the driver's pipes) and `crawl_session` (`Crawler` and `Run`). `smith run lint typecheck test -p
+opencraw-ruby`; the specs start the repository's built driver and replay its conformance transcripts.

data/lib/opencraw/crawl_session/crawl_session.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+require_relative 'crawler_use_case'

data/lib/opencraw/crawl_session/crawler_use_case.rb

@@ -0,0 +1,226 @@
+# frozen_string_literal: true
+
+require_relative '../driver_channel/driver_channel'
+require_relative '../driver_runtime/driver_runtime'
+
+module OpenCraw
+  # OpenCraw's engine from Ruby: one driver process, runs and worker pools, and the blocks it calls back.
+  module CrawlSession
+    # The driver protocol this client speaks.
+    PROTOCOL_VERSION = '1.2.0'
+
+    # Raised by a hook, captcha solver or access plugin to fail that one call with this message, and nothing else.
+    class CalloutError < StandardError; end
+
+    # A run, or a worker pool, the driver is crawling.
+    class Run
+      attr_reader :id
+
+      def initialize(id, &cancel)
+        @id = id
+        @cancel = cancel
+        @lock = Mutex.new
+        @changed = ConditionVariable.new
+        @records = []
+        @events = []
+        @ended = false
+      end
+
+      # Each record's data, as it is emitted, from the start; the enumerator ends with the run.
+      def records = follow(@records)
+
+      # Every crawl event, as core reports it, from the start; the enumerator ends with the run.
+      def events = follow(@events)
+
+      # The run's report, once it has finished; raises when the run could not run.
+      def report
+        @lock.synchronize do
+          @changed.wait(@lock) until @ended
+          @report or raise(@error || "#{@id} ended without a report")
+        end
+      end
+
+      # Stops the run; it still ends, and report still answers.
+      def cancel = @cancel.call
+
+      # @api private
+      def add(kind, item)
+        @lock.synchronize do
+          (kind == :record ? @records : @events) << item
+          @changed.broadcast
+        end
+      end
+
+      # @api private
+      def finish(params)
+        @lock.synchronize do
+          @report = params['report']
+          @error = params['error']
+          @ended = true
+          @changed.broadcast
+        end
+      end
+
+      private
+
+      def follow(items)
+        Enumerator.new do |yielder|
+          seen = 0
+          loop do
+            fresh, done = @lock.synchronize do
+              @changed.wait(@lock) while seen == items.length && !@ended
+              [items[seen..], @ended]
+            end
+            fresh.each { |item| yielder << item }
+            seen += fresh.length
+            break if done && seen == items.length
+          end
+        end
+      end
+    end
+
+    # Starts the driver once and drives it: validate, run, work, probe, diff and export. Hooks, captcha solvers
+    # and access plugins are blocks registered by name before runs start; what a block returns is its answer,
+    # and raising fails that one call, never the run.
+    class Crawler
+      attr_reader :info
+
+      # The driver is command when given; else OPENCRAW_DRIVER or the downloaded runtime of version (no Node needed).
+      def self.open(**)
+        crawler = new(**)
+        return crawler unless block_given?
+
+        begin
+          yield crawler
+        ensure
+          crawler.close
+        end
+      end
+
+      def initialize(version: nil, command: nil, cache_dir: nil, offline: nil, env: nil)
+        @callouts = { 'hook' => {}, 'captcha' => {}, 'access' => {} }
+        @runs = {}
+        @next_items = {}
+        @lock = Mutex.new
+        @routed = ConditionVariable.new
+        command ||= DriverRuntime.ensure_driver_runtime(version.to_s, cache_dir: cache_dir, offline: offline).command
+        @channel = DriverChannel::JsonRpcChannel.new(command, env: env) { |method, params| handle(method, params) }
+        @info = @channel.request('initialize',
+                                 { clientName: 'opencraw-ruby', clientVersion: '0.0.1',
+                                   protocolVersion: PROTOCOL_VERSION })
+      end
+
+      # Serves the hook name: block(input, args, request).
+      def hook(name, &block) = register('hook', name, block)
+
+      # Serves a captcha solver: block(challenge, request) answering { 'token' => ... } or { 'text' => ... }.
+      def captcha_solver(name, &block) = register('captcha', name, block)
+
+      # Serves an access plugin: block(input, request), a lease (phase "lease") or a release.
+      def access_plugin(name, &block) = register('access', name, block)
+
+      # Checks recipes without running them: the inputs, the hooks they call, every problem with its path.
+      def validate(paths: nil, recipes: nil) = @channel.request('validate', source(paths, recipes))
+
+      # Starts a run; options are the driver's run options (out, only, dryRun, parallel, access, ...).
+      def run(paths: nil, recipes: nil, **options) = start('run', source(paths, recipes).merge(options))
+
+      # Starts a worker pool: next_item gives each window its item ({ 'id', 'vars' }), nil ends the work.
+      def work(next_item, paths: nil, recipes: nil, **options)
+        run = start('work', source(paths, recipes).merge(options))
+        @lock.synchronize do
+          @next_items[run.id] = next_item
+          @routed.broadcast
+        end
+        run
+      end
+
+      # Fetches a page (or renders it, browser: true) and says where its data lives.
+      def probe(url, **)
+        @channel.request('probe', { url: url, **, accessPlugins: @callouts['access'].keys })
+      end
+
+      # Compares two runs' JSON Lines files by key.
+      def diff(previous, current, **)
+        @channel.request('diff', { previous: previous, current: current, ** })
+      end
+
+      # Writes a run's JSON Lines file in another format, with an output recipe's columns.
+      def export(records, output, out) = @channel.request('export', { records: records, output: output, out: out })
+
+      # Asks the driver to shut down and waits for it to exit.
+      def close
+        begin
+          @channel.request('shutdown')
+        rescue DriverChannel::DriverError
+          nil # Already gone: nothing to stop.
+        end
+        @channel.close
+      end
+
+      private
+
+      def register(kind, name, block)
+        @lock.synchronize { @callouts[kind][name.to_s] = block }
+      end
+
+      def source(paths, recipes)
+        raise ArgumentError, 'give exactly one of paths or recipes' if paths.nil? == recipes.nil?
+
+        paths ? { paths: paths.map(&:to_s) } : { recipes: recipes }
+      end
+
+      def start(method, params)
+        served = @lock.synchronize do
+          { hooks: @callouts['hook'].keys, captchaSolvers: @callouts['captcha'].keys,
+            accessPlugins: @callouts['access'].keys }
+        end
+        id = @channel.request(method, params.merge(served))['runId']
+        run = Run.new(id) { @channel.request('cancel', { runId: id })['cancelled'] }
+        @lock.synchronize do
+          @runs[id] = run
+          @routed.broadcast
+        end
+        run
+      end
+
+      # A run's first messages can arrive with its answer, before start has stored it.
+      def wait_for(&lookup)
+        @lock.synchronize do
+          @routed.wait(@lock) until (found = lookup.call)
+          found
+        end
+      end
+
+      def handle(method, params)
+        return answer_callout(params['request']) if method == 'callout'
+
+        id = params['runId']
+        run = wait_for { @runs[id] }
+        case method
+        when 'work.next' then { item: wait_for { @next_items[id] }.call }
+        when 'event' then run.add(:event, params['event'])
+        when 'record' then run.add(:record, params['data'])
+        when 'finished'
+          @lock.synchronize { @runs.delete(id) }
+          run.finish(params)
+        end
+      end
+
+      def answer_callout(call)
+        block = @lock.synchronize { @callouts.dig(call['kind'], call['name']) }
+        return { status: 'error', error: "this client serves no #{call['kind']} #{call['name'].inspect}" } unless block
+
+        output = if call['kind'] == 'hook'
+                   block.call(call['input'], call['args'] || {},
+                              call)
+                 else
+                   block.call(call['input'], call)
+                 end
+        { status: 'ok', output: output }
+      rescue StandardError => e
+        { status: 'error', error: e.is_a?(CalloutError) ? e.message : "#{e.class}: #{e.message}" }
+      end
+    end
+  end
+end

data/lib/opencraw/driver_channel/driver_channel.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+require_relative 'json_rpc_channel_client'

data/lib/opencraw/driver_channel/json_rpc_channel_client.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'open3'
+
+module OpenCraw
+  # The driver's protocol: JSON-RPC 2.0, one message per line, over the driver process's pipes.
+  module DriverChannel
+    # The driver answered a request with an error: JSON-RPC's code and message.
+    class DriverError < StandardError
+      attr_reader :code
+
+      def initialize(code, message)
+        super(message)
+        @code = code
+      end
+    end
+
+    # A started driver and the protocol over its stdin and stdout. A reader thread settles answers and hands
+    # the driver's notifications to the handler in order; each request from the driver (a callout, work.next)
+    # runs on its own thread, so a slow hook never holds the channel. Writes are one line each, never interleaved.
+    class JsonRpcChannel
+      def initialize(command, env: nil, &handler)
+        args = env ? [env, *command, { unsetenv_others: true }] : command
+        @stdin, @stdout, @stderr, @process = Open3.popen3(*args)
+        @handler = handler
+        @pending = {}
+        @next_id = 1
+        @lock = Mutex.new
+        @write = Mutex.new
+        @closed = false
+        Thread.new { @stderr.each_line { nil } }
+        @reader = Thread.new { read }
+      end
+
+      # Sends a request and waits for its result; raises DriverError with the driver's error.
+      def request(method, params = nil)
+        reply = Queue.new
+        id = @lock.synchronize do
+          raise DriverError.new(-32_000, 'the driver has exited') if @closed
+
+          @pending[@next_id] = reply
+          (@next_id += 1) - 1
+        end
+        write(jsonrpc: '2.0', id: id, method: method, params: params)
+        answer = reply.pop
+        raise answer if answer.is_a?(DriverError)
+
+        answer
+      end
+
+      # Closes the driver's stdin and waits for it to exit.
+      def close
+        @stdin.close unless @stdin.closed?
+        @reader.join
+        @process.value
+      end
+
+      private
+
+      def write(message)
+        @write.synchronize do
+          @stdin.puts(JSON.generate(message))
+          @stdin.flush
+        end
+      end
+
+      def answer(message)
+        result = @handler.call(message['method'], message['params'] || {})
+        write(jsonrpc: '2.0', id: message['id'], result: result)
+      rescue StandardError => e
+        write(jsonrpc: '2.0', id: message['id'], error: { code: -32_000, message: e.message })
+      end
+
+      def read
+        @stdout.each_line do |line|
+          next if line.strip.empty?
+
+          message = JSON.parse(line)
+          if message['method']
+            if message.key?('id')
+              Thread.new { answer(message) }
+            else
+              # In order: a run's last events land before its finished.
+              @handler.call(message['method'], message['params'] || {})
+            end
+            next
+          end
+          reply = @lock.synchronize { @pending.delete(message['id']) }
+          next unless reply
+
+          error = message['error']
+          reply << (error ? DriverError.new(error['code'], error['message']) : message['result'])
+        end
+      ensure
+        @lock.synchronize do
+          @closed = true
+          @pending.each_value { |reply| reply << DriverError.new(-32_000, 'the driver has exited') }
+          @pending.clear
+        end
+      end
+    end
+  end
+end

data/lib/opencraw/driver_runtime/driver_runtime.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+require_relative 'ensure_driver_runtime_use_case'
+require_relative 'runtime_manifest_contract'

data/lib/opencraw/driver_runtime/ensure_driver_runtime_use_case.rb

@@ -0,0 +1,124 @@
+# frozen_string_literal: true
+
+require 'digest'
+require 'fileutils'
+require 'json'
+require 'net/http'
+require 'rbconfig'
+require 'rubygems/package'
+require 'securerandom'
+require 'uri'
+require 'zlib'
+require_relative 'runtime_manifest_contract'
+
+module OpenCraw
+  # The driver's runtime: one archive per platform, downloaded once, so a Ruby program needs no Node.
+  module DriverRuntime
+    # Where the driver's releases are: <base>/driver@<version>/driver-runtime.json and the archives beside it.
+    RELEASES_URL = 'https://github.com/Open-Craw/OpenCraw/releases/download'
+
+    # A runtime ready to start: its folder, and the command, absolute.
+    Runtime = Struct.new(:folder, :command, keyword_init: true)
+
+    # This machine as the runtime names platforms: Node's process.platform-process.arch.
+    def self.current_platform
+      host = RbConfig::CONFIG['host_os']
+      system = if host.match?(/mswin|mingw|cygwin/) then 'win32'
+               elsif host.include?('darwin') then 'darwin'
+               else 'linux'
+               end
+      arch = RbConfig::CONFIG['host_cpu'].match?(/arm64|aarch64/) ? 'arm64' : 'x64'
+      "#{system}-#{arch}"
+    end
+
+    # The driver's runtime for this platform, downloaded once into a per-user cache. OPENCRAW_DRIVER names an
+    # unpacked runtime to use as it is; a cached one is used as it is. Otherwise, unless offline, the release's
+    # driver-runtime.json names this platform's archive: it is downloaded, refused when its SHA-256 differs,
+    # unpacked into a temporary folder (refusing links and entries outside it) and moved into the cache in one
+    # step, so a half-written runtime is never used.
+    def self.ensure_driver_runtime(version, cache_dir: nil, offline: nil, platform: nil, base_url: RELEASES_URL,
+                                   env: ENV, fetch: nil)
+      named = env['OPENCRAW_DRIVER'].to_s
+      return runtime_in(File.expand_path(named)) unless named.empty?
+
+      where = platform || current_platform
+      cache = cache_dir || File.join(Dir.home, '.cache', 'opencraw', 'driver')
+      folder = File.join(cache, version, where)
+      return runtime_in(folder) if File.exist?(File.join(folder, 'runtime.json'))
+
+      offline = env['OPENCRAW_DRIVER_OFFLINE'] == '1' if offline.nil?
+      if offline
+        raise "driver #{version} for #{where} is not in #{cache}, and downloading is off (OPENCRAW_DRIVER_OFFLINE)"
+      end
+
+      get = fetch || method(:download)
+      release = "#{base_url}/#{URI.encode_www_form_component("driver@#{version}")}"
+      manifest = parse_manifest(JSON.parse(get.call("#{release}/driver-runtime.json")))
+      archive = manifest.archives[where]
+      raise "driver #{version} has no runtime for #{where}: it has #{manifest.archives.keys.join(', ')}" unless archive
+
+      bytes = get.call("#{release}/#{archive.file}")
+      sha256 = Digest::SHA256.hexdigest(bytes)
+      if sha256 != archive.sha256
+        raise "#{archive.file} does not match driver-runtime.json " \
+              "(sha256 #{sha256}, expected #{archive.sha256}): refusing it"
+      end
+
+      FileUtils.mkdir_p(File.join(cache, version))
+      unpacking = File.join(cache, version, ".#{where}-#{SecureRandom.hex(4)}")
+      begin
+        untar(bytes, unpacking)
+        File.rename(unpacking, folder)
+      rescue StandardError
+        FileUtils.rm_rf(unpacking)
+        # Another process may have put the same runtime there first: that one is as good.
+        return runtime_in(folder) if File.exist?(File.join(folder, 'runtime.json'))
+
+        raise
+      end
+      runtime_in(folder)
+    end
+
+    def self.download(url, limit = 5)
+      response = Net::HTTP.get_response(URI(url))
+      return download(response['location'], limit - 1) if response.is_a?(Net::HTTPRedirection) && limit.positive?
+      raise "GET #{url}: #{response.code}" unless response.is_a?(Net::HTTPSuccess)
+
+      response.body.b
+    end
+
+    # Unpacks a .tar.gz into folder, refusing any link and any entry that would land outside it.
+    def self.untar(bytes, folder)
+      FileUtils.mkdir_p(folder)
+      root = "#{File.expand_path(folder)}/"
+      Zlib::GzipReader.wrap(StringIO.new(bytes)) do |zipped|
+        Gem::Package::TarReader.new(zipped) do |entries|
+          entries.each do |entry|
+            target = File.expand_path(entry.full_name, folder)
+            unless "#{target}/".start_with?(root)
+              raise "the archive's #{entry.full_name.inspect} is outside the runtime's folder: refusing it"
+            end
+            if entry.symlink? || entry.header.typeflag == '1'
+              raise "the archive's #{entry.full_name.inspect} is a link: refusing it"
+            end
+
+            if entry.directory?
+              FileUtils.mkdir_p(target)
+            elsif entry.file?
+              FileUtils.mkdir_p(File.dirname(target))
+              File.binwrite(target, entry.read.to_s)
+              File.chmod((entry.header.mode & 0o777) | 0o600, target)
+            end
+          end
+        end
+      end
+    end
+
+    def self.runtime_in(folder)
+      described = JSON.parse(File.read(File.join(folder, 'runtime.json')))
+      Runtime.new(folder: folder, command: described.fetch('command').map { |part| File.join(folder, part) })
+    end
+
+    private_class_method :download, :untar, :runtime_in
+  end
+end

data/lib/opencraw/driver_runtime/runtime_manifest_contract.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module OpenCraw
+  # The driver's runtime: its release manifest's shape.
+  module DriverRuntime
+    # The platforms the runtime is built for.
+    PLATFORMS = %w[win32-x64 darwin-arm64 darwin-x64 linux-x64 linux-arm64].freeze
+
+    # One platform's archive: its file name beside the manifest, its SHA-256 and its size.
+    RuntimeArchive = Struct.new(:file, :sha256, :byte_size, keyword_init: true)
+
+    # driver-runtime.json, the driver's schemas/driver-runtime.schema.json: every platform's archive of one release.
+    RuntimeManifest = Struct.new(:driver_version, :protocol_version, :node, :archives, keyword_init: true)
+
+    # The manifest, checked the way its schema checks it.
+    def self.parse_manifest(value)
+      raise ArgumentError, 'driver-runtime.json is not an object' unless value.is_a?(Hash)
+
+      %w[driverVersion protocolVersion node].each do |field|
+        raise ArgumentError, "driver-runtime.json: #{field} is required" unless value[field].is_a?(String)
+      end
+      raise ArgumentError, 'driver-runtime.json: archives is required' unless value['archives'].is_a?(Hash)
+
+      archives = value['archives'].to_h do |platform, archive|
+        raise ArgumentError, "driver-runtime.json: #{platform} is not a platform" unless PLATFORMS.include?(platform)
+
+        valid = archive.is_a?(Hash) && archive['file'].to_s.match?(/\Aopencraw-driver-[\w.-]+\.tar\.gz\z/) &&
+                archive['sha256'].to_s.match?(/\A[0-9a-f]{64}\z/)
+        unless valid
+          raise ArgumentError,
+                "driver-runtime.json: the archive for #{platform} has no valid file and sha256"
+        end
+
+        archived = RuntimeArchive.new(file: archive['file'], sha256: archive['sha256'], byte_size: archive['size'].to_i)
+        [platform, archived]
+      end
+      RuntimeManifest.new(driver_version: value['driverVersion'], protocol_version: value['protocolVersion'],
+                          node: value['node'], archives: archives)
+    end
+  end
+end

data/lib/opencraw.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require_relative 'opencraw/crawl_session/crawl_session'
+
+# Crawl with OpenCraw from Ruby, with no Node to install: OpenCraw::Crawler downloads the driver's runtime for
+# this platform once, starts it, and drives it over its pipes. Its code lives in the slices under opencraw/.
+module OpenCraw
+  Crawler = CrawlSession::Crawler
+  Run = CrawlSession::Run
+  CalloutError = CrawlSession::CalloutError
+  DriverError = DriverChannel::DriverError
+end

metadata

@@ -0,0 +1,49 @@
+--- !ruby/object:Gem::Specification
+name: opencraw
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- opencraw
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies: []
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- lib/opencraw.rb
+- lib/opencraw/crawl_session/crawl_session.rb
+- lib/opencraw/crawl_session/crawler_use_case.rb
+- lib/opencraw/driver_channel/driver_channel.rb
+- lib/opencraw/driver_channel/json_rpc_channel_client.rb
+- lib/opencraw/driver_runtime/driver_runtime.rb
+- lib/opencraw/driver_runtime/ensure_driver_runtime_use_case.rb
+- lib/opencraw/driver_runtime/runtime_manifest_contract.rb
+homepage: https://github.com/Open-Craw/OpenCraw
+licenses:
+- MIT
+metadata:
+  rubygems_mfa_required: 'true'
+  smith_kind: package
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.2'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.9
+specification_version: 4
+summary: 'Crawl with OpenCraw from Ruby, with no Node to install: the engine through
+  its driver.'
+test_files: []