nous 0.3.0 → 0.5.0

data/vendor/defuddle.mjs.LEGAL.txt

@@ -0,0 +1,10 @@
+Bundled license information:
+
+@mixmark-io/domino/lib/style_parser.js:
+  /**
+   * @license
+   * Copyright Google LLC All Rights Reserved.
+   *
+   * Use of this source code is governed by an MIT-style license that can be
+   * found in the LICENSE file at https://angular.io/license
+   */

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: nous
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Dan Frenette
@@ -214,14 +214,24 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".agents/skills/verify-nous/SKILL.md"
 - ".rspec"
 - ".ruby-version"
 - ".standard.yml"
 - CHANGELOG.md
+- CODING_STANDARDS.md
 - LICENSE.txt
 - README.md
 - Rakefile
+- docs/setup/defuddle.md
+- docs/setup/verification.md
+- docs/verification/README.md
+- docs/verification/defuddle.md
+- docs/verification/failure-propagation.md
+- docs/verification/retained-html.md
 - exe/nous
+- javascript/build.mjs
+- javascript/defuddle.mjs
 - lib/nous.rb
 - lib/nous/cli.rb
 - lib/nous/command.rb
@@ -234,22 +244,33 @@ files:
 - lib/nous/crawler/redirect_follower.rb
 - lib/nous/crawler/single_page_fetcher.rb
 - lib/nous/crawler/url_filter.rb
+- lib/nous/decode_html.rb
+- lib/nous/extract_html.rb
 - lib/nous/extractor/default.rb
 - lib/nous/extractor/default/client.rb
-- lib/nous/extractor/jina.rb
-- lib/nous/extractor/jina/client.rb
+- lib/nous/extractor/defuddle.rb
+- lib/nous/extractor/defuddle/client.rb
+- lib/nous/extractor/defuddle/run_process.rb
+- lib/nous/extractor/extraction_error.rb
 - lib/nous/fetcher.rb
 - lib/nous/fetcher/extraction_runner.rb
 - lib/nous/fetcher/page_extractor.rb
 - lib/nous/primitives/configuration.rb
 - lib/nous/primitives/extracted_content.rb
+- lib/nous/primitives/fetch_record.rb
+- lib/nous/primitives/fetch_result.rb
 - lib/nous/primitives/page.rb
-- lib/nous/primitives/raw_page.rb
 - lib/nous/primitives/url.rb
 - lib/nous/serializer.rb
 - lib/nous/url_resolver.rb
 - lib/nous/version.rb
+- package-lock.json
+- package.json
 - sig/nous.rbs
+- tasks/verification.rake
+- vendor/THIRD_PARTY_LICENSES.txt
+- vendor/defuddle.mjs
+- vendor/defuddle.mjs.LEGAL.txt
 homepage: https://github.com/danfrenette/nous
 licenses:
 - MIT

data/lib/nous/extractor/jina/client.rb

@@ -1,59 +0,0 @@
-# frozen_string_literal: true
-
-require "faraday"
-require "faraday/retry"
-require "json"
-
-module Nous
-  module Extractor
-    class Jina
-      class Client
-        class RequestError < StandardError; end
-
-        BASE_URL = "https://r.jina.ai"
-        RETRYABLE_STATUSES = [429, 500, 502, 503, 504].freeze
-        MAX_RETRIES = 3
-
-        def initialize(api_key: nil, timeout: 30, retry_interval: 1)
-          @connection = build_connection(api_key:, timeout:, retry_interval:)
-        end
-
-        def get(url)
-          response = connection.get("/#{url}")
-          parse(response.body)
-        rescue Faraday::Error => e
-          raise RequestError, e.message
-        end
-
-        private
-
-        attr_reader :connection
-
-        def build_connection(api_key:, timeout:, retry_interval:)
-          Faraday.new(url: BASE_URL) do |f|
-            f.response :raise_error
-
-            f.request :retry,
-              max: MAX_RETRIES,
-              interval: retry_interval,
-              backoff_factor: 2,
-              retry_statuses: RETRYABLE_STATUSES
-
-            f.headers["Accept"] = "application/json"
-            f.headers["X-No-Cache"] = "true"
-            f.headers["Authorization"] = "Bearer #{api_key}" if api_key
-
-            f.options.timeout = timeout
-            f.options.open_timeout = timeout
-          end
-        end
-
-        def parse(body)
-          JSON.parse(body)
-        rescue JSON::ParserError => e
-          raise RequestError, "invalid JSON: #{e.message}"
-        end
-      end
-    end
-  end
-end

data/lib/nous/extractor/jina.rb

@@ -1,26 +0,0 @@
-# frozen_string_literal: true
-
-module Nous
-  module Extractor
-    class Jina < Command
-      def initialize(api_key: nil, timeout: 30, **client_options)
-        @client = Client.new(api_key: api_key || ENV["JINA_API_KEY"], timeout:, **client_options)
-      end
-
-      def extract(raw_page)
-        body = client.get(raw_page.url)
-
-        success(payload: ExtractedContent.new(
-          title: body.dig("data", "title") || "",
-          content: body.dig("data", "content") || ""
-        ))
-      rescue Client::RequestError => e
-        failure(ExtractionError.new(e.message))
-      end
-
-      private
-
-      attr_reader :client
-    end
-  end
-end

data/lib/nous/primitives/raw_page.rb

@@ -1,5 +0,0 @@
-# frozen_string_literal: true
-
-module Nous
-  RawPage = Data.define(:url, :pathname, :html)
-end