huginn_datacircle_agent 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 6a022db3f1ff492afbaf28b1cc4a935a271a758877912cee8ffed3cd6dc25b71
+  data.tar.gz: 550324c4aa6de94ac365cf6e056b45e4784ee89a730a0da4c6ecf47e793a3deb
+SHA512:
+  metadata.gz: cf53385e9fc84cdd2247aadfce4dc26c2a04c1013ce1b45bec614fe71a3fb7c84103080f92c824f850e73984f91ffbdf6cf6b5c7bbeeeb3c16c752a824a27b43
+  data.tar.gz: 46165f536cf5754b45d16ca972a8bb9d3e4de62008b47e0dd6bba7d01c37f8e298bb722db2e37782d907af65f0cb3b70a72abdf900691ca7e022654df4806d1b

data/LICENSE.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Datacircle
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,93 @@
+# Datacircle agent for Huginn
+
+Find the best GTM APIs. We benchmark all the GTM APIs and we put the results on our website: https://datacircle.dev
+
+This gem adds the Datacircle Agent to [Huginn](https://github.com/huginn/huginn). The agent emits the best plans of an API, like an email finder or a SERP API, as Huginn events. It reads Datacircle's open API, with no key and no account.
+
+## Use the agent to
+
+- Get the best email finder plans each morning, and an email when the first one changes.
+- Find every plan of one provider, or the cheapest plans of an API.
+- Follow one provider's rank on each API it sells.
+
+## Its calls
+
+Set `call` to one of these. Each one is a call of the API, at https://api.datacircle.dev/v1:
+
+| `call` | API call | Events |
+|---|---|---|
+| `best_plans` | `GET /apis/{slug}/best` | the plans of one API (`api`), best first |
+| `search_plans` | `GET /plans` | plans by API, provider or plan name (`api`, `query`), sorted by `score`, `provider` or `price` |
+| `list_apis` | `GET /apis` | every API on Datacircle, with the API's slug and best plan |
+| `provider` | `GET /providers/{slug}` | one provider's rank on each API (`provider`) |
+
+`limit` is how many plans `best_plans` and `search_plans` emit, 1 to 50. Each plan has a rank, a score out of 100, a price per 1,000 and a link to the plan's page on datacircle.dev. That page lists every field.
+
+Each option takes Liquid. So a Datacircle Agent can read the event of another agent: `"api": "{{ slug }}"` reads the best plans of each API a `list_apis` agent emits.
+
+The spec: https://api.datacircle.dev/v1/openapi.json. At most 60 calls a minute from one address. The agent waits and calls again when the API asks it to.
+
+## Install
+
+Add the gem to `ADDITIONAL_GEMS` in Huginn's `.env`:
+
+```
+ADDITIONAL_GEMS=huginn_datacircle_agent
+```
+
+With Huginn's docker image, pass it as a variable:
+
+```bash
+docker run -it -p 3000:3000 -e ADDITIONAL_GEMS=huginn_datacircle_agent ghcr.io/huginn/huginn
+```
+
+Then run `bundle` and restart Huginn. The Datacircle Agent is in the agent types of "New Agent".
+
+## Options
+
+```json
+{
+  "call": "best_plans",
+  "api": "email-finder",
+  "query": "",
+  "sort": "score",
+  "provider": "",
+  "limit": "10",
+  "offset": "0",
+  "expected_update_period_in_days": "2"
+}
+```
+
+- `api`: the API's slug, the words of its sheet's address on datacircle.dev, like `email-finder` for https://datacircle.dev/email-finder-api-pricing. `search_plans` also takes the API's name.
+- `query`: words that must all appear in the provider's or plan's name.
+- `provider`: the provider's slug, its name in lowercase with hyphens, like `bright-data`.
+- `offset`: how many plans `search_plans` skips first.
+
+## An event
+
+A plan of `best_plans`, shortened. Each plan has 83 fields:
+
+```json
+{
+  "rank": 1,
+  "provider": "GetProspect",
+  "plan": "Growth 50k",
+  "api": "Employee email",
+  "score": 79.7,
+  "perThousand": 7.98,
+  "pay": 399,
+  "currency": "USD",
+  "billing": "monthly",
+  "page": "https://datacircle.dev/find-email-apis/getprospect/free-post-v2-email-find"
+}
+```
+
+## Links
+
+- Datacircle: https://datacircle.dev
+- What each field means: https://datacircle.dev/fields
+- The API's spec: https://api.datacircle.dev/v1/openapi.json
+
+## License
+
+MIT

data/lib/huginn_datacircle_agent/datacircle_agent.rb

@@ -0,0 +1,180 @@
+module Agents
+  class DatacircleAgent < Agent
+    include WebRequestConcern
+    include FormConfigurable
+
+    API = 'https://api.datacircle.dev/v1'.freeze
+    CALLS = %w[best_plans search_plans list_apis provider].freeze
+    SORTS = %w[score provider price].freeze
+
+    can_dry_run!
+    default_schedule 'every_1d'
+
+    description <<~MD
+      The Datacircle Agent reads Datacircle's open API and emits an event for each plan, API or rank it gets back. Datacircle benchmarks the GTM APIs (email finders, LinkedIn profile APIs, SERP APIs and more) and puts the results on [datacircle.dev](https://datacircle.dev). The API needs no key and no account.
+
+      Set `call` to one of:
+
+      * `best_plans`: the plans of one API, best first (`GET /apis/{slug}/best`). `api` is the API's slug, like `email-finder`.
+      * `search_plans`: the plans whose provider or plan name holds every word of `query` (`GET /plans`). `api` keeps one API's plans only. `sort` is `score`, `provider` or `price`. `offset` skips the first plans found.
+      * `list_apis`: every API on Datacircle, with its slug and its best plan (`GET /apis`).
+      * `provider`: one provider's rank on each API it sells (`GET /providers/{slug}`). `provider` is its slug, like `bright-data`.
+
+      `limit` is how many plans `best_plans` and `search_plans` emit, 1 to 50 (10 when left empty).
+
+      Each option takes [Liquid](https://github.com/huginn/huginn/wiki/Formatting-Events-using-Liquid), filled by each event the agent receives. So `"api": "{{ slug }}"` reads the best plans of each API a `list_apis` agent emits.
+
+      Set `expected_update_period_in_days` to the most days that should pass between two events of this agent.
+
+      What each field of a plan means: [datacircle.dev/fields](https://datacircle.dev/fields). The spec: [api.datacircle.dev/v1/openapi.json](https://api.datacircle.dev/v1/openapi.json). At most 60 calls a minute from one address.
+    MD
+
+    event_description <<~MD
+      A plan of `best_plans` or `search_plans` looks like this, shortened (each plan has 83 fields, [datacircle.dev/fields](https://datacircle.dev/fields) says what each means):
+
+          {
+            "rank": 1,
+            "provider": "GetProspect",
+            "plan": "Growth 50k",
+            "api": "Employee email",
+            "score": 79.7,
+            "perThousand": 7.98,
+            "pay": 399,
+            "currency": "USD",
+            "billing": "monthly",
+            "page": "https://datacircle.dev/find-email-apis/getprospect/free-post-v2-email-find"
+          }
+
+      `list_apis` emits one event for each API: `api`, `slug`, `plans`, `providers`, `sheetPage`, `bestPage` and its `best` plan.
+
+      `provider` emits one event for each API the provider sells: `provider`, `providerSlug`, `website`, `api`, `slug`, `rank`, `sheetPage`, `bestPage` and the provider's `plans` on that API.
+    MD
+
+    form_configurable :call, type: :array, values: CALLS
+    form_configurable :api
+    form_configurable :query
+    form_configurable :sort, type: :array, values: SORTS
+    form_configurable :provider
+    form_configurable :limit
+    form_configurable :offset
+    form_configurable :expected_update_period_in_days
+
+    def default_options
+      {
+        'call' => 'best_plans',
+        'api' => 'email-finder',
+        'query' => '',
+        'sort' => 'score',
+        'provider' => '',
+        'limit' => '10',
+        'offset' => '0',
+        'expected_update_period_in_days' => '2'
+      }
+    end
+
+    def validate_options
+      call = options['call'].to_s
+      unless CALLS.include?(call) || liquid?(call)
+        errors.add(:base, "call must be one of #{CALLS.join(', ')}")
+      end
+      errors.add(:base, 'api is required for best_plans') if call == 'best_plans' && options['api'].blank?
+      errors.add(:base, 'provider is required for provider') if call == 'provider' && options['provider'].blank?
+
+      sort = options['sort'].to_s
+      if sort.present? && !SORTS.include?(sort) && !liquid?(sort)
+        errors.add(:base, "sort must be one of #{SORTS.join(', ')}")
+      end
+      { 'limit' => 1..50, 'offset' => 0.. }.each do |key, range|
+        value = options[key].to_s
+        next if value.blank? || liquid?(value)
+        unless value =~ /\A\d+\z/ && range.cover?(value.to_i)
+          errors.add(:base, "#{key} must be a whole number from #{range.begin}#{range.end ? " to #{range.end}" : ' up'}")
+        end
+      end
+
+      errors.add(:base, 'expected_update_period_in_days is required') if options['expected_update_period_in_days'].blank?
+      validate_web_request_options!
+    end
+
+    def working?
+      event_created_within?(interpolated['expected_update_period_in_days']) && !recent_error_logs?
+    end
+
+    def check
+      emit_answer(interpolated)
+    end
+
+    def receive(incoming_events)
+      incoming_events.each do |event|
+        interpolate_with(event) do
+          emit_answer(interpolated)
+        end
+      end
+    end
+
+    private
+
+    def liquid?(value)
+      value.include?('{{') || value.include?('{%')
+    end
+
+    def emit_answer(opts)
+      call = opts['call'].to_s
+      path, params = request_for(call, opts)
+      return error("call must be one of #{CALLS.join(', ')}, not #{call.inspect}") unless path
+
+      response = get(path, params)
+      body = begin
+        JSON.parse(response.body)
+      rescue JSON::ParserError
+        nil
+      end
+      unless response.success? && body
+        reason = body.is_a?(Hash) && body['error'] ? body['error'] : response.body.to_s[0, 300]
+        return error("Datacircle answered #{response.status} to #{path}: #{reason}")
+      end
+
+      items_of(call, body).each do |item|
+        create_event payload: item
+      end
+    end
+
+    def request_for(call, opts)
+      case call
+      when 'best_plans'
+        ["/apis/#{ERB::Util.url_encode(opts['api'].to_s.strip)}/best", { 'limit' => opts['limit'] }]
+      when 'search_plans'
+        ['/plans', opts.slice('api', 'query', 'sort', 'limit', 'offset')]
+      when 'list_apis'
+        ['/apis', {}]
+      when 'provider'
+        ["/providers/#{ERB::Util.url_encode(opts['provider'].to_s.strip)}", {}]
+      end
+    end
+
+    # One call, and once more after the wait the API asks for when more than 60 calls a minute came from this address.
+    def get(path, params)
+      params = params.transform_values { |value| value.to_s.strip }.reject { |_, value| value.empty? }
+      response = faraday.get("#{API}#{path}", params, 'Accept' => 'application/json')
+      return response unless response.status == 429
+
+      wait = response.headers['Retry-After'].to_i.clamp(1, 60)
+      log "Datacircle asked to wait #{wait} s (60 calls a minute at most), calling again then"
+      sleep wait
+      faraday.get("#{API}#{path}", params, 'Accept' => 'application/json')
+    end
+
+    def items_of(call, body)
+      case call
+      when 'best_plans', 'search_plans'
+        body['plans']
+      when 'list_apis'
+        body
+      when 'provider'
+        body['apis'].map do |api|
+          { 'provider' => body['provider'], 'providerSlug' => body['slug'], 'website' => body['website'] }.merge(api)
+        end
+      end
+    end
+  end
+end

data/lib/huginn_datacircle_agent.rb

@@ -0,0 +1,3 @@
+require 'huginn_agent'
+
+HuginnAgent.register 'huginn_datacircle_agent/datacircle_agent'

metadata

@@ -0,0 +1,64 @@
+--- !ruby/object:Gem::Specification
+name: huginn_datacircle_agent
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Wayne Hamadi
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: huginn_agent
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 'A Huginn agent that emits the best plans of each GTM API, plan searches
+  and provider ranks from Datacircle''s open API, with no key. We benchmark all the
+  GTM APIs and we put the results on our website: https://datacircle.dev'
+email:
+- wayne@datacircle.dev
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE.txt
+- README.md
+- lib/huginn_datacircle_agent.rb
+- lib/huginn_datacircle_agent/datacircle_agent.rb
+homepage: https://datacircle.dev
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://datacircle.dev
+  source_code_uri: https://github.com/waynehamadi/huginn_datacircle_agent
+  documentation_uri: https://github.com/waynehamadi/huginn_datacircle_agent#readme
+  bug_tracker_uri: https://github.com/waynehamadi/huginn_datacircle_agent/issues
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.7'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 4.0.24
+specification_version: 4
+summary: Find the best GTM APIs, in Huginn.
+test_files: []