pikuri-web 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/README.md +68 -0
- data/lib/pikuri/web/fetch.rb +153 -0
- data/lib/pikuri/web/read_cached.rb +96 -0
- data/lib/pikuri/web/scrape.rb +107 -0
- data/lib/pikuri/web/scraper.rb +188 -0
- data/lib/pikuri/web/search/brave.rb +298 -0
- data/lib/pikuri/web/search/duckduckgo.rb +181 -0
- data/lib/pikuri/web/search/engines.rb +430 -0
- data/lib/pikuri/web/search/exa.rb +243 -0
- data/lib/pikuri/web/search/page_store.rb +142 -0
- data/lib/pikuri/web/search/rate_limiter.rb +114 -0
- data/lib/pikuri/web/search/result.rb +42 -0
- data/lib/pikuri/web/search.rb +229 -0
- data/lib/pikuri/web/url_cache/in_memory.rb +96 -0
- data/lib/pikuri/web/url_cache/on_disk.rb +80 -0
- data/lib/pikuri/web/url_cache.rb +63 -0
- data/lib/pikuri-web.rb +36 -0
- metadata +138 -0
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'digest'
|
|
4
|
+
require 'fileutils'
|
|
5
|
+
|
|
6
|
+
module Pikuri
|
|
7
|
+
module Web
|
|
8
|
+
module UrlCache
|
|
9
|
+
# The on-disk store: one file per entry, +<dir>/<sha256 of key>.txt+,
|
|
10
|
+
# freshness by mtime, no sidecar metadata. Survives a restart and is shared
|
|
11
|
+
# by every pikuri process. Part of {UrlCache} — see it for the API.
|
|
12
|
+
#
|
|
13
|
+
# CACHE = UrlCache::OnDisk.new(ttl: UrlCache::DEFAULT_TTL, dir: "#{UrlCache::ROOT_DIR}/fetch")
|
|
14
|
+
#
|
|
15
|
+
# A stale entry stays on disk until a {#write} with its key replaces it;
|
|
16
|
+
# +rm -rf+ the directory to clear it. Never use it for a payload whose terms
|
|
17
|
+
# forbid storing it — that is {InMemory}'s job.
|
|
18
|
+
#
|
|
19
|
+
# Sharing: +P_shared_benign+ — an instance holds only its TTL and directory,
|
|
20
|
+
# and every owner keeps it in a class-level constant, so all agents (and
|
|
21
|
+
# other pikuri processes) hit the same files. Two callers racing on one cold
|
|
22
|
+
# key both compute and both write the same bytes — the accepted tradeoff for
|
|
23
|
+
# keeping this a few dozen lines (worst case: a duplicate fetch). The
|
|
24
|
+
# cross-agent flow is deliberate: a public URL's content is nobody's private
|
|
25
|
+
# context.
|
|
26
|
+
class OnDisk
|
|
27
|
+
# @param ttl [Numeric] freshness window in seconds; an entry with an older
|
|
28
|
+
# mtime is a miss. +Float::INFINITY+ never expires.
|
|
29
|
+
# @param dir [String] where the files live; created lazily on first write
|
|
30
|
+
def initialize(ttl:, dir:)
|
|
31
|
+
@ttl = ttl
|
|
32
|
+
@dir = dir
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
# @param key [String] a URL or any opaque string
|
|
36
|
+
# @return [String, nil] the stored payload, or +nil+ when absent or stale
|
|
37
|
+
def read(key)
|
|
38
|
+
path = path_for(key)
|
|
39
|
+
File.read(path) if fresh?(path)
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# @param key [String] a URL or any opaque string
|
|
43
|
+
# @param value [String] payload to store, replacing any entry
|
|
44
|
+
# @return [String] +value+
|
|
45
|
+
def write(key, value)
|
|
46
|
+
FileUtils.mkdir_p(@dir)
|
|
47
|
+
File.write(path_for(key), value)
|
|
48
|
+
value
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
# Return the stored payload for +key+ if fresh, else yield to compute it,
|
|
52
|
+
# store and return it. A raising block stores nothing (errors aren't cached).
|
|
53
|
+
#
|
|
54
|
+
# @param key [String] a URL or any opaque string
|
|
55
|
+
# @yieldreturn [String] payload to store and return on a miss
|
|
56
|
+
# @return [String] cached or freshly-computed payload
|
|
57
|
+
def fetch(key)
|
|
58
|
+
cached = read(key)
|
|
59
|
+
return cached unless cached.nil?
|
|
60
|
+
|
|
61
|
+
write(key, yield)
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# @param key [String]
|
|
65
|
+
# @return [String] absolute path of the cache file for +key+
|
|
66
|
+
def path_for(key)
|
|
67
|
+
File.join(@dir, "#{Digest::SHA256.hexdigest(key)}.txt")
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
private
|
|
71
|
+
|
|
72
|
+
# @param path [String]
|
|
73
|
+
# @return [Boolean] true when +path+ exists and was written within the TTL
|
|
74
|
+
def fresh?(path)
|
|
75
|
+
File.exist?(path) && Time.now - File.mtime(path) < @ttl
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
end
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pikuri
|
|
4
|
+
module Web
|
|
5
|
+
# Text payloads cached by key within a TTL, so a tool doesn't redo a request.
|
|
6
|
+
# Two stores share one duck-typed API:
|
|
7
|
+
#
|
|
8
|
+
# cache = UrlCache::OnDisk.new(ttl: UrlCache::DEFAULT_TTL, dir: "#{UrlCache::ROOT_DIR}/web_scrape")
|
|
9
|
+
# cache = UrlCache::InMemory.new(ttl: UrlCache::DEFAULT_TTL, max_entries: 100)
|
|
10
|
+
#
|
|
11
|
+
# cache.fetch(key) { compute } # yield on a miss, store, return; a raise stores nothing
|
|
12
|
+
# cache.read(key) # => String, or nil on a miss or stale entry
|
|
13
|
+
# cache.write(key, value) # => value; replaces any entry
|
|
14
|
+
#
|
|
15
|
+
# Pick the store by what the payload's source allows. {OnDisk} survives a
|
|
16
|
+
# restart and is shared by every pikuri process; {InMemory} lives only as
|
|
17
|
+
# long as the Ruby VM, for results whose terms forbid storing them — a search
|
|
18
|
+
# provider says which via +disk_cacheable?+. Each user holds its own
|
|
19
|
+
# instance in a constant, never a shared one: the dir, TTL and size are per
|
|
20
|
+
# use. {NULL} stands in for either.
|
|
21
|
+
module UrlCache
|
|
22
|
+
# Root dir for the {OnDisk} subdirs, per XDG:
|
|
23
|
+
# +$XDG_CACHE_HOME/pikuri/url_cache+ or +~/.cache/pikuri/url_cache+. Each
|
|
24
|
+
# user picks its own subdir (+"#{ROOT_DIR}/web_scrape"+) so keys can't
|
|
25
|
+
# collide.
|
|
26
|
+
#
|
|
27
|
+
# @return [String]
|
|
28
|
+
ROOT_DIR = Paths.cache.join('url_cache').to_path
|
|
29
|
+
|
|
30
|
+
# Default freshness window: 2 hours. Long enough to cover an interactive
|
|
31
|
+
# session (revisiting a page or re-running a search within the working
|
|
32
|
+
# window hits the cache), short enough that resuming the next day doesn't
|
|
33
|
+
# serve stale results. (opencode caches nothing; +pi-web-fetch+ uses 15 min,
|
|
34
|
+
# +pi-web-search+ 5.)
|
|
35
|
+
#
|
|
36
|
+
# @return [Integer]
|
|
37
|
+
DEFAULT_TTL = 2 * 60 * 60
|
|
38
|
+
|
|
39
|
+
# Null cache: always misses, never stores — a drop-in for either store in
|
|
40
|
+
# tests, or anywhere caching should be off.
|
|
41
|
+
#
|
|
42
|
+
# UrlCache::NULL.fetch('k') { 'x' } # => "x", yields every time
|
|
43
|
+
# UrlCache::NULL.read('k') # => nil
|
|
44
|
+
#
|
|
45
|
+
# @return [Object]
|
|
46
|
+
NULL = Object.new
|
|
47
|
+
# @param _key [String] ignored
|
|
48
|
+
# @return [nil] always a miss
|
|
49
|
+
def NULL.read(_key) = nil
|
|
50
|
+
|
|
51
|
+
# @param _key [String] ignored
|
|
52
|
+
# @param value [String] returned, never stored
|
|
53
|
+
# @return [String] +value+
|
|
54
|
+
def NULL.write(_key, value) = value
|
|
55
|
+
|
|
56
|
+
# @param _key [String] ignored
|
|
57
|
+
# @yieldreturn [String] freshly-computed payload
|
|
58
|
+
# @return [String] whatever the block returned
|
|
59
|
+
def NULL.fetch(_key) = yield
|
|
60
|
+
NULL.freeze
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
end
|
data/lib/pikuri-web.rb
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'pikuri-core'
|
|
4
|
+
require 'pikuri-workspace'
|
|
5
|
+
require 'pikuri-html'
|
|
6
|
+
|
|
7
|
+
# Entry file for the pikuri-web gem: mounts a per-gem Zeitwerk loader at this
|
|
8
|
+
# +lib/+ and eager-loads it, so the +ALL_CAPS+ tool constants ({Pikuri::Web::FETCH}
|
|
9
|
+
# and friends), which Zeitwerk's filename↔CamelCase rule won't auto-load, are
|
|
10
|
+
# defined at boot. Requiring pikuri-html registers the HTML extractor the scraper
|
|
11
|
+
# renders pages through.
|
|
12
|
+
module Pikuri
|
|
13
|
+
# The tools that reach the internet: search, scrape, fetch, and a reader for a
|
|
14
|
+
# search's stored page text.
|
|
15
|
+
#
|
|
16
|
+
# c.add_tool Pikuri::Web::Search.from_h(examples_config['search'])
|
|
17
|
+
# c.add_tool Pikuri::Web::SCRAPE
|
|
18
|
+
# c.add_tool Pikuri::Web::FETCH
|
|
19
|
+
#
|
|
20
|
+
# Every tool here brings untrusted text in, and all but {READ_CACHED} send
|
|
21
|
+
# bytes out — {SCRAPE} and {FETCH} to any host the model names. That is why
|
|
22
|
+
# they live outside pikuri-core: an agent that needs none of them never loads
|
|
23
|
+
# them, and wiring one is a visible choice on the trifecta map.
|
|
24
|
+
#
|
|
25
|
+
# For a search-only agent, pair a +page_text+ {Search.build} with
|
|
26
|
+
# {READ_CACHED} rather than {SCRAPE}; see {Search.build}.
|
|
27
|
+
module Web
|
|
28
|
+
LOADER = Zeitwerk::Loader.new
|
|
29
|
+
LOADER.tag = 'pikuri-web'
|
|
30
|
+
LOADER.push_dir(File.expand_path('.', __dir__))
|
|
31
|
+
LOADER.ignore(__FILE__)
|
|
32
|
+
LOADER.inflector.inflect('duckduckgo' => 'DuckDuckGo')
|
|
33
|
+
LOADER.setup
|
|
34
|
+
LOADER.eager_load
|
|
35
|
+
end
|
|
36
|
+
end
|
metadata
ADDED
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
|
2
|
+
name: pikuri-web
|
|
3
|
+
version: !ruby/object:Gem::Version
|
|
4
|
+
version: 0.2.0
|
|
5
|
+
platform: ruby
|
|
6
|
+
authors:
|
|
7
|
+
- Martin Vysny
|
|
8
|
+
bindir: bin
|
|
9
|
+
cert_chain: []
|
|
10
|
+
date: 1980-01-02 00:00:00.000000000 Z
|
|
11
|
+
dependencies:
|
|
12
|
+
- !ruby/object:Gem::Dependency
|
|
13
|
+
name: pikuri-core
|
|
14
|
+
requirement: !ruby/object:Gem::Requirement
|
|
15
|
+
requirements:
|
|
16
|
+
- - '='
|
|
17
|
+
- !ruby/object:Gem::Version
|
|
18
|
+
version: 0.2.0
|
|
19
|
+
type: :runtime
|
|
20
|
+
prerelease: false
|
|
21
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
22
|
+
requirements:
|
|
23
|
+
- - '='
|
|
24
|
+
- !ruby/object:Gem::Version
|
|
25
|
+
version: 0.2.0
|
|
26
|
+
- !ruby/object:Gem::Dependency
|
|
27
|
+
name: pikuri-workspace
|
|
28
|
+
requirement: !ruby/object:Gem::Requirement
|
|
29
|
+
requirements:
|
|
30
|
+
- - '='
|
|
31
|
+
- !ruby/object:Gem::Version
|
|
32
|
+
version: 0.2.0
|
|
33
|
+
type: :runtime
|
|
34
|
+
prerelease: false
|
|
35
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
36
|
+
requirements:
|
|
37
|
+
- - '='
|
|
38
|
+
- !ruby/object:Gem::Version
|
|
39
|
+
version: 0.2.0
|
|
40
|
+
- !ruby/object:Gem::Dependency
|
|
41
|
+
name: pikuri-html
|
|
42
|
+
requirement: !ruby/object:Gem::Requirement
|
|
43
|
+
requirements:
|
|
44
|
+
- - '='
|
|
45
|
+
- !ruby/object:Gem::Version
|
|
46
|
+
version: 0.2.0
|
|
47
|
+
type: :runtime
|
|
48
|
+
prerelease: false
|
|
49
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
50
|
+
requirements:
|
|
51
|
+
- - '='
|
|
52
|
+
- !ruby/object:Gem::Version
|
|
53
|
+
version: 0.2.0
|
|
54
|
+
- !ruby/object:Gem::Dependency
|
|
55
|
+
name: faraday
|
|
56
|
+
requirement: !ruby/object:Gem::Requirement
|
|
57
|
+
requirements:
|
|
58
|
+
- - "~>"
|
|
59
|
+
- !ruby/object:Gem::Version
|
|
60
|
+
version: '2.9'
|
|
61
|
+
type: :runtime
|
|
62
|
+
prerelease: false
|
|
63
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
64
|
+
requirements:
|
|
65
|
+
- - "~>"
|
|
66
|
+
- !ruby/object:Gem::Version
|
|
67
|
+
version: '2.9'
|
|
68
|
+
- !ruby/object:Gem::Dependency
|
|
69
|
+
name: nokogiri
|
|
70
|
+
requirement: !ruby/object:Gem::Requirement
|
|
71
|
+
requirements:
|
|
72
|
+
- - "~>"
|
|
73
|
+
- !ruby/object:Gem::Version
|
|
74
|
+
version: '1.19'
|
|
75
|
+
type: :runtime
|
|
76
|
+
prerelease: false
|
|
77
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
78
|
+
requirements:
|
|
79
|
+
- - "~>"
|
|
80
|
+
- !ruby/object:Gem::Version
|
|
81
|
+
version: '1.19'
|
|
82
|
+
description: |
|
|
83
|
+
pikuri-web gives a pikuri-core agent the tools that reach the internet:
|
|
84
|
+
+web_search+ (DuckDuckGo, plus Brave and Exa when the host supplies a
|
|
85
|
+
key), +web_scrape+ (a page as readable Markdown), +fetch+ (a URL's body
|
|
86
|
+
verbatim) and +web_read_cached+ (a search result's page text, with no
|
|
87
|
+
request of its own). Each brings untrusted text into the conversation
|
|
88
|
+
and most send bytes out, so they live outside pikuri-core: an agent
|
|
89
|
+
that doesn't wire them never loads them.
|
|
90
|
+
email:
|
|
91
|
+
- martin@vysny.me
|
|
92
|
+
executables: []
|
|
93
|
+
extensions: []
|
|
94
|
+
extra_rdoc_files: []
|
|
95
|
+
files:
|
|
96
|
+
- README.md
|
|
97
|
+
- lib/pikuri-web.rb
|
|
98
|
+
- lib/pikuri/web/fetch.rb
|
|
99
|
+
- lib/pikuri/web/read_cached.rb
|
|
100
|
+
- lib/pikuri/web/scrape.rb
|
|
101
|
+
- lib/pikuri/web/scraper.rb
|
|
102
|
+
- lib/pikuri/web/search.rb
|
|
103
|
+
- lib/pikuri/web/search/brave.rb
|
|
104
|
+
- lib/pikuri/web/search/duckduckgo.rb
|
|
105
|
+
- lib/pikuri/web/search/engines.rb
|
|
106
|
+
- lib/pikuri/web/search/exa.rb
|
|
107
|
+
- lib/pikuri/web/search/page_store.rb
|
|
108
|
+
- lib/pikuri/web/search/rate_limiter.rb
|
|
109
|
+
- lib/pikuri/web/search/result.rb
|
|
110
|
+
- lib/pikuri/web/url_cache.rb
|
|
111
|
+
- lib/pikuri/web/url_cache/in_memory.rb
|
|
112
|
+
- lib/pikuri/web/url_cache/on_disk.rb
|
|
113
|
+
homepage: https://github.com/mvysny/pikuri
|
|
114
|
+
licenses:
|
|
115
|
+
- MIT
|
|
116
|
+
metadata:
|
|
117
|
+
source_code_uri: https://github.com/mvysny/pikuri/tree/master
|
|
118
|
+
changelog_uri: https://github.com/mvysny/pikuri/blob/master/CHANGELOG.md
|
|
119
|
+
bug_tracker_uri: https://github.com/mvysny/pikuri/issues
|
|
120
|
+
rubygems_mfa_required: 'true'
|
|
121
|
+
rdoc_options: []
|
|
122
|
+
require_paths:
|
|
123
|
+
- lib
|
|
124
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
|
125
|
+
requirements:
|
|
126
|
+
- - ">="
|
|
127
|
+
- !ruby/object:Gem::Version
|
|
128
|
+
version: '3.3'
|
|
129
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
130
|
+
requirements:
|
|
131
|
+
- - ">="
|
|
132
|
+
- !ruby/object:Gem::Version
|
|
133
|
+
version: '0'
|
|
134
|
+
requirements: []
|
|
135
|
+
rubygems_version: 3.6.7
|
|
136
|
+
specification_version: 4
|
|
137
|
+
summary: Web search, scrape and fetch tools for pikuri.
|
|
138
|
+
test_files: []
|