pikuri-web 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,80 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'digest'
4
+ require 'fileutils'
5
+
6
+ module Pikuri
7
+ module Web
8
+ module UrlCache
9
+ # The on-disk store: one file per entry, +<dir>/<sha256 of key>.txt+,
10
+ # freshness by mtime, no sidecar metadata. Survives a restart and is shared
11
+ # by every pikuri process. Part of {UrlCache} — see it for the API.
12
+ #
13
+ # CACHE = UrlCache::OnDisk.new(ttl: UrlCache::DEFAULT_TTL, dir: "#{UrlCache::ROOT_DIR}/fetch")
14
+ #
15
+ # A stale entry stays on disk until a {#write} with its key replaces it;
16
+ # +rm -rf+ the directory to clear it. Never use it for a payload whose terms
17
+ # forbid storing it — that is {InMemory}'s job.
18
+ #
19
+ # Sharing: +P_shared_benign+ — an instance holds only its TTL and directory,
20
+ # and every owner keeps it in a class-level constant, so all agents (and
21
+ # other pikuri processes) hit the same files. Two callers racing on one cold
22
+ # key both compute and both write the same bytes — the accepted tradeoff for
23
+ # keeping this a few dozen lines (worst case: a duplicate fetch). The
24
+ # cross-agent flow is deliberate: a public URL's content is nobody's private
25
+ # context.
26
+ class OnDisk
27
+ # @param ttl [Numeric] freshness window in seconds; an entry with an older
28
+ # mtime is a miss. +Float::INFINITY+ never expires.
29
+ # @param dir [String] where the files live; created lazily on first write
30
+ def initialize(ttl:, dir:)
31
+ @ttl = ttl
32
+ @dir = dir
33
+ end
34
+
35
+ # @param key [String] a URL or any opaque string
36
+ # @return [String, nil] the stored payload, or +nil+ when absent or stale
37
+ def read(key)
38
+ path = path_for(key)
39
+ File.read(path) if fresh?(path)
40
+ end
41
+
42
+ # @param key [String] a URL or any opaque string
43
+ # @param value [String] payload to store, replacing any entry
44
+ # @return [String] +value+
45
+ def write(key, value)
46
+ FileUtils.mkdir_p(@dir)
47
+ File.write(path_for(key), value)
48
+ value
49
+ end
50
+
51
+ # Return the stored payload for +key+ if fresh, else yield to compute it,
52
+ # store and return it. A raising block stores nothing (errors aren't cached).
53
+ #
54
+ # @param key [String] a URL or any opaque string
55
+ # @yieldreturn [String] payload to store and return on a miss
56
+ # @return [String] cached or freshly-computed payload
57
+ def fetch(key)
58
+ cached = read(key)
59
+ return cached unless cached.nil?
60
+
61
+ write(key, yield)
62
+ end
63
+
64
+ # @param key [String]
65
+ # @return [String] absolute path of the cache file for +key+
66
+ def path_for(key)
67
+ File.join(@dir, "#{Digest::SHA256.hexdigest(key)}.txt")
68
+ end
69
+
70
+ private
71
+
72
+ # @param path [String]
73
+ # @return [Boolean] true when +path+ exists and was written within the TTL
74
+ def fresh?(path)
75
+ File.exist?(path) && Time.now - File.mtime(path) < @ttl
76
+ end
77
+ end
78
+ end
79
+ end
80
+ end
@@ -0,0 +1,63 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Pikuri
4
+ module Web
5
+ # Text payloads cached by key within a TTL, so a tool doesn't redo a request.
6
+ # Two stores share one duck-typed API:
7
+ #
8
+ # cache = UrlCache::OnDisk.new(ttl: UrlCache::DEFAULT_TTL, dir: "#{UrlCache::ROOT_DIR}/web_scrape")
9
+ # cache = UrlCache::InMemory.new(ttl: UrlCache::DEFAULT_TTL, max_entries: 100)
10
+ #
11
+ # cache.fetch(key) { compute } # yield on a miss, store, return; a raise stores nothing
12
+ # cache.read(key) # => String, or nil on a miss or stale entry
13
+ # cache.write(key, value) # => value; replaces any entry
14
+ #
15
+ # Pick the store by what the payload's source allows. {OnDisk} survives a
16
+ # restart and is shared by every pikuri process; {InMemory} lives only as
17
+ # long as the Ruby VM, for results whose terms forbid storing them — a search
18
+ # provider says which via +disk_cacheable?+. Each user holds its own
19
+ # instance in a constant, never a shared one: the dir, TTL and size are per
20
+ # use. {NULL} stands in for either.
21
+ module UrlCache
22
+ # Root dir for the {OnDisk} subdirs, per XDG:
23
+ # +$XDG_CACHE_HOME/pikuri/url_cache+ or +~/.cache/pikuri/url_cache+. Each
24
+ # user picks its own subdir (+"#{ROOT_DIR}/web_scrape"+) so keys can't
25
+ # collide.
26
+ #
27
+ # @return [String]
28
+ ROOT_DIR = Paths.cache.join('url_cache').to_path
29
+
30
+ # Default freshness window: 2 hours. Long enough to cover an interactive
31
+ # session (revisiting a page or re-running a search within the working
32
+ # window hits the cache), short enough that resuming the next day doesn't
33
+ # serve stale results. (opencode caches nothing; +pi-web-fetch+ uses 15 min,
34
+ # +pi-web-search+ 5.)
35
+ #
36
+ # @return [Integer]
37
+ DEFAULT_TTL = 2 * 60 * 60
38
+
39
+ # Null cache: always misses, never stores — a drop-in for either store in
40
+ # tests, or anywhere caching should be off.
41
+ #
42
+ # UrlCache::NULL.fetch('k') { 'x' } # => "x", yields every time
43
+ # UrlCache::NULL.read('k') # => nil
44
+ #
45
+ # @return [Object]
46
+ NULL = Object.new
47
+ # @param _key [String] ignored
48
+ # @return [nil] always a miss
49
+ def NULL.read(_key) = nil
50
+
51
+ # @param _key [String] ignored
52
+ # @param value [String] returned, never stored
53
+ # @return [String] +value+
54
+ def NULL.write(_key, value) = value
55
+
56
+ # @param _key [String] ignored
57
+ # @yieldreturn [String] freshly-computed payload
58
+ # @return [String] whatever the block returned
59
+ def NULL.fetch(_key) = yield
60
+ NULL.freeze
61
+ end
62
+ end
63
+ end
data/lib/pikuri-web.rb ADDED
@@ -0,0 +1,36 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'pikuri-core'
4
+ require 'pikuri-workspace'
5
+ require 'pikuri-html'
6
+
7
+ # Entry file for the pikuri-web gem: mounts a per-gem Zeitwerk loader at this
8
+ # +lib/+ and eager-loads it, so the +ALL_CAPS+ tool constants ({Pikuri::Web::FETCH}
9
+ # and friends), which Zeitwerk's filename↔CamelCase rule won't auto-load, are
10
+ # defined at boot. Requiring pikuri-html registers the HTML extractor the scraper
11
+ # renders pages through.
12
+ module Pikuri
13
+ # The tools that reach the internet: search, scrape, fetch, and a reader for a
14
+ # search's stored page text.
15
+ #
16
+ # c.add_tool Pikuri::Web::Search.from_h(examples_config['search'])
17
+ # c.add_tool Pikuri::Web::SCRAPE
18
+ # c.add_tool Pikuri::Web::FETCH
19
+ #
20
+ # Every tool here brings untrusted text in, and all but {READ_CACHED} send
21
+ # bytes out — {SCRAPE} and {FETCH} to any host the model names. That is why
22
+ # they live outside pikuri-core: an agent that needs none of them never loads
23
+ # them, and wiring one is a visible choice on the trifecta map.
24
+ #
25
+ # For a search-only agent, pair a +page_text+ {Search.build} with
26
+ # {READ_CACHED} rather than {SCRAPE}; see {Search.build}.
27
+ module Web
28
+ LOADER = Zeitwerk::Loader.new
29
+ LOADER.tag = 'pikuri-web'
30
+ LOADER.push_dir(File.expand_path('.', __dir__))
31
+ LOADER.ignore(__FILE__)
32
+ LOADER.inflector.inflect('duckduckgo' => 'DuckDuckGo')
33
+ LOADER.setup
34
+ LOADER.eager_load
35
+ end
36
+ end
metadata ADDED
@@ -0,0 +1,138 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: pikuri-web
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.2.0
5
+ platform: ruby
6
+ authors:
7
+ - Martin Vysny
8
+ bindir: bin
9
+ cert_chain: []
10
+ date: 1980-01-02 00:00:00.000000000 Z
11
+ dependencies:
12
+ - !ruby/object:Gem::Dependency
13
+ name: pikuri-core
14
+ requirement: !ruby/object:Gem::Requirement
15
+ requirements:
16
+ - - '='
17
+ - !ruby/object:Gem::Version
18
+ version: 0.2.0
19
+ type: :runtime
20
+ prerelease: false
21
+ version_requirements: !ruby/object:Gem::Requirement
22
+ requirements:
23
+ - - '='
24
+ - !ruby/object:Gem::Version
25
+ version: 0.2.0
26
+ - !ruby/object:Gem::Dependency
27
+ name: pikuri-workspace
28
+ requirement: !ruby/object:Gem::Requirement
29
+ requirements:
30
+ - - '='
31
+ - !ruby/object:Gem::Version
32
+ version: 0.2.0
33
+ type: :runtime
34
+ prerelease: false
35
+ version_requirements: !ruby/object:Gem::Requirement
36
+ requirements:
37
+ - - '='
38
+ - !ruby/object:Gem::Version
39
+ version: 0.2.0
40
+ - !ruby/object:Gem::Dependency
41
+ name: pikuri-html
42
+ requirement: !ruby/object:Gem::Requirement
43
+ requirements:
44
+ - - '='
45
+ - !ruby/object:Gem::Version
46
+ version: 0.2.0
47
+ type: :runtime
48
+ prerelease: false
49
+ version_requirements: !ruby/object:Gem::Requirement
50
+ requirements:
51
+ - - '='
52
+ - !ruby/object:Gem::Version
53
+ version: 0.2.0
54
+ - !ruby/object:Gem::Dependency
55
+ name: faraday
56
+ requirement: !ruby/object:Gem::Requirement
57
+ requirements:
58
+ - - "~>"
59
+ - !ruby/object:Gem::Version
60
+ version: '2.9'
61
+ type: :runtime
62
+ prerelease: false
63
+ version_requirements: !ruby/object:Gem::Requirement
64
+ requirements:
65
+ - - "~>"
66
+ - !ruby/object:Gem::Version
67
+ version: '2.9'
68
+ - !ruby/object:Gem::Dependency
69
+ name: nokogiri
70
+ requirement: !ruby/object:Gem::Requirement
71
+ requirements:
72
+ - - "~>"
73
+ - !ruby/object:Gem::Version
74
+ version: '1.19'
75
+ type: :runtime
76
+ prerelease: false
77
+ version_requirements: !ruby/object:Gem::Requirement
78
+ requirements:
79
+ - - "~>"
80
+ - !ruby/object:Gem::Version
81
+ version: '1.19'
82
+ description: |
83
+ pikuri-web gives a pikuri-core agent the tools that reach the internet:
84
+ +web_search+ (DuckDuckGo, plus Brave and Exa when the host supplies a
85
+ key), +web_scrape+ (a page as readable Markdown), +fetch+ (a URL's body
86
+ verbatim) and +web_read_cached+ (a search result's page text, with no
87
+ request of its own). Each brings untrusted text into the conversation
88
+ and most send bytes out, so they live outside pikuri-core: an agent
89
+ that doesn't wire them never loads them.
90
+ email:
91
+ - martin@vysny.me
92
+ executables: []
93
+ extensions: []
94
+ extra_rdoc_files: []
95
+ files:
96
+ - README.md
97
+ - lib/pikuri-web.rb
98
+ - lib/pikuri/web/fetch.rb
99
+ - lib/pikuri/web/read_cached.rb
100
+ - lib/pikuri/web/scrape.rb
101
+ - lib/pikuri/web/scraper.rb
102
+ - lib/pikuri/web/search.rb
103
+ - lib/pikuri/web/search/brave.rb
104
+ - lib/pikuri/web/search/duckduckgo.rb
105
+ - lib/pikuri/web/search/engines.rb
106
+ - lib/pikuri/web/search/exa.rb
107
+ - lib/pikuri/web/search/page_store.rb
108
+ - lib/pikuri/web/search/rate_limiter.rb
109
+ - lib/pikuri/web/search/result.rb
110
+ - lib/pikuri/web/url_cache.rb
111
+ - lib/pikuri/web/url_cache/in_memory.rb
112
+ - lib/pikuri/web/url_cache/on_disk.rb
113
+ homepage: https://github.com/mvysny/pikuri
114
+ licenses:
115
+ - MIT
116
+ metadata:
117
+ source_code_uri: https://github.com/mvysny/pikuri/tree/master
118
+ changelog_uri: https://github.com/mvysny/pikuri/blob/master/CHANGELOG.md
119
+ bug_tracker_uri: https://github.com/mvysny/pikuri/issues
120
+ rubygems_mfa_required: 'true'
121
+ rdoc_options: []
122
+ require_paths:
123
+ - lib
124
+ required_ruby_version: !ruby/object:Gem::Requirement
125
+ requirements:
126
+ - - ">="
127
+ - !ruby/object:Gem::Version
128
+ version: '3.3'
129
+ required_rubygems_version: !ruby/object:Gem::Requirement
130
+ requirements:
131
+ - - ">="
132
+ - !ruby/object:Gem::Version
133
+ version: '0'
134
+ requirements: []
135
+ rubygems_version: 3.6.7
136
+ specification_version: 4
137
+ summary: Web search, scrape and fetch tools for pikuri.
138
+ test_files: []