pikuri-core 0.0.7 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +17 -30
- data/lib/pikuri/agent/chat_transport.rb +168 -107
- data/lib/pikuri/agent/configurator.rb +194 -173
- data/lib/pikuri/agent/control/cancellable.rb +100 -70
- data/lib/pikuri/agent/control/handoff.rb +75 -0
- data/lib/pikuri/agent/control/interloper.rb +107 -118
- data/lib/pikuri/agent/control.rb +13 -35
- data/lib/pikuri/agent/event.rb +144 -197
- data/lib/pikuri/agent/extension.rb +83 -95
- data/lib/pikuri/agent/history.rb +247 -0
- data/lib/pikuri/agent/listener/in_memory_event_list.rb +1 -1
- data/lib/pikuri/agent/listener/terminal.rb +169 -136
- data/lib/pikuri/agent/listener/token_log.rb +84 -157
- data/lib/pikuri/agent/listener.rb +24 -44
- data/lib/pikuri/agent/listener_list.rb +32 -47
- data/lib/pikuri/agent/request_instrumenter.rb +102 -0
- data/lib/pikuri/agent.rb +1095 -577
- data/lib/pikuri/attachment.rb +65 -0
- data/lib/pikuri/bundler_env.rb +68 -0
- data/lib/pikuri/extractor/passthrough.rb +24 -31
- data/lib/pikuri/extractor.rb +125 -155
- data/lib/pikuri/file_type.rb +64 -136
- data/lib/pikuri/finalizers.rb +32 -47
- data/lib/pikuri/paths.rb +104 -13
- data/lib/pikuri/sanitizer.rb +46 -67
- data/lib/pikuri/subprocess.rb +86 -118
- data/lib/pikuri/testing.rb +369 -0
- data/lib/pikuri/tool/calculator.rb +168 -74
- data/lib/pikuri/tool/execute_context.rb +42 -0
- data/lib/pikuri/tool/parameters.rb +31 -29
- data/lib/pikuri/tool/result.rb +164 -0
- data/lib/pikuri/tool/trifecta_legs.rb +248 -0
- data/lib/pikuri/tool.rb +416 -67
- data/lib/pikuri/trifecta/contribution.rb +43 -0
- data/lib/pikuri/trifecta/node.rb +47 -0
- data/lib/pikuri/trifecta/report.rb +228 -0
- data/lib/pikuri/trifecta.rb +126 -0
- data/lib/pikuri/version.rb +1 -1
- data/lib/pikuri-core.rb +70 -92
- data/prompts/agent-loop.txt +5 -0
- data/prompts/pikuri-chat.txt +3 -12
- metadata +26 -72
- data/lib/pikuri/agent/context_window_detector.rb +0 -222
- data/lib/pikuri/agent/control/step_limit.rb +0 -111
- data/lib/pikuri/agent/extension_context.rb +0 -147
- data/lib/pikuri/agent/listener/rate_limited.rb +0 -172
- data/lib/pikuri/agent/synthesizer.rb +0 -171
- data/lib/pikuri/extractor/html.rb +0 -303
- data/lib/pikuri/tool/fetch.rb +0 -172
- data/lib/pikuri/tool/scraper.rb +0 -186
- data/lib/pikuri/tool/search/brave.rb +0 -198
- data/lib/pikuri/tool/search/duckduckgo.rb +0 -207
- data/lib/pikuri/tool/search/engines.rb +0 -186
- data/lib/pikuri/tool/search/exa.rb +0 -229
- data/lib/pikuri/tool/search/rate_limiter.rb +0 -92
- data/lib/pikuri/tool/search/result.rb +0 -29
- data/lib/pikuri/tool/web_scrape.rb +0 -121
- data/lib/pikuri/tool/web_search.rb +0 -57
- data/lib/pikuri/url_cache.rb +0 -112
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'stringio'
|
|
4
|
+
|
|
5
|
+
module Pikuri
|
|
6
|
+
# A file shown to the model beside text — the image +read+ answers, a
|
|
7
|
+
# screenshot the user attaches — held as its bytes:
|
|
8
|
+
#
|
|
9
|
+
# png = File.binread(path)
|
|
10
|
+
# Pikuri::Attachment.new(mime: Pikuri::FileType.detect_mime(png), filename: 'shot.png', bytes: png)
|
|
11
|
+
#
|
|
12
|
+
# It is the only attachment type pikuri's API carries ({Tool::Result#attachments},
|
|
13
|
+
# {Agent::History::Message#attachments}); {#to_ruby_llm} builds ruby_llm's own
|
|
14
|
+
# at the wire. Bytes rather than a path is the point: ruby_llm reads a path
|
|
15
|
+
# String at construction, fetches a URL one, and may read a path again at send
|
|
16
|
+
# time, so a file edited in between reaches the model unchecked. An
|
|
17
|
+
# +Attachment+ is what the model saw, and compares by content.
|
|
18
|
+
class Attachment < Data.define(:mime, :filename, :bytes)
|
|
19
|
+
# @return [Integer] the largest image a tool or host attaches: Anthropic's
|
|
20
|
+
# per-image limit, the same order on OpenAI and Gemini. Refuse a bigger
|
|
21
|
+
# one with an error the model can react to, rather than send a payload
|
|
22
|
+
# the provider rejects.
|
|
23
|
+
MAX_IMAGE_BYTES = 5 * 1024 * 1024
|
|
24
|
+
|
|
25
|
+
# Rebuild one from ruby_llm's type — the way back from a live chat.
|
|
26
|
+
#
|
|
27
|
+
# @param attachment [RubyLLM::Attachment] path- or IO-backed; a URL-backed
|
|
28
|
+
# one is fetched, which pikuri never builds.
|
|
29
|
+
# @return [Attachment]
|
|
30
|
+
def self.from_ruby_llm(attachment)
|
|
31
|
+
new(mime: attachment.mime_type, filename: attachment.filename.to_s, bytes: attachment.content)
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
# @param mime [String] e.g. +"image/png"+
|
|
35
|
+
# @param filename [String] the name the model is shown
|
|
36
|
+
# @param bytes [String] the file's contents; re-tagged binary, since the
|
|
37
|
+
# same image arrives as UTF-8 from a source file and as ASCII-8BIT from
|
|
38
|
+
# base64, and two differently-tagged copies of one PNG are not +==+.
|
|
39
|
+
def initialize(mime:, filename:, bytes:)
|
|
40
|
+
super(mime: mime, filename: filename, bytes: bytes.b.freeze)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# ruby_llm's attachment over these bytes, for the wire. {#mime} crosses as
|
|
44
|
+
# stated, so ruby_llm detects nothing: the provider is told the type pikuri
|
|
45
|
+
# checked, never one guessed from a sender-chosen {#filename}.
|
|
46
|
+
#
|
|
47
|
+
# @return [RubyLLM::Attachment]
|
|
48
|
+
def to_ruby_llm
|
|
49
|
+
RubyLLM::Attachment.new(StringIO.new(bytes), filename: filename, mime_type: mime)
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# The line that stands in for this attachment when the model can't take
|
|
53
|
+
# it ({Agent::ChatTransport#modalities}), so the model learns why it sees
|
|
54
|
+
# nothing:
|
|
55
|
+
#
|
|
56
|
+
# png.withheld_note('model m takes no vision input')
|
|
57
|
+
# # => "[shot.png not shown: model m takes no vision input]"
|
|
58
|
+
#
|
|
59
|
+
# @param reason [String]
|
|
60
|
+
# @return [String]
|
|
61
|
+
def withheld_note(reason)
|
|
62
|
+
"[#{filename} not shown: #{reason}]"
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
end
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pikuri
|
|
4
|
+
# The environment a child process should see when pikuri itself booted under
|
|
5
|
+
# Bundler. Merge it under whatever the child's own configuration asks for:
|
|
6
|
+
#
|
|
7
|
+
# Open3.popen3(BundlerEnv.clean_delta.merge(entry.env), *entry.command)
|
|
8
|
+
#
|
|
9
|
+
# Bundler implements +bundler/setup+ by rewriting the process environment —
|
|
10
|
+
# it injects +BUNDLE_GEMFILE+ and +RUBYOPT+, repoints +GEM_HOME+ /
|
|
11
|
+
# +GEM_PATH+, prepends the bundle's bin dir to +PATH+, and stashes each
|
|
12
|
+
# original under +BUNDLER_ORIG_<NAME>+. A child inherits all of it, and for a
|
|
13
|
+
# child that is itself Ruby that reads as the child being broken: a
|
|
14
|
+
# +bundle exec+ the agent runs in another project resolves against *pikuri's*
|
|
15
|
+
# Gemfile, and +ruby-lsp+ refuses to start at all ("can't find executable
|
|
16
|
+
# ruby-lsp for gem ruby-lsp") unless pikuri's Gemfile happens to list it. Such
|
|
17
|
+
# a child serves the *target project*, so it must see the environment the user
|
|
18
|
+
# had before pikuri started.
|
|
19
|
+
#
|
|
20
|
+
# == Implementation details
|
|
21
|
+
#
|
|
22
|
+
# A delta rather than a scrub, for two reasons. A spawn's +env:+ is additive,
|
|
23
|
+
# so removal has to be spelled as an explicit +nil+; and Bundler *changed*
|
|
24
|
+
# variables it did not invent, so the fix restores each +BUNDLER_ORIG_*+
|
|
25
|
+
# backup — a host-level +GEM_HOME+ from an apt-installed Ruby passes through,
|
|
26
|
+
# where a blanket +GEM_*+ wipe would have eaten it.
|
|
27
|
+
#
|
|
28
|
+
# This does not break the "environment is not a secret store" seam, which
|
|
29
|
+
# says pikuri never filters +ENV+ before spawning: nothing the user set is
|
|
30
|
+
# removed, only pikuri's own boot undone.
|
|
31
|
+
module BundlerEnv
|
|
32
|
+
# The delta that, merged onto the current (bundler-polluted) +ENV+,
|
|
33
|
+
# reconstructs +Bundler.unbundled_env+.
|
|
34
|
+
#
|
|
35
|
+
# Empty when pikuri runs without Bundler (gem-installed, or a host that
|
|
36
|
+
# never required +bundler/setup+), so the child inherits +ENV+ unchanged
|
|
37
|
+
# and no caller needs a branch of its own.
|
|
38
|
+
#
|
|
39
|
+
# @return [Hash{String=>(String,nil)}] an +env:+ for
|
|
40
|
+
# {Pikuri::Subprocess.spawn}, +Open3.popen3+ or +Process.spawn+; a +nil+
|
|
41
|
+
# value unsets that variable in the child.
|
|
42
|
+
def self.clean_delta
|
|
43
|
+
return {} unless defined?(Bundler) && Bundler.respond_to?(:unbundled_env)
|
|
44
|
+
|
|
45
|
+
env_delta(current: ENV.to_h, clean: Bundler.unbundled_env)
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# The additive delta that turns +current+ into +clean+ when passed as a
|
|
49
|
+
# spawn's +env:+.
|
|
50
|
+
#
|
|
51
|
+
# env_delta(current: { 'BUNDLE_GEMFILE' => '/pikuri/Gemfile', 'PATH' => '/bin' },
|
|
52
|
+
# clean: { 'PATH' => '/bin' })
|
|
53
|
+
# # => { 'BUNDLE_GEMFILE' => nil }
|
|
54
|
+
#
|
|
55
|
+
# Keys equal in both are omitted, so a child's inherited environment is
|
|
56
|
+
# otherwise untouched.
|
|
57
|
+
#
|
|
58
|
+
# @param current [Hash{String=>String}]
|
|
59
|
+
# @param clean [Hash{String=>String}]
|
|
60
|
+
# @return [Hash{String=>(String,nil)}] +nil+ where +clean+ lacks the key.
|
|
61
|
+
def self.env_delta(current:, clean:)
|
|
62
|
+
delta = {}
|
|
63
|
+
clean.each { |k, v| delta[k] = v unless current[k] == v }
|
|
64
|
+
current.each_key { |k| delta[k] = nil unless clean.key?(k) }
|
|
65
|
+
delta
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
end
|
|
@@ -2,24 +2,21 @@
|
|
|
2
2
|
|
|
3
3
|
module Pikuri
|
|
4
4
|
module Extractor
|
|
5
|
-
# The terminal plain-text arm of the registry: content that *is*
|
|
6
|
-
#
|
|
7
|
-
#
|
|
8
|
-
# deal with, matching what +File.read+ with a UTF-8 encoding does).
|
|
9
|
-
# Markdown, source files, JSON, robots.txt all land here.
|
|
5
|
+
# The terminal plain-text arm of the registry: content that *is* already
|
|
6
|
+
# text passes through verbatim (forced to UTF-8, invalid bytes left in, as
|
|
7
|
+
# +File.read+ does). Markdown, source files, JSON, robots.txt land here.
|
|
10
8
|
#
|
|
11
|
-
# Matching
|
|
12
|
-
# content-type:
|
|
9
|
+
# Matching splits on whether the transport gave a content-type:
|
|
13
10
|
#
|
|
14
|
-
# * With
|
|
15
|
-
#
|
|
16
|
-
#
|
|
17
|
-
#
|
|
18
|
-
#
|
|
19
|
-
#
|
|
20
|
-
#
|
|
21
|
-
#
|
|
22
|
-
#
|
|
11
|
+
# * With one (the web path): claim +text/*+ only — a non-text type no
|
|
12
|
+
# earlier extractor claimed is *not* second-guessed by sniffing
|
|
13
|
+
# (+application/octet-stream+ gets the {Unsupported} refusal).
|
|
14
|
+
# * Without one (the local-file path, {FileType.detect_mime} → +nil+): claim
|
|
15
|
+
# anything passing {FileType.binary?} on the sample; opaque binaries stay
|
|
16
|
+
# unclaimed and surface as {Unsupported}. A format {FileType.detect_mime}
|
|
17
|
+
# recognises is never text, however ASCII its head: a PDF's first 4 KB of
|
|
18
|
+
# dictionaries passes {FileType.binary?}, and would reach the model raw on
|
|
19
|
+
# a host without a PDF extractor.
|
|
23
20
|
module Passthrough
|
|
24
21
|
# @return [Symbol] {Page#kind} tag.
|
|
25
22
|
def self.kind
|
|
@@ -33,29 +30,25 @@ module Pikuri
|
|
|
33
30
|
def self.matches?(sample:, content_type:)
|
|
34
31
|
return content_type.start_with?('text/') unless content_type.nil?
|
|
35
32
|
|
|
36
|
-
!FileType.binary?(sample)
|
|
33
|
+
FileType.detect_mime(sample).nil? && !FileType.binary?(sample)
|
|
37
34
|
end
|
|
38
35
|
|
|
39
36
|
# @param io [IO, StringIO] IO over the text content.
|
|
40
|
-
# @return [String] the content, tagged UTF-8. Deliberately NOT
|
|
41
|
-
#
|
|
42
|
-
#
|
|
43
|
-
# join of chomped lines would silently normalize away.
|
|
37
|
+
# @return [String] the content, tagged UTF-8. Deliberately NOT derived
|
|
38
|
+
# from {.extract_lines} — a passthrough stays verbatim (trailing
|
|
39
|
+
# newline, CRLF), which joining chomped lines would normalize away.
|
|
44
40
|
def self.extract(io)
|
|
45
41
|
io.read.force_encoding(Encoding::UTF_8)
|
|
46
42
|
end
|
|
47
43
|
|
|
48
|
-
#
|
|
49
|
-
#
|
|
50
|
-
#
|
|
51
|
-
#
|
|
52
|
-
# this stream's tail for an exact +total_lines+ (see
|
|
53
|
-
# {Extractor.extract_paged}).
|
|
44
|
+
# Lazy line stream for {Extractor.extract_paged}: read line-by-line, so a
|
|
45
|
+
# window over the head of a gigabyte log never loads the rest. The whole
|
|
46
|
+
# stream is a cheap sequential read — which is why paging counts this
|
|
47
|
+
# tail for an exact +total_lines+.
|
|
54
48
|
#
|
|
55
|
-
# @param io [IO, StringIO] IO over the text content; must
|
|
56
|
-
#
|
|
57
|
-
# @return [Enumerator::Lazy<String>] chomped lines, tagged
|
|
58
|
-
# UTF-8.
|
|
49
|
+
# @param io [IO, StringIO] IO over the text content; must stay open while
|
|
50
|
+
# the enumerator is consumed.
|
|
51
|
+
# @return [Enumerator::Lazy<String>] chomped lines, tagged UTF-8.
|
|
59
52
|
def self.extract_lines(io)
|
|
60
53
|
io.each_line.lazy.map { |raw| raw.chomp.force_encoding(Encoding::UTF_8) }
|
|
61
54
|
end
|
data/lib/pikuri/extractor.rb
CHANGED
|
@@ -1,82 +1,84 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
module Pikuri
|
|
4
|
-
# The format→text extraction seam: one registry of extractors
|
|
5
|
-
#
|
|
6
|
-
#
|
|
7
|
-
# pikuri-extractors
|
|
8
|
-
#
|
|
4
|
+
# The format→text extraction seam: one ordered registry of extractors
|
|
5
|
+
# turning an +IO+ of a recognised format (plain text in core; HTML via
|
|
6
|
+
# pikuri-html, PDF via pikuri-pdf, office formats via pikuri-workshop's
|
|
7
|
+
# pikuri-extractors) into
|
|
8
|
+
# Markdown-flavoured UTF-8, behind two front doors:
|
|
9
9
|
#
|
|
10
|
-
# * {.extract} — the whole document as one String
|
|
11
|
-
#
|
|
12
|
-
#
|
|
13
|
-
# * {.extract_paged} — the LLM-tool shape: the same extraction,
|
|
14
|
-
# windowed to a line range with a byte cap, returned as a {Page}
|
|
15
|
-
# the caller renders. Backs +Workspace::Read+ and
|
|
16
|
-
# +VectorDb::Tools::Read+ (via the {FileType} path wrappers) so
|
|
17
|
-
# the offset/limit/byte-cap logic lives in one tested place.
|
|
10
|
+
# * {.extract} — the whole document as one String (no windowing).
|
|
11
|
+
# * {.extract_paged} — the same, windowed to a line range with a byte
|
|
12
|
+
# cap, as a {Page} the caller renders.
|
|
18
13
|
#
|
|
19
|
-
#
|
|
20
|
-
#
|
|
21
|
-
#
|
|
22
|
-
# set of format truths and "support a new format" is a registry
|
|
23
|
-
# entry (pikuri-pdf and pikuri-extractors plug PDF and office
|
|
24
|
-
# formats in without pikuri-core knowing), not a new special case
|
|
25
|
-
# in two dispatchers.
|
|
14
|
+
# pikuri-web's scraper (dispatching on the HTTP +Content-Type+) and {FileType}
|
|
15
|
+
# (resolving local paths) both route through this registry, so "support a
|
|
16
|
+
# new format" is a registry entry, not a new dispatcher arm.
|
|
26
17
|
#
|
|
27
18
|
# == The extractor duck type
|
|
28
19
|
#
|
|
29
|
-
# Each {.registry} entry implements
|
|
20
|
+
# Each {.registry} entry implements:
|
|
30
21
|
#
|
|
31
|
-
# * +matches?(sample:, content_type:)+ → +Boolean+ — claim the
|
|
32
|
-
#
|
|
33
|
-
#
|
|
34
|
-
# +
|
|
35
|
-
#
|
|
36
|
-
#
|
|
37
|
-
# * +
|
|
38
|
-
#
|
|
39
|
-
#
|
|
40
|
-
# * +kind+ → +Symbol+ — a short tag (+:text+ / +:pdf+ / +:html+)
|
|
41
|
-
# carried on {Page#kind} so rendering callers can word
|
|
42
|
-
# format-specific trailers ("End of PDF", the scanned-image
|
|
43
|
-
# hint) without re-sniffing.
|
|
22
|
+
# * +matches?(sample:, content_type:)+ → +Boolean+ — claim the content.
|
|
23
|
+
# +sample+ is the leading {FileType::SAMPLE_BYTES} (magic-byte sniff);
|
|
24
|
+
# +content_type+ is the normalized HTTP/{FileType.detect_mime} type, or
|
|
25
|
+
# +nil+ ("no metadata — sniff if you can").
|
|
26
|
+
# * +extract(io)+ → +String+ — whole document as Markdown-flavoured UTF-8.
|
|
27
|
+
# Raises {Error} on content it claimed but can't parse.
|
|
28
|
+
# * +kind+ → +Symbol+ — a short tag (+:text+ / +:pdf+ / +:html+) carried
|
|
29
|
+
# on {Page#kind} so callers word format-specific trailers without
|
|
30
|
+
# re-sniffing.
|
|
44
31
|
#
|
|
45
|
-
# plus one *optional* method
|
|
46
|
-
# produced incrementally:
|
|
32
|
+
# plus one *optional* method:
|
|
47
33
|
#
|
|
48
|
-
# * +extract_lines(io)+ → +Enumerator<String>+ — the same content
|
|
49
|
-
#
|
|
50
|
-
# {.extract_paged} prefers
|
|
51
|
-
# the moment the window fills, so the
|
|
52
|
-
#
|
|
53
|
-
#
|
|
54
|
-
#
|
|
55
|
-
#
|
|
56
|
-
# {Error} mid-iteration. Extractors that need the whole document
|
|
57
|
-
# to produce anything ({HTML}: Readability walks the full DOM —
|
|
58
|
-
# true of any subprocess-based extractor too) simply omit it;
|
|
59
|
-
# {.extract_paged} then extracts in full and windows the result.
|
|
34
|
+
# * +extract_lines(io)+ → +Enumerator<String>+ — the same content as a
|
|
35
|
+
# lazy stream of +chomp+ed lines, numbered exactly as +extract+'s (see
|
|
36
|
+
# == Lines). {.extract_paged} prefers it and stops
|
|
37
|
+
# the moment the window fills, so the document's tail is never parsed
|
|
38
|
+
# (pikuri-pdf parses pages on access). Must be consumed while +io+ is
|
|
39
|
+
# open; may raise {Error} mid-iteration. Extractors needing the whole
|
|
40
|
+
# document (pikuri-web's HTML, any subprocess-based one) omit it, and
|
|
41
|
+
# {.extract_paged} extracts in full then windows.
|
|
60
42
|
#
|
|
61
|
-
# Windowing
|
|
62
|
-
#
|
|
63
|
-
#
|
|
64
|
-
# genuinely format-specific half of paging.
|
|
43
|
+
# Windowing (offset / limit / byte cap / truncation) is presentation and
|
|
44
|
+
# lives once in {.extract_paged}, never per extractor — +extract_lines+ is
|
|
45
|
+
# line *production*, the only format-specific half of paging.
|
|
65
46
|
#
|
|
66
|
-
# ==
|
|
47
|
+
# == Lines
|
|
48
|
+
#
|
|
49
|
+
# The one definition of "line N" for everything that pages, cites or
|
|
50
|
+
# indexes extracted text — a reader's +offset:+, a search hit's line range,
|
|
51
|
+
# a host program counting lines itself:
|
|
52
|
+
#
|
|
53
|
+
# "a\r\nb\rc\n\nd\n" # 4 lines: "a", "b\rc", "", "d"
|
|
67
54
|
#
|
|
68
|
-
#
|
|
69
|
-
#
|
|
55
|
+
# * 1-based; a range is inclusive at both ends.
|
|
56
|
+
# * LF is the only terminator: CRLF numbers like LF, a lone CR is *not* a
|
|
57
|
+
# break, a final LF opens no extra line. Line N is the text after the
|
|
58
|
+
# (N-1)th +"\n"+.
|
|
59
|
+
# * Lines of the *extracted* text. For {Passthrough} formats (plain text,
|
|
60
|
+
# Markdown, source) they are the file's own lines; for PDF / office /
|
|
61
|
+
# HTML they number the extractor's output, +"--- Page N ---"+ markers
|
|
62
|
+
# included — reproducible only by running the same extractor.
|
|
70
63
|
#
|
|
71
|
-
#
|
|
72
|
-
#
|
|
73
|
-
#
|
|
74
|
-
#
|
|
64
|
+
# Every reader paging through {.extract_paged} numbers lines this way, so
|
|
65
|
+
# a line number one of them reports (a search hit's +lines a-b+, say) is a
|
|
66
|
+
# valid +offset:+ for any other over the same file, the workspace +read+
|
|
67
|
+
# included.
|
|
75
68
|
#
|
|
76
|
-
#
|
|
77
|
-
# +
|
|
78
|
-
#
|
|
79
|
-
#
|
|
69
|
+
# A host counting a file's lines must split on LF alone. +grep -n+, +sed+,
|
|
70
|
+
# +awk+ and Ruby's +each_line+ agree; Python's +str.splitlines()+ and
|
|
71
|
+
# text-mode +open()+ and Java's +BufferedReader.readLine()+ /
|
|
72
|
+
# +Files.readAllLines()+ also break on a lone CR (+splitlines+ on +\v+,
|
|
73
|
+
# +\f+, U+2028 and more too), and every number after one is silently off.
|
|
74
|
+
#
|
|
75
|
+
# == Errors
|
|
76
|
+
#
|
|
77
|
+
# Both modes are failures the caller's LLM can react to, under one
|
|
78
|
+
# rescuable root:
|
|
79
|
+
#
|
|
80
|
+
# * {Unsupported} — nothing in {.registry} claimed the content.
|
|
81
|
+
# * {Error} (the root) — an extractor claimed it but the parse failed.
|
|
80
82
|
module Extractor
|
|
81
83
|
module_function
|
|
82
84
|
|
|
@@ -108,71 +110,50 @@ module Pikuri
|
|
|
108
110
|
# {PAGE_MAX_LINE_LENGTH}.
|
|
109
111
|
PAGE_LINE_TRUNCATION_MARKER = "... (line truncated to #{PAGE_MAX_LINE_LENGTH} chars)"
|
|
110
112
|
|
|
111
|
-
# One windowed slice of a document, returned by {.extract_paged}
|
|
112
|
-
#
|
|
113
|
-
# everything a trailer needs without the caller re-reading the
|
|
114
|
-
# document.
|
|
113
|
+
# One windowed slice of a document, returned by {.extract_paged},
|
|
114
|
+
# carrying everything a trailer needs without re-reading the document.
|
|
115
115
|
#
|
|
116
|
-
#
|
|
116
|
+
# * +lines+ — +Array<String>+, the collected window; already per-line
|
|
117
|
+
# truncated (with {PAGE_LINE_TRUNCATION_MARKER}), *not* line-numbered
|
|
118
|
+
# (numbering is the caller's presentation). Includes any +"--- Page N
|
|
119
|
+
# ---"+ marker lines, which count toward +limit+/the byte cap.
|
|
120
|
+
# * +start_line+ — line number of +lines.first+ (the requested +offset+),
|
|
121
|
+
# per {Extractor}'s == Lines.
|
|
122
|
+
# * +total_lines+ — +Integer+ document line count when known, else +nil+.
|
|
123
|
+
# Known at EOF, when extracted in full (HTML), or on a cheap-to-count
|
|
124
|
+
# stream (text); +nil+ when a lazy stream stopped early (byte cap, or a
|
|
125
|
+
# PDF window filled before the last page — counting the rest would
|
|
126
|
+
# parse every page).
|
|
127
|
+
# * +more+ — +true+ if content remains past this window.
|
|
128
|
+
# * +byte_capped+ — +true+ if the byte cap (not the line limit) stopped it.
|
|
129
|
+
# * +kind+ — the matched extractor's +kind+ tag.
|
|
117
130
|
#
|
|
118
|
-
#
|
|
119
|
-
#
|
|
120
|
-
#
|
|
121
|
-
# a PDF the array includes the +"--- Page N ---"+ marker lines
|
|
122
|
-
# pikuri-pdf's extractor emits, which count toward +limit+ / the
|
|
123
|
-
# byte cap like any other line.
|
|
124
|
-
# * +start_line+ — the 1-indexed line number of +lines.first+
|
|
125
|
-
# (i.e. the +offset+ the caller asked for). +lines.last+ is at
|
|
126
|
-
# +start_line + lines.length - 1+.
|
|
127
|
-
# * +total_lines+ — total line count of the document when known,
|
|
128
|
-
# else +nil+. Known when the read reached EOF, when the format
|
|
129
|
-
# was extracted in full (no +extract_lines+ — e.g. HTML), or
|
|
130
|
-
# when the lazy stream is cheap enough to count to the end
|
|
131
|
-
# (plain text). +nil+ when a lazy stream stopped early — the
|
|
132
|
-
# byte cap fired, or a PDF filled the window before its last
|
|
133
|
-
# page (counting the rest would mean parsing every page,
|
|
134
|
-
# defeating the laziness).
|
|
135
|
-
# * +more+ — +true+ if content remains past this window (the
|
|
136
|
-
# caller should offer +offset = start_line + lines.length+).
|
|
137
|
-
# * +byte_capped+ — +true+ if the byte cap (not the line limit)
|
|
138
|
-
# was the stopping criterion.
|
|
139
|
-
# * +kind+ — the matched extractor's +kind+ tag (+:text+ /
|
|
140
|
-
# +:pdf+ / +:html+); lets the caller word format-specific
|
|
141
|
-
# trailers and the empty-document message.
|
|
142
|
-
#
|
|
143
|
-
# An empty document yields +lines: []+, +total_lines: 0+; an
|
|
144
|
-
# +offset+ past EOF yields +lines: []+ with +total_lines+ set to
|
|
145
|
-
# the real (non-zero) count — the caller distinguishes the two.
|
|
131
|
+
# Empty document → +lines: []+, +total_lines: 0+; +offset+ past EOF →
|
|
132
|
+
# +lines: []+ with +total_lines+ at the real count — the caller
|
|
133
|
+
# distinguishes the two.
|
|
146
134
|
Page = Data.define(:lines, :start_line, :total_lines, :more, :byte_capped, :kind)
|
|
147
135
|
|
|
148
|
-
# The extractor registry, consulted in order — first match wins.
|
|
149
|
-
#
|
|
150
|
-
#
|
|
151
|
-
#
|
|
152
|
-
# a
|
|
153
|
-
#
|
|
154
|
-
#
|
|
155
|
-
#
|
|
156
|
-
# terminal entry (+registry.insert(-2, X)+ — pikuri-extractors
|
|
157
|
-
# does this).
|
|
136
|
+
# The extractor registry, consulted in order — first match wins. Core
|
|
137
|
+
# ships only the terminal {Passthrough} plain-text arm. A gem picks its
|
|
138
|
+
# insertion point by claim strength: a magic-byte sniff that never
|
|
139
|
+
# misfires on text goes at the *front* to beat a content-type claimer
|
|
140
|
+
# under a lying header (+registry.unshift(X)+ — pikuri-pdf); a
|
|
141
|
+
# content-type/weaker claimer inserts before the terminal entry
|
|
142
|
+
# (+registry.insert(-2, X)+ — pikuri-web's HTML, pikuri-workshop's office
|
|
143
|
+
# extractors).
|
|
158
144
|
#
|
|
159
|
-
# @return [Array<#matches?>] mutable, deliberately —
|
|
160
|
-
# plug-in seam.
|
|
145
|
+
# @return [Array<#matches?>] mutable, deliberately — the plug-in seam.
|
|
161
146
|
def registry
|
|
162
|
-
@registry ||= [
|
|
147
|
+
@registry ||= [Passthrough]
|
|
163
148
|
end
|
|
164
149
|
|
|
165
|
-
# Extract the whole document behind +io+ as one Markdown-flavoured
|
|
166
|
-
#
|
|
167
|
-
# with no extractable text).
|
|
150
|
+
# Extract the whole document behind +io+ as one Markdown-flavoured UTF-8
|
|
151
|
+
# String. May be empty (empty text file, scanned-image PDF).
|
|
168
152
|
#
|
|
169
|
-
# @param io [IO, StringIO] seekable IO
|
|
170
|
-
#
|
|
171
|
-
#
|
|
172
|
-
#
|
|
173
|
-
# the transport supplies one (HTTP header, {FileType.detect_mime}
|
|
174
|
-
# result); +nil+ when unknown — extractors then rely on their
|
|
175
|
-
# byte sniffs.
|
|
153
|
+
# @param io [IO, StringIO] seekable IO at the content start; reads a
|
|
154
|
+
# leading sample for the sniff and rewinds before extracting.
|
|
155
|
+
# @param content_type [String, nil] normalized content-type when the
|
|
156
|
+
# transport supplies one; +nil+ when unknown (extractors sniff).
|
|
176
157
|
# @return [String]
|
|
177
158
|
# @raise [Unsupported] when no registry entry claims the content.
|
|
178
159
|
# @raise [Error] when the matched extractor cannot parse it.
|
|
@@ -180,41 +161,34 @@ module Pikuri
|
|
|
180
161
|
extractor_for(io, content_type).extract(io)
|
|
181
162
|
end
|
|
182
163
|
|
|
183
|
-
# Extract +io+
|
|
184
|
-
# +
|
|
185
|
-
# +max_bytes+ is reached, with over-long lines truncated at
|
|
164
|
+
# Extract +io+ into a windowed {Page}: lines from +offset+ (1-indexed) up
|
|
165
|
+
# to +limit+, stopping early at +max_bytes+, over-long lines truncated at
|
|
186
166
|
# +max_line_length+.
|
|
187
167
|
#
|
|
188
|
-
# Lazy where the format allows: extractors
|
|
189
|
-
#
|
|
190
|
-
#
|
|
191
|
-
#
|
|
192
|
-
#
|
|
193
|
-
# in full and then windowed, which is also what makes their
|
|
194
|
-
# +total_lines+ always exact.
|
|
168
|
+
# Lazy where the format allows: +extract_lines+ extractors (text,
|
|
169
|
+
# pikuri-pdf) are consumed only until the window fills — a 500-page PDF's
|
|
170
|
+
# first window parses a handful of pages. Extractors without it (HTML) are
|
|
171
|
+
# extracted in full then windowed, which is why their +total_lines+ is
|
|
172
|
+
# always exact.
|
|
195
173
|
#
|
|
196
|
-
# @param io [IO, StringIO] seekable IO
|
|
174
|
+
# @param io [IO, StringIO] seekable IO at the start.
|
|
197
175
|
# @param content_type [String, nil] as for {.extract}.
|
|
198
|
-
# @param offset [Integer] 1-indexed first line
|
|
199
|
-
#
|
|
200
|
-
# @param limit [Integer] maximum lines to collect. Caller
|
|
201
|
-
# validates +limit >= 1+.
|
|
176
|
+
# @param offset [Integer] 1-indexed first line; caller validates +>= 1+.
|
|
177
|
+
# @param limit [Integer] max lines to collect; caller validates +>= 1+.
|
|
202
178
|
# @param max_bytes [Integer] hard byte cap on collected content.
|
|
203
179
|
# @param max_line_length [Integer] per-line truncation threshold.
|
|
204
|
-
# @return [Page]
|
|
180
|
+
# @return [Page]
|
|
205
181
|
# @raise [Unsupported] when no registry entry claims the content.
|
|
206
182
|
# @raise [Error] when the matched extractor cannot parse it.
|
|
207
183
|
def extract_paged(io, content_type: nil, offset: 1, limit: PAGE_DEFAULT_LIMIT,
|
|
208
184
|
max_bytes: PAGE_MAX_BYTES, max_line_length: PAGE_MAX_LINE_LENGTH)
|
|
209
185
|
extractor = extractor_for(io, content_type)
|
|
210
186
|
if extractor.respond_to?(:extract_lines)
|
|
211
|
-
# count_tail is a per-format economics call: once the window
|
|
212
|
-
#
|
|
213
|
-
#
|
|
214
|
-
#
|
|
215
|
-
#
|
|
216
|
-
# (pikuri-pdf's included) get the conservative default (stop
|
|
217
|
-
# early, total unknown).
|
|
187
|
+
# count_tail is a per-format economics call: once the window fills,
|
|
188
|
+
# counting the rest of a text stream is a cheap sequential read (so
|
|
189
|
+
# the trailer says "of N"), but for a PDF it would parse every
|
|
190
|
+
# remaining page — what extract_lines exists to avoid. Plugged-in
|
|
191
|
+
# extractors get the conservative default (stop early, total unknown).
|
|
218
192
|
window(extractor.extract_lines(io),
|
|
219
193
|
offset: offset, limit: limit, max_bytes: max_bytes,
|
|
220
194
|
max_line_length: max_line_length, kind: extractor.kind,
|
|
@@ -227,11 +201,10 @@ module Pikuri
|
|
|
227
201
|
end
|
|
228
202
|
end
|
|
229
203
|
|
|
230
|
-
#
|
|
231
|
-
#
|
|
232
|
-
# each extractor in order.
|
|
204
|
+
# First registry entry claiming +io+'s content: read the leading
|
|
205
|
+
# {FileType::SAMPLE_BYTES} for the sniff, rewind, ask each in order.
|
|
233
206
|
#
|
|
234
|
-
# @param io [IO, StringIO] seekable IO
|
|
207
|
+
# @param io [IO, StringIO] seekable IO at the start.
|
|
235
208
|
# @param content_type [String, nil]
|
|
236
209
|
# @return [#extract] the matched extractor.
|
|
237
210
|
# @raise [Unsupported] when nothing matches.
|
|
@@ -244,15 +217,12 @@ module Pikuri
|
|
|
244
217
|
end
|
|
245
218
|
private_class_method :extractor_for
|
|
246
219
|
|
|
247
|
-
# Collect a {Page} window
|
|
248
|
-
#
|
|
249
|
-
#
|
|
250
|
-
#
|
|
251
|
-
#
|
|
252
|
-
#
|
|
253
|
-
# when consuming the rest of the stream is cheap; without it the
|
|
254
|
-
# loop breaks and leaves the total unknown. The byte cap always
|
|
255
|
-
# aborts the count.
|
|
220
|
+
# Collect a {Page} window from +lines+ (Array or lazy Enumerator of
|
|
221
|
+
# +chomp+ed lines). +known_total+ is the full count when extracted up
|
|
222
|
+
# front (Array), +nil+ for a lazy stream — then +total_lines+ is exact
|
|
223
|
+
# only if iteration reached EOF: +count_tail+ keeps counting past the
|
|
224
|
+
# limit when the tail is cheap, else the loop breaks and the total stays
|
|
225
|
+
# unknown. The byte cap always aborts the count.
|
|
256
226
|
#
|
|
257
227
|
# @param lines [Enumerable<String>]
|
|
258
228
|
# @param known_total [Integer, nil]
|