pikuri-core 0.0.7 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +17 -30
  3. data/lib/pikuri/agent/chat_transport.rb +168 -107
  4. data/lib/pikuri/agent/configurator.rb +194 -173
  5. data/lib/pikuri/agent/control/cancellable.rb +100 -70
  6. data/lib/pikuri/agent/control/handoff.rb +75 -0
  7. data/lib/pikuri/agent/control/interloper.rb +107 -118
  8. data/lib/pikuri/agent/control.rb +13 -35
  9. data/lib/pikuri/agent/event.rb +144 -197
  10. data/lib/pikuri/agent/extension.rb +83 -95
  11. data/lib/pikuri/agent/history.rb +247 -0
  12. data/lib/pikuri/agent/listener/in_memory_event_list.rb +1 -1
  13. data/lib/pikuri/agent/listener/terminal.rb +169 -136
  14. data/lib/pikuri/agent/listener/token_log.rb +84 -157
  15. data/lib/pikuri/agent/listener.rb +24 -44
  16. data/lib/pikuri/agent/listener_list.rb +32 -47
  17. data/lib/pikuri/agent/request_instrumenter.rb +102 -0
  18. data/lib/pikuri/agent.rb +1095 -577
  19. data/lib/pikuri/attachment.rb +65 -0
  20. data/lib/pikuri/bundler_env.rb +68 -0
  21. data/lib/pikuri/extractor/passthrough.rb +24 -31
  22. data/lib/pikuri/extractor.rb +125 -155
  23. data/lib/pikuri/file_type.rb +64 -136
  24. data/lib/pikuri/finalizers.rb +32 -47
  25. data/lib/pikuri/paths.rb +104 -13
  26. data/lib/pikuri/sanitizer.rb +46 -67
  27. data/lib/pikuri/subprocess.rb +86 -118
  28. data/lib/pikuri/testing.rb +369 -0
  29. data/lib/pikuri/tool/calculator.rb +168 -74
  30. data/lib/pikuri/tool/execute_context.rb +42 -0
  31. data/lib/pikuri/tool/parameters.rb +31 -29
  32. data/lib/pikuri/tool/result.rb +164 -0
  33. data/lib/pikuri/tool/trifecta_legs.rb +248 -0
  34. data/lib/pikuri/tool.rb +416 -67
  35. data/lib/pikuri/trifecta/contribution.rb +43 -0
  36. data/lib/pikuri/trifecta/node.rb +47 -0
  37. data/lib/pikuri/trifecta/report.rb +228 -0
  38. data/lib/pikuri/trifecta.rb +126 -0
  39. data/lib/pikuri/version.rb +1 -1
  40. data/lib/pikuri-core.rb +70 -92
  41. data/prompts/agent-loop.txt +5 -0
  42. data/prompts/pikuri-chat.txt +3 -12
  43. metadata +26 -72
  44. data/lib/pikuri/agent/context_window_detector.rb +0 -222
  45. data/lib/pikuri/agent/control/step_limit.rb +0 -111
  46. data/lib/pikuri/agent/extension_context.rb +0 -147
  47. data/lib/pikuri/agent/listener/rate_limited.rb +0 -172
  48. data/lib/pikuri/agent/synthesizer.rb +0 -171
  49. data/lib/pikuri/extractor/html.rb +0 -303
  50. data/lib/pikuri/tool/fetch.rb +0 -172
  51. data/lib/pikuri/tool/scraper.rb +0 -186
  52. data/lib/pikuri/tool/search/brave.rb +0 -198
  53. data/lib/pikuri/tool/search/duckduckgo.rb +0 -207
  54. data/lib/pikuri/tool/search/engines.rb +0 -186
  55. data/lib/pikuri/tool/search/exa.rb +0 -229
  56. data/lib/pikuri/tool/search/rate_limiter.rb +0 -92
  57. data/lib/pikuri/tool/search/result.rb +0 -29
  58. data/lib/pikuri/tool/web_scrape.rb +0 -121
  59. data/lib/pikuri/tool/web_search.rb +0 -57
  60. data/lib/pikuri/url_cache.rb +0 -112
@@ -0,0 +1,65 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'stringio'
4
+
5
+ module Pikuri
6
+ # A file shown to the model beside text — the image +read+ answers, a
7
+ # screenshot the user attaches — held as its bytes:
8
+ #
9
+ # png = File.binread(path)
10
+ # Pikuri::Attachment.new(mime: Pikuri::FileType.detect_mime(png), filename: 'shot.png', bytes: png)
11
+ #
12
+ # It is the only attachment type pikuri's API carries ({Tool::Result#attachments},
13
+ # {Agent::History::Message#attachments}); {#to_ruby_llm} builds ruby_llm's own
14
+ # at the wire. Bytes rather than a path is the point: ruby_llm reads a path
15
+ # String at construction, fetches a URL one, and may read a path again at send
16
+ # time, so a file edited in between reaches the model unchecked. An
17
+ # +Attachment+ is what the model saw, and compares by content.
18
+ class Attachment < Data.define(:mime, :filename, :bytes)
19
+ # @return [Integer] the largest image a tool or host attaches: Anthropic's
20
+ # per-image limit, the same order on OpenAI and Gemini. Refuse a bigger
21
+ # one with an error the model can react to, rather than send a payload
22
+ # the provider rejects.
23
+ MAX_IMAGE_BYTES = 5 * 1024 * 1024
24
+
25
+ # Rebuild one from ruby_llm's type — the way back from a live chat.
26
+ #
27
+ # @param attachment [RubyLLM::Attachment] path- or IO-backed; a URL-backed
28
+ # one is fetched, which pikuri never builds.
29
+ # @return [Attachment]
30
+ def self.from_ruby_llm(attachment)
31
+ new(mime: attachment.mime_type, filename: attachment.filename.to_s, bytes: attachment.content)
32
+ end
33
+
34
+ # @param mime [String] e.g. +"image/png"+
35
+ # @param filename [String] the name the model is shown
36
+ # @param bytes [String] the file's contents; re-tagged binary, since the
37
+ # same image arrives as UTF-8 from a source file and as ASCII-8BIT from
38
+ # base64, and two differently-tagged copies of one PNG are not +==+.
39
+ def initialize(mime:, filename:, bytes:)
40
+ super(mime: mime, filename: filename, bytes: bytes.b.freeze)
41
+ end
42
+
43
+ # ruby_llm's attachment over these bytes, for the wire. {#mime} crosses as
44
+ # stated, so ruby_llm detects nothing: the provider is told the type pikuri
45
+ # checked, never one guessed from a sender-chosen {#filename}.
46
+ #
47
+ # @return [RubyLLM::Attachment]
48
+ def to_ruby_llm
49
+ RubyLLM::Attachment.new(StringIO.new(bytes), filename: filename, mime_type: mime)
50
+ end
51
+
52
+ # The line that stands in for this attachment when the model can't take
53
+ # it ({Agent::ChatTransport#modalities}), so the model learns why it sees
54
+ # nothing:
55
+ #
56
+ # png.withheld_note('model m takes no vision input')
57
+ # # => "[shot.png not shown: model m takes no vision input]"
58
+ #
59
+ # @param reason [String]
60
+ # @return [String]
61
+ def withheld_note(reason)
62
+ "[#{filename} not shown: #{reason}]"
63
+ end
64
+ end
65
+ end
@@ -0,0 +1,68 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Pikuri
4
+ # The environment a child process should see when pikuri itself booted under
5
+ # Bundler. Merge it under whatever the child's own configuration asks for:
6
+ #
7
+ # Open3.popen3(BundlerEnv.clean_delta.merge(entry.env), *entry.command)
8
+ #
9
+ # Bundler implements +bundler/setup+ by rewriting the process environment —
10
+ # it injects +BUNDLE_GEMFILE+ and +RUBYOPT+, repoints +GEM_HOME+ /
11
+ # +GEM_PATH+, prepends the bundle's bin dir to +PATH+, and stashes each
12
+ # original under +BUNDLER_ORIG_<NAME>+. A child inherits all of it, and for a
13
+ # child that is itself Ruby that reads as the child being broken: a
14
+ # +bundle exec+ the agent runs in another project resolves against *pikuri's*
15
+ # Gemfile, and +ruby-lsp+ refuses to start at all ("can't find executable
16
+ # ruby-lsp for gem ruby-lsp") unless pikuri's Gemfile happens to list it. Such
17
+ # a child serves the *target project*, so it must see the environment the user
18
+ # had before pikuri started.
19
+ #
20
+ # == Implementation details
21
+ #
22
+ # A delta rather than a scrub, for two reasons. A spawn's +env:+ is additive,
23
+ # so removal has to be spelled as an explicit +nil+; and Bundler *changed*
24
+ # variables it did not invent, so the fix restores each +BUNDLER_ORIG_*+
25
+ # backup — a host-level +GEM_HOME+ from an apt-installed Ruby passes through,
26
+ # where a blanket +GEM_*+ wipe would have eaten it.
27
+ #
28
+ # This does not break the "environment is not a secret store" seam, which
29
+ # says pikuri never filters +ENV+ before spawning: nothing the user set is
30
+ # removed, only pikuri's own boot undone.
31
+ module BundlerEnv
32
+ # The delta that, merged onto the current (bundler-polluted) +ENV+,
33
+ # reconstructs +Bundler.unbundled_env+.
34
+ #
35
+ # Empty when pikuri runs without Bundler (gem-installed, or a host that
36
+ # never required +bundler/setup+), so the child inherits +ENV+ unchanged
37
+ # and no caller needs a branch of its own.
38
+ #
39
+ # @return [Hash{String=>(String,nil)}] an +env:+ for
40
+ # {Pikuri::Subprocess.spawn}, +Open3.popen3+ or +Process.spawn+; a +nil+
41
+ # value unsets that variable in the child.
42
+ def self.clean_delta
43
+ return {} unless defined?(Bundler) && Bundler.respond_to?(:unbundled_env)
44
+
45
+ env_delta(current: ENV.to_h, clean: Bundler.unbundled_env)
46
+ end
47
+
48
+ # The additive delta that turns +current+ into +clean+ when passed as a
49
+ # spawn's +env:+.
50
+ #
51
+ # env_delta(current: { 'BUNDLE_GEMFILE' => '/pikuri/Gemfile', 'PATH' => '/bin' },
52
+ # clean: { 'PATH' => '/bin' })
53
+ # # => { 'BUNDLE_GEMFILE' => nil }
54
+ #
55
+ # Keys equal in both are omitted, so a child's inherited environment is
56
+ # otherwise untouched.
57
+ #
58
+ # @param current [Hash{String=>String}]
59
+ # @param clean [Hash{String=>String}]
60
+ # @return [Hash{String=>(String,nil)}] +nil+ where +clean+ lacks the key.
61
+ def self.env_delta(current:, clean:)
62
+ delta = {}
63
+ clean.each { |k, v| delta[k] = v unless current[k] == v }
64
+ current.each_key { |k| delta[k] = nil unless clean.key?(k) }
65
+ delta
66
+ end
67
+ end
68
+ end
@@ -2,24 +2,21 @@
2
2
 
3
3
  module Pikuri
4
4
  module Extractor
5
- # The terminal plain-text arm of the registry: content that *is*
6
- # already text needs no extraction, so it passes through verbatim
7
- # (forced to UTF-8 — invalid bytes are left in for downstream to
8
- # deal with, matching what +File.read+ with a UTF-8 encoding does).
9
- # Markdown, source files, JSON, robots.txt all land here.
5
+ # The terminal plain-text arm of the registry: content that *is* already
6
+ # text passes through verbatim (forced to UTF-8, invalid bytes left in, as
7
+ # +File.read+ does). Markdown, source files, JSON, robots.txt land here.
10
8
  #
11
- # Matching is split by whether the transport supplied a
12
- # content-type:
9
+ # Matching splits on whether the transport gave a content-type:
13
10
  #
14
- # * With a content-type (the web path): claim +text/*+ only.
15
- # A non-text type that no earlier extractor claimed is *not*
16
- # second-guessed by sniffing — a server declaring
17
- # +application/octet-stream+ gets the {Unsupported} refusal the
18
- # LLM can react to, same as before this registry existed.
19
- # * Without one (the local-file path, where {FileType.detect_mime}
20
- # returned +nil+ for "unrecognised"): claim anything that passes
21
- # the {FileType.binary?} heuristic on the sample. Opaque
22
- # binaries stay unclaimed and surface as {Unsupported}.
11
+ # * With one (the web path): claim +text/*+ only — a non-text type no
12
+ # earlier extractor claimed is *not* second-guessed by sniffing
13
+ # (+application/octet-stream+ gets the {Unsupported} refusal).
14
+ # * Without one (the local-file path, {FileType.detect_mime} → +nil+): claim
15
+ # anything passing {FileType.binary?} on the sample; opaque binaries stay
16
+ # unclaimed and surface as {Unsupported}. A format {FileType.detect_mime}
17
+ # recognises is never text, however ASCII its head: a PDF's first 4 KB of
18
+ # dictionaries passes {FileType.binary?}, and would reach the model raw on
19
+ # a host without a PDF extractor.
23
20
  module Passthrough
24
21
  # @return [Symbol] {Page#kind} tag.
25
22
  def self.kind
@@ -33,29 +30,25 @@ module Pikuri
33
30
  def self.matches?(sample:, content_type:)
34
31
  return content_type.start_with?('text/') unless content_type.nil?
35
32
 
36
- !FileType.binary?(sample)
33
+ FileType.detect_mime(sample).nil? && !FileType.binary?(sample)
37
34
  end
38
35
 
39
36
  # @param io [IO, StringIO] IO over the text content.
40
- # @return [String] the content, tagged UTF-8. Deliberately NOT
41
- # derived from {.extract_lines} — a passthrough must stay
42
- # verbatim (trailing newline, CRLF line endings), which a
43
- # join of chomped lines would silently normalize away.
37
+ # @return [String] the content, tagged UTF-8. Deliberately NOT derived
38
+ # from {.extract_lines} — a passthrough stays verbatim (trailing
39
+ # newline, CRLF), which joining chomped lines would normalize away.
44
40
  def self.extract(io)
45
41
  io.read.force_encoding(Encoding::UTF_8)
46
42
  end
47
43
 
48
- # The lazy line stream for {Extractor.extract_paged}: the IO is
49
- # read line-by-line, so a window over the head of a gigabyte
50
- # log never loads the rest. Consuming the whole stream is a
51
- # cheap sequential read — which is why the paging window counts
52
- # this stream's tail for an exact +total_lines+ (see
53
- # {Extractor.extract_paged}).
44
+ # Lazy line stream for {Extractor.extract_paged}: read line-by-line, so a
45
+ # window over the head of a gigabyte log never loads the rest. The whole
46
+ # stream is a cheap sequential read — which is why paging counts this
47
+ # tail for an exact +total_lines+.
54
48
  #
55
- # @param io [IO, StringIO] IO over the text content; must
56
- # remain open while the enumerator is consumed.
57
- # @return [Enumerator::Lazy<String>] chomped lines, tagged
58
- # UTF-8.
49
+ # @param io [IO, StringIO] IO over the text content; must stay open while
50
+ # the enumerator is consumed.
51
+ # @return [Enumerator::Lazy<String>] chomped lines, tagged UTF-8.
59
52
  def self.extract_lines(io)
60
53
  io.each_line.lazy.map { |raw| raw.chomp.force_encoding(Encoding::UTF_8) }
61
54
  end
@@ -1,82 +1,84 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Pikuri
4
- # The format→text extraction seam: one registry of extractors that
5
- # turn an +IO+ of some recognised format (HTML and plain text out
6
- # of the box; PDF / office formats via the pikuri-pdf /
7
- # pikuri-extractors plug-in gems) into Markdown-flavoured UTF-8
8
- # text, consumed through two front doors:
4
+ # The format→text extraction seam: one ordered registry of extractors
5
+ # turning an +IO+ of a recognised format (plain text in core; HTML via
6
+ # pikuri-html, PDF via pikuri-pdf, office formats via pikuri-workshop's
7
+ # pikuri-extractors) into
8
+ # Markdown-flavoured UTF-8, behind two front doors:
9
9
  #
10
- # * {.extract} — the whole document as one String. The shape the
11
- # indexing / caching callers want ({Pikuri::VectorDb}'s indexer,
12
- # {Tool::WebScrape}'s URL cache): no windowing, no presentation.
13
- # * {.extract_paged} — the LLM-tool shape: the same extraction,
14
- # windowed to a line range with a byte cap, returned as a {Page}
15
- # the caller renders. Backs +Workspace::Read+ and
16
- # +VectorDb::Tools::Read+ (via the {FileType} path wrappers) so
17
- # the offset/limit/byte-cap logic lives in one tested place.
10
+ # * {.extract} — the whole document as one String (no windowing).
11
+ # * {.extract_paged} — the same, windowed to a line range with a byte
12
+ # cap, as a {Page} the caller renders.
18
13
  #
19
- # Both front doors — +Tool::Scraper+ dispatching on the HTTP
20
- # +Content-Type+ header for the web tools, and {FileType} resolving
21
- # local paths — route through this one registry, so both share one
22
- # set of format truths and "support a new format" is a registry
23
- # entry (pikuri-pdf and pikuri-extractors plug PDF and office
24
- # formats in without pikuri-core knowing), not a new special case
25
- # in two dispatchers.
14
+ # pikuri-web's scraper (dispatching on the HTTP +Content-Type+) and {FileType}
15
+ # (resolving local paths) both route through this registry, so "support a
16
+ # new format" is a registry entry, not a new dispatcher arm.
26
17
  #
27
18
  # == The extractor duck type
28
19
  #
29
- # Each {.registry} entry implements three methods:
20
+ # Each {.registry} entry implements:
30
21
  #
31
- # * +matches?(sample:, content_type:)+ → +Boolean+ — claim the
32
- # content. +sample+ is the leading {FileType::SAMPLE_BYTES} bytes
33
- # (for magic-byte sniffs); +content_type+ is the normalized HTTP
34
- # +Content-Type+ for web content, the {FileType.detect_mime}
35
- # result for local files, and may be +nil+ ("no transport
36
- # metadata — sniff if you can").
37
- # * +extract(io)+ → +String+ — the whole document as
38
- # Markdown-flavoured UTF-8 text. Raises {Error} on content the
39
- # extractor claimed but cannot parse (malformed PDF, ...).
40
- # * +kind+ → +Symbol+ — a short tag (+:text+ / +:pdf+ / +:html+)
41
- # carried on {Page#kind} so rendering callers can word
42
- # format-specific trailers ("End of PDF", the scanned-image
43
- # hint) without re-sniffing.
22
+ # * +matches?(sample:, content_type:)+ → +Boolean+ — claim the content.
23
+ # +sample+ is the leading {FileType::SAMPLE_BYTES} (magic-byte sniff);
24
+ # +content_type+ is the normalized HTTP/{FileType.detect_mime} type, or
25
+ # +nil+ ("no metadata — sniff if you can").
26
+ # * +extract(io)+ → +String+ — whole document as Markdown-flavoured UTF-8.
27
+ # Raises {Error} on content it claimed but can't parse.
28
+ # * +kind+ → +Symbol+ — a short tag (+:text+ / +:pdf+ / +:html+) carried
29
+ # on {Page#kind} so callers word format-specific trailers without
30
+ # re-sniffing.
44
31
  #
45
- # plus one *optional* method for formats whose lines can be
46
- # produced incrementally:
32
+ # plus one *optional* method:
47
33
  #
48
- # * +extract_lines(io)+ → +Enumerator<String>+ — the same content
49
- # as +extract+, as a lazy stream of already-+chomp+ed lines.
50
- # {.extract_paged} prefers this when present and stops consuming
51
- # the moment the window fills, so the rest of the document is
52
- # never parsed (pikuri-pdf's extractor: pdf-reader's page list
53
- # parses on access; {Passthrough}: the IO is read line-by-line).
54
- # The enumerator
55
- # must be consumed while +io+ is still open, and may raise
56
- # {Error} mid-iteration. Extractors that need the whole document
57
- # to produce anything ({HTML}: Readability walks the full DOM —
58
- # true of any subprocess-based extractor too) simply omit it;
59
- # {.extract_paged} then extracts in full and windows the result.
34
+ # * +extract_lines(io)+ → +Enumerator<String>+ — the same content as a
35
+ # lazy stream of +chomp+ed lines, numbered exactly as +extract+'s (see
36
+ # == Lines). {.extract_paged} prefers it and stops
37
+ # the moment the window fills, so the document's tail is never parsed
38
+ # (pikuri-pdf parses pages on access). Must be consumed while +io+ is
39
+ # open; may raise {Error} mid-iteration. Extractors needing the whole
40
+ # document (pikuri-web's HTML, any subprocess-based one) omit it, and
41
+ # {.extract_paged} extracts in full then windows.
60
42
  #
61
- # Windowing itself (offset / limit / byte cap / line truncation) is
62
- # presentation and deliberately lives once in {.extract_paged}, not
63
- # per extractor — +extract_lines+ is line *production*, the only
64
- # genuinely format-specific half of paging.
43
+ # Windowing (offset / limit / byte cap / truncation) is presentation and
44
+ # lives once in {.extract_paged}, never per extractor — +extract_lines+ is
45
+ # line *production*, the only format-specific half of paging.
65
46
  #
66
- # == Errors
47
+ # == Lines
48
+ #
49
+ # The one definition of "line N" for everything that pages, cites or
50
+ # indexes extracted text — a reader's +offset:+, a search hit's line range,
51
+ # a host program counting lines itself:
52
+ #
53
+ # "a\r\nb\rc\n\nd\n" # 4 lines: "a", "b\rc", "", "d"
67
54
  #
68
- # Both failure modes are failures the *caller's* LLM can react to,
69
- # so they share one rescuable root:
55
+ # * 1-based; a range is inclusive at both ends.
56
+ # * LF is the only terminator: CRLF numbers like LF, a lone CR is *not* a
57
+ # break, a final LF opens no extra line. Line N is the text after the
58
+ # (N-1)th +"\n"+.
59
+ # * Lines of the *extracted* text. For {Passthrough} formats (plain text,
60
+ # Markdown, source) they are the file's own lines; for PDF / office /
61
+ # HTML they number the extractor's output, +"--- Page N ---"+ markers
62
+ # included — reproducible only by running the same extractor.
70
63
  #
71
- # * {Unsupported} — nothing in {.registry} claimed the content
72
- # (opaque binary, an unhandled content-type).
73
- # * {Error} (the root) — an extractor claimed the content but the
74
- # parse failed (malformed PDF, ...).
64
+ # Every reader paging through {.extract_paged} numbers lines this way, so
65
+ # a line number one of them reports (a search hit's +lines a-b+, say) is a
66
+ # valid +offset:+ for any other over the same file, the workspace +read+
67
+ # included.
75
68
  #
76
- # Callers map them to their own conventions:
77
- # +Tool::Scraper+ re-raises both as +FetchError+;
78
- # {FileType.read_as_text} maps {Unsupported} to the +ArgumentError+
79
- # binary refusal and {Error} to a +RuntimeError+ carrying the path.
69
+ # A host counting a file's lines must split on LF alone. +grep -n+, +sed+,
70
+ # +awk+ and Ruby's +each_line+ agree; Python's +str.splitlines()+ and
71
+ # text-mode +open()+ and Java's +BufferedReader.readLine()+ /
72
+ # +Files.readAllLines()+ also break on a lone CR (+splitlines+ on +\v+,
73
+ # +\f+, U+2028 and more too), and every number after one is silently off.
74
+ #
75
+ # == Errors
76
+ #
77
+ # Both modes are failures the caller's LLM can react to, under one
78
+ # rescuable root:
79
+ #
80
+ # * {Unsupported} — nothing in {.registry} claimed the content.
81
+ # * {Error} (the root) — an extractor claimed it but the parse failed.
80
82
  module Extractor
81
83
  module_function
82
84
 
@@ -108,71 +110,50 @@ module Pikuri
108
110
  # {PAGE_MAX_LINE_LENGTH}.
109
111
  PAGE_LINE_TRUNCATION_MARKER = "... (line truncated to #{PAGE_MAX_LINE_LENGTH} chars)"
110
112
 
111
- # One windowed slice of a document, returned by {.extract_paged}.
112
- # The caller turns this into an observation; this struct carries
113
- # everything a trailer needs without the caller re-reading the
114
- # document.
113
+ # One windowed slice of a document, returned by {.extract_paged},
114
+ # carrying everything a trailer needs without re-reading the document.
115
115
  #
116
- # == Fields
116
+ # * +lines+ — +Array<String>+, the collected window; already per-line
117
+ # truncated (with {PAGE_LINE_TRUNCATION_MARKER}), *not* line-numbered
118
+ # (numbering is the caller's presentation). Includes any +"--- Page N
119
+ # ---"+ marker lines, which count toward +limit+/the byte cap.
120
+ # * +start_line+ — line number of +lines.first+ (the requested +offset+),
121
+ # per {Extractor}'s == Lines.
122
+ # * +total_lines+ — +Integer+ document line count when known, else +nil+.
123
+ # Known at EOF, when extracted in full (HTML), or on a cheap-to-count
124
+ # stream (text); +nil+ when a lazy stream stopped early (byte cap, or a
125
+ # PDF window filled before the last page — counting the rest would
126
+ # parse every page).
127
+ # * +more+ — +true+ if content remains past this window.
128
+ # * +byte_capped+ — +true+ if the byte cap (not the line limit) stopped it.
129
+ # * +kind+ — the matched extractor's +kind+ tag.
117
130
  #
118
- # * +lines+ — +Array<String>+, the collected window. Already
119
- # per-line truncated (with {PAGE_LINE_TRUNCATION_MARKER}); *not*
120
- # line-numbered — numbering is presentation the caller adds. For
121
- # a PDF the array includes the +"--- Page N ---"+ marker lines
122
- # pikuri-pdf's extractor emits, which count toward +limit+ / the
123
- # byte cap like any other line.
124
- # * +start_line+ — the 1-indexed line number of +lines.first+
125
- # (i.e. the +offset+ the caller asked for). +lines.last+ is at
126
- # +start_line + lines.length - 1+.
127
- # * +total_lines+ — total line count of the document when known,
128
- # else +nil+. Known when the read reached EOF, when the format
129
- # was extracted in full (no +extract_lines+ — e.g. HTML), or
130
- # when the lazy stream is cheap enough to count to the end
131
- # (plain text). +nil+ when a lazy stream stopped early — the
132
- # byte cap fired, or a PDF filled the window before its last
133
- # page (counting the rest would mean parsing every page,
134
- # defeating the laziness).
135
- # * +more+ — +true+ if content remains past this window (the
136
- # caller should offer +offset = start_line + lines.length+).
137
- # * +byte_capped+ — +true+ if the byte cap (not the line limit)
138
- # was the stopping criterion.
139
- # * +kind+ — the matched extractor's +kind+ tag (+:text+ /
140
- # +:pdf+ / +:html+); lets the caller word format-specific
141
- # trailers and the empty-document message.
142
- #
143
- # An empty document yields +lines: []+, +total_lines: 0+; an
144
- # +offset+ past EOF yields +lines: []+ with +total_lines+ set to
145
- # the real (non-zero) count — the caller distinguishes the two.
131
+ # Empty document → +lines: []+, +total_lines: 0+; +offset+ past EOF →
132
+ # +lines: []+ with +total_lines+ at the real count — the caller
133
+ # distinguishes the two.
146
134
  Page = Data.define(:lines, :start_line, :total_lines, :more, :byte_capped, :kind)
147
135
 
148
- # The extractor registry, consulted in order — first match wins.
149
- # Core ships two entries: {HTML} matches on content-type, and
150
- # {Passthrough} is the terminal plain-text arm. A gem adding a
151
- # format picks its insertion point by the strength of its claim:
152
- # a magic-byte sniff that never misfires on text goes at the
153
- # *front* so it beats {HTML}'s content-type match even under a
154
- # lying header (+registry.unshift(X)+ — pikuri-pdf does this);
155
- # a content-type / weaker-sniff claimer inserts before the
156
- # terminal entry (+registry.insert(-2, X)+ — pikuri-extractors
157
- # does this).
136
+ # The extractor registry, consulted in order — first match wins. Core
137
+ # ships only the terminal {Passthrough} plain-text arm. A gem picks its
138
+ # insertion point by claim strength: a magic-byte sniff that never
139
+ # misfires on text goes at the *front* to beat a content-type claimer
140
+ # under a lying header (+registry.unshift(X)+ — pikuri-pdf); a
141
+ # content-type/weaker claimer inserts before the terminal entry
142
+ # (+registry.insert(-2, X)+ — pikuri-web's HTML, pikuri-workshop's office
143
+ # extractors).
158
144
  #
159
- # @return [Array<#matches?>] mutable, deliberately — this is the
160
- # plug-in seam.
145
+ # @return [Array<#matches?>] mutable, deliberately — the plug-in seam.
161
146
  def registry
162
- @registry ||= [HTML, Passthrough]
147
+ @registry ||= [Passthrough]
163
148
  end
164
149
 
165
- # Extract the whole document behind +io+ as one Markdown-flavoured
166
- # UTF-8 String. May be empty (empty text file, scanned-image PDF
167
- # with no extractable text).
150
+ # Extract the whole document behind +io+ as one Markdown-flavoured UTF-8
151
+ # String. May be empty (empty text file, scanned-image PDF).
168
152
  #
169
- # @param io [IO, StringIO] seekable IO positioned at the start of
170
- # the content; this method reads a leading sample for the
171
- # +matches?+ sniff and rewinds before extracting.
172
- # @param content_type [String, nil] normalized content-type when
173
- # the transport supplies one (HTTP header, {FileType.detect_mime}
174
- # result); +nil+ when unknown — extractors then rely on their
175
- # byte sniffs.
153
+ # @param io [IO, StringIO] seekable IO at the content start; reads a
154
+ # leading sample for the sniff and rewinds before extracting.
155
+ # @param content_type [String, nil] normalized content-type when the
156
+ # transport supplies one; +nil+ when unknown (extractors sniff).
176
157
  # @return [String]
177
158
  # @raise [Unsupported] when no registry entry claims the content.
178
159
  # @raise [Error] when the matched extractor cannot parse it.
@@ -180,41 +161,34 @@ module Pikuri
180
161
  extractor_for(io, content_type).extract(io)
181
162
  end
182
163
 
183
- # Extract +io+ and return a windowed {Page}: the lines from
184
- # +offset+ (1-indexed) up to +limit+ of them, stopping early if
185
- # +max_bytes+ is reached, with over-long lines truncated at
164
+ # Extract +io+ into a windowed {Page}: lines from +offset+ (1-indexed) up
165
+ # to +limit+, stopping early at +max_bytes+, over-long lines truncated at
186
166
  # +max_line_length+.
187
167
  #
188
- # Lazy where the format allows: extractors that implement
189
- # +extract_lines+ (plain text; pikuri-pdf's PDF) are consumed
190
- # only until the window fills — reading the first window of a
191
- # 500-page PDF parses a handful of pages, and the first page of
192
- # a gigabyte log never loads it. Extractors without it (HTML) are extracted
193
- # in full and then windowed, which is also what makes their
194
- # +total_lines+ always exact.
168
+ # Lazy where the format allows: +extract_lines+ extractors (text,
169
+ # pikuri-pdf) are consumed only until the window fills — a 500-page PDF's
170
+ # first window parses a handful of pages. Extractors without it (HTML) are
171
+ # extracted in full then windowed, which is why their +total_lines+ is
172
+ # always exact.
195
173
  #
196
- # @param io [IO, StringIO] seekable IO positioned at the start.
174
+ # @param io [IO, StringIO] seekable IO at the start.
197
175
  # @param content_type [String, nil] as for {.extract}.
198
- # @param offset [Integer] 1-indexed first line to include. The
199
- # caller is responsible for validating +offset >= 1+.
200
- # @param limit [Integer] maximum lines to collect. Caller
201
- # validates +limit >= 1+.
176
+ # @param offset [Integer] 1-indexed first line; caller validates +>= 1+.
177
+ # @param limit [Integer] max lines to collect; caller validates +>= 1+.
202
178
  # @param max_bytes [Integer] hard byte cap on collected content.
203
179
  # @param max_line_length [Integer] per-line truncation threshold.
204
- # @return [Page] the windowed slice.
180
+ # @return [Page]
205
181
  # @raise [Unsupported] when no registry entry claims the content.
206
182
  # @raise [Error] when the matched extractor cannot parse it.
207
183
  def extract_paged(io, content_type: nil, offset: 1, limit: PAGE_DEFAULT_LIMIT,
208
184
  max_bytes: PAGE_MAX_BYTES, max_line_length: PAGE_MAX_LINE_LENGTH)
209
185
  extractor = extractor_for(io, content_type)
210
186
  if extractor.respond_to?(:extract_lines)
211
- # count_tail is a per-format economics call: once the window
212
- # fills, counting the rest of a plain-text stream is a cheap
213
- # sequential read (so the trailer can say "of N"), while for
214
- # a PDF it would mean parsing every remaining page — exactly
215
- # what extract_lines exists to avoid. Plugged-in extractors
216
- # (pikuri-pdf's included) get the conservative default (stop
217
- # early, total unknown).
187
+ # count_tail is a per-format economics call: once the window fills,
188
+ # counting the rest of a text stream is a cheap sequential read (so
189
+ # the trailer says "of N"), but for a PDF it would parse every
190
+ # remaining page — what extract_lines exists to avoid. Plugged-in
191
+ # extractors get the conservative default (stop early, total unknown).
218
192
  window(extractor.extract_lines(io),
219
193
  offset: offset, limit: limit, max_bytes: max_bytes,
220
194
  max_line_length: max_line_length, kind: extractor.kind,
@@ -227,11 +201,10 @@ module Pikuri
227
201
  end
228
202
  end
229
203
 
230
- # Find the first registry entry claiming +io+'s content: read the
231
- # leading {FileType::SAMPLE_BYTES} for the sniff, rewind, and ask
232
- # each extractor in order.
204
+ # First registry entry claiming +io+'s content: read the leading
205
+ # {FileType::SAMPLE_BYTES} for the sniff, rewind, ask each in order.
233
206
  #
234
- # @param io [IO, StringIO] seekable IO positioned at the start.
207
+ # @param io [IO, StringIO] seekable IO at the start.
235
208
  # @param content_type [String, nil]
236
209
  # @return [#extract] the matched extractor.
237
210
  # @raise [Unsupported] when nothing matches.
@@ -244,15 +217,12 @@ module Pikuri
244
217
  end
245
218
  private_class_method :extractor_for
246
219
 
247
- # Collect a {Page} window out of +lines+ (an Array or a lazy
248
- # Enumerator of already-+chomp+ed lines). +known_total+ is the
249
- # full line count when the caller extracted everything up front
250
- # (Array case), +nil+ for a lazy stream — then +total_lines+ is
251
- # exact only if the iteration reached EOF: +count_tail+ keeps
252
- # the loop counting (without collecting) past the line limit
253
- # when consuming the rest of the stream is cheap; without it the
254
- # loop breaks and leaves the total unknown. The byte cap always
255
- # aborts the count.
220
+ # Collect a {Page} window from +lines+ (Array or lazy Enumerator of
221
+ # +chomp+ed lines). +known_total+ is the full count when extracted up
222
+ # front (Array), +nil+ for a lazy stream — then +total_lines+ is exact
223
+ # only if iteration reached EOF: +count_tail+ keeps counting past the
224
+ # limit when the tail is cheap, else the loop breaks and the total stays
225
+ # unknown. The byte cap always aborts the count.
256
226
  #
257
227
  # @param lines [Enumerable<String>]
258
228
  # @param known_total [Integer, nil]