langextract 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +9 -0
- data/LICENSE.txt +21 -0
- data/README.md +250 -0
- data/lib/langextract/config.rb +20 -0
- data/lib/langextract/core/annotation.rb +36 -0
- data/lib/langextract/core/chunking.rb +116 -0
- data/lib/langextract/core/data.rb +271 -0
- data/lib/langextract/core/errors.rb +13 -0
- data/lib/langextract/core/format_handler.rb +154 -0
- data/lib/langextract/core/prompt_validation.rb +52 -0
- data/lib/langextract/core/prompting.rb +72 -0
- data/lib/langextract/core/resolver.rb +261 -0
- data/lib/langextract/core/tokenizer.rb +50 -0
- data/lib/langextract/core/types.rb +21 -0
- data/lib/langextract/errors.rb +13 -0
- data/lib/langextract/extractor.rb +158 -0
- data/lib/langextract/factory.rb +61 -0
- data/lib/langextract/io.rb +35 -0
- data/lib/langextract/providers/base.rb +21 -0
- data/lib/langextract/providers/router.rb +35 -0
- data/lib/langextract/providers/ruby_llm.rb +37 -0
- data/lib/langextract/version.rb +5 -0
- data/lib/langextract/visualization.rb +106 -0
- data/lib/langextract.rb +53 -0
- metadata +74 -0
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "erb"
|
|
4
|
+
|
|
5
|
+
require_relative "io"
|
|
6
|
+
|
|
7
|
+
module LangExtract
|
|
8
|
+
class Visualization
|
|
9
|
+
def render(input)
|
|
10
|
+
annotated_documents = coerce_documents(input)
|
|
11
|
+
|
|
12
|
+
render_page(annotated_documents)
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
private
|
|
16
|
+
|
|
17
|
+
def render_page(annotated_documents)
|
|
18
|
+
<<~HTML
|
|
19
|
+
<!doctype html>
|
|
20
|
+
<html lang="en">
|
|
21
|
+
<head>
|
|
22
|
+
<meta charset="utf-8">
|
|
23
|
+
<title>LangExtract Visualization</title>
|
|
24
|
+
<style>
|
|
25
|
+
body { font-family: system-ui, sans-serif; line-height: 1.55; margin: 2rem; color: #1f2933; }
|
|
26
|
+
main { max-width: 960px; margin: 0 auto; }
|
|
27
|
+
mark { background: #ffe08a; border-radius: 0.2rem; padding: 0.05rem 0.15rem; }
|
|
28
|
+
mark[data-status="fuzzy"] { background: #b7e4ff; }
|
|
29
|
+
.label { color: #52606d; font-size: 0.85em; margin-left: 0.2rem; }
|
|
30
|
+
pre { white-space: pre-wrap; font: inherit; }
|
|
31
|
+
</style>
|
|
32
|
+
</head>
|
|
33
|
+
<body>
|
|
34
|
+
<main>
|
|
35
|
+
#{annotated_documents.map { |annotated| render_document(annotated) }.join("\n")}
|
|
36
|
+
</main>
|
|
37
|
+
</body>
|
|
38
|
+
</html>
|
|
39
|
+
HTML
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def render_document(annotated)
|
|
43
|
+
highlights = valid_extractions(annotated.extractions)
|
|
44
|
+
|
|
45
|
+
<<~HTML
|
|
46
|
+
<section>
|
|
47
|
+
<h1>#{escape(annotated.id || 'Document')}</h1>
|
|
48
|
+
<pre>#{highlight_text(annotated.text, highlights)}</pre>
|
|
49
|
+
</section>
|
|
50
|
+
HTML
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def coerce_documents(input)
|
|
54
|
+
return [input] if input.is_a?(Core::AnnotatedDocument)
|
|
55
|
+
return input.to_a if document_collection?(input)
|
|
56
|
+
|
|
57
|
+
loaded = LangExtract::IO.load_annotated_documents_jsonl(input.to_s)
|
|
58
|
+
raise Core::IOFailure, "no annotated documents found in #{input}" if loaded.empty?
|
|
59
|
+
|
|
60
|
+
loaded
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def document_collection?(input)
|
|
64
|
+
input.respond_to?(:to_a) && input.to_a.all?(Core::AnnotatedDocument)
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def valid_extractions(extractions)
|
|
68
|
+
occupied = []
|
|
69
|
+
|
|
70
|
+
sorted = extractions.select(&:grounded?).sort_by { |extraction| extraction.char_interval.start_pos }
|
|
71
|
+
|
|
72
|
+
sorted.filter_map do |extraction|
|
|
73
|
+
next if occupied.any? { |interval| interval.overlaps?(extraction.char_interval) }
|
|
74
|
+
|
|
75
|
+
occupied << extraction.char_interval
|
|
76
|
+
extraction
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def highlight_text(text, extractions)
|
|
81
|
+
cursor = 0
|
|
82
|
+
output = +""
|
|
83
|
+
|
|
84
|
+
extractions.each do |extraction|
|
|
85
|
+
interval = extraction.char_interval
|
|
86
|
+
output << escape(text[cursor...interval.start_pos])
|
|
87
|
+
output << render_mark(extraction, text[interval.start_pos...interval.end_pos])
|
|
88
|
+
cursor = interval.end_pos
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
output << escape(text[cursor..])
|
|
92
|
+
output
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
def render_mark(extraction, content)
|
|
96
|
+
label = extraction.description || extraction.extraction_class || "extraction"
|
|
97
|
+
<<~HTML.delete("\n")
|
|
98
|
+
<mark data-status="#{escape(extraction.alignment_status)}">#{escape(content)}<span class="label">#{escape(label)}</span></mark>
|
|
99
|
+
HTML
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
def escape(value)
|
|
103
|
+
ERB::Util.html_escape(value.to_s)
|
|
104
|
+
end
|
|
105
|
+
end
|
|
106
|
+
end
|
data/lib/langextract.rb
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "langextract/version"
|
|
4
|
+
require_relative "langextract/config"
|
|
5
|
+
require_relative "langextract/errors"
|
|
6
|
+
require_relative "langextract/core/errors"
|
|
7
|
+
require_relative "langextract/core/data"
|
|
8
|
+
require_relative "langextract/core/tokenizer"
|
|
9
|
+
require_relative "langextract/core/chunking"
|
|
10
|
+
require_relative "langextract/core/format_handler"
|
|
11
|
+
require_relative "langextract/core/prompt_validation"
|
|
12
|
+
require_relative "langextract/core/prompting"
|
|
13
|
+
require_relative "langextract/core/resolver"
|
|
14
|
+
require_relative "langextract/core/annotation"
|
|
15
|
+
require_relative "langextract/extractor"
|
|
16
|
+
require_relative "langextract/io"
|
|
17
|
+
require_relative "langextract/visualization"
|
|
18
|
+
require_relative "langextract/factory"
|
|
19
|
+
|
|
20
|
+
module LangExtract
|
|
21
|
+
CharInterval = Core::CharInterval
|
|
22
|
+
TokenInterval = Core::TokenInterval
|
|
23
|
+
Extraction = Core::Extraction
|
|
24
|
+
Document = Core::Document
|
|
25
|
+
AnnotatedDocument = Core::AnnotatedDocument
|
|
26
|
+
ExampleData = Core::ExampleData
|
|
27
|
+
AlignmentStatus = Core::AlignmentStatus
|
|
28
|
+
|
|
29
|
+
class << self
|
|
30
|
+
def config
|
|
31
|
+
@config ||= Config.new
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def configure
|
|
35
|
+
yield(config)
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def reset_configuration!
|
|
39
|
+
@config = nil
|
|
40
|
+
Factory.reset_router!
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def extract(**kwargs)
|
|
44
|
+
text = kwargs.delete(:text)
|
|
45
|
+
documents = kwargs.delete(:documents)
|
|
46
|
+
Extractor.new(**kwargs).extract(text: text, documents: documents)
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def visualize(input)
|
|
50
|
+
Visualization.new.render(input)
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
metadata
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
|
2
|
+
name: langextract
|
|
3
|
+
version: !ruby/object:Gem::Version
|
|
4
|
+
version: 0.1.0
|
|
5
|
+
platform: ruby
|
|
6
|
+
authors:
|
|
7
|
+
- David Paluy
|
|
8
|
+
bindir: exe
|
|
9
|
+
cert_chain: []
|
|
10
|
+
date: 1980-01-02 00:00:00.000000000 Z
|
|
11
|
+
dependencies: []
|
|
12
|
+
description: Extract structured information from text with source grounding, deterministic
|
|
13
|
+
serialization, and HTML visualization.
|
|
14
|
+
email:
|
|
15
|
+
- dpaluy@users.noreply.github.com
|
|
16
|
+
executables: []
|
|
17
|
+
extensions: []
|
|
18
|
+
extra_rdoc_files:
|
|
19
|
+
- CHANGELOG.md
|
|
20
|
+
- LICENSE.txt
|
|
21
|
+
- README.md
|
|
22
|
+
files:
|
|
23
|
+
- CHANGELOG.md
|
|
24
|
+
- LICENSE.txt
|
|
25
|
+
- README.md
|
|
26
|
+
- lib/langextract.rb
|
|
27
|
+
- lib/langextract/config.rb
|
|
28
|
+
- lib/langextract/core/annotation.rb
|
|
29
|
+
- lib/langextract/core/chunking.rb
|
|
30
|
+
- lib/langextract/core/data.rb
|
|
31
|
+
- lib/langextract/core/errors.rb
|
|
32
|
+
- lib/langextract/core/format_handler.rb
|
|
33
|
+
- lib/langextract/core/prompt_validation.rb
|
|
34
|
+
- lib/langextract/core/prompting.rb
|
|
35
|
+
- lib/langextract/core/resolver.rb
|
|
36
|
+
- lib/langextract/core/tokenizer.rb
|
|
37
|
+
- lib/langextract/core/types.rb
|
|
38
|
+
- lib/langextract/errors.rb
|
|
39
|
+
- lib/langextract/extractor.rb
|
|
40
|
+
- lib/langextract/factory.rb
|
|
41
|
+
- lib/langextract/io.rb
|
|
42
|
+
- lib/langextract/providers/base.rb
|
|
43
|
+
- lib/langextract/providers/router.rb
|
|
44
|
+
- lib/langextract/providers/ruby_llm.rb
|
|
45
|
+
- lib/langextract/version.rb
|
|
46
|
+
- lib/langextract/visualization.rb
|
|
47
|
+
homepage: https://github.com/dpaluy/langextract
|
|
48
|
+
licenses:
|
|
49
|
+
- MIT
|
|
50
|
+
metadata:
|
|
51
|
+
homepage_uri: https://github.com/dpaluy/langextract
|
|
52
|
+
documentation_uri: https://rubydoc.info/gems/langextract
|
|
53
|
+
source_code_uri: https://github.com/dpaluy/langextract
|
|
54
|
+
changelog_uri: https://github.com/dpaluy/langextract/blob/main/CHANGELOG.md
|
|
55
|
+
bug_tracker_uri: https://github.com/dpaluy/langextract/issues
|
|
56
|
+
rubygems_mfa_required: 'true'
|
|
57
|
+
rdoc_options: []
|
|
58
|
+
require_paths:
|
|
59
|
+
- lib
|
|
60
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
|
61
|
+
requirements:
|
|
62
|
+
- - ">="
|
|
63
|
+
- !ruby/object:Gem::Version
|
|
64
|
+
version: 3.4.5
|
|
65
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
66
|
+
requirements:
|
|
67
|
+
- - ">="
|
|
68
|
+
- !ruby/object:Gem::Version
|
|
69
|
+
version: '0'
|
|
70
|
+
requirements: []
|
|
71
|
+
rubygems_version: 3.6.9
|
|
72
|
+
specification_version: 4
|
|
73
|
+
summary: Ruby port of LangExtract for source-grounded structured extraction.
|
|
74
|
+
test_files: []
|