giblish 0.8.2 → 2.0.0.pre.alpha1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/unit_tests.yml +30 -0
- data/.gitignore +7 -3
- data/.ruby-version +1 -1
- data/Changelog.adoc +59 -0
- data/README.adoc +261 -0
- data/docs/concepts/text_search.adoc +213 -0
- data/docs/concepts/text_search_im/cgi-search_request.puml +35 -0
- data/docs/concepts/text_search_im/cgi-search_request.svg +397 -0
- data/docs/concepts/text_search_im/search_request.puml +40 -0
- data/docs/concepts/text_search_im/search_request.svg +408 -0
- data/docs/howtos/trigger_generation.adoc +180 -0
- data/docs/{setup_server_assets → howtos/trigger_generation_im}/Render Documents.png +0 -0
- data/docs/{setup_server_assets → howtos/trigger_generation_im}/View Documents.png +0 -0
- data/docs/{setup_server_assets → howtos/trigger_generation_im}/deploy_with_hooks.graphml +0 -0
- data/docs/{setup_server_assets → howtos/trigger_generation_im}/deploy_with_hooks.svg +0 -0
- data/docs/{setup_server_assets → howtos/trigger_generation_im}/deploy_with_jenkins.graphml +0 -0
- data/docs/{setup_server_assets → howtos/trigger_generation_im}/deploy_with_jenkins.svg +0 -0
- data/docs/howtos/trigger_generation_im/docgen_github.puml +51 -0
- data/docs/{setup_server_assets → howtos/trigger_generation_im}/giblish_deployment.graphml +0 -0
- data/docs/howtos/trigger_generation_im/post-receive-example.sh +50 -0
- data/docs/reference/box_flow_spec.adoc +22 -0
- data/docs/reference/search_spec.adoc +185 -0
- data/giblish.gemspec +54 -32
- data/lib/giblish/adocsrc_providers.rb +23 -0
- data/lib/giblish/application.rb +214 -41
- data/lib/giblish/cmdline.rb +273 -259
- data/lib/giblish/config_utils.rb +41 -0
- data/lib/giblish/configurator.rb +163 -0
- data/lib/giblish/conversion_info.rb +120 -0
- data/lib/giblish/docattr_providers.rb +125 -0
- data/lib/giblish/docid/docid.rb +181 -0
- data/lib/giblish/github_trigger/webhook_manager.rb +64 -0
- data/lib/giblish/gitrepos/checkoutmanager.rb +124 -0
- data/lib/giblish/{gititf.rb → gitrepos/gititf.rb} +30 -4
- data/lib/giblish/gitrepos/gitsummary.erb +61 -0
- data/lib/giblish/gitrepos/gitsummaryprovider.rb +78 -0
- data/lib/giblish/gitrepos/history_pb.rb +41 -0
- data/lib/giblish/indexbuilders/d3treegraph.rb +88 -0
- data/lib/giblish/indexbuilders/depgraphbuilder.rb +109 -0
- data/lib/giblish/indexbuilders/dotdigraphadoc.rb +174 -0
- data/lib/giblish/indexbuilders/standard_index.erb +10 -0
- data/lib/giblish/indexbuilders/subtree_indices.rb +132 -0
- data/lib/giblish/indexbuilders/templates/circles.html.erb +111 -0
- data/lib/giblish/indexbuilders/templates/flame.html.erb +61 -0
- data/lib/giblish/indexbuilders/templates/tree.html.erb +366 -0
- data/lib/giblish/indexbuilders/templates/treemap.html.erb +127 -0
- data/lib/giblish/indexbuilders/verbatimtree.rb +94 -0
- data/lib/giblish/pathtree.rb +473 -74
- data/lib/giblish/resourcepaths.rb +150 -0
- data/lib/giblish/search/expand_adoc.rb +55 -0
- data/lib/giblish/search/headingindexer.rb +312 -0
- data/lib/giblish/search/request_manager.rb +110 -0
- data/lib/giblish/search/searchquery.rb +68 -0
- data/lib/giblish/search/textsearcher.rb +349 -0
- data/lib/giblish/subtreeinfobuilder.rb +77 -0
- data/lib/giblish/treeconverter.rb +272 -0
- data/lib/giblish/utils.rb +142 -294
- data/lib/giblish/version.rb +1 -1
- data/lib/giblish.rb +10 -7
- data/scripts/hooks/post-receive.example +66 -0
- data/{docgen/scripts/githook_examples → scripts/hooks}/post-update.example +0 -0
- data/{docgen → scripts}/resources/css/adoc-colony.css +0 -0
- data/scripts/resources/css/giblish-serif.css +419 -0
- data/scripts/resources/css/giblish.css +1979 -419
- data/{docgen → scripts}/resources/fonts/Ubuntu-B.ttf +0 -0
- data/{docgen → scripts}/resources/fonts/Ubuntu-BI.ttf +0 -0
- data/{docgen → scripts}/resources/fonts/Ubuntu-R.ttf +0 -0
- data/{docgen → scripts}/resources/fonts/Ubuntu-RI.ttf +0 -0
- data/{docgen → scripts}/resources/fonts/mplus1p-regular-fallback.ttf +0 -0
- data/{docgen → scripts}/resources/images/giblish_logo.png +0 -0
- data/{docgen → scripts}/resources/images/giblish_logo.svg +0 -0
- data/{docgen → scripts}/resources/themes/giblish.yml +0 -0
- data/scripts/wserv_development.rb +32 -0
- data/web_apps/cgi_search/gibsearch.rb +43 -0
- data/web_apps/gh_webhook_trigger/config.ru +2 -0
- data/web_apps/gh_webhook_trigger/gh_webhook_trigger.rb +73 -0
- data/web_apps/gh_webhook_trigger/public/dummy.txt +3 -0
- data/web_apps/sinatra_search/config.ru +2 -0
- data/web_apps/sinatra_search/public/dummy.txt +3 -0
- data/web_apps/sinatra_search/sinatra_search.rb +34 -0
- data/web_apps/sinatra_search/tmp/restart.txt +0 -0
- metadata +188 -85
- data/.rubocop.yml +0 -7
- data/.travis.yml +0 -3
- data/Changelog +0 -16
- data/Gemfile +0 -4
- data/README.adoc +0 -1
- data/Rakefile +0 -41
- data/bin/console +0 -14
- data/bin/setup +0 -8
- data/data/testdocs/malformed/no_header.adoc +0 -5
- data/data/testdocs/toplevel.adoc +0 -19
- data/data/testdocs/wellformed/adorned_purpose.adoc +0 -17
- data/data/testdocs/wellformed/docidtest/docid_1.adoc +0 -24
- data/data/testdocs/wellformed/docidtest/docid_2.adoc +0 -8
- data/data/testdocs/wellformed/simple.adoc +0 -14
- data/data/testdocs/wellformed/source_highlighting/highlight_source.adoc +0 -38
- data/docgen/resources/css/giblish.css +0 -1979
- data/docgen/scripts/Jenkinsfile +0 -18
- data/docgen/scripts/gen_adoc_org.sh +0 -58
- data/docs/README.adoc +0 -387
- data/docs/setup_server.adoc +0 -202
- data/lib/giblish/buildgraph.rb +0 -216
- data/lib/giblish/buildindex.rb +0 -459
- data/lib/giblish/core.rb +0 -451
- data/lib/giblish/docconverter.rb +0 -308
- data/lib/giblish/docid.rb +0 -180
- data/lib/giblish/docinfo.rb +0 -75
- data/lib/giblish/indexheadings.rb +0 -251
- data/lib/giblish-search.cgi +0 -459
- data/scripts/hooks/post-receive +0 -57
- data/scripts/publish_html.sh +0 -99
@@ -0,0 +1,349 @@
|
|
1
|
+
require "pathname"
|
2
|
+
require "json"
|
3
|
+
require_relative "searchquery"
|
4
|
+
require_relative "../pathtree"
|
5
|
+
|
6
|
+
module Giblish
|
7
|
+
# reads all lines in the given file at instantiation and
|
8
|
+
# washes the text from some adoc formatting sequences.
|
9
|
+
class LoadAdocSrcFromFile
|
10
|
+
attr_reader :src_lines
|
11
|
+
|
12
|
+
def initialize(filepath)
|
13
|
+
@src_lines = []
|
14
|
+
File.readlines(filepath.to_s, chomp: true, encoding: "UTF-8").each do |line|
|
15
|
+
@src_lines << wash_line(line)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
private
|
20
|
+
|
21
|
+
def wash_line(line)
|
22
|
+
# remove some asciidoctor format sequences
|
23
|
+
# '::', '^|===', '^==', '^--, ':myvar: ...'
|
24
|
+
r = Regexp.new(/(::+|^[=|]+|^--+|^:\w+:.*$)/)
|
25
|
+
line.gsub(r, "")
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
# Encapsulates raw data and information deducable from one
|
30
|
+
# search query
|
31
|
+
class SearchParameters
|
32
|
+
# a hash with { param => value } of all query parameters from the URI.
|
33
|
+
attr_reader :parameters
|
34
|
+
attr_reader :uri
|
35
|
+
|
36
|
+
def self.from_uri(uri_str, uri_mappings: {"/" => "/var/www/html/"})
|
37
|
+
q = SearchQuery.new(uri: uri_str)
|
38
|
+
SearchParameters.new(query: q, uri_mappings: uri_mappings)
|
39
|
+
end
|
40
|
+
|
41
|
+
def self.from_hash(h, uri_mappings: {"/" => "/var/www/html/"})
|
42
|
+
q = SearchQuery.new(query_params: h)
|
43
|
+
SearchParameters.new(query: q, uri_mappings: uri_mappings)
|
44
|
+
end
|
45
|
+
|
46
|
+
# the fragment of the calling doc's URI
|
47
|
+
# eg: www.mysite.com/my/doc/path/subdir1/doc1.html -> uri_path = my/doc/path/subdir1/doc1.html
|
48
|
+
def uri_path
|
49
|
+
URI(calling_url).path
|
50
|
+
end
|
51
|
+
|
52
|
+
# the URI path to the search asset directory
|
53
|
+
# eg: www.mysite.com/my/doc/path/subdir1/doc1.html could be assets_uri_path = my/doc/path/gibsearch_assets
|
54
|
+
def assets_uri_path
|
55
|
+
@assets_uri_path ||= Pathname.new(uri_path).dirname.join(search_assets_top_rel).cleanpath
|
56
|
+
@assets_uri_path
|
57
|
+
end
|
58
|
+
|
59
|
+
# the URI path to the css file to use for styling the search result
|
60
|
+
# eg: www.mysite.com/my/doc/path/subdir1/doc1.html could be css_path = my/doc/path/web/mystyle.css
|
61
|
+
def css_path
|
62
|
+
return nil if @query.css_path.nil?
|
63
|
+
|
64
|
+
return @query.css_path if @query.css_path.absolute?
|
65
|
+
|
66
|
+
# css paths from the query is relative, add it to the
|
67
|
+
@css_path ||= Pathname.new(uri_path).dirname.join(@query.css_path).cleanpath
|
68
|
+
@css_path
|
69
|
+
end
|
70
|
+
|
71
|
+
# return:: the uri path pointing to the doc repo top dir
|
72
|
+
def uri_path_repo_top
|
73
|
+
Pathname.new(uri_path).join(search_assets_top_rel.dirname).dirname
|
74
|
+
end
|
75
|
+
|
76
|
+
# return:: the absolute Pathname of the file system path to the
|
77
|
+
# search assets top dir.
|
78
|
+
def assets_fs_path
|
79
|
+
uri_to_fs(assets_uri_path)
|
80
|
+
end
|
81
|
+
|
82
|
+
def method_missing(meth, *args, &block)
|
83
|
+
return @query.send(meth, *args, &block) if respond_to_missing?(meth)
|
84
|
+
|
85
|
+
super(meth, args, &block)
|
86
|
+
end
|
87
|
+
|
88
|
+
def respond_to_missing?(meth, include_private = false)
|
89
|
+
@query.respond_to?(meth)
|
90
|
+
end
|
91
|
+
|
92
|
+
# return:: the relative path from the doc top dir to the file
|
93
|
+
def repo_file_path
|
94
|
+
Pathname.new(uri_path).relative_path_from(uri_path_repo_top)
|
95
|
+
end
|
96
|
+
|
97
|
+
# repo_filepath:: the filepath from the repo top to a given file
|
98
|
+
# fragment:: the fragment id or nil
|
99
|
+
#
|
100
|
+
# return:: the access url for a given section in a given src file
|
101
|
+
def url(repo_filepath, fragment = nil)
|
102
|
+
p = Pathname.new(repo_filepath)
|
103
|
+
|
104
|
+
# create result by replacing relevant parts of the original uri
|
105
|
+
res = URI(@query.calling_url)
|
106
|
+
res.query = nil
|
107
|
+
res.fragment = fragment
|
108
|
+
res.path = uri_path_repo_top.join(p.sub_ext(".html")).cleanpath.to_s
|
109
|
+
res
|
110
|
+
end
|
111
|
+
|
112
|
+
private
|
113
|
+
|
114
|
+
# Search input:
|
115
|
+
#
|
116
|
+
# query:: a SearchQuery instance
|
117
|
+
# uri_mappings:: mappings between uri.path prefix and an absolute path in the local
|
118
|
+
# file system. Ex {"/my/doc" => "/var/www/html/doc/repos"}. The default
|
119
|
+
# is { "/" => "/var/www/html" }
|
120
|
+
#
|
121
|
+
# ex URI = www.example.com/search/action?calling-uri=www.example.com/my/doc/repo/subdir/file1.html&search-assets-top-rel=../my/docs&search-phrase=hejsan
|
122
|
+
def initialize(query:, uri_mappings: {"/" => "/var/www/html/"})
|
123
|
+
@query = query
|
124
|
+
|
125
|
+
# convert keys and values to Pathnames
|
126
|
+
@uri_mappings = uri_mappings.map { |k, v| [Pathname.new(k).cleanpath, Pathname.new(v).cleanpath] }.to_h
|
127
|
+
validate_parameters
|
128
|
+
end
|
129
|
+
|
130
|
+
# return:: a Pathname where the prefix of an uri path has been replaced with the
|
131
|
+
# corresponding fs mapping, if one exists. Returns the original pathname
|
132
|
+
# if no corresponding mapping exists.
|
133
|
+
# if more than one mapping match, the longest is used.
|
134
|
+
def uri_to_fs(uri_path)
|
135
|
+
up = Pathname.new(uri_path).cleanpath
|
136
|
+
matches = {}
|
137
|
+
|
138
|
+
@uri_mappings.each do |key, value|
|
139
|
+
key_length = key.to_s.length
|
140
|
+
# we must treat '/' specially since its the only case where
|
141
|
+
# the key ends with a '/'
|
142
|
+
s = key.root? ? "" : key
|
143
|
+
tmp = up.sub(s.to_s, value.to_s).cleanpath
|
144
|
+
matches[key_length] = tmp if tmp != up
|
145
|
+
end
|
146
|
+
return up if matches.empty?
|
147
|
+
|
148
|
+
# return longest matching key
|
149
|
+
matches.max { |item| item[0] }[1]
|
150
|
+
end
|
151
|
+
|
152
|
+
# Require that:
|
153
|
+
#
|
154
|
+
# - a relative asset path is included
|
155
|
+
# - a search phrase is included
|
156
|
+
# - the uri_mappings map an absolute uri path to an absolute, and existing file
|
157
|
+
# system path.
|
158
|
+
def validate_parameters
|
159
|
+
# asset_top_rel
|
160
|
+
raise ArgumentError, "Asset top must be relative, found '#{search_assets_top_rel}'" if search_assets_top_rel.absolute?
|
161
|
+
|
162
|
+
# uri_mapping
|
163
|
+
@uri_mappings.each do |k, v|
|
164
|
+
raise ArgumentError, "The uri path in the uri_mapping must be absolute, found: '#{k}'" unless k.absolute?
|
165
|
+
raise ArgumentError, "The file system directory path must be absolute, found: '#{v}'" unless v.absolute?
|
166
|
+
raise ArgumentError, "The file system diretory does not exist: '#{v}'" unless v.exist?
|
167
|
+
raise ArgumentError, "The uri_mapping must be a directory, found file: '#{v}'" unless v.directory?
|
168
|
+
end
|
169
|
+
end
|
170
|
+
end
|
171
|
+
|
172
|
+
# Provides access to all search related info for one tree
|
173
|
+
# of adoc src docs.
|
174
|
+
class SearchDataRepo
|
175
|
+
attr_reader :search_db, :src_tree, :db_mod_time
|
176
|
+
|
177
|
+
SEARCH_DB_BASENAME = "heading_db.json"
|
178
|
+
|
179
|
+
# assets_uri_path:: a Pathname to the top dir of the search asset folder
|
180
|
+
def initialize(assets_fs_path)
|
181
|
+
@assets_fs_path = assets_fs_path
|
182
|
+
@search_db = read_search_db
|
183
|
+
@src_tree = build_src_tree
|
184
|
+
end
|
185
|
+
|
186
|
+
# find section with closest lower line_no to line_info
|
187
|
+
# NOTE: line_no in is 1-based
|
188
|
+
def in_section(filepath, line_no)
|
189
|
+
i = info(filepath)
|
190
|
+
i[:sections].reverse.find { |section| line_no >= section[:line_no] }
|
191
|
+
end
|
192
|
+
|
193
|
+
# return:: the info from the repo for the given filepath or nil if no info
|
194
|
+
# exists
|
195
|
+
def info(filepath)
|
196
|
+
@search_db[:fileinfos].find { |info| info[:filepath] == filepath.to_s }
|
197
|
+
end
|
198
|
+
|
199
|
+
def is_stale
|
200
|
+
@db_mod_time != File.stat(db_filepath.to_s).mtime
|
201
|
+
end
|
202
|
+
|
203
|
+
private
|
204
|
+
|
205
|
+
def build_src_tree
|
206
|
+
# setup the tree of source files and pro-actively read in all text
|
207
|
+
# into memory
|
208
|
+
src_tree = PathTree.build_from_fs(@assets_fs_path, prune: false) do |p|
|
209
|
+
p.extname.downcase == ".adoc"
|
210
|
+
end
|
211
|
+
src_tree.traverse_preorder do |level, node|
|
212
|
+
next unless node.leaf?
|
213
|
+
|
214
|
+
node.data = LoadAdocSrcFromFile.new(node.pathname)
|
215
|
+
end
|
216
|
+
src_tree
|
217
|
+
end
|
218
|
+
|
219
|
+
def db_filepath
|
220
|
+
@assets_fs_path.join(SEARCH_DB_BASENAME)
|
221
|
+
end
|
222
|
+
|
223
|
+
def read_search_db
|
224
|
+
# read the heading_db from file
|
225
|
+
@db_mod_time = File.stat(db_filepath.to_s).mtime
|
226
|
+
json = File.read(db_filepath.to_s)
|
227
|
+
JSON.parse(json, symbolize_names: true)
|
228
|
+
end
|
229
|
+
end
|
230
|
+
|
231
|
+
# Caches a number of SearchDataRepo instances in memory and returns the
|
232
|
+
# one corresponding to the given SearchParameters instance.
|
233
|
+
class SearchRepoCache
|
234
|
+
def initialize
|
235
|
+
@repos = {}
|
236
|
+
end
|
237
|
+
|
238
|
+
# search_parameters:: a SearchParameters instance
|
239
|
+
#
|
240
|
+
# returns:: the SearchDataRepo corresponding to the given search parameters
|
241
|
+
def repo(search_parameters)
|
242
|
+
ap = search_parameters.assets_fs_path
|
243
|
+
|
244
|
+
# check if we shall read a new repo from disk
|
245
|
+
if !@repos.key?(ap) || @repos[ap].is_stale
|
246
|
+
# Uncomment for debugging...
|
247
|
+
# puts "read from disk for ap: #{ap}.."
|
248
|
+
# puts "is stale" if @repos.key?(ap) && @repos[ap].is_stale
|
249
|
+
@repos[ap] = SearchDataRepo.new(ap)
|
250
|
+
end
|
251
|
+
|
252
|
+
@repos[ap]
|
253
|
+
end
|
254
|
+
end
|
255
|
+
|
256
|
+
# Provides text search capability for the given source repository.
|
257
|
+
class TextSearcher
|
258
|
+
def initialize(repo_cache)
|
259
|
+
@repo_cache = repo_cache
|
260
|
+
end
|
261
|
+
|
262
|
+
# transform the output from grep_tree to an Array of hashes according to:
|
263
|
+
#
|
264
|
+
# {Pathname("subdir/file1.adoc") => {
|
265
|
+
# doc_title: "My doc!!",
|
266
|
+
# sections: [{
|
267
|
+
# url: URI("http://my.site.com/docs/repo1/file1.html#section_id_1"),
|
268
|
+
# title: "Purpose",
|
269
|
+
# lines: [
|
270
|
+
# "this is the line with matching text",
|
271
|
+
# "this is another line with matching text"
|
272
|
+
# ]
|
273
|
+
# }]
|
274
|
+
# }}
|
275
|
+
# search_params:: a SearchParameters instance
|
276
|
+
def search(search_params)
|
277
|
+
repo = @repo_cache.repo(search_params)
|
278
|
+
|
279
|
+
grep_results = grep_tree(repo, search_params)
|
280
|
+
|
281
|
+
search_result(repo, grep_results, search_params)
|
282
|
+
end
|
283
|
+
|
284
|
+
# result = {
|
285
|
+
# filepath => [{
|
286
|
+
# line_no: nil,
|
287
|
+
# line: ""
|
288
|
+
# }]
|
289
|
+
# }
|
290
|
+
def grep_tree(repo, search_params)
|
291
|
+
result = Hash.new { |h, k| h[k] = [] }
|
292
|
+
|
293
|
+
# handle case-sensitivity and input as regex pattern or string
|
294
|
+
r_flags = search_params.consider_case? ? 0 : Regexp::IGNORECASE
|
295
|
+
r_str = search_params.as_regexp? ? search_params.search_phrase : Regexp.escape(search_params.search_phrase)
|
296
|
+
r = Regexp.new(r_str, r_flags)
|
297
|
+
|
298
|
+
# find all matching lines in the src tree
|
299
|
+
repo.src_tree.traverse_preorder do |level, node|
|
300
|
+
next unless node.leaf?
|
301
|
+
|
302
|
+
relative_path = node.relative_path_from(repo.src_tree)
|
303
|
+
line_no = 0
|
304
|
+
node.data.src_lines.each do |line|
|
305
|
+
line_no += 1
|
306
|
+
next if line.empty? || !r.match?(line)
|
307
|
+
|
308
|
+
# replace match with an embedded rule that can
|
309
|
+
# be styled
|
310
|
+
result[relative_path] << {
|
311
|
+
line_no: line_no,
|
312
|
+
line: line.gsub(r, '[.red]##*_\0_*##')
|
313
|
+
}
|
314
|
+
end
|
315
|
+
end
|
316
|
+
result
|
317
|
+
end
|
318
|
+
|
319
|
+
# returns:: a hash described in the 'search' method doc
|
320
|
+
def search_result(repo, grep_result, search_params)
|
321
|
+
result = Hash.new { |h, k| h[k] = [] }
|
322
|
+
|
323
|
+
grep_result.each do |filepath, matches|
|
324
|
+
db = repo.info(filepath)
|
325
|
+
next if db.nil?
|
326
|
+
|
327
|
+
sect_to_match = Hash.new { |h, k| h[k] = [] }
|
328
|
+
matches.each do |match|
|
329
|
+
s = repo.in_section(filepath, match[:line_no])
|
330
|
+
sect_to_match[s] << match
|
331
|
+
end
|
332
|
+
|
333
|
+
sections = sect_to_match.collect do |section, matches|
|
334
|
+
{
|
335
|
+
url: search_params.url(filepath, section[:id]),
|
336
|
+
title: section[:title],
|
337
|
+
lines: matches.collect { |match| match[:line].chomp }
|
338
|
+
}
|
339
|
+
end
|
340
|
+
|
341
|
+
result[filepath] = {
|
342
|
+
doc_title: db[:title],
|
343
|
+
sections: sections
|
344
|
+
}
|
345
|
+
end
|
346
|
+
result
|
347
|
+
end
|
348
|
+
end
|
349
|
+
end
|
@@ -0,0 +1,77 @@
|
|
1
|
+
require "pathname"
|
2
|
+
require_relative "pathtree"
|
3
|
+
|
4
|
+
module Giblish
|
5
|
+
class SubtreeSrcItf
|
6
|
+
attr_reader :adoc_source
|
7
|
+
def initialize(dst_node, output_basename)
|
8
|
+
raise NotImplementedError
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
class SubtreeInfoBuilder
|
13
|
+
attr_accessor :docattr_provider
|
14
|
+
|
15
|
+
DEFAULT_BASENAME = "index"
|
16
|
+
|
17
|
+
# docattr_provider:: an object implementing the DocAttributesBase interface
|
18
|
+
# api_opt_provider:: an object implementing the api_options(dst_top) interface
|
19
|
+
# adoc_src_provider:: a Class or object implementing the SubtreeSrcItf interface
|
20
|
+
# basename:: the name of the output file that is generated in each directory
|
21
|
+
def initialize(docattr_provider = nil, api_opt_provider = nil, adoc_src_provider = nil, basename = DEFAULT_BASENAME)
|
22
|
+
@docattr_provider = docattr_provider
|
23
|
+
@api_opt_provider = api_opt_provider
|
24
|
+
@adoc_src_provider = adoc_src_provider || SubtreeIndexBase
|
25
|
+
@basename = basename
|
26
|
+
@adoc_source = nil
|
27
|
+
end
|
28
|
+
|
29
|
+
def document_attributes(src_node, dst_node, dst_top)
|
30
|
+
@docattr_provider.nil? ? {} : @docattr_provider.document_attributes(src_node, dst_node, dst_top)
|
31
|
+
end
|
32
|
+
|
33
|
+
def api_options(src_node, dst_node, dst_top)
|
34
|
+
@api_opt_provider.nil? ? {} : @api_opt_provider.api_options(dst_top)
|
35
|
+
end
|
36
|
+
|
37
|
+
def adoc_source(src_node, dst_node, dst_top)
|
38
|
+
@adoc_source
|
39
|
+
end
|
40
|
+
|
41
|
+
# Called from TreeConverter during post build phase
|
42
|
+
#
|
43
|
+
# adds a 'index' node for each directory in the source tree
|
44
|
+
# and convert that index using the options from the provider
|
45
|
+
# objects given at instantiation of this object
|
46
|
+
def on_postbuild(src_tree, dst_tree, converter)
|
47
|
+
dst_tree.traverse_preorder do |level, dst_node|
|
48
|
+
# we only care about directories
|
49
|
+
next if dst_node.leaf?
|
50
|
+
|
51
|
+
# get the relative path to the index dir from the top dir
|
52
|
+
index_dir = dst_node.pathname.relative_path_from(dst_tree.pathname).cleanpath
|
53
|
+
Giblog.logger.debug { "Creating #{@basename} under #{index_dir}" }
|
54
|
+
|
55
|
+
# get the adoc source from the provider (Class or instance)
|
56
|
+
@adoc_source = if @adoc_src_provider.is_a?(Class)
|
57
|
+
@adoc_src_provider.new(dst_node, @basename).adoc_source
|
58
|
+
else
|
59
|
+
@adoc_src_provider.adoc_source
|
60
|
+
end
|
61
|
+
|
62
|
+
# add a virtual 'index.adoc' node as the only node in a source tree
|
63
|
+
# with this object as source for conversion options
|
64
|
+
# and adoc_source
|
65
|
+
v_path = Pathname.new("/virtual") / index_dir / "#{@basename}.adoc"
|
66
|
+
v_tree = PathTree.new(v_path, self)
|
67
|
+
src_node = v_tree.node(v_path, from_root: true)
|
68
|
+
|
69
|
+
# add the destination node where the converted file will be stored
|
70
|
+
i_node = dst_node.add_descendants(@basename)
|
71
|
+
|
72
|
+
# do the conversion
|
73
|
+
converter.convert(src_node, i_node, dst_tree)
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
@@ -0,0 +1,272 @@
|
|
1
|
+
require "asciidoctor"
|
2
|
+
require "asciidoctor-pdf"
|
3
|
+
require_relative "pathtree"
|
4
|
+
require_relative "conversion_info"
|
5
|
+
require_relative "utils"
|
6
|
+
|
7
|
+
module Giblish
|
8
|
+
# Converts all nodes in the supplied src PathTree from adoc to the format
|
9
|
+
# given by the user.
|
10
|
+
#
|
11
|
+
# Requires that all leaf nodes has a 'data' member that can receive an
|
12
|
+
# 'adoc_source' method that returns a string with the source to be converted.
|
13
|
+
#
|
14
|
+
# implements three phases with user hooks:
|
15
|
+
# pre_build -> build -> post_build
|
16
|
+
#
|
17
|
+
# Prebuild::
|
18
|
+
# add a pre_builder object that responds to:
|
19
|
+
# def run(src_tree, dst_tree, converter)
|
20
|
+
# where
|
21
|
+
# src_tree:: the node in a PathTree corresponding to the top of the
|
22
|
+
# src directory
|
23
|
+
# dst_tree:: the node in a PathTree corresponding to the top of the
|
24
|
+
# dst directory
|
25
|
+
# converter:: the specific converter used to convert the adoc source to
|
26
|
+
# the desired destination format.
|
27
|
+
|
28
|
+
class TreeConverter
|
29
|
+
attr_reader :dst_tree, :pre_builders, :post_builders, :converter
|
30
|
+
|
31
|
+
class << self
|
32
|
+
# register all asciidoctor extensions given at instantiation
|
33
|
+
#
|
34
|
+
# adoc_ext::
|
35
|
+
# { preprocessor: [], ... }
|
36
|
+
#
|
37
|
+
# see https://docs.asciidoctor.org/asciidoctor/latest/extensions/register/
|
38
|
+
def register_adoc_extensions(adoc_ext)
|
39
|
+
return if adoc_ext.nil?
|
40
|
+
|
41
|
+
%i[preprocessor tree_processor postprocessor docinfo_processor block
|
42
|
+
block_macro inline_macro include_processor].each do |e|
|
43
|
+
next unless adoc_ext.key?(e)
|
44
|
+
|
45
|
+
Array(adoc_ext[e])&.each do |c|
|
46
|
+
Giblog.logger.debug { "Register #{c.class} as #{e}" }
|
47
|
+
Asciidoctor::Extensions.register { send(e, c) }
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
def unregister_adoc_extenstions
|
53
|
+
Asciidoctor::Extensions.unregister_all
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
# opts:
|
58
|
+
# logger: the logger used internally by this instance (default nil)
|
59
|
+
# adoc_log_level - the log level when logging messages emitted by asciidoctor
|
60
|
+
# (default Logger::Severity::WARN)
|
61
|
+
# pre_builders
|
62
|
+
# post_builders
|
63
|
+
# adoc_api_opts
|
64
|
+
# adoc_doc_attribs
|
65
|
+
# conversion_cb {success: Proc(src,dst,adoc) fail: Proc(src,dst,exc)
|
66
|
+
def initialize(src_top, dst_top, opts = {})
|
67
|
+
# setup logging
|
68
|
+
@logger = opts.fetch(:logger, Giblog.logger)
|
69
|
+
@adoc_log_level = opts.fetch(:adoc_log_level, Logger::Severity::WARN)
|
70
|
+
|
71
|
+
# get the top-most node of the source and destination trees
|
72
|
+
@src_tree = src_top
|
73
|
+
@dst_tree = PathTree.new(dst_top).node(dst_top, from_root: true)
|
74
|
+
|
75
|
+
# setup build-phase callback objects
|
76
|
+
@pre_builders = Array(opts.fetch(:pre_builders, []))
|
77
|
+
@post_builders = Array(opts.fetch(:post_builders, []))
|
78
|
+
@converter = DefaultConverter.new(@logger, opts)
|
79
|
+
@adoc_ext = opts.fetch(:adoc_extensions, nil)
|
80
|
+
end
|
81
|
+
|
82
|
+
# abort_on_exc:: if true, an exception lower down the chain will
|
83
|
+
# abort the conversion and raised to the caller. If false, exceptions
|
84
|
+
# will be swallowed. In both cases, an 'error' log entry is created.
|
85
|
+
def run(abort_on_exc: true)
|
86
|
+
TreeConverter.register_adoc_extensions(@adoc_ext)
|
87
|
+
pre_build(abort_on_exc: abort_on_exc)
|
88
|
+
build(abort_on_exc: abort_on_exc)
|
89
|
+
post_build(abort_on_exc: abort_on_exc)
|
90
|
+
ensure
|
91
|
+
TreeConverter.unregister_adoc_extenstions
|
92
|
+
end
|
93
|
+
|
94
|
+
def pre_build(abort_on_exc: true)
|
95
|
+
@pre_builders.each do |pb|
|
96
|
+
pb.on_prebuild(@src_tree, @dst_tree, @converter)
|
97
|
+
rescue => ex
|
98
|
+
@logger&.error { ex.message.to_s }
|
99
|
+
raise ex if abort_on_exc
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
def build(abort_on_exc: true)
|
104
|
+
@src_tree.traverse_preorder do |level, n|
|
105
|
+
next unless n.leaf?
|
106
|
+
|
107
|
+
# create the destination node, using the correct suffix depending on conversion backend
|
108
|
+
rel_path = n.relative_path_from(@src_tree)
|
109
|
+
Giblog.logger.debug { "Creating dst node: #{rel_path}" }
|
110
|
+
dst_node = @dst_tree.add_descendants(rel_path)
|
111
|
+
|
112
|
+
# perform the conversion
|
113
|
+
@converter.convert(n, dst_node, @dst_tree)
|
114
|
+
rescue => exc
|
115
|
+
@logger&.error { "#{n.pathname} - #{exc.message}" }
|
116
|
+
raise exc if abort_on_exc
|
117
|
+
end
|
118
|
+
end
|
119
|
+
|
120
|
+
def post_build(abort_on_exc: true)
|
121
|
+
@post_builders.each do |pb|
|
122
|
+
pb.on_postbuild(@src_tree, @dst_tree, @converter)
|
123
|
+
rescue => exc
|
124
|
+
raise exc if abort_on_exc
|
125
|
+
@logger&.error { exc.message.to_s }
|
126
|
+
end
|
127
|
+
end
|
128
|
+
|
129
|
+
# the default callback will tie a 'SuccessfulConversion' instance
|
130
|
+
# to the destination node as its data
|
131
|
+
def self.on_success(src_node, dst_node, dst_tree, doc, adoc_log_str)
|
132
|
+
dst_node.data = DataDelegator.new(SuccessfulConversion.new(
|
133
|
+
src_node: src_node, dst_node: dst_node, dst_top: dst_tree, adoc: doc, adoc_stderr: adoc_log_str
|
134
|
+
))
|
135
|
+
end
|
136
|
+
|
137
|
+
# the default callback will tie a 'FailedConversion' instance
|
138
|
+
# to the destination node as its data
|
139
|
+
def self.on_failure(src_node, dst_node, dst_tree, ex, adoc_log_str)
|
140
|
+
Giblog.logger.error { ex.message }
|
141
|
+
dst_node.data = DataDelegator.new(FailedConversion.new(
|
142
|
+
src_node: src_node, dst_node: dst_node, dst_top: dst_tree, error_msg: ex.message
|
143
|
+
))
|
144
|
+
end
|
145
|
+
end
|
146
|
+
|
147
|
+
class DefaultConverter
|
148
|
+
attr_accessor :adoc_api_opts
|
149
|
+
# see https://docs.asciidoctor.org/asciidoc/latest/attributes/document-attributes-reference/
|
150
|
+
DEFAULT_ADOC_DOC_ATTRIBS = {
|
151
|
+
"data-uri" => true,
|
152
|
+
"hide-uri-scheme" => true,
|
153
|
+
"xrefstyle" => "short",
|
154
|
+
"source-highlighter" => "rouge",
|
155
|
+
"source-linenums-option" => true
|
156
|
+
}
|
157
|
+
|
158
|
+
# see https://docs.asciidoctor.org/asciidoctor/latest/api/options/
|
159
|
+
DEFAULT_ADOC_OPTS = {
|
160
|
+
backend: "html5",
|
161
|
+
# base_dir:
|
162
|
+
catalog_assets: false,
|
163
|
+
# converter:
|
164
|
+
doctype: "article",
|
165
|
+
# eruby:
|
166
|
+
# ignore extention stuff
|
167
|
+
header_only: false,
|
168
|
+
# logger:
|
169
|
+
mkdirs: false,
|
170
|
+
parse: true,
|
171
|
+
safe: :unsafe,
|
172
|
+
sourcemap: false,
|
173
|
+
# template stuff TBD,
|
174
|
+
# to_file:
|
175
|
+
# to_dir:
|
176
|
+
standalone: true
|
177
|
+
}
|
178
|
+
|
179
|
+
# opts:
|
180
|
+
# adoc_log_level
|
181
|
+
# adoc_api_opts
|
182
|
+
# adoc_doc_attribs
|
183
|
+
# conversion_cb {
|
184
|
+
# success: lambda(src, dst, dst_rel_path, doc, logstr)
|
185
|
+
# fail: lambda(src,dst,exc)
|
186
|
+
# }
|
187
|
+
def initialize(logger, opts)
|
188
|
+
@logger = logger
|
189
|
+
@adoc_log_level = opts.fetch(:adoc_log_level, Logger::Severity::WARN)
|
190
|
+
@conv_cb = opts.fetch(:conversion_cb, {
|
191
|
+
success: ->(src, dst, dst_rel_path, doc, logstr) { TreeConverter.on_success(src, dst, dst_rel_path, doc, logstr) },
|
192
|
+
failure: ->(src, dst, dst_rel_path, ex, logstr) { TreeConverter.on_failure(src, dst, dst_rel_path, ex, logstr) }
|
193
|
+
})
|
194
|
+
|
195
|
+
# merge user's options with the default, giving preference
|
196
|
+
# to the user
|
197
|
+
@adoc_api_opts = DEFAULT_ADOC_OPTS.dup
|
198
|
+
.merge!(opts.fetch(:adoc_api_opts, {}))
|
199
|
+
@adoc_api_opts[:attributes] = DEFAULT_ADOC_DOC_ATTRIBS.dup
|
200
|
+
.merge!(opts.fetch(:adoc_doc_attribs, {}))
|
201
|
+
end
|
202
|
+
|
203
|
+
# require the following methods to be available from the src node:
|
204
|
+
# adoc_source
|
205
|
+
#
|
206
|
+
# the following methods will be called if supported:
|
207
|
+
# document_attributes
|
208
|
+
# api_options
|
209
|
+
#
|
210
|
+
# src_node:: the PathTree node containing the info on adoc source and any
|
211
|
+
# added api_options or doc_attributes
|
212
|
+
# dst_node:: the PathTree node where conversion info is to be stored
|
213
|
+
# dst_top:: the PathTree node representing the top dir of the destination
|
214
|
+
# under which all converted files are written.
|
215
|
+
def convert(src_node, dst_node, dst_top)
|
216
|
+
@logger&.info { "Converting #{src_node.pathname} and store result under #{dst_node.parent.pathname}" }
|
217
|
+
|
218
|
+
# merge the common api opts with node specific
|
219
|
+
api_opts = @adoc_api_opts.dup
|
220
|
+
api_opts.merge!(src_node.api_options(src_node, dst_node, dst_top)) if src_node.respond_to?(:api_options)
|
221
|
+
api_opts[:attributes].merge!(src_node.document_attributes(src_node, dst_node, dst_top)) if src_node.respond_to?(:document_attributes)
|
222
|
+
|
223
|
+
# use a new logger instance for each conversion
|
224
|
+
adoc_logger = Giblish::AsciidoctorLogger.new(@logger, @adoc_log_level)
|
225
|
+
|
226
|
+
begin
|
227
|
+
# load the source to enable access to doc properties
|
228
|
+
#
|
229
|
+
# NOTE: the 'parse: false' is needed to prevent preprocessor extensions to be run as part
|
230
|
+
# of loading the document. We want them to run during the 'convert' call later when
|
231
|
+
# doc attribs have been amended.
|
232
|
+
doc = Asciidoctor.load(src_node.adoc_source(src_node, dst_node, dst_top), api_opts.merge(
|
233
|
+
{
|
234
|
+
parse: false,
|
235
|
+
logger: adoc_logger
|
236
|
+
}
|
237
|
+
))
|
238
|
+
|
239
|
+
# piggy-back our own info on the doc attributes hash so that
|
240
|
+
# asciidoctor extensions can use this info later on
|
241
|
+
doc.attributes["giblish-info"] = {
|
242
|
+
src_node: src_node,
|
243
|
+
dst_node: dst_node,
|
244
|
+
dst_top: dst_top
|
245
|
+
}
|
246
|
+
|
247
|
+
# update the destination node with the correct file suffix. This is dependent
|
248
|
+
# on the type of conversion performed
|
249
|
+
dst_node.name = dst_node.name.sub_ext(doc.attributes["outfilesuffix"])
|
250
|
+
d = dst_node.pathname
|
251
|
+
|
252
|
+
# make sure the dst dir exists
|
253
|
+
d.dirname.mkpath
|
254
|
+
|
255
|
+
# write the converted doc to the file
|
256
|
+
output = doc.convert(api_opts.merge({logger: adoc_logger}))
|
257
|
+
doc.write(output, d.to_s)
|
258
|
+
|
259
|
+
# give user the opportunity to eg store the result of the conversion
|
260
|
+
# as data in the destination node
|
261
|
+
@conv_cb[:success]&.call(src_node, dst_node, dst_top, doc, adoc_logger.in_mem_storage.string)
|
262
|
+
true
|
263
|
+
rescue => ex
|
264
|
+
@logger&.error { "Conversion failed for #{src_node.pathname}" }
|
265
|
+
@logger&.error { ex.message }
|
266
|
+
@logger&.error { ex.backtrace }
|
267
|
+
@conv_cb[:failure]&.call(src_node, dst_node, dst_top, ex, adoc_logger.in_mem_storage.string)
|
268
|
+
false
|
269
|
+
end
|
270
|
+
end
|
271
|
+
end
|
272
|
+
end
|