giblish 0.8.2 → 2.0.0.pre.alpha1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/unit_tests.yml +30 -0
  3. data/.gitignore +7 -3
  4. data/.ruby-version +1 -1
  5. data/Changelog.adoc +59 -0
  6. data/README.adoc +261 -0
  7. data/docs/concepts/text_search.adoc +213 -0
  8. data/docs/concepts/text_search_im/cgi-search_request.puml +35 -0
  9. data/docs/concepts/text_search_im/cgi-search_request.svg +397 -0
  10. data/docs/concepts/text_search_im/search_request.puml +40 -0
  11. data/docs/concepts/text_search_im/search_request.svg +408 -0
  12. data/docs/howtos/trigger_generation.adoc +180 -0
  13. data/docs/{setup_server_assets → howtos/trigger_generation_im}/Render Documents.png +0 -0
  14. data/docs/{setup_server_assets → howtos/trigger_generation_im}/View Documents.png +0 -0
  15. data/docs/{setup_server_assets → howtos/trigger_generation_im}/deploy_with_hooks.graphml +0 -0
  16. data/docs/{setup_server_assets → howtos/trigger_generation_im}/deploy_with_hooks.svg +0 -0
  17. data/docs/{setup_server_assets → howtos/trigger_generation_im}/deploy_with_jenkins.graphml +0 -0
  18. data/docs/{setup_server_assets → howtos/trigger_generation_im}/deploy_with_jenkins.svg +0 -0
  19. data/docs/howtos/trigger_generation_im/docgen_github.puml +51 -0
  20. data/docs/{setup_server_assets → howtos/trigger_generation_im}/giblish_deployment.graphml +0 -0
  21. data/docs/howtos/trigger_generation_im/post-receive-example.sh +50 -0
  22. data/docs/reference/box_flow_spec.adoc +22 -0
  23. data/docs/reference/search_spec.adoc +185 -0
  24. data/giblish.gemspec +54 -32
  25. data/lib/giblish/adocsrc_providers.rb +23 -0
  26. data/lib/giblish/application.rb +214 -41
  27. data/lib/giblish/cmdline.rb +273 -259
  28. data/lib/giblish/config_utils.rb +41 -0
  29. data/lib/giblish/configurator.rb +163 -0
  30. data/lib/giblish/conversion_info.rb +120 -0
  31. data/lib/giblish/docattr_providers.rb +125 -0
  32. data/lib/giblish/docid/docid.rb +181 -0
  33. data/lib/giblish/github_trigger/webhook_manager.rb +64 -0
  34. data/lib/giblish/gitrepos/checkoutmanager.rb +124 -0
  35. data/lib/giblish/{gititf.rb → gitrepos/gititf.rb} +30 -4
  36. data/lib/giblish/gitrepos/gitsummary.erb +61 -0
  37. data/lib/giblish/gitrepos/gitsummaryprovider.rb +78 -0
  38. data/lib/giblish/gitrepos/history_pb.rb +41 -0
  39. data/lib/giblish/indexbuilders/d3treegraph.rb +88 -0
  40. data/lib/giblish/indexbuilders/depgraphbuilder.rb +109 -0
  41. data/lib/giblish/indexbuilders/dotdigraphadoc.rb +174 -0
  42. data/lib/giblish/indexbuilders/standard_index.erb +10 -0
  43. data/lib/giblish/indexbuilders/subtree_indices.rb +132 -0
  44. data/lib/giblish/indexbuilders/templates/circles.html.erb +111 -0
  45. data/lib/giblish/indexbuilders/templates/flame.html.erb +61 -0
  46. data/lib/giblish/indexbuilders/templates/tree.html.erb +366 -0
  47. data/lib/giblish/indexbuilders/templates/treemap.html.erb +127 -0
  48. data/lib/giblish/indexbuilders/verbatimtree.rb +94 -0
  49. data/lib/giblish/pathtree.rb +473 -74
  50. data/lib/giblish/resourcepaths.rb +150 -0
  51. data/lib/giblish/search/expand_adoc.rb +55 -0
  52. data/lib/giblish/search/headingindexer.rb +312 -0
  53. data/lib/giblish/search/request_manager.rb +110 -0
  54. data/lib/giblish/search/searchquery.rb +68 -0
  55. data/lib/giblish/search/textsearcher.rb +349 -0
  56. data/lib/giblish/subtreeinfobuilder.rb +77 -0
  57. data/lib/giblish/treeconverter.rb +272 -0
  58. data/lib/giblish/utils.rb +142 -294
  59. data/lib/giblish/version.rb +1 -1
  60. data/lib/giblish.rb +10 -7
  61. data/scripts/hooks/post-receive.example +66 -0
  62. data/{docgen/scripts/githook_examples → scripts/hooks}/post-update.example +0 -0
  63. data/{docgen → scripts}/resources/css/adoc-colony.css +0 -0
  64. data/scripts/resources/css/giblish-serif.css +419 -0
  65. data/scripts/resources/css/giblish.css +1979 -419
  66. data/{docgen → scripts}/resources/fonts/Ubuntu-B.ttf +0 -0
  67. data/{docgen → scripts}/resources/fonts/Ubuntu-BI.ttf +0 -0
  68. data/{docgen → scripts}/resources/fonts/Ubuntu-R.ttf +0 -0
  69. data/{docgen → scripts}/resources/fonts/Ubuntu-RI.ttf +0 -0
  70. data/{docgen → scripts}/resources/fonts/mplus1p-regular-fallback.ttf +0 -0
  71. data/{docgen → scripts}/resources/images/giblish_logo.png +0 -0
  72. data/{docgen → scripts}/resources/images/giblish_logo.svg +0 -0
  73. data/{docgen → scripts}/resources/themes/giblish.yml +0 -0
  74. data/scripts/wserv_development.rb +32 -0
  75. data/web_apps/cgi_search/gibsearch.rb +43 -0
  76. data/web_apps/gh_webhook_trigger/config.ru +2 -0
  77. data/web_apps/gh_webhook_trigger/gh_webhook_trigger.rb +73 -0
  78. data/web_apps/gh_webhook_trigger/public/dummy.txt +3 -0
  79. data/web_apps/sinatra_search/config.ru +2 -0
  80. data/web_apps/sinatra_search/public/dummy.txt +3 -0
  81. data/web_apps/sinatra_search/sinatra_search.rb +34 -0
  82. data/web_apps/sinatra_search/tmp/restart.txt +0 -0
  83. metadata +188 -85
  84. data/.rubocop.yml +0 -7
  85. data/.travis.yml +0 -3
  86. data/Changelog +0 -16
  87. data/Gemfile +0 -4
  88. data/README.adoc +0 -1
  89. data/Rakefile +0 -41
  90. data/bin/console +0 -14
  91. data/bin/setup +0 -8
  92. data/data/testdocs/malformed/no_header.adoc +0 -5
  93. data/data/testdocs/toplevel.adoc +0 -19
  94. data/data/testdocs/wellformed/adorned_purpose.adoc +0 -17
  95. data/data/testdocs/wellformed/docidtest/docid_1.adoc +0 -24
  96. data/data/testdocs/wellformed/docidtest/docid_2.adoc +0 -8
  97. data/data/testdocs/wellformed/simple.adoc +0 -14
  98. data/data/testdocs/wellformed/source_highlighting/highlight_source.adoc +0 -38
  99. data/docgen/resources/css/giblish.css +0 -1979
  100. data/docgen/scripts/Jenkinsfile +0 -18
  101. data/docgen/scripts/gen_adoc_org.sh +0 -58
  102. data/docs/README.adoc +0 -387
  103. data/docs/setup_server.adoc +0 -202
  104. data/lib/giblish/buildgraph.rb +0 -216
  105. data/lib/giblish/buildindex.rb +0 -459
  106. data/lib/giblish/core.rb +0 -451
  107. data/lib/giblish/docconverter.rb +0 -308
  108. data/lib/giblish/docid.rb +0 -180
  109. data/lib/giblish/docinfo.rb +0 -75
  110. data/lib/giblish/indexheadings.rb +0 -251
  111. data/lib/giblish-search.cgi +0 -459
  112. data/scripts/hooks/post-receive +0 -57
  113. data/scripts/publish_html.sh +0 -99
@@ -0,0 +1,349 @@
1
+ require "pathname"
2
+ require "json"
3
+ require_relative "searchquery"
4
+ require_relative "../pathtree"
5
+
6
+ module Giblish
7
+ # reads all lines in the given file at instantiation and
8
+ # washes the text from some adoc formatting sequences.
9
+ class LoadAdocSrcFromFile
10
+ attr_reader :src_lines
11
+
12
+ def initialize(filepath)
13
+ @src_lines = []
14
+ File.readlines(filepath.to_s, chomp: true, encoding: "UTF-8").each do |line|
15
+ @src_lines << wash_line(line)
16
+ end
17
+ end
18
+
19
+ private
20
+
21
+ def wash_line(line)
22
+ # remove some asciidoctor format sequences
23
+ # '::', '^|===', '^==', '^--, ':myvar: ...'
24
+ r = Regexp.new(/(::+|^[=|]+|^--+|^:\w+:.*$)/)
25
+ line.gsub(r, "")
26
+ end
27
+ end
28
+
29
+ # Encapsulates raw data and information deducable from one
30
+ # search query
31
+ class SearchParameters
32
+ # a hash with { param => value } of all query parameters from the URI.
33
+ attr_reader :parameters
34
+ attr_reader :uri
35
+
36
+ def self.from_uri(uri_str, uri_mappings: {"/" => "/var/www/html/"})
37
+ q = SearchQuery.new(uri: uri_str)
38
+ SearchParameters.new(query: q, uri_mappings: uri_mappings)
39
+ end
40
+
41
+ def self.from_hash(h, uri_mappings: {"/" => "/var/www/html/"})
42
+ q = SearchQuery.new(query_params: h)
43
+ SearchParameters.new(query: q, uri_mappings: uri_mappings)
44
+ end
45
+
46
+ # the fragment of the calling doc's URI
47
+ # eg: www.mysite.com/my/doc/path/subdir1/doc1.html -> uri_path = my/doc/path/subdir1/doc1.html
48
+ def uri_path
49
+ URI(calling_url).path
50
+ end
51
+
52
+ # the URI path to the search asset directory
53
+ # eg: www.mysite.com/my/doc/path/subdir1/doc1.html could be assets_uri_path = my/doc/path/gibsearch_assets
54
+ def assets_uri_path
55
+ @assets_uri_path ||= Pathname.new(uri_path).dirname.join(search_assets_top_rel).cleanpath
56
+ @assets_uri_path
57
+ end
58
+
59
+ # the URI path to the css file to use for styling the search result
60
+ # eg: www.mysite.com/my/doc/path/subdir1/doc1.html could be css_path = my/doc/path/web/mystyle.css
61
+ def css_path
62
+ return nil if @query.css_path.nil?
63
+
64
+ return @query.css_path if @query.css_path.absolute?
65
+
66
+ # css paths from the query is relative, add it to the
67
+ @css_path ||= Pathname.new(uri_path).dirname.join(@query.css_path).cleanpath
68
+ @css_path
69
+ end
70
+
71
+ # return:: the uri path pointing to the doc repo top dir
72
+ def uri_path_repo_top
73
+ Pathname.new(uri_path).join(search_assets_top_rel.dirname).dirname
74
+ end
75
+
76
+ # return:: the absolute Pathname of the file system path to the
77
+ # search assets top dir.
78
+ def assets_fs_path
79
+ uri_to_fs(assets_uri_path)
80
+ end
81
+
82
+ def method_missing(meth, *args, &block)
83
+ return @query.send(meth, *args, &block) if respond_to_missing?(meth)
84
+
85
+ super(meth, args, &block)
86
+ end
87
+
88
+ def respond_to_missing?(meth, include_private = false)
89
+ @query.respond_to?(meth)
90
+ end
91
+
92
+ # return:: the relative path from the doc top dir to the file
93
+ def repo_file_path
94
+ Pathname.new(uri_path).relative_path_from(uri_path_repo_top)
95
+ end
96
+
97
+ # repo_filepath:: the filepath from the repo top to a given file
98
+ # fragment:: the fragment id or nil
99
+ #
100
+ # return:: the access url for a given section in a given src file
101
+ def url(repo_filepath, fragment = nil)
102
+ p = Pathname.new(repo_filepath)
103
+
104
+ # create result by replacing relevant parts of the original uri
105
+ res = URI(@query.calling_url)
106
+ res.query = nil
107
+ res.fragment = fragment
108
+ res.path = uri_path_repo_top.join(p.sub_ext(".html")).cleanpath.to_s
109
+ res
110
+ end
111
+
112
+ private
113
+
114
+ # Search input:
115
+ #
116
+ # query:: a SearchQuery instance
117
+ # uri_mappings:: mappings between uri.path prefix and an absolute path in the local
118
+ # file system. Ex {"/my/doc" => "/var/www/html/doc/repos"}. The default
119
+ # is { "/" => "/var/www/html" }
120
+ #
121
+ # ex URI = www.example.com/search/action?calling-uri=www.example.com/my/doc/repo/subdir/file1.html&search-assets-top-rel=../my/docs&search-phrase=hejsan
122
+ def initialize(query:, uri_mappings: {"/" => "/var/www/html/"})
123
+ @query = query
124
+
125
+ # convert keys and values to Pathnames
126
+ @uri_mappings = uri_mappings.map { |k, v| [Pathname.new(k).cleanpath, Pathname.new(v).cleanpath] }.to_h
127
+ validate_parameters
128
+ end
129
+
130
+ # return:: a Pathname where the prefix of an uri path has been replaced with the
131
+ # corresponding fs mapping, if one exists. Returns the original pathname
132
+ # if no corresponding mapping exists.
133
+ # if more than one mapping match, the longest is used.
134
+ def uri_to_fs(uri_path)
135
+ up = Pathname.new(uri_path).cleanpath
136
+ matches = {}
137
+
138
+ @uri_mappings.each do |key, value|
139
+ key_length = key.to_s.length
140
+ # we must treat '/' specially since its the only case where
141
+ # the key ends with a '/'
142
+ s = key.root? ? "" : key
143
+ tmp = up.sub(s.to_s, value.to_s).cleanpath
144
+ matches[key_length] = tmp if tmp != up
145
+ end
146
+ return up if matches.empty?
147
+
148
+ # return longest matching key
149
+ matches.max { |item| item[0] }[1]
150
+ end
151
+
152
+ # Require that:
153
+ #
154
+ # - a relative asset path is included
155
+ # - a search phrase is included
156
+ # - the uri_mappings map an absolute uri path to an absolute, and existing file
157
+ # system path.
158
+ def validate_parameters
159
+ # asset_top_rel
160
+ raise ArgumentError, "Asset top must be relative, found '#{search_assets_top_rel}'" if search_assets_top_rel.absolute?
161
+
162
+ # uri_mapping
163
+ @uri_mappings.each do |k, v|
164
+ raise ArgumentError, "The uri path in the uri_mapping must be absolute, found: '#{k}'" unless k.absolute?
165
+ raise ArgumentError, "The file system directory path must be absolute, found: '#{v}'" unless v.absolute?
166
+ raise ArgumentError, "The file system diretory does not exist: '#{v}'" unless v.exist?
167
+ raise ArgumentError, "The uri_mapping must be a directory, found file: '#{v}'" unless v.directory?
168
+ end
169
+ end
170
+ end
171
+
172
+ # Provides access to all search related info for one tree
173
+ # of adoc src docs.
174
+ class SearchDataRepo
175
+ attr_reader :search_db, :src_tree, :db_mod_time
176
+
177
+ SEARCH_DB_BASENAME = "heading_db.json"
178
+
179
+ # assets_uri_path:: a Pathname to the top dir of the search asset folder
180
+ def initialize(assets_fs_path)
181
+ @assets_fs_path = assets_fs_path
182
+ @search_db = read_search_db
183
+ @src_tree = build_src_tree
184
+ end
185
+
186
+ # find section with closest lower line_no to line_info
187
+ # NOTE: line_no in is 1-based
188
+ def in_section(filepath, line_no)
189
+ i = info(filepath)
190
+ i[:sections].reverse.find { |section| line_no >= section[:line_no] }
191
+ end
192
+
193
+ # return:: the info from the repo for the given filepath or nil if no info
194
+ # exists
195
+ def info(filepath)
196
+ @search_db[:fileinfos].find { |info| info[:filepath] == filepath.to_s }
197
+ end
198
+
199
+ def is_stale
200
+ @db_mod_time != File.stat(db_filepath.to_s).mtime
201
+ end
202
+
203
+ private
204
+
205
+ def build_src_tree
206
+ # setup the tree of source files and pro-actively read in all text
207
+ # into memory
208
+ src_tree = PathTree.build_from_fs(@assets_fs_path, prune: false) do |p|
209
+ p.extname.downcase == ".adoc"
210
+ end
211
+ src_tree.traverse_preorder do |level, node|
212
+ next unless node.leaf?
213
+
214
+ node.data = LoadAdocSrcFromFile.new(node.pathname)
215
+ end
216
+ src_tree
217
+ end
218
+
219
+ def db_filepath
220
+ @assets_fs_path.join(SEARCH_DB_BASENAME)
221
+ end
222
+
223
+ def read_search_db
224
+ # read the heading_db from file
225
+ @db_mod_time = File.stat(db_filepath.to_s).mtime
226
+ json = File.read(db_filepath.to_s)
227
+ JSON.parse(json, symbolize_names: true)
228
+ end
229
+ end
230
+
231
+ # Caches a number of SearchDataRepo instances in memory and returns the
232
+ # one corresponding to the given SearchParameters instance.
233
+ class SearchRepoCache
234
+ def initialize
235
+ @repos = {}
236
+ end
237
+
238
+ # search_parameters:: a SearchParameters instance
239
+ #
240
+ # returns:: the SearchDataRepo corresponding to the given search parameters
241
+ def repo(search_parameters)
242
+ ap = search_parameters.assets_fs_path
243
+
244
+ # check if we shall read a new repo from disk
245
+ if !@repos.key?(ap) || @repos[ap].is_stale
246
+ # Uncomment for debugging...
247
+ # puts "read from disk for ap: #{ap}.."
248
+ # puts "is stale" if @repos.key?(ap) && @repos[ap].is_stale
249
+ @repos[ap] = SearchDataRepo.new(ap)
250
+ end
251
+
252
+ @repos[ap]
253
+ end
254
+ end
255
+
256
+ # Provides text search capability for the given source repository.
257
+ class TextSearcher
258
+ def initialize(repo_cache)
259
+ @repo_cache = repo_cache
260
+ end
261
+
262
+ # transform the output from grep_tree to an Array of hashes according to:
263
+ #
264
+ # {Pathname("subdir/file1.adoc") => {
265
+ # doc_title: "My doc!!",
266
+ # sections: [{
267
+ # url: URI("http://my.site.com/docs/repo1/file1.html#section_id_1"),
268
+ # title: "Purpose",
269
+ # lines: [
270
+ # "this is the line with matching text",
271
+ # "this is another line with matching text"
272
+ # ]
273
+ # }]
274
+ # }}
275
+ # search_params:: a SearchParameters instance
276
+ def search(search_params)
277
+ repo = @repo_cache.repo(search_params)
278
+
279
+ grep_results = grep_tree(repo, search_params)
280
+
281
+ search_result(repo, grep_results, search_params)
282
+ end
283
+
284
+ # result = {
285
+ # filepath => [{
286
+ # line_no: nil,
287
+ # line: ""
288
+ # }]
289
+ # }
290
+ def grep_tree(repo, search_params)
291
+ result = Hash.new { |h, k| h[k] = [] }
292
+
293
+ # handle case-sensitivity and input as regex pattern or string
294
+ r_flags = search_params.consider_case? ? 0 : Regexp::IGNORECASE
295
+ r_str = search_params.as_regexp? ? search_params.search_phrase : Regexp.escape(search_params.search_phrase)
296
+ r = Regexp.new(r_str, r_flags)
297
+
298
+ # find all matching lines in the src tree
299
+ repo.src_tree.traverse_preorder do |level, node|
300
+ next unless node.leaf?
301
+
302
+ relative_path = node.relative_path_from(repo.src_tree)
303
+ line_no = 0
304
+ node.data.src_lines.each do |line|
305
+ line_no += 1
306
+ next if line.empty? || !r.match?(line)
307
+
308
+ # replace match with an embedded rule that can
309
+ # be styled
310
+ result[relative_path] << {
311
+ line_no: line_no,
312
+ line: line.gsub(r, '[.red]##*_\0_*##')
313
+ }
314
+ end
315
+ end
316
+ result
317
+ end
318
+
319
+ # returns:: a hash described in the 'search' method doc
320
+ def search_result(repo, grep_result, search_params)
321
+ result = Hash.new { |h, k| h[k] = [] }
322
+
323
+ grep_result.each do |filepath, matches|
324
+ db = repo.info(filepath)
325
+ next if db.nil?
326
+
327
+ sect_to_match = Hash.new { |h, k| h[k] = [] }
328
+ matches.each do |match|
329
+ s = repo.in_section(filepath, match[:line_no])
330
+ sect_to_match[s] << match
331
+ end
332
+
333
+ sections = sect_to_match.collect do |section, matches|
334
+ {
335
+ url: search_params.url(filepath, section[:id]),
336
+ title: section[:title],
337
+ lines: matches.collect { |match| match[:line].chomp }
338
+ }
339
+ end
340
+
341
+ result[filepath] = {
342
+ doc_title: db[:title],
343
+ sections: sections
344
+ }
345
+ end
346
+ result
347
+ end
348
+ end
349
+ end
@@ -0,0 +1,77 @@
1
+ require "pathname"
2
+ require_relative "pathtree"
3
+
4
+ module Giblish
5
+ class SubtreeSrcItf
6
+ attr_reader :adoc_source
7
+ def initialize(dst_node, output_basename)
8
+ raise NotImplementedError
9
+ end
10
+ end
11
+
12
+ class SubtreeInfoBuilder
13
+ attr_accessor :docattr_provider
14
+
15
+ DEFAULT_BASENAME = "index"
16
+
17
+ # docattr_provider:: an object implementing the DocAttributesBase interface
18
+ # api_opt_provider:: an object implementing the api_options(dst_top) interface
19
+ # adoc_src_provider:: a Class or object implementing the SubtreeSrcItf interface
20
+ # basename:: the name of the output file that is generated in each directory
21
+ def initialize(docattr_provider = nil, api_opt_provider = nil, adoc_src_provider = nil, basename = DEFAULT_BASENAME)
22
+ @docattr_provider = docattr_provider
23
+ @api_opt_provider = api_opt_provider
24
+ @adoc_src_provider = adoc_src_provider || SubtreeIndexBase
25
+ @basename = basename
26
+ @adoc_source = nil
27
+ end
28
+
29
+ def document_attributes(src_node, dst_node, dst_top)
30
+ @docattr_provider.nil? ? {} : @docattr_provider.document_attributes(src_node, dst_node, dst_top)
31
+ end
32
+
33
+ def api_options(src_node, dst_node, dst_top)
34
+ @api_opt_provider.nil? ? {} : @api_opt_provider.api_options(dst_top)
35
+ end
36
+
37
+ def adoc_source(src_node, dst_node, dst_top)
38
+ @adoc_source
39
+ end
40
+
41
+ # Called from TreeConverter during post build phase
42
+ #
43
+ # adds a 'index' node for each directory in the source tree
44
+ # and convert that index using the options from the provider
45
+ # objects given at instantiation of this object
46
+ def on_postbuild(src_tree, dst_tree, converter)
47
+ dst_tree.traverse_preorder do |level, dst_node|
48
+ # we only care about directories
49
+ next if dst_node.leaf?
50
+
51
+ # get the relative path to the index dir from the top dir
52
+ index_dir = dst_node.pathname.relative_path_from(dst_tree.pathname).cleanpath
53
+ Giblog.logger.debug { "Creating #{@basename} under #{index_dir}" }
54
+
55
+ # get the adoc source from the provider (Class or instance)
56
+ @adoc_source = if @adoc_src_provider.is_a?(Class)
57
+ @adoc_src_provider.new(dst_node, @basename).adoc_source
58
+ else
59
+ @adoc_src_provider.adoc_source
60
+ end
61
+
62
+ # add a virtual 'index.adoc' node as the only node in a source tree
63
+ # with this object as source for conversion options
64
+ # and adoc_source
65
+ v_path = Pathname.new("/virtual") / index_dir / "#{@basename}.adoc"
66
+ v_tree = PathTree.new(v_path, self)
67
+ src_node = v_tree.node(v_path, from_root: true)
68
+
69
+ # add the destination node where the converted file will be stored
70
+ i_node = dst_node.add_descendants(@basename)
71
+
72
+ # do the conversion
73
+ converter.convert(src_node, i_node, dst_tree)
74
+ end
75
+ end
76
+ end
77
+ end
@@ -0,0 +1,272 @@
1
+ require "asciidoctor"
2
+ require "asciidoctor-pdf"
3
+ require_relative "pathtree"
4
+ require_relative "conversion_info"
5
+ require_relative "utils"
6
+
7
+ module Giblish
8
+ # Converts all nodes in the supplied src PathTree from adoc to the format
9
+ # given by the user.
10
+ #
11
+ # Requires that all leaf nodes has a 'data' member that can receive an
12
+ # 'adoc_source' method that returns a string with the source to be converted.
13
+ #
14
+ # implements three phases with user hooks:
15
+ # pre_build -> build -> post_build
16
+ #
17
+ # Prebuild::
18
+ # add a pre_builder object that responds to:
19
+ # def run(src_tree, dst_tree, converter)
20
+ # where
21
+ # src_tree:: the node in a PathTree corresponding to the top of the
22
+ # src directory
23
+ # dst_tree:: the node in a PathTree corresponding to the top of the
24
+ # dst directory
25
+ # converter:: the specific converter used to convert the adoc source to
26
+ # the desired destination format.
27
+
28
+ class TreeConverter
29
+ attr_reader :dst_tree, :pre_builders, :post_builders, :converter
30
+
31
+ class << self
32
+ # register all asciidoctor extensions given at instantiation
33
+ #
34
+ # adoc_ext::
35
+ # { preprocessor: [], ... }
36
+ #
37
+ # see https://docs.asciidoctor.org/asciidoctor/latest/extensions/register/
38
+ def register_adoc_extensions(adoc_ext)
39
+ return if adoc_ext.nil?
40
+
41
+ %i[preprocessor tree_processor postprocessor docinfo_processor block
42
+ block_macro inline_macro include_processor].each do |e|
43
+ next unless adoc_ext.key?(e)
44
+
45
+ Array(adoc_ext[e])&.each do |c|
46
+ Giblog.logger.debug { "Register #{c.class} as #{e}" }
47
+ Asciidoctor::Extensions.register { send(e, c) }
48
+ end
49
+ end
50
+ end
51
+
52
+ def unregister_adoc_extenstions
53
+ Asciidoctor::Extensions.unregister_all
54
+ end
55
+ end
56
+
57
+ # opts:
58
+ # logger: the logger used internally by this instance (default nil)
59
+ # adoc_log_level - the log level when logging messages emitted by asciidoctor
60
+ # (default Logger::Severity::WARN)
61
+ # pre_builders
62
+ # post_builders
63
+ # adoc_api_opts
64
+ # adoc_doc_attribs
65
+ # conversion_cb {success: Proc(src,dst,adoc) fail: Proc(src,dst,exc)
66
+ def initialize(src_top, dst_top, opts = {})
67
+ # setup logging
68
+ @logger = opts.fetch(:logger, Giblog.logger)
69
+ @adoc_log_level = opts.fetch(:adoc_log_level, Logger::Severity::WARN)
70
+
71
+ # get the top-most node of the source and destination trees
72
+ @src_tree = src_top
73
+ @dst_tree = PathTree.new(dst_top).node(dst_top, from_root: true)
74
+
75
+ # setup build-phase callback objects
76
+ @pre_builders = Array(opts.fetch(:pre_builders, []))
77
+ @post_builders = Array(opts.fetch(:post_builders, []))
78
+ @converter = DefaultConverter.new(@logger, opts)
79
+ @adoc_ext = opts.fetch(:adoc_extensions, nil)
80
+ end
81
+
82
+ # abort_on_exc:: if true, an exception lower down the chain will
83
+ # abort the conversion and raised to the caller. If false, exceptions
84
+ # will be swallowed. In both cases, an 'error' log entry is created.
85
+ def run(abort_on_exc: true)
86
+ TreeConverter.register_adoc_extensions(@adoc_ext)
87
+ pre_build(abort_on_exc: abort_on_exc)
88
+ build(abort_on_exc: abort_on_exc)
89
+ post_build(abort_on_exc: abort_on_exc)
90
+ ensure
91
+ TreeConverter.unregister_adoc_extenstions
92
+ end
93
+
94
+ def pre_build(abort_on_exc: true)
95
+ @pre_builders.each do |pb|
96
+ pb.on_prebuild(@src_tree, @dst_tree, @converter)
97
+ rescue => ex
98
+ @logger&.error { ex.message.to_s }
99
+ raise ex if abort_on_exc
100
+ end
101
+ end
102
+
103
+ def build(abort_on_exc: true)
104
+ @src_tree.traverse_preorder do |level, n|
105
+ next unless n.leaf?
106
+
107
+ # create the destination node, using the correct suffix depending on conversion backend
108
+ rel_path = n.relative_path_from(@src_tree)
109
+ Giblog.logger.debug { "Creating dst node: #{rel_path}" }
110
+ dst_node = @dst_tree.add_descendants(rel_path)
111
+
112
+ # perform the conversion
113
+ @converter.convert(n, dst_node, @dst_tree)
114
+ rescue => exc
115
+ @logger&.error { "#{n.pathname} - #{exc.message}" }
116
+ raise exc if abort_on_exc
117
+ end
118
+ end
119
+
120
+ def post_build(abort_on_exc: true)
121
+ @post_builders.each do |pb|
122
+ pb.on_postbuild(@src_tree, @dst_tree, @converter)
123
+ rescue => exc
124
+ raise exc if abort_on_exc
125
+ @logger&.error { exc.message.to_s }
126
+ end
127
+ end
128
+
129
+ # the default callback will tie a 'SuccessfulConversion' instance
130
+ # to the destination node as its data
131
+ def self.on_success(src_node, dst_node, dst_tree, doc, adoc_log_str)
132
+ dst_node.data = DataDelegator.new(SuccessfulConversion.new(
133
+ src_node: src_node, dst_node: dst_node, dst_top: dst_tree, adoc: doc, adoc_stderr: adoc_log_str
134
+ ))
135
+ end
136
+
137
+ # the default callback will tie a 'FailedConversion' instance
138
+ # to the destination node as its data
139
+ def self.on_failure(src_node, dst_node, dst_tree, ex, adoc_log_str)
140
+ Giblog.logger.error { ex.message }
141
+ dst_node.data = DataDelegator.new(FailedConversion.new(
142
+ src_node: src_node, dst_node: dst_node, dst_top: dst_tree, error_msg: ex.message
143
+ ))
144
+ end
145
+ end
146
+
147
+ class DefaultConverter
148
+ attr_accessor :adoc_api_opts
149
+ # see https://docs.asciidoctor.org/asciidoc/latest/attributes/document-attributes-reference/
150
+ DEFAULT_ADOC_DOC_ATTRIBS = {
151
+ "data-uri" => true,
152
+ "hide-uri-scheme" => true,
153
+ "xrefstyle" => "short",
154
+ "source-highlighter" => "rouge",
155
+ "source-linenums-option" => true
156
+ }
157
+
158
+ # see https://docs.asciidoctor.org/asciidoctor/latest/api/options/
159
+ DEFAULT_ADOC_OPTS = {
160
+ backend: "html5",
161
+ # base_dir:
162
+ catalog_assets: false,
163
+ # converter:
164
+ doctype: "article",
165
+ # eruby:
166
+ # ignore extention stuff
167
+ header_only: false,
168
+ # logger:
169
+ mkdirs: false,
170
+ parse: true,
171
+ safe: :unsafe,
172
+ sourcemap: false,
173
+ # template stuff TBD,
174
+ # to_file:
175
+ # to_dir:
176
+ standalone: true
177
+ }
178
+
179
+ # opts:
180
+ # adoc_log_level
181
+ # adoc_api_opts
182
+ # adoc_doc_attribs
183
+ # conversion_cb {
184
+ # success: lambda(src, dst, dst_rel_path, doc, logstr)
185
+ # fail: lambda(src,dst,exc)
186
+ # }
187
+ def initialize(logger, opts)
188
+ @logger = logger
189
+ @adoc_log_level = opts.fetch(:adoc_log_level, Logger::Severity::WARN)
190
+ @conv_cb = opts.fetch(:conversion_cb, {
191
+ success: ->(src, dst, dst_rel_path, doc, logstr) { TreeConverter.on_success(src, dst, dst_rel_path, doc, logstr) },
192
+ failure: ->(src, dst, dst_rel_path, ex, logstr) { TreeConverter.on_failure(src, dst, dst_rel_path, ex, logstr) }
193
+ })
194
+
195
+ # merge user's options with the default, giving preference
196
+ # to the user
197
+ @adoc_api_opts = DEFAULT_ADOC_OPTS.dup
198
+ .merge!(opts.fetch(:adoc_api_opts, {}))
199
+ @adoc_api_opts[:attributes] = DEFAULT_ADOC_DOC_ATTRIBS.dup
200
+ .merge!(opts.fetch(:adoc_doc_attribs, {}))
201
+ end
202
+
203
+ # require the following methods to be available from the src node:
204
+ # adoc_source
205
+ #
206
+ # the following methods will be called if supported:
207
+ # document_attributes
208
+ # api_options
209
+ #
210
+ # src_node:: the PathTree node containing the info on adoc source and any
211
+ # added api_options or doc_attributes
212
+ # dst_node:: the PathTree node where conversion info is to be stored
213
+ # dst_top:: the PathTree node representing the top dir of the destination
214
+ # under which all converted files are written.
215
+ def convert(src_node, dst_node, dst_top)
216
+ @logger&.info { "Converting #{src_node.pathname} and store result under #{dst_node.parent.pathname}" }
217
+
218
+ # merge the common api opts with node specific
219
+ api_opts = @adoc_api_opts.dup
220
+ api_opts.merge!(src_node.api_options(src_node, dst_node, dst_top)) if src_node.respond_to?(:api_options)
221
+ api_opts[:attributes].merge!(src_node.document_attributes(src_node, dst_node, dst_top)) if src_node.respond_to?(:document_attributes)
222
+
223
+ # use a new logger instance for each conversion
224
+ adoc_logger = Giblish::AsciidoctorLogger.new(@logger, @adoc_log_level)
225
+
226
+ begin
227
+ # load the source to enable access to doc properties
228
+ #
229
+ # NOTE: the 'parse: false' is needed to prevent preprocessor extensions to be run as part
230
+ # of loading the document. We want them to run during the 'convert' call later when
231
+ # doc attribs have been amended.
232
+ doc = Asciidoctor.load(src_node.adoc_source(src_node, dst_node, dst_top), api_opts.merge(
233
+ {
234
+ parse: false,
235
+ logger: adoc_logger
236
+ }
237
+ ))
238
+
239
+ # piggy-back our own info on the doc attributes hash so that
240
+ # asciidoctor extensions can use this info later on
241
+ doc.attributes["giblish-info"] = {
242
+ src_node: src_node,
243
+ dst_node: dst_node,
244
+ dst_top: dst_top
245
+ }
246
+
247
+ # update the destination node with the correct file suffix. This is dependent
248
+ # on the type of conversion performed
249
+ dst_node.name = dst_node.name.sub_ext(doc.attributes["outfilesuffix"])
250
+ d = dst_node.pathname
251
+
252
+ # make sure the dst dir exists
253
+ d.dirname.mkpath
254
+
255
+ # write the converted doc to the file
256
+ output = doc.convert(api_opts.merge({logger: adoc_logger}))
257
+ doc.write(output, d.to_s)
258
+
259
+ # give user the opportunity to eg store the result of the conversion
260
+ # as data in the destination node
261
+ @conv_cb[:success]&.call(src_node, dst_node, dst_top, doc, adoc_logger.in_mem_storage.string)
262
+ true
263
+ rescue => ex
264
+ @logger&.error { "Conversion failed for #{src_node.pathname}" }
265
+ @logger&.error { ex.message }
266
+ @logger&.error { ex.backtrace }
267
+ @conv_cb[:failure]&.call(src_node, dst_node, dst_top, ex, adoc_logger.in_mem_storage.string)
268
+ false
269
+ end
270
+ end
271
+ end
272
+ end