relaton 3.1.3 → 3.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,4 +1,6 @@
1
1
  require "fileutils"
2
+ require "json"
3
+ require "yaml"
2
4
  require_relative "../bibdata"
3
5
  require_relative "../bibcollection"
4
6
  require_relative "xml_to_html_renderer"
@@ -34,15 +36,28 @@ module Relaton
34
36
  # @param template [String] The liquid tempalte directory
35
37
  #
36
38
  # @return [String] HTML
37
- def self.to_html(file, style = nil, template = nil)
39
+ def self.to_html(file, style = nil, template = nil, context = nil)
38
40
  new(
39
41
  file,
40
42
  style: style || File.join(File.dirname(__FILE__), "templates/index-style.css"),
41
43
  template: template || File.join(File.dirname(__FILE__), "templates/"),
42
- extension: "html"
44
+ extension: "html",
45
+ context: context && load_context(context)
43
46
  ).to_html
44
47
  end
45
48
 
49
+ # A YAML or JSON file whose contents merge into the Liquid template
50
+ # context (#455).
51
+ #
52
+ # @param file [String] path to the context file
53
+ # @return [Hash]
54
+ def self.load_context(file)
55
+ case File.extname(file)
56
+ when ".json" then JSON.parse(File.read(file, encoding: "utf-8"))
57
+ else YAML.safe_load(File.read(file, encoding: "utf-8"))
58
+ end
59
+ end
60
+
46
61
  private
47
62
 
48
63
  attr_reader :file, :outdir, :options, :writable, :overwrite
@@ -52,7 +67,8 @@ module Relaton
52
67
  Relaton::Cli::XmlToHtmlRenderer.render(
53
68
  xml_content(file),
54
69
  stylesheet: options[:style],
55
- liquid_dir: options[:template]
70
+ liquid_dir: options[:template],
71
+ context: options[:context]
56
72
  )
57
73
  end
58
74
 
@@ -0,0 +1,171 @@
1
+ require "fileutils"
2
+ require "time"
3
+ require "yaml"
4
+ require "lutaml/store"
5
+ require_relative "../bibcollection"
6
+ require_relative "util"
7
+
8
+ module Relaton
9
+ module Cli
10
+ # The collections of `relaton collection` (relaton#549): one lutaml-store
11
+ # FileSystem store under `<dir>/v2/`, keyed by the collection name the
12
+ # user types. Each value is the collection's YAML (`{"root" => …}`, as
13
+ # `Bibcollection#to_h` makes it) with a top-level `updated` time, so the
14
+ # data file is the same YAML an older relaton wrote.
15
+ #
16
+ # lutaml-store is the only storage layer, as for `Db::Cache` (relaton#548):
17
+ # its lock makes a read-modify-write atomic between threads and processes,
18
+ # and its rename makes a write atomic. Nothing here builds a path from a
19
+ # name or takes a lock of its own. The one raw IO is the one-time move of
20
+ # the flat files an older relaton wrote directly in `<dir>`: each is
21
+ # imported, then moved to `<dir>-v1.bak/`, never deleted.
22
+ class CollectionStore
23
+ LAYOUT = "v2".freeze
24
+ UPDATED = "updated".freeze
25
+
26
+ attr_reader :dir
27
+
28
+ # @param dir [String] the collections directory
29
+ def initialize(dir)
30
+ @dir = dir
31
+ @store = Lutaml::Store::BasicStore.new(
32
+ adapter_type: :filesystem,
33
+ adapter_options: { path: File.join(dir, LAYOUT), extension: ".yaml",
34
+ integrity_checks: false,
35
+ create_directories: false },
36
+ cache: { enabled: false },
37
+ )
38
+ move_flat_files
39
+ end
40
+
41
+ # @param pattern [String] a glob over the names (`--collection`)
42
+ # @return [Array<String>] the matching names, sorted
43
+ def names(pattern = "*")
44
+ @store.keys.select { |name| File.fnmatch pattern, name }.sort
45
+ end
46
+
47
+ # @param name [String]
48
+ def exist?(name)
49
+ @store.exists? name
50
+ end
51
+
52
+ # @param name [String]
53
+ # @return [Relaton::Bibcollection, nil]
54
+ def get(name)
55
+ hash = read name
56
+ Relaton::Bibcollection.new hash["root"] if hash
57
+ end
58
+
59
+ # @param name [String]
60
+ # @return [Time, nil] when the collection was last written
61
+ def updated(name)
62
+ time = read(name)&.dig UPDATED
63
+ Time.iso8601 time if time
64
+ end
65
+
66
+ # @param name [String]
67
+ # @return [Integer, nil] the size of the stored collection, in bytes
68
+ def size(name)
69
+ @store.get(name)&.bytesize
70
+ end
71
+
72
+ # @param name [String]
73
+ # @param coll [Relaton::Bibcollection]
74
+ # @return [Boolean] false when the store already holds the name
75
+ def create(name, coll)
76
+ @store.adapter.transaction do
77
+ next false if @store.exists? name
78
+
79
+ @store.set name, encode(coll.to_h)
80
+ true
81
+ end
82
+ end
83
+
84
+ # Atomic read-modify-write under the store's exclusive lock.
85
+ #
86
+ # @param name [String]
87
+ # @yieldparam coll [Relaton::Bibcollection, nil] nil when it is missing
88
+ # @yieldreturn [Relaton::Bibcollection]
89
+ def update(name)
90
+ @store.update(name) do |value|
91
+ hash = decode value if value
92
+ coll = hash && Relaton::Bibcollection.new(hash["root"])
93
+ encode yield(coll).to_h
94
+ end
95
+ end
96
+
97
+ private
98
+
99
+ # @return [Hash, nil]
100
+ def read(name)
101
+ value = @store.get name
102
+ decode value if value
103
+ end
104
+
105
+ def decode(value)
106
+ YAML.safe_load value, permitted_classes: [Date, Time, Symbol],
107
+ aliases: true
108
+ end
109
+
110
+ def encode(hash, updated = Time.now)
111
+ hash.merge(UPDATED => updated.utc.iso8601).to_yaml
112
+ end
113
+
114
+ # A flat collection file is imported unless the store already holds its
115
+ # name, and then moved to the backup either way, under the store's lock
116
+ # so two processes do not move one file. The scan runs again inside the
117
+ # lock: another process can have moved the files first.
118
+ def move_flat_files
119
+ return if flat_collections.empty?
120
+
121
+ @store.adapter.transaction do
122
+ flat_collections(warn: false).each do |name, path, hash|
123
+ if @store.exists? name
124
+ Util.warn "collections #{dir}: `#{name}` is in the store; " \
125
+ "the flat file is not imported, moved to #{backup path}"
126
+ else
127
+ @store.set name, encode(hash, File.mtime(path))
128
+ Util.info "collections #{dir}: `#{name}` imported, the flat " \
129
+ "file moved to #{backup path}"
130
+ end
131
+ end
132
+ end
133
+ end
134
+
135
+ # @param warn [Boolean] warn about a file that is not valid YAML
136
+ # @return [Array<Array(String, String, Hash)>] name, path and YAML of
137
+ # each file directly in the directory that holds a collection
138
+ def flat_collections(warn: true)
139
+ return [] unless Dir.exist? dir
140
+
141
+ Dir.children(dir).sort.filter_map do |name|
142
+ path = File.join dir, name
143
+ next unless File.file?(path) && !name.start_with?(".")
144
+
145
+ hash = YAML.safe_load File.read(path, encoding: "UTF-8"),
146
+ permitted_classes: [Date, Time, Symbol],
147
+ aliases: true
148
+ [name, path, hash] if hash.is_a?(Hash) && hash["root"].is_a?(Hash)
149
+ rescue Psych::Exception, ArgumentError => e
150
+ Util.warn "the file `#{path}` isn't a collection: #{e.message}" if warn
151
+ nil
152
+ rescue Errno::ENOENT
153
+ nil # another process moved it
154
+ end
155
+ end
156
+
157
+ # Moves a file to `<dir>-v1.bak/`; a name the backup already holds
158
+ # takes a numeric suffix.
159
+ # @return [String] the backup path
160
+ def backup(path)
161
+ bak = "#{File.expand_path dir}-v1.bak" # `dir/` must not put it inside
162
+ FileUtils.mkdir_p bak
163
+ target = File.join bak, File.basename(path)
164
+ n = 0
165
+ target = File.join(bak, "#{File.basename path}.#{n += 1}") while File.exist? target
166
+ FileUtils.mv path, target
167
+ target
168
+ end
169
+ end
170
+ end
171
+ end
@@ -2,6 +2,7 @@ require "date"
2
2
  require "tmpdir"
3
3
  require "shellwords"
4
4
  require "pubid"
5
+ require_relative "util"
5
6
  require_relative "relaton_file"
6
7
  require_relative "xml_convertor"
7
8
  require_relative "yaml_convertor"
@@ -58,10 +59,14 @@ module Relaton
58
59
  option :"publication-date-after",
59
60
  desc: "Fetch only documents published after the specified date " \
60
61
  "(e.g. 2002, 2002-01, or 2002-01-01)"
62
+ option :api, type: :boolean, desc: Relaton::Cli::API_OPTION_DESC
61
63
 
62
64
  def fetch(code)
65
+ api_was = apply_api_option options
63
66
  io = IO.new($stdout.fcntl(::Fcntl::F_DUPFD), mode: "w:UTF-8")
64
67
  io.puts(fetch_document(code, options) || supported_type_message)
68
+ ensure
69
+ restore_api_option api_was
65
70
  end
66
71
 
67
72
  desc "Metanorma-XML-File / Directory Relaton-XML-Directory",
@@ -134,9 +139,14 @@ module Relaton
134
139
  "rendering Relaton items and collection"
135
140
  option :overwrite, aliases: :f, type: :boolean, default: false,
136
141
  desc: "Overwrite the existing file"
142
+ option :context, aliases: :c,
143
+ desc: "YAML or JSON file whose contents merge into " \
144
+ "the Liquid template context; the renderer's " \
145
+ "own keys cannot be overridden"
137
146
 
138
147
  def xml2html(file, style = nil, template = nil)
139
- Relaton::Cli::XMLConvertor.to_html(file, style, template)
148
+ Relaton::Cli::XMLConvertor.to_html(file, style, template,
149
+ options[:context])
140
150
  end
141
151
 
142
152
  desc "yaml2html RELATON-INDEX-YAML", "Concatenate Relaton Collection " \
@@ -147,9 +157,14 @@ module Relaton
147
157
  "rendering Relaton items and collection"
148
158
  option :overwrite, aliases: :f, type: :boolean, default: false,
149
159
  desc: "Overwrite the existing file"
160
+ option :context, aliases: :c,
161
+ desc: "YAML or JSON file whose contents merge into " \
162
+ "the Liquid template context; the renderer's " \
163
+ "own keys cannot be overridden"
150
164
 
151
165
  def yaml2html(file, style = nil, template = nil)
152
- Relaton::Cli::YAMLConvertor.to_html(file, style, template)
166
+ Relaton::Cli::YAMLConvertor.to_html(file, style, template,
167
+ options[:context])
153
168
  end
154
169
 
155
170
  desc "index [DATA-DIR]",
@@ -343,6 +358,26 @@ module Relaton
343
358
  "Invalid date range: --publication-date-after (#{date_after}) must be before --publication-date-before (#{date_before})."
344
359
  end
345
360
 
361
+ # `--api` / `--no-api` (relaton#535) sets the API tier for this command;
362
+ # no flag keeps the configuration (RELATON_USE_API, else off). Offline
363
+ # mode still makes no request. The command restores the value it
364
+ # returns (#restore_api_option), so a caller that runs the CLI in its
365
+ # own process keeps its configuration.
366
+ #
367
+ # @return [Boolean, nil] the configured value, to restore
368
+ def apply_api_option(options)
369
+ config = Relaton::Db.configuration
370
+ was = config.use_api
371
+ config.use_api = options[:api] unless options[:api].nil?
372
+ was
373
+ end
374
+
375
+ # @param was [Boolean, nil] what #apply_api_option returned; nil when
376
+ # the command stopped before it
377
+ def restore_api_option(was)
378
+ Relaton::Db.configuration.use_api = was unless was.nil?
379
+ end
380
+
346
381
  # @param code [String]
347
382
  # @param options [Hash]
348
383
  # @option options [String] :type
@@ -351,7 +386,9 @@ module Relaton
351
386
  # @return [String, nil]
352
387
  def fetch_document(code, options) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity,Metrics/AbcSize,Metrics/MethodLength
353
388
  year = options[:year]&.to_s
389
+ # `api` is the CLI's switch (#apply_api_option), not a fetch option.
354
390
  dup_opts = options.dup.transform_keys { |k| k.to_s.gsub("-", "_").to_sym }
391
+ dup_opts.delete :api
355
392
  %i[publication_date_before publication_date_after].each do |key|
356
393
  dup_opts[key] = parse_date_option(dup_opts[key], key.to_s.tr("_", "-").prepend("--")) if dup_opts[key]
357
394
  end
@@ -1,5 +1,7 @@
1
1
  require "pubid"
2
+ require_relative "util"
2
3
  require_relative "full_text_search"
4
+ require_relative "collection_store"
3
5
 
4
6
  module Relaton
5
7
  module Cli
@@ -15,15 +17,10 @@ module Relaton
15
17
  option :doctype, desc: "Documents type"
16
18
 
17
19
  def create(file)
18
- dir = directory
19
- file_path = File.join dir, file
20
20
  col = Relaton::Bibcollection.new options
21
- if File.exist? file_path
22
- Util.warn "Collection `#{file}` aready exist"
23
- else
24
- FileUtils.mkdir_p dir # unless Dir.exist? dir
25
- File.write file_path, col.to_yaml, encoding: "UTF-8"
26
- end
21
+ return if store.create file, col
22
+
23
+ Util.warn "Collection `#{file}` aready exist"
27
24
  end
28
25
 
29
26
  desc "info COLLECTION", "View collection information"
@@ -31,11 +28,12 @@ module Relaton
31
28
  "Default is $HOME/.relaton/collections."
32
29
 
33
30
  def info(file) # rubocop:disable Metrics/AbcSize
34
- path = File.join directory, file
35
- puts "Collection: #{File.basename path}"
36
- puts "Last updated: #{File.mtime path}"
37
- puts "File size: #{File.size path}"
38
- col = Relaton::Bibcollection.new YAML.load_file(path)["root"]
31
+ col = store.get file
32
+ return Util.warn "Collection `#{file}` does not exist" unless col
33
+
34
+ puts "Collection: #{file}"
35
+ puts "Last updated: #{store.updated(file)&.localtime}"
36
+ puts "File size: #{store.size file}"
39
37
  puts "Number of items: #{col.items.size}"
40
38
  puts "Author: #{col.author}"
41
39
  puts "Title: #{col.title}"
@@ -47,12 +45,9 @@ module Relaton
47
45
  option :entries, aliases: :e, type: :boolean, desc: "Show entries"
48
46
 
49
47
  def list
50
- Dir[File.join(directory, "*")].each do |f|
51
- yml = read_yaml f
52
- if yml && yml["root"]
53
- puts File.basename f
54
- puts_entries yml
55
- end
48
+ store.names.each do |name|
49
+ puts name
50
+ puts_entries store.get(name)
56
51
  end
57
52
  end
58
53
 
@@ -90,7 +85,7 @@ module Relaton
90
85
  searcher = Relaton::FullTextSeatch.new(col[:collection])
91
86
  searcher.search text
92
87
  if searcher.any?
93
- puts "Collection: #{File.basename(col[:file])}"
88
+ puts "Collection: #{col[:file]}"
94
89
  searcher.print_results
95
90
  end
96
91
  end
@@ -113,8 +108,10 @@ module Relaton
113
108
  desc: "Collection to store a document"
114
109
  option :dir, aliases: :d, desc: "Directory with collections. Default is " \
115
110
  "$HOME/.relaton/collections."
111
+ option :api, type: :boolean, desc: Relaton::Cli::API_OPTION_DESC
116
112
 
117
113
  def fetch(code) # rubocop:disable Metrics/AbcSize
114
+ api_was = apply_api_option options
118
115
  opts = {}
119
116
  if options[:"publication-date-before"]
120
117
  opts[:publication_date_before] = parse_date_option(options[:"publication-date-before"], "--publication-date-before")
@@ -124,11 +121,7 @@ module Relaton
124
121
  end
125
122
  validate_date_range opts[:publication_date_after], opts[:publication_date_before]
126
123
  doc = Relaton.db.fetch(code, options[:year]&.to_s, **opts)
127
- if doc
128
- colfile = File.join directory, options[:collection]
129
- coll = read_collection colfile
130
- coll << doc
131
- File.write colfile, coll.to_yaml, encoding: "UTF-8"
124
+ if doc then add_to_collection doc
132
125
  else Util.info "No matching bibliographic entry found"
133
126
  end
134
127
  rescue Relaton::UnknownReferenceError, Relaton::NotInLocalStoreError,
@@ -139,6 +132,8 @@ module Relaton
139
132
  Util.warn e.message
140
133
  rescue Pubid::Errors::Error
141
134
  Util.error %("#{code}" is not a recognized standards identifier)
135
+ ensure
136
+ restore_api_option api_was
142
137
  end
143
138
 
144
139
  desc "import FILE", "Import document or collection from an XML file " \
@@ -149,23 +144,19 @@ module Relaton
149
144
  option :dir, aliases: :d, desc: "Directory with collections. Default is " \
150
145
  "$HOME/.relaton/collections."
151
146
 
152
- def import(file) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
153
- collfile = File.join directory, options[:collection]
154
- coll = read_collection collfile
155
- xml = Nokogiri::XML File.read(file, encoding: "UTF-8")
156
- xml.remove_namespaces!
147
+ def import(file)
148
+ xml = read_xml file
157
149
  if xml.at "relaton-collection"
158
150
  imported = import_collection(xml)
159
- if coll
151
+ store.update(options[:collection]) do |coll|
152
+ next imported unless coll # a new collection is the imported one
153
+
160
154
  coll << imported
161
- else
162
- coll = imported
155
+ coll
163
156
  end
164
157
  else
165
- coll ||= Relaton::Bibcollection.new({})
166
- coll << Relaton::Bibdata.from_xml(xml)
158
+ add_to_collection Relaton::Bibdata.from_xml(xml)
167
159
  end
168
- File.write collfile, coll.to_yaml, encoding: "UTF-8"
169
160
  end
170
161
 
171
162
  desc "export COLLECTION", "Export collection into XML file"
@@ -173,7 +164,9 @@ module Relaton
173
164
  "$HOME/.relaton/collections."
174
165
 
175
166
  def export(file)
176
- coll = read_collection File.join(directory, file)
167
+ coll = store.get file
168
+ return Util.warn "Collection `#{file}` does not exist" unless coll
169
+
177
170
  outfile = "#{file.sub(/\.\w+$/, '')}.xml"
178
171
  File.write outfile, coll.to_xml(bibdata: true), encoding: "UTF-8"
179
172
  end
@@ -201,44 +194,44 @@ module Relaton
201
194
  options.fetch :dir, File.join(Dir.home, ".relaton/collections")
202
195
  end
203
196
 
204
- # @param file [String]
205
- # @return [Hash, nil]
206
- def read_yaml(file)
207
- YAML.load_file file if File.file? file
208
- rescue Psych::SyntaxError
209
- Util.error "the file `#{file}` isn't a collection."
210
- nil
197
+ # The collections, in a lutaml-store store under `<dir>/v2` (#549).
198
+ # @return [Relaton::Cli::CollectionStore]
199
+ def store
200
+ @store ||= CollectionStore.new directory
211
201
  end
212
202
 
213
- # @param file [String]
214
- # @return [Relaton::Bibcollection, nil]
215
- def read_collection(file)
216
- return unless File.file?(file)
203
+ # Adds an item to the `--collection` in one atomic read-modify-write,
204
+ # and creates the collection when it is missing.
205
+ def add_to_collection(item)
206
+ store.update(options[:collection]) do |coll|
207
+ coll ||= Relaton::Bibcollection.new({})
208
+ coll << item
209
+ coll
210
+ end
211
+ end
217
212
 
218
- Relaton::Bibcollection.new YAML.load_file(file)["root"]
213
+ # The XML file `import` reads: user input, not a collection.
214
+ # @return [Nokogiri::XML::Document]
215
+ def read_xml(file)
216
+ xml = Nokogiri::XML File.read(file, encoding: "UTF-8")
217
+ xml.remove_namespaces!
218
+ xml
219
219
  end
220
220
 
221
221
  # @return [Array<Hash>]
222
222
  def collections
223
- file = options.fetch :collection, "*"
224
- Dir[File.join directory, file].reduce([]) do |m, f|
225
- yml = read_yaml f
226
- if yml && yml["root"]
227
- m << { collection: Relaton::Bibcollection.new(yml["root"]),
228
- file: f }
229
- end
230
- m
223
+ store.names(options.fetch(:collection, "*")).filter_map do |name|
224
+ coll = store.get name
225
+ { collection: coll, file: name } if coll
231
226
  end
232
227
  end
233
228
 
234
- # Puts document IDs for each item in tthe cokllection
235
- # @param hash [Hash] Relaton collection
236
- def puts_entries(hash)
229
+ # Puts document IDs for each item in the collection
230
+ # @param coll [Relaton::Bibcollection]
231
+ def puts_entries(coll)
237
232
  return unless options[:entries]
238
233
 
239
- Relaton::Bibcollection.new(hash["root"]).items.each do |b|
240
- puts " #{b.docidentifier}"
241
- end
234
+ coll.items.each { |b| puts " #{b.docidentifier}" }
242
235
  end
243
236
 
244
237
  # @param item [Relaton::Bibdata]
@@ -1,5 +1,11 @@
1
1
  module Relaton
2
2
  module Cli
3
+ # The `--api` option of `fetch` and `collection fetch` (relaton#535).
4
+ # Here, not in command.rb: a command's class body reads it.
5
+ API_OPTION_DESC = "Ask api.relaton.org before the flavor's own " \
6
+ "source; --no-api does not (default: RELATON_USE_API, " \
7
+ "else off)".freeze
8
+
3
9
  module Util
4
10
  extend Relaton::Bib::Util
5
11
  PROGNAME = "relaton-cli".freeze
@@ -3,9 +3,14 @@ require "liquid"
3
3
 
4
4
  module Relaton::Cli
5
5
  class XmlToHtmlRenderer
6
- def initialize(liquid_dir: nil, stylesheet: nil)
6
+ def initialize(liquid_dir: nil, stylesheet: nil, context: nil)
7
7
  @liquid_dir = liquid_dir
8
8
  @stylesheet = read_file(stylesheet)
9
+ # Extra template data (#455). Merged under the renderer's own keys:
10
+ # a context entry can add to the Liquid document (a deployed site's
11
+ # domain for canonical-URL construction) but never clobber the
12
+ # engine's document model (title/css/date/author/documents).
13
+ @context = context || {}
9
14
  init_liquid_template_and_filesystem
10
15
  end
11
16
 
@@ -47,12 +52,14 @@ module Relaton::Cli
47
52
  def build_liquid_document(source)
48
53
  bibcollection = build_bibcollection(source)
49
54
  hash_to_liquid(
50
- depth: 2,
51
- css: stylesheet,
52
- title: bibcollection.title,
53
- date: DateTime.now.to_s,
54
- author: bibcollection.author,
55
- documents: document_items(bibcollection)
55
+ @context.merge(
56
+ depth: 2,
57
+ css: stylesheet,
58
+ title: bibcollection.title,
59
+ date: DateTime.now.to_s,
60
+ author: bibcollection.author,
61
+ documents: document_items(bibcollection)
62
+ )
56
63
  )
57
64
  end
58
65