metanorma-plugin-lutaml 0.2.4 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fedcbe218c7a2e414eeac2dff54a798c0fcb2df287f8c516c81cb2cf8484848b
-  data.tar.gz: 343ba55ae29367376bae9c0175698fe241e8d2c533d420d0becab9cc2f1fdfe9
+  metadata.gz: a69cd9fc268fc930ee6a904fded7ff1b9a8e8bc9d168692723a8e288c153a1b9
+  data.tar.gz: e0c0f656845153953ea5b157a896a2c080701bd4201807bd588b98813aa5a408
 SHA512:
-  metadata.gz: 38554d8cef5c12bb9650bbbbf270be3f34f4f4672868a6625966fc2c356bd686bb8e0b585cd9cb466acdc765abc7facce5ef6752ba632014910f1c7f0bc333b1
-  data.tar.gz: 46a0bd0c726fdb101783b376a79b633db26ece0b10272d934f4a349a62e8d4f99d5fe881728ed1713599d9e33dcc706ddfcd51205708a4762a1f44f1e236fbeb
+  metadata.gz: 47ae06e3671be20040af10541394b7876be02fa955dbecd8eab8983b2350fa12e62b1375276e3ff48671b0e38bc91bc8b7c758cdb6cc162d7fa4c35a6a665288
+  data.tar.gz: da04ecfba6f48f522294d733e6b16d0199657130c69f6407e77c702b8f94dceda641c528d323987f5b3e61b90ace50963e878c48c7f3aa6dda83313fd378e0d2

data/.github/workflows/rake.yml

@@ -1,8 +1,10 @@
+# Auto-generated by Cimas: Do not edit it manually!
+# See https://github.com/metanorma/cimas
 name: rake
 
 on:
   push:
-    branches: [ main ]
+    branches: [ master, main  ]
     tags: [ v* ]
   pull_request:
 
@@ -14,52 +16,38 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.6', '2.5', '2.4' ]
+        ruby: [ '2.7', '2.6', '2.5', '2.4' ]
         os: [ ubuntu-latest, windows-latest, macos-latest ]
         experimental: [ false ]
         include:
-          - ruby: '2.7'
+          - ruby: '3.0'
             os: 'ubuntu-latest'
             experimental: true
-          - ruby: '2.7'
+          - ruby: '3.0'
             os: 'windows-latest'
             experimental: true
-          - ruby: '2.7'
+          - ruby: '3.0'
             os: 'macos-latest'
             experimental: true
     steps:
       - uses: actions/checkout@master
 
-      - name: Use Ruby
-        uses: ruby/setup-ruby@v1
+      - uses: ruby/setup-ruby@v1
         with:
           ruby-version: ${{ matrix.ruby }}
           bundler-cache: true
 
-      - name: Update gems
-        run: bundle install --jobs 4 --retry 3
+      - uses: metanorma/metanorma-build-scripts/plantuml-setup-action@master
 
-      - name: Install Grpahviz Ubuntu
-        if: matrix.os == 'ubuntu-latest'
-        run: sudo apt-get install graphviz
+      - run: bundle exec rake
 
-      - name: Install Grpahviz macOS
-        if: matrix.os == 'macos-latest'
-        run: brew install graphviz
-
-      - name: Install Grpahviz Windows
-        if: matrix.os == 'windows-latest'
-        uses: nick-invision/retry@v1
+  tests-passed:
+    needs: rake
+    runs-on: ubuntu-latest
+    steps:
+      - uses: peter-evans/repository-dispatch@v1
         with:
-          polling_interval_seconds: 5
-          timeout_minutes: 5
-          max_attempts: 3
-          command: choco install --no-progress graphviz
-
-      - name: Check dot command
-        if: matrix.os == 'windows-latest'
-        run: |
-          dot -?
-
-      - name: Run specs
-        run: bundle exec rake
+          token: ${{ secrets.METANORMA_CI_PAT_TOKEN || secrets.GITHUB_TOKEN }}
+          repository: ${{ github.repository }}
+          event-type: notify
+          client-payload: '{"ref": "${{ github.ref }}", "sha": "${{ github.sha }}"}'

data/Dockerfile

@@ -0,0 +1,4 @@
+FROM ruby:2.6
+COPY . .
+RUN gem install bundler:2.1.4
+RUN bundle install

data/Makefile

@@ -0,0 +1,2 @@
+fix-lint-staged:
+	git status --short | egrep '^(A|M)' | awk '{ print $$2}' | grep -v db/schema.rb | xargs bundle exec rubocop -a

data/README.adoc

@@ -83,6 +83,36 @@ Will produce this output:
 
 This macro also supports `.lutaml` files.
 
+Instead of using the direct path to the file one can use `lutaml-express-index` document attribute to supply directory with express files or yaml index file to parse as well as the cache file location. The syntax is as follows:
+
+[source,adoc]
+-----
+:lutaml-express-index: my_custom_name; dir_or_index_path[; cache=cache_path]
+-----
+
+Where:
+
+* `my_custom_name` - is name of the parsed express files context for the later use with lutaml macro
+
+* `dir_or_index_path` - location of directory with express files or path to yaml index file to parse
+
+* `cache_path` - optional, location of the expressir cache file to use
+
+Example of real usage:
+
+[source,adoc]
+-----
+= Document title
+Author
+:lutaml-express-index: my_custom_name; /path/to/express_files; cache=/path/to/cache_file.yaml
+[lutaml,my_custom_name,my_context]
+----
+{% for schema in my_context.schemas %}
+== {{schema.id}}
+{% endfor %}
+----
+-----
+
 === Usage, `lutaml_uml_attributes_table` macro
 
 This macro allows to quickly render datamodel attributes/values tables. Given `example.lutaml` file with the content:

data/lib/metanorma/plugin/lutaml/liquid/custom_filters.rb

@@ -0,0 +1,15 @@
+require "reverse_adoc"
+
+module Metanorma
+  module Plugin
+    module Lutaml
+      module Liquid
+        module CustomFilters
+          def html2adoc(input)
+            ReverseAdoc.convert(input)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/metanorma/plugin/lutaml/lutaml_preprocessor.rb

@@ -76,18 +76,23 @@ module Metanorma
           file_paths = []
           result = contexts_names.each_with_object([]) do |path, res|
             if express_indexes[path]
-              res.push(*express_indexes[path])
+              res.push(express_indexes[path])
             else
               file_paths.push(path)
             end
           end
           if !file_paths.empty?
             from_files = content_from_files(document, file_paths)
-                          .map do |n|
-                            n.to_liquid.map { |j| j.merge('relative_path_prefix' => Utils.relative_file_path(document, File.dirname(j['file']))) }
-                          end
-                          .flatten
-            result += from_files
+            # TODO: decide how to handle expressir multiply file parse as one object and lutaml
+            if from_files.is_a?(Array)
+              result.push(*from_files.map(&:to_liquid))
+            else
+              from_files = from_files.to_liquid
+              from_files["schemas"] = from_files["schemas"].map do |n|
+                n.merge("relative_path_prefix" => Utils.relative_file_path(document, File.dirname(n["file"])))
+              end
+              result.push(from_files)
+            end
           end
           result
         end
@@ -96,15 +101,20 @@ module Metanorma
           options = parse_options(block_match[3])
           contexts_items(block_match, document, express_indexes)
             .map do |items|
-              opts = options.merge('relative_path_prefix' => items['relative_path_prefix'])
+              if items["schemas"]
+                items["schemas"] = items["schemas"].map do |j|
+                  opts = options.merge("relative_path_prefix" => j["relative_path_prefix"])
+                  decorate_context_items(j, opts)
+                end
+              end
               parse_context_block(document: document,
                                   context_lines: current_block,
-                                  context_items: decorate_context_items(items, opts),
+                                  context_items: items,
                                   context_name: block_match[2].strip)
             end.flatten
-        # rescue StandardError => e
-        #   document.logger.warn("Failed to parse lutaml block: #{e.message}")
-        #   []
+          rescue StandardError => e
+            document.logger.warn("Failed to parse lutaml block: #{e.message}")
+            []
         end
 
         def parse_options(options_string)
@@ -144,7 +154,8 @@ module Metanorma
           render_result, errors = Utils.render_liquid_string(
             template_string: context_lines.join("\n"),
             context_items: context_items,
-            context_name: context_name
+            context_name: context_name,
+            document: document
           )
           Utils.notify_render_errors(document, errors)
           render_result.split("\n")

data/lib/metanorma/plugin/lutaml/lutaml_uml_attributes_table_preprocessor.rb

@@ -60,7 +60,8 @@ module Metanorma
           render_result, errors = Utils.render_liquid_string(
             template_string: table_template,
             context_items: entity_definition,
-            context_name: "definition"
+            context_name: "definition",
+            document: document
           )
           Utils.notify_render_errors(document, errors)
           render_result.split("\n")

data/lib/metanorma/plugin/lutaml/utils.rb

@@ -1,8 +1,15 @@
+require "expressir/express/cache"
+require "metanorma/plugin/lutaml/liquid/custom_filters"
+
+::Liquid::Template.register_filter(Metanorma::Plugin::Lutaml::Liquid::CustomFilters)
+
 module Metanorma
   module Plugin
     module Lutaml
       # Helpers for lutaml macroses
       module Utils
+        LUTAML_EXP_IDX_TAG = /^:lutaml-express-index:(?<index_name>.+?);(?<index_path>.+?);?(\s*cache=(?<cache_path>.+))?$/.freeze
+
         module_function
 
         def relative_file_path(document, file_path)
@@ -15,8 +22,10 @@ module Metanorma
         end
 
         def render_liquid_string(template_string:, context_items:,
-                                 context_name:)
-          liquid_template = Liquid::Template.parse(template_string)
+                                 context_name:, document:)
+          liquid_template = ::Liquid::Template.parse(template_string)
+          # Allow includes for the template
+          liquid_template.registers[:file_system] = ::Liquid::LocalFileSystem.new(Utils.relative_file_path(document, ""))
           rendered_string = liquid_template
             .render(context_name => context_items,
                     strict_variables: true,
@@ -32,20 +41,76 @@ module Metanorma
           end
         end
 
-        def process_express_index(folder, name, idxs, document)
-          idxs[name] = []
-          Dir["#{Utils.relative_file_path(document, folder)}/*"].each do |path|
-            next if [".", ".."].include?(path)
-
-            begin
-              parsed = ::Lutaml::Parser.parse(File.new(path, encoding: "UTF-8"))
-              parsed = parsed.map { |n| n.to_liquid.map { |j| j.merge('relative_path_prefix' => Utils.relative_file_path(document, File.dirname(j['file']))) } }.flatten
-              idxs[name]
-                .push(*parsed)
-            rescue StandardError => e
-              document.logger.warn("Failed to load #{path}: #{e.message}")
+        def process_express_index(path, cache_path, document, force_read = false)
+          cache_full_path = Utils.relative_file_path(document, cache_path) if cache_path
+          if !force_read && cache_full_path && File.file?(cache_full_path)
+            return express_from_cache(cache_full_path)
+          end
+
+          full_path = Utils.relative_file_path(document, path)
+          wrapper = express_from_path(document, full_path)
+          if cache_full_path && !File.file?(cache_full_path)
+            express_write_cache(cache_full_path, wrapper.original_document, document)
+          end
+          wrapper
+        rescue Expressir::Error
+          FileUtils.rm_rf(cache_full_path)
+          process_express_index(path, cache_path, document, true)
+        rescue StandardError => e
+          document.logger.warn("Failed to load #{full_path}: #{e.message}")
+          nil
+        end
+
+        def express_from_cache(path)
+          ::Lutaml::Parser
+            .parse(File.new(path), ::Lutaml::Parser::EXPRESS_CACHE_PARSE_TYPE)
+        end
+
+        def express_write_cache(path, repository, document)
+          root_path = Pathname.new(relative_file_path(document, ""))
+          Expressir::Express::Cache
+            .to_file(path,
+                     repository,
+                     root_path: root_path)
+        end
+
+        def express_from_path(document, path)
+          if File.directory?(path)
+            return express_from_folder(path)
+          end
+
+          express_from_index(document, path)
+        end
+
+        def express_from_folder(folder)
+          files = Dir["#{folder}/*.exp"].map { |nested_path| File.new(nested_path, encoding: "UTF-8") }
+          ::Lutaml::Parser.parse(files)
+        end
+
+        def express_decorate_wrapper(wrapper, document)
+          serialized = wrapper.to_liquid
+          serialized["schemas"] = serialized["schemas"].map do |j|
+            j.merge("relative_path_prefix" => Utils.relative_file_path(document, File.dirname(j["file"])))
+          end
+          serialized
+        end
+
+        def express_from_index(document, path)
+          yaml_content = YAML.safe_load(File.read(path))
+          root_path = yaml_content["path"]
+          schemas_paths = yaml_content
+            .fetch("schemas")
+            .map do |(schema_name, schema_values)|
+            if schema_values["path"]
+              schema_values["path"]
+            else
+              File.join(root_path.to_s, "#{schema_name}.exp")
             end
           end
+          files_to_load = schemas_paths.map do |path|
+            File.new(Utils.relative_file_path(document, path), encoding: "UTF-8")
+          end
+          ::Lutaml::Parser.parse(files_to_load)
         end
 
         def parse_document_express_indexes(document, input_lines)
@@ -54,14 +119,15 @@ module Metanorma
             line = input_lines.next
             break if line.length.zero?
 
-            _, name, folder = line.match(/^:lutaml-express-index:(.+?);(.+?)$/)&.to_a
-            if folder && name
-              process_express_index(
-                folder.strip.gsub(";", ""),
-                name.strip,
-                express_indexes,
-                document
-              )
+            match = line.match(LUTAML_EXP_IDX_TAG)
+            if match
+              name = match[:index_name]
+              path = match[:index_path]
+              cache = match[:cache_path]
+              repositories = process_express_index(path.strip, cache, document)
+              if repositories
+                express_indexes[name.strip] = express_decorate_wrapper(repositories, document)
+              end
             end
           end
           express_indexes

data/lib/metanorma/plugin/lutaml/version.rb

@@ -1,7 +1,7 @@
 module Metanorma
   module Plugin
     module Lutaml
-      VERSION = "0.2.4".freeze
+      VERSION = "0.4.0".freeze
     end
   end
 end

data/metanorma-plugin-lutaml.gemspec

@@ -24,10 +24,10 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency "liquid"
-  spec.add_dependency "lutaml", "0.4.1.pre.alpha.2"
-  spec.add_dependency "lutaml-uml", "~> 0.2.0"
+  spec.add_dependency "lutaml"
   spec.add_dependency "metanorma"
   spec.add_dependency "relaton-cli"
+  spec.add_dependency "reverse_adoc"
 
   spec.add_development_dependency "byebug"
   spec.add_development_dependency "equivalent-xml"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: metanorma-plugin-lutaml
 version: !ruby/object:Gem::Version
-  version: 0.2.4
+  version: 0.4.0
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-03-05 00:00:00.000000000 Z
+date: 2021-04-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: liquid
@@ -28,32 +28,32 @@ dependencies:
   name: lutaml
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.4.1.pre.alpha.2
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.4.1.pre.alpha.2
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: lutaml-uml
+  name: metanorma
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.2.0
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.2.0
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: metanorma
+  name: relaton-cli
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -67,7 +67,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: relaton-cli
+  name: reverse_adoc
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -244,14 +244,17 @@ files:
 - ".github/workflows/rake.yml"
 - ".gitignore"
 - CODE_OF_CONDUCT.md
+- Dockerfile
 - Gemfile
 - LICENSE
+- Makefile
 - README.adoc
 - Rakefile
 - bin/console
 - bin/setup
 - lib/metanorma-plugin-lutaml.rb
 - lib/metanorma/plugin/lutaml/express_remarks_decorator.rb
+- lib/metanorma/plugin/lutaml/liquid/custom_filters.rb
 - lib/metanorma/plugin/lutaml/lutaml_diagram_block.rb
 - lib/metanorma/plugin/lutaml/lutaml_preprocessor.rb
 - lib/metanorma/plugin/lutaml/lutaml_uml_attributes_table_preprocessor.rb