etna 0.1.49 → 0.1.51

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4e6abb600dd6cd4dd0986021317933f8b782b94f03d30315383bc74b052613ab
-  data.tar.gz: 909e5c67319a91dd11db4d081ec721f9438b13b8fb15dbc9d19cef2e5a5197b1
+  metadata.gz: be19e727959bd06d5af503f7a3bbc30fde48fe7aa0db200b93c32f656bd1a642
+  data.tar.gz: 179736a0638b7e200bb9a55aeae79fddf852a9d5e0fb33c7650a11e06bfc11f7
 SHA512:
-  metadata.gz: 4ef5fff5973c32b570306cab9292b7107e16c9f158dc6e3828fb19e6e4d8441f315e075091d39ce13641f81f3ecbdb428f3add61238d9c4043527e6139212702
-  data.tar.gz: 66d797cdfc76684f325aa37299e82de323b966f337d3b7e68b5ed8542cc1cb766d6b05c740256c2da63a586d240a8a2dbe9c8a1af9df3b9d0712b1a59f66ef0b
+  metadata.gz: a7ac565d1132f522d6f955f250c0e23762909294ede0370f5d2e3ff3fad503f78c4e26de5cd95d035defff96ab8c533ca72fd339d998ce2306e48b1cf955cc9a
+  data.tar.gz: a765da090dafc236beace6473fb6c18b56060406ed3d6f0f50b92cc2c06cde29cb1e86b35a0f14015cdcdfaa28c6aa598cb4881a8f54be6ac607c7da61925137

data/lib/commands.rb

@@ -250,9 +250,10 @@ class EtnaApp
       string_flags << "--log-file"
       string_flags << "--log-level"
       string_flags << "--concurrency"
+      string_flags << "--page-size"
+      boolean_flags << "--list-only"
 
-
-      def execute(project_name:, log_file:'/dev/stdout', log_level: ::Logger::INFO, concurrency: 1)
+      def execute(project_name:, log_file:'/dev/stdout', log_level: ::Logger::INFO, concurrency: 1, page_size: 20, list_only: false)
         logger = Etna::Logger.new(log_file, 0, 1048576)
 
         logger.level = log_level
@@ -266,6 +267,8 @@ class EtnaApp
             logger: logger,
             project_name: project_name,
             model_name: 'project', filesystem: filesystem,
+            list_only: list_only,
+            page_size: page_size.to_i,
             concurrency: concurrency.to_i)
 
         workflow.materialize_all(project_name)

data/lib/etna/clients/magma/workflows/materialize_magma_record_files_workflow.rb

@@ -9,10 +9,10 @@ module Etna
           :metis_client, :magma_client, :project_name,
           :model_name, :model_filters, :model_attributes_mask,
           :filesystem, :logger, :stub_files, :concurrency,
-          :record_names, keyword_init: true)
+          :page_size, :record_names, :list_only, keyword_init: true)
 
         def initialize(**kwds)
-          super(**({filesystem: Etna::Filesystem.new, concurrency: 10, record_names: "all"}.update(kwds)))
+          super(**({filesystem: Etna::Filesystem.new, page_size: 20, concurrency: 10, record_names: "all", list_only: false}.update(kwds)))
         end
 
         def magma_crud
@@ -34,7 +34,7 @@ module Etna
               record_names,
               model_attributes_mask: model_attributes_mask,
               model_filters: model_filters,
-              page_size: 20,
+              page_size: page_size,
           ) do |template, document|
             logger&.info("Materializing #{template.name}##{document[template.identifier]}")
             templates[template.name] = template
@@ -108,17 +108,27 @@ module Etna
             end
 
             dest_file = File.join(dest_dir, metadata_file_name(record_name: record[template.identifier], record_model_name: template.name, ext: "_#{attr_name}_#{idx}#{File.extname(filename)}"))
-            filesystem.mkdir_p(File.dirname(dest_file))
-            sync_metis_data_workflow.copy_file(dest: dest_file, url: url, stub: stub_files)
-            record_to_serialize[attr_name] << {file: dest_file, original_filename: filename}
+
+            if list_only
+              puts dest_file
+            else
+              filesystem.mkdir_p(File.dirname(dest_file))
+              sync_metis_data_workflow.copy_file(dest: dest_file, url: url, stub: stub_files)
+              record_to_serialize[attr_name] << {file: dest_file, original_filename: filename}
+            end
           end
 
           dest_file = File.join(dest_dir, metadata_file_name(record_name: record[template.identifier], record_model_name: template.name, ext: '.json'))
-          filesystem.mkdir_p(File.dirname(dest_file))
-          json = record_to_serialize.to_json
 
-          filesystem.with_writeable(dest_file, "w", size_hint: json.bytes.length) do |io|
-            io.write(json)
+          if list_only
+            puts dest_file
+          else
+            filesystem.mkdir_p(File.dirname(dest_file))
+            json = record_to_serialize.to_json
+
+            filesystem.with_writeable(dest_file, "w", size_hint: json.bytes.length) do |io|
+              io.write(json)
+            end
           end
         end
 

data/lib/etna/clients/magma/workflows/walk_model_tree_workflow.rb

@@ -63,12 +63,21 @@ module Etna
             page_size: 100,
             &block)
           q = [ { model_name: model_name, from: nil, record_names: record_names } ]
-          seen = Set.new
+          seen = {}
 
           while (path = q.pop)
             model_name = path[:model_name]
-            next if seen.include?([path[:from], model_name])
-            seen.add([path[:from], model_name])
+            seen[model_name] ||= Set.new
+
+            new_record_names = path[:record_names]
+
+            if new_record_names.is_a?(Array)
+              new_record_names -= seen[model_name].to_a
+
+              next if new_record_names.empty?
+
+              seen[model_name].merge(new_record_names)
+            end
 
             template = template_for(model_name)
             query_attributes, walk_attributes = masked_attributes(
@@ -80,7 +89,7 @@ module Etna
             request = RetrievalRequest.new(
                 project_name: magma_crud.project_name,
                 model_name: model_name,
-                record_names: path[:record_names],
+                record_names: new_record_names,
                 filter: model_filters[model_name],
                 attribute_names: query_attributes,
                 page_size: page_size, page: 1
@@ -96,6 +105,8 @@ module Etna
               attributes = []
               model = response.models.model(model_name)
 
+              seen[model_name].merge( model.documents.document_keys )
+
               template.attributes.attribute_keys.each do |attr_name|
                 attr = template.attributes.attribute(attr_name)
                 if attr.attribute_type == AttributeType::TABLE && attribute_included?(walk_attributes, attr_name)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: etna
 version: !ruby/object:Gem::Version
-  version: 0.1.49
+  version: 0.1.51
 platform: ruby
 authors:
 - Saurabh Asthana
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-02-13 00:00:00.000000000 Z
+date: 2023-03-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rack