etna 0.1.50 → 0.1.51

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2d4a9a5034898ac9abf629e6ecbf0bdc3856c2ed9a0d43afe03147b2d6ff777d
-  data.tar.gz: c4d65c138d401605aba76e1e0eff1a4e860778d982cb6a7c04d834e829fa3df2
+  metadata.gz: be19e727959bd06d5af503f7a3bbc30fde48fe7aa0db200b93c32f656bd1a642
+  data.tar.gz: 179736a0638b7e200bb9a55aeae79fddf852a9d5e0fb33c7650a11e06bfc11f7
 SHA512:
-  metadata.gz: 20505cf23f06bea3343194a4f956b7ffec44786cef9a92b5a78f48aa7793dec746aa710b8007c96f9d4d9ba5001c2e9e3464d464458873fcb4d43bed43dbee09
-  data.tar.gz: 8bc2f47aaa9daa3a211ce40ce5b3649e82d435225370533e8d34628b3c8d302a60051a68cc99a35b760a6d021df087fdd37d200c8062a933f5f10d599af9ad53
+  metadata.gz: a7ac565d1132f522d6f955f250c0e23762909294ede0370f5d2e3ff3fad503f78c4e26de5cd95d035defff96ab8c533ca72fd339d998ce2306e48b1cf955cc9a
+  data.tar.gz: a765da090dafc236beace6473fb6c18b56060406ed3d6f0f50b92cc2c06cde29cb1e86b35a0f14015cdcdfaa28c6aa598cb4881a8f54be6ac607c7da61925137

data/lib/commands.rb

@@ -250,10 +250,10 @@ class EtnaApp
       string_flags << "--log-file"
       string_flags << "--log-level"
       string_flags << "--concurrency"
-      string_flags << "--page_size"
+      string_flags << "--page-size"
+      boolean_flags << "--list-only"
 
-
-      def execute(project_name:, log_file:'/dev/stdout', log_level: ::Logger::INFO, concurrency: 1, page_size: 20)
+      def execute(project_name:, log_file:'/dev/stdout', log_level: ::Logger::INFO, concurrency: 1, page_size: 20, list_only: false)
         logger = Etna::Logger.new(log_file, 0, 1048576)
 
         logger.level = log_level
@@ -267,6 +267,7 @@ class EtnaApp
             logger: logger,
             project_name: project_name,
             model_name: 'project', filesystem: filesystem,
+            list_only: list_only,
             page_size: page_size.to_i,
             concurrency: concurrency.to_i)
 

data/lib/etna/clients/magma/workflows/materialize_magma_record_files_workflow.rb

@@ -9,10 +9,10 @@ module Etna
           :metis_client, :magma_client, :project_name,
           :model_name, :model_filters, :model_attributes_mask,
           :filesystem, :logger, :stub_files, :concurrency,
-          :page_size, :record_names, keyword_init: true)
+          :page_size, :record_names, :list_only, keyword_init: true)
 
         def initialize(**kwds)
-          super(**({filesystem: Etna::Filesystem.new, page_size: 20, concurrency: 10, record_names: "all"}.update(kwds)))
+          super(**({filesystem: Etna::Filesystem.new, page_size: 20, concurrency: 10, record_names: "all", list_only: false}.update(kwds)))
         end
 
         def magma_crud
@@ -108,17 +108,27 @@ module Etna
             end
 
             dest_file = File.join(dest_dir, metadata_file_name(record_name: record[template.identifier], record_model_name: template.name, ext: "_#{attr_name}_#{idx}#{File.extname(filename)}"))
-            filesystem.mkdir_p(File.dirname(dest_file))
-            sync_metis_data_workflow.copy_file(dest: dest_file, url: url, stub: stub_files)
-            record_to_serialize[attr_name] << {file: dest_file, original_filename: filename}
+
+            if list_only
+              puts dest_file
+            else
+              filesystem.mkdir_p(File.dirname(dest_file))
+              sync_metis_data_workflow.copy_file(dest: dest_file, url: url, stub: stub_files)
+              record_to_serialize[attr_name] << {file: dest_file, original_filename: filename}
+            end
           end
 
           dest_file = File.join(dest_dir, metadata_file_name(record_name: record[template.identifier], record_model_name: template.name, ext: '.json'))
-          filesystem.mkdir_p(File.dirname(dest_file))
-          json = record_to_serialize.to_json
 
-          filesystem.with_writeable(dest_file, "w", size_hint: json.bytes.length) do |io|
-            io.write(json)
+          if list_only
+            puts dest_file
+          else
+            filesystem.mkdir_p(File.dirname(dest_file))
+            json = record_to_serialize.to_json
+
+            filesystem.with_writeable(dest_file, "w", size_hint: json.bytes.length) do |io|
+              io.write(json)
+            end
           end
         end
 

data/lib/etna/clients/magma/workflows/walk_model_tree_workflow.rb

@@ -63,12 +63,21 @@ module Etna
             page_size: 100,
             &block)
           q = [ { model_name: model_name, from: nil, record_names: record_names } ]
-          seen = Set.new
+          seen = {}
 
           while (path = q.pop)
             model_name = path[:model_name]
-            next if seen.include?([path[:from], model_name])
-            seen.add([path[:from], model_name])
+            seen[model_name] ||= Set.new
+
+            new_record_names = path[:record_names]
+
+            if new_record_names.is_a?(Array)
+              new_record_names -= seen[model_name].to_a
+
+              next if new_record_names.empty?
+
+              seen[model_name].merge(new_record_names)
+            end
 
             template = template_for(model_name)
             query_attributes, walk_attributes = masked_attributes(
@@ -80,7 +89,7 @@ module Etna
             request = RetrievalRequest.new(
                 project_name: magma_crud.project_name,
                 model_name: model_name,
-                record_names: path[:record_names],
+                record_names: new_record_names,
                 filter: model_filters[model_name],
                 attribute_names: query_attributes,
                 page_size: page_size, page: 1
@@ -96,6 +105,8 @@ module Etna
               attributes = []
               model = response.models.model(model_name)
 
+              seen[model_name].merge( model.documents.document_keys )
+
               template.attributes.attribute_keys.each do |attr_name|
                 attr = template.attributes.attribute(attr_name)
                 if attr.attribute_type == AttributeType::TABLE && attribute_included?(walk_attributes, attr_name)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: etna
 version: !ruby/object:Gem::Version
-  version: 0.1.50
+  version: 0.1.51
 platform: ruby
 authors:
 - Saurabh Asthana
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-02-16 00:00:00.000000000 Z
+date: 2023-03-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rack