exwiw 0.1.6 → 0.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c3fe2fb0c2cff1c962899799f4cfe22981a7b730759d456dc01ab8d19254dbec
-  data.tar.gz: 7bf651780e74074b50e9b0345a7cdcb9ec0281cd3d60ff04764cf03c1804b32e
+  metadata.gz: 4d94b3d27454accfa118d2ee8196f8df53ad026de2cce65c23d783b80ff9320d
+  data.tar.gz: 4292c5dca37b34d9a40892440603df62e77e8de62b68375e2990102da83c08f6
 SHA512:
-  metadata.gz: a8ee7520812cb8a655cd4f9b8dffa3afdd622311c6b39c5b7942aae5743c3642cb44a7b6b2fdd56ffdf967a8225c220866691a8a001bf75606c70871d4d731ca
-  data.tar.gz: 10182f0c69f0d773e1a515f932148c7b8795a51ba64e46141d0630459c815594e470ac62e4d082a695426ec254430dcac6c3cdbaff74b47e34c49f0dfa9a901a
+  metadata.gz: 0cae5f397aff3258f7115625e2828d17579b754982287117384257d9858c5867d063b200954d978e54a95e86c5edf919203084f0913fdfdc2a156fde3f71d1cc
+  data.tar.gz: 41705e1dbcb3a9664e4fdeeaacd1da6b49f35131b1d03191adfdcc8c101bd348bc22d96fc4ebc6daf68fa67b4b70ce68a8f29eecdd986021b0ef8e9685191331

data/CHANGELOG.md

@@ -2,6 +2,14 @@
 
 ## [Unreleased]
 
+## [0.1.7] - 2026-05-14
+
+### Added
+
+- Add embedded document support to the MongoDB adapter via `embedded_in: { collection_name, path }`. Embedded configs are not dumped as their own jsonl; their `replace_with` rules apply to subdocuments (Array or Hash, with multi-level nesting) inside the parent collection.
+
+## [0.1.6] - 2026-03-14
+
 ### Added
 
 - Add `bulk_insert_chunk_size` table config to split the generated `INSERT` statement into chunks of the specified size. ([#8](https://github.com/riseshia/exwiw/pull/8))

data/README.md

@@ -186,6 +186,44 @@ The MongoDB adapter is experimental. To use it:
 - `raw_sql` is not supported (the `MongodbField` schema does not declare it; any `raw_sql` keys in scenario JSON are silently dropped on load). Use `replace_with` for masking.
 - The MongoDB adapter does not support the collection-level `filter` field (it raises `NotImplementedError` if set, since the SQL-string filter cannot be applied to MongoDB).
 
+#### Embedded documents
+
+MongoDB models often store one-to-many relationships as embedded subdocument arrays (e.g. `users` documents with a `posts: [...]` field). To mask fields inside embedded subdocuments, declare a separate config with `embedded_in`:
+
+```jsonc
+// scenario/users.json — top-level collection
+{
+  "name": "users",
+  "primary_key": "_id",
+  "belongs_tos": [{ "table_name": "shops", "foreign_key": "shop_id" }],
+  "fields": [
+    { "name": "_id" },
+    { "name": "name", "replace_with": "masked{_id}" },
+    { "name": "shop_id" }
+  ]
+}
+
+// scenario/posts.json — embedded under users.posts
+{
+  "name": "posts",
+  "primary_key": "_id",
+  "embedded_in": { "collection_name": "users", "path": "posts" },
+  "belongs_tos": [],
+  "fields": [
+    { "name": "_id" },
+    { "name": "title", "replace_with": "masked-{_id}" }
+  ]
+}
+```
+
+At runtime:
+
+- `posts` is **not** dumped as its own jsonl file. Its `replace_with` rules are applied to the subdocuments inside the parent `users` document at the path `posts`.
+- `path` accepts dot-separated paths for nested fields (e.g. `"profile.contacts"`).
+- Both arrays of subdocuments and a single Hash subdocument at `path` are supported. Multiple levels of nesting work via embedded chains.
+- Cross-collection references from inside an embedded subdocument (`belongs_tos` on an embedded config) are not supported and raise `ArgumentError` on load.
+- Specifying an embedded config as `--target-table` raises `NotImplementedError`; pass the top-level collection name instead.
+
 ## How it works
 
 - Load the table information from the specified config file.

data/lib/exwiw/adapter/mongodb_adapter.rb

@@ -3,10 +3,9 @@
 require 'json'
 
 # NOTE: This adapter consumes MongodbCollectionConfig (`fields` instead of
-# the SQL adapters' `columns`). It assumes a "flat" document schema where
-# references between collections are expressed as scalar foreign keys
-# (e.g. `shop_id` on `users`); the forward fan-out strategy here cannot
-# follow references that live inside embedded structures.
+# `columns`, plus `embedded_in`). Top-level collections are dumped as one
+# jsonl per collection; configs marked `embedded_in` are not dumped on their
+# own — their masking rules apply to subdocuments inside the parent.
 module Exwiw
   module Adapter
     class MongodbAdapter < Base
@@ -19,8 +18,32 @@ module Exwiw
         @state = {}
       end
 
-      def build_query(config, dump_target, _config_by_name)
+      def dumpable?(config)
+        !config.embedded?
+      end
+
+      def validate_as_dump_target!(config)
+        return unless config.embedded?
+
+        raise NotImplementedError,
+              "dump_target '#{config.name}' is an embedded MongodbCollectionConfig; " \
+              "specify a top-level collection instead."
+      end
+
+      def build_query(config, dump_target, config_by_name)
+        if config.embedded?
+          raise NotImplementedError,
+                "MongodbAdapter#build_query was called with embedded config '#{config.name}'. " \
+                "Embedded configs are masked through the parent collection."
+        end
+
         reject_filter!(config)
+        # Stash the embedded-children index for the matching to_bulk_insert call
+        # below. The Adapter contract does not pass config_by_name to
+        # to_bulk_insert (SQL adapters don't need it), so we rely on the Runner
+        # invariant that build_query is always called before to_bulk_insert for
+        # the same config.
+        @embedded_children_by_parent = index_embedded_children(config_by_name)
 
         filter =
           if config.name == dump_target.table_name
@@ -57,9 +80,15 @@ module Exwiw
         docs
       end
 
+      # NOTE: relies on @embedded_children_by_parent set by a prior build_query
+      # call for the same config. This implicit ordering exists because the
+      # Adapter contract intentionally does not thread config_by_name through
+      # to_bulk_insert (SQL adapters don't need it). Safe in Runner, fragile in
+      # tests — call build_query first.
       def to_bulk_insert(rows, config)
         rows.map do |doc|
           apply_replace_with!(doc, config)
+          apply_embedded_masking!(doc, config)
           JSON.generate(extended_json(doc))
         end.join("\n")
       end
@@ -96,6 +125,14 @@ module Exwiw
               "collection-level `filter` is not supported by MongodbAdapter (collection: #{config.name})"
       end
 
+      private def index_embedded_children(config_by_name)
+        config_by_name.each_value.with_object({}) do |child, acc|
+          next unless child.embedded?
+
+          (acc[child.embedded_in.collection_name] ||= []) << child
+        end
+      end
+
       private def build_projection(config)
         projection = {}
         # Always include primary key so masking templates referencing it work,
@@ -104,6 +141,11 @@ module Exwiw
         config.fields.each do |field|
           projection[field.name] = 1
         end
+        # Pull in paths owned by configs that mark themselves embedded in this
+        # collection, so the masker sees the subdocuments.
+        embedded_children_of(config).each do |child|
+          projection[child.embedded_in.path] = 1
+        end
         projection
       end
 
@@ -118,6 +160,32 @@ module Exwiw
         end
       end
 
+      private def apply_embedded_masking!(doc, parent_config)
+        embedded_children_of(parent_config).each do |child|
+          walk(doc, child.embedded_in.path) do |subdoc|
+            apply_replace_with!(subdoc, child)
+            apply_embedded_masking!(subdoc, child)
+          end
+        end
+      end
+
+      private def embedded_children_of(parent_config)
+        @embedded_children_by_parent.fetch(parent_config.name, [])
+      end
+
+      private def walk(doc, dotted_path)
+        segments = dotted_path.split(".")
+        *prefix, last = segments
+        container = prefix.reduce(doc) { |acc, seg| acc.is_a?(Hash) ? acc[seg] : nil }
+        return unless container.is_a?(Hash)
+
+        value = container[last]
+        case value
+        when Array then value.each { |sub| yield sub if sub.is_a?(Hash) }
+        when Hash  then yield value
+        end
+      end
+
       private def extended_json(doc)
         if doc.respond_to?(:as_extended_json)
           doc.as_extended_json(mode: :relaxed)

data/lib/exwiw/adapter.rb

@@ -34,6 +34,18 @@ module Exwiw
       def supports_bulk_delete?
         true
       end
+
+      # Whether the given config produces its own dump output and needs an
+      # independent processing pass. SQL adapters always do; non-SQL adapters
+      # may exclude e.g. embedded subdocument configs.
+      def dumpable?(_config)
+        true
+      end
+
+      # Hook for adapter-specific validation when this config is used as the
+      # dump_target. Default: nothing to validate.
+      def validate_as_dump_target!(_config)
+      end
     end
 
     # @params [Exwiw::QueryAst] query_ast

data/lib/exwiw/embedded_in.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Exwiw
+  class EmbeddedIn
+    include Serdes
+
+    attribute :collection_name, String
+    attribute :path, String
+
+    def self.from_symbol_keys(hash)
+      from(hash.transform_keys(&:to_s))
+    end
+  end
+end

data/lib/exwiw/mongodb_collection_config.rb

@@ -14,8 +14,34 @@ module Exwiw
     attribute :fields, array(MongodbField)
     attribute :bulk_insert_chunk_size, optional(Integer), skip_serializing_if_nil: true
 
+    # Marks this config as physically embedded inside another collection's
+    # documents. When set, this config is not processed as a standalone dump
+    # unit; its masking rules are applied to the parent's subdocuments at
+    # `path`.
+    attribute :embedded_in, optional(EmbeddedIn), skip_serializing_if_nil: true
+
+    def self.from(obj)
+      instance = super
+      instance.__send__(:validate_embedded!)
+      instance
+    end
+
     def self.from_symbol_keys(hash)
       from(JSON.parse(hash.to_json))
     end
+
+    def embedded?
+      !embedded_in.nil?
+    end
+
+    private def validate_embedded!
+      return unless embedded?
+      return if belongs_tos.empty?
+
+      raise ArgumentError,
+            "MongodbCollectionConfig '#{name}' is embedded_in '#{embedded_in.collection_name}'; " \
+            "belongs_tos must be empty (cross-collection refs from inside embedded arrays " \
+            "are not supported)."
+    end
   end
 end

data/lib/exwiw/runner.rb

@@ -20,17 +20,20 @@ module Exwiw
 
     def run
       adapter = Adapter.build(@connection_config, @logger)
-      tables = load_table_config(adapter.class.table_config_class)
+      configs = load_table_config(adapter.class.table_config_class)
+
+      table_by_name = configs.each_with_object({}) { |config, hash| hash[config.name] = config }
+
+      target = table_by_name[@dump_target.table_name]
+      adapter.validate_as_dump_target!(target) if target
 
       @logger.info("Determining table processing order...")
-      ordered_table_names = DetermineTableProcessingOrder.run(tables)
+      ordered_table_names = DetermineTableProcessingOrder.run(configs.select { |c| adapter.dumpable?(c) })
 
       if !Dir.exist?(@output_dir)
         FileUtils.mkdir_p(@output_dir)
       end
 
-      table_by_name = tables.each_with_object({}) { |table, hash| hash[table.name] = table }
-
       total_size = ordered_table_names.size
       ordered_table_names.each_with_index do |table_name, idx|
         @logger.info("Processing table '#{table_name}'... (#{idx + 1}/#{total_size})")

data/lib/exwiw/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Exwiw
-  VERSION = "0.1.6"
+  VERSION = "0.1.7"
 end

data/lib/exwiw.rb

@@ -8,6 +8,7 @@ require "serdes"
 require_relative "exwiw/belongs_to"
 require_relative "exwiw/table_column"
 require_relative "exwiw/table_config"
+require_relative "exwiw/embedded_in"
 require_relative "exwiw/mongodb_field"
 require_relative "exwiw/mongodb_collection_config"
 require_relative "exwiw/adapter"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: exwiw
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.1.7
 platform: ruby
 authors:
 - Shia
@@ -32,7 +32,6 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- ".rubocop.yml"
 - CHANGELOG.md
 - LICENSE.txt
 - README.md
@@ -46,6 +45,7 @@ files:
 - lib/exwiw/belongs_to.rb
 - lib/exwiw/cli.rb
 - lib/exwiw/determine_table_processing_order.rb
+- lib/exwiw/embedded_in.rb
 - lib/exwiw/mongo_query.rb
 - lib/exwiw/mongodb_collection_config.rb
 - lib/exwiw/mongodb_field.rb

data/.rubocop.yml

@@ -1,10 +0,0 @@
-plugins:
-  - rubocop-greppable_rails
-
-AllCops:
-  TargetRubyVersion: 3.3
-  DisabledByDefault: true
-  SuggestExtensions: false
-
-GreppableRails/UseInlineAccessModifier:
-  Enabled: true