wikidata-diff-analyzer 0.1.0 → 2.0.0

data/lib/wikidata/diff/description_analyzer.rb

@@ -1,49 +1,71 @@
-class DescriptionAnalyzer
+class DescriptionAnalyzer  
     def self.isolate_descriptions_differences(current_content, parent_content)
         return {
-        changed: [],
-        removed: [],
-        added: []
+        changed_descriptions: [],
+        removed_descriptions: [],
+        added_descriptions: []
         } if current_content.nil? && parent_content.nil?
-    
-        current_descriptions = current_content['descriptions'] || {}
-        parent_descriptions = parent_content['descriptions'] || {}
 
+        if current_content
+            current_descriptions = current_content['descriptions']
+            if current_descriptions.nil? || current_descriptions.is_a?(Array)
+                current_descriptions = {}
+            end
+        else
+            current_descriptions = {}
+        end
+    
+        
+        if parent_content
+            parent_descriptions = parent_content['descriptions']
+            if parent_descriptions.nil? || parent_descriptions.is_a?(Array)
+                parent_descriptions = {}
+            end
+        else
+            parent_descriptions = {}
+        end
     
         changed_descriptions = []    # Initialize as an array
         removed_descriptions = []    # Initialize as an array
         added_descriptions = []      # Initialize as an array
-    
-        if !current_descriptions.is_a?(Hash) || !parent_descriptions.is_a?(Hash)
-        return{
-            changed: changed_descriptions,
-            removed: removed_descriptions,
-            added: added_descriptions
-        }
-        end
 
-        # Iterate over each language in the current descriptions
-        (current_descriptions || {}).each do |lang, current_description|
-        parent_description = parent_descriptions[lang]
-    
-        if parent_description.nil?
-            added_descriptions << { lang: lang }
-        elsif current_description != parent_description
-            changed_descriptions << { lang: lang }
-        end
-        end
-    
-        # Iterate over each language in the parent descriptions to find removed descriptions
-        (parent_descriptions || {}).each do |lang, parent_description|
-        if current_descriptions[lang].nil?
-            removed_descriptions << { lang: lang }
-        end
+
+        # if parentid is 0, add all current description as added and return it
+        if parent_content.nil?
+            if !current_descriptions.empty?
+                current_descriptions.each do |lang, description|
+                    added_descriptions << { lang: lang }
+                end
+            end
+            return {
+                changed_descriptions: changed_descriptions,
+                removed_descriptions: removed_descriptions,
+                added_descriptions: added_descriptions
+            }
+        else
+            # Iterate over each language in the current descriptions
+            (current_descriptions).each do |lang, current_description|
+                # checking if the parent descriptions is empty
+                if parent_descriptions.empty?
+                    added_descriptions << { lang: lang }
+                elsif parent_descriptions[lang].nil?
+                    added_descriptions << { lang: lang }
+                elsif current_description != parent_descriptions[lang]
+                    changed_descriptions << { lang: lang }
+                end
+            end
+            
+                # Iterate over each language in the parent descriptions to find removed descriptions
+            (parent_descriptions).each do |lang, parent_description|
+                if current_descriptions.empty?
+                    removed_descriptions << { lang: lang }
+                end
+            end
         end
-    
         {
-        changed: changed_descriptions,
-        removed: removed_descriptions,
-        added: added_descriptions
+            changed_descriptions: changed_descriptions,
+            removed_descriptions: removed_descriptions,
+            added_descriptions: added_descriptions
         }
     end
 end

data/lib/wikidata/diff/form_analyzer.rb

@@ -0,0 +1,67 @@
+require_relative 'representation_analyzer'
+require_relative 'inside_claim_analyzer'
+class FormAnalyzer
+    def self.isolate_forms_differences(current_content, parent_content)
+        added_forms = []
+        removed_forms = []
+        changed_forms = []
+        added_representations = []
+        removed_representations = []
+        changed_representations = []
+        added_formclaims = []
+        removed_formclaims = []
+        changed_formclaims = []
+      
+        current_forms = current_content&.fetch("forms", []) || []
+        parent_forms = parent_content&.fetch("forms", []) || []
+      
+        current_forms.each_with_index do |current_form, index|
+          parent_form = parent_forms[index]
+      
+          if parent_form.nil?
+            # Claim was added
+            added_forms << { index: index }
+          elsif current_form
+            # Claim was changed
+            changed_forms << { index: index }
+          end
+      
+          representations = RepresentationAnalyzer.isolate_representation_differences(current_form, parent_form)
+          added_representations += representations[:added]
+          removed_representations += representations[:removed]
+          changed_representations += representations[:changed]
+      
+          formclaims = InsideClaimAnalyzer.isolate_inside_claim_differences(current_form, parent_form)
+          added_formclaims += formclaims[:added]
+          removed_formclaims += formclaims[:removed]
+          changed_formclaims += formclaims[:changed]
+        end
+      
+        parent_forms.each_with_index do |parent_form, index|
+          current_form = current_forms[index]
+      
+          if current_form.nil?
+            # Claim was removed
+            removed_forms << { index: index }
+      
+            representations = RepresentationAnalyzer.isolate_representation_differences(nil, parent_form)
+            removed_representations += representations[:removed]
+      
+            formclaims = InsideClaimAnalyzer.isolate_inside_claim_differences(nil, parent_form)
+            removed_formclaims += formclaims[:removed]
+          end
+        end
+      
+        {
+          added_forms: added_forms,
+          removed_forms: removed_forms,
+          changed_forms: changed_forms,
+          added_representations: added_representations,
+          removed_representations: removed_representations,
+          changed_representations: changed_representations,
+          added_formclaims: added_formclaims,
+          removed_formclaims: removed_formclaims,
+          changed_formclaims: changed_formclaims
+        }
+      end
+end

data/lib/wikidata/diff/gloss_analyzer.rb

@@ -0,0 +1,71 @@
+class GlossAnalyzer
+    def self.isolate_gloss_differences(current_content, parent_content)
+        return {
+        changed: [],
+        removed: [],
+        added: []
+        } if current_content.nil? && parent_content.nil?
+    
+        if current_content
+            current_glosses = current_content['glosses']
+            if current_glosses.nil? || current_glosses.is_a?(Array)
+                current_glosses = {}
+            end
+        else
+            current_glosses = {}
+        end
+
+        if parent_content
+            parent_glosses = parent_content['glosses'] 
+            if parent_glosses.nil? || parent_glosses.is_a?(Array)
+                parent_glosses = {}
+            end
+        else
+            parent_glosses = {}
+        end
+
+
+
+        changed = []
+        removed = []
+        added = []
+
+        # if parentid is 0, then add all labels as added and return it
+        if parent_content.nil?
+            current_glosses.each do |lang, label|
+                added << { lang: lang }
+            end
+            return {
+                changed: changed,
+                removed: removed,
+                added: added
+            }
+        else
+
+        
+            # Iterate over each language in the current labels
+            (current_glosses || {}).each do |lang, current_gloss|
+                parent_gloss = parent_glosses[lang]
+            
+                if parent_gloss.nil?
+                    added << { lang: lang }
+                elsif current_gloss != parent_gloss
+                    changed << { lang: lang }
+                end
+            end
+            
+                # Iterate over each language in the parent labels to find removed labels
+            (parent_glosses || {}).each do |lang, parent_gloss|
+                if current_glosses[lang].nil?
+                    removed << { lang: lang }
+                end
+            end
+        end
+    
+        {
+        changed: changed,
+        removed: removed,
+        added: added
+        }
+    end
+end

data/lib/wikidata/diff/inside_claim_analyzer.rb

@@ -0,0 +1,84 @@
+class InsideClaimAnalyzer
+    def self.isolate_inside_claim_differences(current_content, parent_content)
+        # Initialize empty arrays to store the added, removed, and changed claims
+        added = []
+        removed = []
+        changed = []
+
+        if current_content.nil?
+            current_content_claims = {}
+        else
+            current_content_claims = current_content["claims"]
+            if !current_content_claims.is_a?(Hash)
+                current_content_claims = {}
+            end
+        end
+        
+
+        if parent_content.nil?
+            parent_content_claims = {}
+        else
+            parent_content_claims = parent_content["claims"]
+            if !parent_content_claims.is_a?(Hash)
+                parent_content_claims = {}
+            end
+        end
+        
+        # if parentid is 0, add all current claims as added claims and return it
+        if parent_content.nil?
+            current_content_claims.each do |claim_key, current_claims|
+                current_claims.each_with_index do |current_claim, index|
+                    added << { key: claim_key, index: index }
+                end
+            end
+        else
+            # Iterate over each claim key in the current content
+            current_content_claims.each do |claim_key, current_claims|
+            # Check if the claim key exists in the parent content
+            if parent_content_claims.key?(claim_key)
+                parent_claims = parent_content_claims[claim_key]
+                # Iterate over each claim in the current and parent content
+                current_claims.each_with_index do |current_claim, index|
+                    parent_claim = parent_claims[index]
+                    if parent_claim.nil?
+                        # Claim was added
+                        added << { key: claim_key, index: index }
+
+                    elsif current_claim != parent_claim
+                        # Claim was changed
+                        changed << { key: claim_key, index: index }
+                    end
+                end
+                # Check for removed claims
+                parent_claims.each_with_index do |parent_claim, index|
+                    current_claim = current_claims[index]
+                    if current_claim.nil?
+                        # Claim was removed
+                        removed << { key: claim_key, index: index }
+                    end
+                end
+            else
+                # All claims in current content with this key were added
+                current_claims.each_index do |index|
+                    added << { key: claim_key, index: index }
+                end
+            end
+            end
+
+            parent_content_claims.each do |claim_key, parent_claims|
+                # current content[claims] can be nil
+                parent_claims.each_index do |index|
+                    if current_content_claims.nil? || !current_content_claims.key?(claim_key)
+                        removed << { key: claim_key, index: index }
+                    end
+                end
+            end
+        end
+
+        {
+            added: added,
+            removed: removed,
+            changed: changed
+        }
+    end
+end

data/lib/wikidata/diff/label_analyzer.rb

@@ -1,53 +1,75 @@
-class LabelAnalyzer
+class LabelAnalyzer 
+      
     def self.isolate_labels_differences(current_content, parent_content)
         return {
-        changed: [],
-        removed: [],
-        added: []
+        changed_labels: [],
+        removed_labels: [],
+        added_labels: []
         } if current_content.nil? && parent_content.nil?
-    
-        current_labels = current_content['labels'] || {}
-        parent_labels = parent_content['labels'] || {}
-
-        changed_labels = []
-        removed_labels = []
-        added_labels = []
-
-        if current_labels.is_a?(Array) || parent_labels.is_a?(Array)
-        return {
-        changed: changed_labels,
-        removed: removed_labels,
-        added: added_labels
-        }
+        
+        if current_content
+            current_labels = current_content['labels']
+            if current_labels.nil? || current_labels.is_a?(Array)
+                current_labels = {}
+            end
+        else
+            current_labels = {}
         end
 
-    
-        # Iterate over each language in the current labels
-        (current_labels || {}).each do |lang, current_label|
-        parent_label = parent_labels[lang]
-    
-        if parent_label.nil?
-            added_labels << { lang: lang }
-        elsif current_label != parent_label
-            changed_labels << { lang: lang }
-        end
-        end
-    
-        # Iterate over each language in the parent labels to find removed labels
-        (parent_labels || {}).each do |lang, parent_label|
-        if current_labels[lang].nil?
-            removed_labels << { lang: lang }
-        end
+        if parent_content
+            parent_labels = parent_content['labels']
+            if parent_labels.nil? || parent_labels.is_a?(Array)
+                parent_labels = {}
+            end
+        else
+            parent_labels = {}
         end
 
-        # puts "Changed labels: #{changed_labels}"
-        # puts "Removed labels: #{removed_labels}"
-        # puts "Added labels: #{added_labels}"
+
+
+        changed_labels_labels = []
+        removed_labels_labels = []
+        added_labels_labels = []
+
+        # if parentid is 0, then add all labels as added_labels and return it
+        if parent_content.nil?  
+            if !current_labels.empty?
+                current_labels.each do |lang, label|
+                    added_labels_labels << { lang: lang }
+                end
+            end
+            return {
+                changed_labels: changed_labels_labels,
+                removed_labels: removed_labels_labels,
+                added_labels: added_labels_labels
+            }
+        else
+            # Iterate over each language in the current labels
+            (current_labels).each do |lang, current_label|
+                if parent_labels.empty?
+                    added_labels_labels << { lang: lang }
+                else
+                    parent_label = parent_labels[lang]
+                    if parent_label.nil?
+                        added_labels_labels << { lang: lang }
+                    elsif current_label != parent_label
+                        changed_labels_labels << { lang: lang }
+                    end
+                end
+            end
+            
+                # Iterate over each language in the parent labels to find removed_labels labels
+                (parent_labels).each do |lang, parent_label|
+                if current_labels.empty?
+                    removed_labels_labels << { lang: lang }
+                end
+            end
+        end
     
         {
-        changed: changed_labels,
-        removed: removed_labels,
-        added: added_labels
+        changed_labels: changed_labels_labels,
+        removed_labels: removed_labels_labels,
+        added_labels: added_labels_labels
         }
     end
 end

data/lib/wikidata/diff/large_batches_analyzer.rb

@@ -5,35 +5,49 @@ class LargeBatchesAnalyzer
     def self.handle_large_batches(revision_ids, batch_size)
         revision_contents = {}
         parent_contents = {}
-    
-    
-        revision_ids_batches = revision_ids.each_slice(batch_size).to_a
-        puts "Handling revision_ids_batches: #{revision_ids_batches.length}"
-        revision_ids_batches.each do |batch|
-            parsed_contents = Api.get_revision_contents(batch)
-            if parsed_contents
-                parent_ids = []
-                revision_contents.merge!(parsed_contents) if parsed_contents
-                parsed_contents.values.each do |data|
-                    parent_id = data[:parentid]
-                    
-                    if parent_id != 0 && !parent_id.nil?
-                        parent_ids << parent_id
-                    end
-                end
-                parent_contents_batch = Api.get_revision_contents(parent_ids)
-                parent_contents.merge!(parent_contents_batch) if parent_contents_batch
+        first_revisions = []
+      
+        revision_ids.each_slice(batch_size) do |batch|
+          parent_ids = []
+          parsed_contents = Api.get_revision_contents(batch)
+          next unless parsed_contents
+
+          # I have to check if any of the revision ids in the parsed content has parentid == 0
+          parsed_contents.each do |revid, data|
+            if data[:parentid] == 0
+              first_revisions << revid
+            else
+              parent_ids << data[:parentid]
             end
+          end
+          revision_contents.merge!(parsed_contents)
+          parent_contents_batch = Api.get_revision_contents(parent_ids)
+          parent_contents.merge!(parent_contents_batch) if parent_contents_batch
         end
-    
+
+        build_result(revision_contents, parent_contents, first_revisions)
+      end
+      
+      def self.build_result(revision_contents, parent_contents, first_revisions)
         result = {}
         revision_contents.each do |revid, data|
-            parentid = data[:parentid]
-            parent_content = parent_contents[parentid] if parentid
-            current = data ? data[:content] : nil
-            parent = parent_content ? parent_content[:content] : nil
-            result[revid] = { current_content: current, parent_content: parent }    
+          parent_content = parent_contents[data[:parentid]]
+          result[revid] = {
+            current_content: data&.fetch(:content, nil),
+            parent_content: parent_content&.fetch(:content, nil),
+            comment: data&.fetch(:comment, nil),
+            model: data&.fetch(:model, nil)
+          }
+        end
+        first_revisions.each do |revid|
+          result[revid] = {
+            current_content: revision_contents[revid]&.fetch(:content, nil),
+            parent_content: nil,
+            comment: revision_contents[revid]&.fetch(:comment, nil),
+            model: revision_contents[revid]&.fetch(:model, nil)
+          }
         end
         result
-    end
+      end
+      
 end

data/lib/wikidata/diff/lemma_analyzer.rb

@@ -0,0 +1,70 @@
+class LemmaAnalyzer
+    def self.isolate_lemmas_differences(current_content, parent_content)
+        return {
+        changed_lemmas: [],
+        removed_lemmas: [],
+        added_lemmas: []
+        } if current_content.nil? && parent_content.nil?
+    
+        
+        if current_content
+            current_labels = current_content['lemmas']
+            if current_labels.nil? || current_labels.is_a?(Array)
+                current_labels = {}
+            end
+        else
+            current_labels = {}
+        end
+        if parent_content
+            parent_labels = parent_content['lemmas'] 
+            if parent_labels.nil? || parent_labels.is_a?(Array)
+                parent_labels = {}
+            end
+        else
+            parent_labels = {}
+        end
+
+        changed_labels = []
+        removed_labels = []
+        added_labels = []
+
+
+        # if parentid is 0, then add all labels as added and return it
+        if parent_content.nil?
+            current_labels.each do |lang, label|
+                added_labels << { lang: lang }
+            end
+            return {
+                changed_lemmas: changed_labels,
+                removed_lemmas: removed_labels,
+                added_lemmas: added_labels
+            }
+        else
+
+        
+            # Iterate over each language in the current labels
+            (current_labels || {}).each do |lang, current_label|
+                parent_label = parent_labels[lang]
+            
+                if parent_label.nil?
+                    added_labels << { lang: lang }
+                elsif current_label != parent_label
+                    changed_labels << { lang: lang }
+                end
+                end
+            
+                # Iterate over each language in the parent labels to find removed labels
+                (parent_labels || {}).each do |lang, parent_label|
+                if current_labels[lang].nil?
+                    removed_labels << { lang: lang }
+                end
+            end
+        end
+    
+        {
+        changed_lemmas: changed_labels,
+        removed_lemmas: removed_labels,
+        added_lemmas: added_labels
+        }
+    end
+end

data/lib/wikidata/diff/qualifier_analyzer.rb

@@ -0,0 +1,83 @@
+class QualifierAnalyzer
+    # helper method for adding qualifiers
+    # handles added and removed qualifiers
+    def self.qualifier_updates(claim, updated_qualifiers, claim_key, claim_index)
+        if claim["qualifiers"]
+            qualifiers = claim["qualifiers"]
+            qualifiers.each do |qualifier_key, qualifier_values|
+                qualifier_values.each_with_index do |qualifier_value, qualifier_index|
+                updated_qualifiers << {
+                claim_key: claim_key,
+                claim_index: claim_index,
+                qualifier_key: qualifier_key,
+                qualifier_index: qualifier_index
+                }
+                end
+            end
+        end
+        updated_qualifiers
+    end
+
+    # helper method for changed qualifiers
+    def self.handle_changed_qualifiers(current_claim, parent_claim, changed_qualifiers, added_qualifiers, removed_qualifiers, claim_key, claim_index)
+        current_qualifiers = current_claim["qualifiers"] ? current_claim["qualifiers"] : {}
+        parent_qualifiers = parent_claim["qualifiers"] ? parent_claim["qualifiers"] : {}
+
+        current_qualifiers.each do |qualifier_key, qualifier_values|
+            qualifier_values.each_with_index do |qualifier_value, qualifier_index|
+                if parent_qualifiers.key?(qualifier_key)
+                parent = parent_qualifiers[qualifier_key]
+                end
+                # Check if the qualifier index exists in the parent content
+                if !parent.nil?
+                parent = parent[qualifier_index]
+                # check if the parent claim was changed by comparing the objects first
+                if parent != qualifier_value
+                    # Claim was changed
+                    changed_qualifiers << {
+                    claim_key: claim_key,
+                    claim_index: claim_index,
+                    qualifier_key: qualifier_key,
+                    qualifier_index: qualifier_index
+                    }
+                end
+                else
+                # Claim was added
+                added_qualifiers << {
+                    claim_key: claim_key,
+                    claim_index: claim_index,
+                    qualifier_key: qualifier_key,
+                    qualifier_index: qualifier_index
+                }
+                end
+            end
+        end
+        # Check for removed claims
+        parent_qualifiers.each do |qualifier_key, qualifier_values|
+            qualifier_values.each_with_index do |qualifier_value, qualifier_index|
+                if current_qualifiers.key?(qualifier_key)
+                current = current_qualifiers[qualifier_key]
+                end
+                # Check if the qualifier index exists in the current content
+                if !current.nil?
+                current = current[qualifier_index]
+                end
+                if current.nil?
+                # Claim was removed
+                removed_qualifiers << {
+                    claim_key: claim_key,
+                    claim_index: claim_index,
+                    qualifier_key: qualifier_key,
+                    qualifier_index: qualifier_index
+                }
+                end
+            end
+        end
+
+        {
+            added_qualifiers: added_qualifiers,
+            removed_qualifiers: removed_qualifiers,
+            changed_qualifiers: changed_qualifiers
+        }
+    end
+end