wikidata-diff-analyzer 0.1.1 → 2.0.1

data/lib/wikidata/diff/claim_analyzer.rb

@@ -1,5 +1,8 @@
+require_relative 'reference_analyzer'
+require_relative 'qualifier_analyzer'
+
 class ClaimAnalyzer
-    def self.isolate_claim_differences(current_content, parent_content)
+    def self.isolate_claims_differences(current_content, parent_content)
         # Initialize empty arrays to store the added, removed, and changed claims
         added_claims = []
         removed_claims = []
@@ -11,234 +14,114 @@ class ClaimAnalyzer
         removed_qualifiers = []
         changed_qualifiers = []
 
-        if !current_content["claims"].is_a?(Hash) || !parent_content["claims"].is_a?(Hash)
-        return {
-            added_claims: added_claims,
-            removed_claims: removed_claims,
-            changed_claims: changed_claims,
-            added_references: added_references,
-            removed_references: removed_references,
-            changed_references: changed_references,
-            added_qualifiers: added_qualifiers,
-            removed_qualifiers: removed_qualifiers,
-            changed_qualifiers: changed_qualifiers
-        }
-        end
-        # Iterate over each claim key in the current content
-        current_content["claims"].each do |claim_key, current_claims|
-        # Check if the claim key exists in the parent content
-        if parent_content["claims"].key?(claim_key)
-            parent_claims = parent_content["claims"][claim_key]
-            # Iterate over each claim in the current and parent content
-            current_claims.each_with_index do |current_claim, index|
-            parent_claim = parent_claims[index]
-            if parent_claim.nil?
-                # Claim was added
-                added_claims << { key: claim_key, index: index }
-                # check if there's any references or qualifiers in this claim
-                added_references = reference_updates(current_claim, added_references, claim_key, index)
-                added_qualifiers = qualifier_updates(current_claim, added_qualifiers, claim_key, index)
-
-            elsif current_claim != parent_claim
-                # Claim was changed
-                changed_claims << { key: claim_key, index: index }
-                # check if there's any references or qualifiers in this claim
-                changed = handle_changed_references(current_claim, parent_claim, changed_references, added_references, removed_references, claim_key, index)
-                added_references = changed[:added_references]
-                removed_references = changed[:removed_references]
-                changed_references = changed[:changed_references]
-                changed_qualifiers = handle_changed_qualifiers(current_claim, parent_claim, changed_qualifiers, added_qualifiers, removed_qualifiers, claim_key, index)
-                added_qualifiers = changed_qualifiers[:added_qualifiers]
-                removed_qualifiers = changed_qualifiers[:removed_qualifiers]
-                changed_qualifiers = changed_qualifiers[:changed_qualifiers]
-            end
-            end
-            # Check for removed claims
-            parent_claims.each_with_index do |parent_claim, index|
-            current_claim = current_claims[index]
-            if current_claim.nil?
-                # Claim was removed
-                removed_claims << { key: claim_key, index: index }
-
-                # check if there's any references or qualifiers in this claim
-                removed_references = reference_updates(parent_claim, removed_references, claim_key, index)
-                removed_qualifiers = qualifier_updates(parent_claim, removed_qualifiers, claim_key, index)
-            end
-            end
+        if current_content.nil?
+            current_content_claims = {}
         else
-            # All claims in current content with this key were added
-            current_claims.each_index do |index|
-            added_claims << { key: claim_key, index: index }
-            # check if there's any references or qualifiers in this claim
-            added_references = reference_updates(current_claims[index], added_references, claim_key, index)
-            added_qualifiers = qualifier_updates(current_claims[index], added_qualifiers, claim_key, index)
+            current_content_claims = current_content["claims"]
+            if !current_content_claims.is_a?(Hash)
+                current_content_claims = {}
             end
         end
-        end
+        
 
-        parent_content["claims"].each do |claim_key, parent_claims|
-        # current content[claims] can be nil
-        parent_claims.each_index do |index|
-            if current_content["claims"].nil? || !current_content["claims"].key?(claim_key)
-            removed_claims << { key: claim_key, index: index }
-            # check if there's any references or qualifiers in this claim
-            removed_references = reference_updates(parent_claims[index], removed_references, claim_key, index)
-            removed_qualifiers = qualifier_updates(parent_claims[index], removed_qualifiers, claim_key, index)
+        if parent_content.nil?
+            parent_content_claims = {}
+        else
+            parent_content_claims = parent_content["claims"]
+            if !parent_content_claims.is_a?(Hash)
+                parent_content_claims = {}
             end
         end
-        end
-    
-        # puts "Added claims: #{added_claims}"
-        # puts "Removed claims: #{removed_claims}"
-        # puts "Changed claims: #{changed_claims}"
-        # puts "Added references: #{added_references}"
-        # puts "Removed references: #{removed_references}"
-        # puts "Changed references: #{changed_references}"
-        # puts "Added qualifiers: #{added_qualifiers}"
-        # puts "Removed qualifiers: #{removed_qualifiers}"
-        # puts "Changed qualifiers: #{changed_qualifiers}"
-
-
-        {
-        added_claims: added_claims,
-        removed_claims: removed_claims,
-        changed_claims: changed_claims,
-        added_references: added_references,
-        removed_references: removed_references,
-        changed_references: changed_references,
-        added_qualifiers: added_qualifiers,
-        removed_qualifiers: removed_qualifiers,
-        changed_qualifiers: changed_qualifiers
-        }
-    end
-
-    # helper method for adding and removing references
-    def self.reference_updates(claim, updated_references, claim_key, claim_index)
-        if claim["references"]
-        claim["references"].each_with_index do |current_ref, ref_index|
-            updated_references << { claim_key: claim_key, claim_index: claim_index, reference_index: ref_index }
-        end
-        end
-        updated_references
-    end
-
-    # helper method for changed references
-    def self.handle_changed_references(current_claim, parent_claim, changed_references, added_references, removed_references, claim_key, claim_index)
-        current_references = current_claim["references"] ? current_claim["references"] : []
-        parent_references = parent_claim["references"] ? parent_claim["references"] : []
-    
-        current_references.each_with_index do |current_ref, ref_index|
-        if parent_references.empty?
-            added_references << { claim_key: claim_key, claim_index: claim_index, reference_index: ref_index }
-        elsif !parent_references.include?(current_ref)
-            added_references << { claim_key: claim_key, claim_index: claim_index, reference_index: ref_index }
-        elsif ref_modified?(current_ref, parent_references)
-            changed_references << { claim_key: claim_key, claim_index: claim_index, reference_index: ref_index }
-        end
-        end
-    
-        parent_references.each_with_index do |parent_ref, ref_index|
-        if !current_references.include?(parent_ref)
-            removed_references << { claim_key: claim_key, claim_index: claim_index, reference_index: ref_index }
-        end
-        end
+        
+        # if parentid is 0, add all current claims as added claims and return it
+        if parent_content.nil?
+            current_content_claims.each do |claim_key, current_claims|
+                current_claims.each_with_index do |current_claim, index|
+                    added_claims << { key: claim_key, index: index }
+                    # check if there's any references or qualifiers in this claim
+                    added_references = ReferenceAnalyzer.reference_updates(current_claim, added_references, claim_key, index)
+                    added_qualifiers = QualifierAnalyzer.qualifier_updates(current_claim, added_qualifiers, claim_key, index)
+                end
+            end
+        else
+            # Iterate over each claim key in the current content
+            current_content_claims.each do |claim_key, current_claims|
+            # Check if the claim key exists in the parent content
+            if parent_content_claims.key?(claim_key)
+                parent_claims = parent_content_claims[claim_key]
+                # Iterate over each claim in the current and parent content
+                current_claims.each_with_index do |current_claim, index|
+                    parent_claim = parent_claims[index]
+                    if parent_claim.nil?
+                        # Claim was added
+                        added_claims << { key: claim_key, index: index }
+                        # check if there's any references or qualifiers in this claim
+                        added_references = ReferenceAnalyzer.reference_updates(current_claim, added_references, claim_key, index)
+                        added_qualifiers = QualifierAnalyzer.qualifier_updates(current_claim, added_qualifiers, claim_key, index)
 
-        {
-        added_references: added_references,
-        removed_references: removed_references,
-        changed_references: changed_references
-        }
-    end
+                    elsif current_claim != parent_claim
+                        # Claim was changed
+                        changed_claims << { key: claim_key, index: index }
+                        # check if there's any references or qualifiers in this claim
+                        changed_references_hash = ReferenceAnalyzer.handle_changed_references(current_claim, parent_claim, changed_references, added_references, removed_references, claim_key, index)
 
-    # helper method for checking if a reference has been modified
-    def self.ref_modified?(current_reference, parent_references)
-        parent_references.each do |parent_reference|
-        if current_reference["snaks"] != parent_reference["snaks"]
-            return true
-        end
-        end
-        false
-    end
+                        added_references = changed_references_hash[:added_references]
+                        removed_references = changed_references_hash[:removed_references]
+                        changed_references = changed_references_hash[:changed_references]
 
-    # helper method for adding qualifiers
-    # handles added and removed qualifiers
-    def self.qualifier_updates(claim, updated_qualifiers, claim_key, claim_index)
-        if claim["qualifiers"]
-        qualifiers = claim["qualifiers"]
-        qualifiers.each do |qualifier_key, qualifier_values|
-            qualifier_values.each_with_index do |qualifier_value, qualifier_index|
-            updated_qualifiers << {
-            claim_key: claim_key,
-            claim_index: claim_index,
-            qualifier_key: qualifier_key,
-            qualifier_index: qualifier_index
-            }
-            end
-        end
-        end
-        updated_qualifiers
-    end
+                        changed_qualifiers_hash = QualifierAnalyzer.handle_changed_qualifiers(current_claim, parent_claim, changed_qualifiers, added_qualifiers, removed_qualifiers, claim_key, index)
 
-    # helper method for changed qualifiers
-    def self.handle_changed_qualifiers(current_claim, parent_claim, changed_qualifiers, added_qualifiers, removed_qualifiers, claim_key, claim_index)
-        current_qualifiers = current_claim["qualifiers"] ? current_claim["qualifiers"] : {}
-        parent_qualifiers = parent_claim["qualifiers"] ? parent_claim["qualifiers"] : {}
+                        added_qualifiers = changed_qualifiers_hash[:added_qualifiers]
+                        removed_qualifiers = changed_qualifiers_hash[:removed_qualifiers]
+                        changed_qualifiers = changed_qualifiers_hash[:changed_qualifiers]
+                    end
+                end
+                # Check for removed claims
+                parent_claims.each_with_index do |parent_claim, index|
+                    current_claim = current_claims[index]
+                    if current_claim.nil?
+                        # Claim was removed
+                        removed_claims << { key: claim_key, index: index }
 
-        current_qualifiers.each do |qualifier_key, qualifier_values|
-        qualifier_values.each_with_index do |qualifier_value, qualifier_index|
-            if parent_qualifiers.key?(qualifier_key)
-            parent = parent_qualifiers[qualifier_key]
-            end
-            # Check if the qualifier index exists in the parent content
-            if !parent.nil?
-            parent = parent[qualifier_index]
-            # check if the parent claim was changed by comparing the objects first
-            if parent != qualifier_value
-                # Claim was changed
-                changed_qualifiers << {
-                claim_key: claim_key,
-                claim_index: claim_index,
-                qualifier_key: qualifier_key,
-                qualifier_index: qualifier_index
-                }
-            end
+                        # check if there's any references or qualifiers in this claim
+                        removed_references = ReferenceAnalyzer.reference_updates(parent_claim, removed_references, claim_key, index)
+                        removed_qualifiers = QualifierAnalyzer.qualifier_updates(parent_claim, removed_qualifiers, claim_key, index)
+                    end
+                end
             else
-            # Claim was added
-            added_qualifiers << {
-                claim_key: claim_key,
-                claim_index: claim_index,
-                qualifier_key: qualifier_key,
-                qualifier_index: qualifier_index
-            }
-            end
-        end
-        end
-        # Check for removed claims
-        parent_qualifiers.each do |qualifier_key, qualifier_values|
-        qualifier_values.each_with_index do |qualifier_value, qualifier_index|
-            if current_qualifiers.key?(qualifier_key)
-            current = current_qualifiers[qualifier_key]
+                # All claims in current content with this key were added
+                current_claims.each_index do |index|
+                    added_claims << { key: claim_key, index: index }
+                    # check if there's any references or qualifiers in this claim
+                    added_references = ReferenceAnalyzer.reference_updates(current_claims[index], added_references, claim_key, index)
+                    added_qualifiers = QualifierAnalyzer.qualifier_updates(current_claims[index], added_qualifiers, claim_key, index)
+                end
             end
-            # Check if the qualifier index exists in the current content
-            if !current.nil?
-            current = current[qualifier_index]
             end
-            if current.nil?
-            # Claim was removed
-            removed_qualifiers << {
-                claim_key: claim_key,
-                claim_index: claim_index,
-                qualifier_key: qualifier_key,
-                qualifier_index: qualifier_index
-            }
+
+            parent_content_claims.each do |claim_key, parent_claims|
+                # current content[claims] can be nil
+                parent_claims.each_index do |index|
+                    if current_content_claims.nil? || !current_content_claims.key?(claim_key)
+                    removed_claims << { key: claim_key, index: index }
+                    # check if there's any references or qualifiers in this claim
+                    removed_references = ReferenceAnalyzer.reference_updates(parent_claims[index], removed_references, claim_key, index)
+                    removed_qualifiers = QualifierAnalyzer.qualifier_updates(parent_claims[index], removed_qualifiers, claim_key, index)
+                    end
+                end
             end
         end
-        end
 
         {
-        added_qualifiers: added_qualifiers,
-        removed_qualifiers: removed_qualifiers,
-        changed_qualifiers: changed_qualifiers
+            added_claims: added_claims,
+            removed_claims: removed_claims,
+            changed_claims: changed_claims,
+            added_references: added_references,
+            removed_references: removed_references,
+            changed_references: changed_references,
+            added_qualifiers: added_qualifiers,
+            removed_qualifiers: removed_qualifiers,
+            changed_qualifiers: changed_qualifiers
         }
     end
 end

data/lib/wikidata/diff/comment_analyzer.rb

@@ -0,0 +1,49 @@
+class CommentAnalyzer
+    def self.isolate_comment_differences(comment)
+    phrases = {
+        'merge_to': 0,
+        'merge_from': 0,
+        'redirect': 0,
+        'undo': 0,
+        'restore': 0,
+        'clear_item': 0,
+        'create_item': 0,
+    }
+
+    if comment.nil?
+        return phrases
+    end
+
+    if comment.include?('wbmergeitems-from')
+        phrases[:merge_from] = 1
+    end
+
+    if comment.include?('wbmergeitems-to')
+        phrases[:merge_to] = 1
+    end
+
+    if comment.include?('wbcreateredirect')
+        phrases[:redirect] = 1
+    end
+
+    if comment.include?('undo:')
+        phrases[:undo] = 1
+    end
+
+    if comment.include?('restore:')
+        phrases[:restore] = 1
+    end
+
+    if comment.include?('wbeditentity-override')
+        phrases[:clear_item] = 1
+    end
+
+    # create-property, create-item, create-lexeme all includes this phrase
+    # so based on content model in revision analyzer, it is decided which one it is
+    if comment.include?('wbeditentity-create')
+        phrases[:create_item] = 1
+    end
+
+    return phrases
+    end
+end

data/lib/wikidata/diff/description_analyzer.rb

@@ -1,49 +1,71 @@
-class DescriptionAnalyzer
+class DescriptionAnalyzer  
     def self.isolate_descriptions_differences(current_content, parent_content)
         return {
-        changed: [],
-        removed: [],
-        added: []
+        changed_descriptions: [],
+        removed_descriptions: [],
+        added_descriptions: []
         } if current_content.nil? && parent_content.nil?
-    
-        current_descriptions = current_content['descriptions'] || {}
-        parent_descriptions = parent_content['descriptions'] || {}
 
+        if current_content
+            current_descriptions = current_content['descriptions']
+            if current_descriptions.nil? || current_descriptions.is_a?(Array)
+                current_descriptions = {}
+            end
+        else
+            current_descriptions = {}
+        end
+    
+        
+        if parent_content
+            parent_descriptions = parent_content['descriptions']
+            if parent_descriptions.nil? || parent_descriptions.is_a?(Array)
+                parent_descriptions = {}
+            end
+        else
+            parent_descriptions = {}
+        end
     
         changed_descriptions = []    # Initialize as an array
         removed_descriptions = []    # Initialize as an array
         added_descriptions = []      # Initialize as an array
-    
-        if !current_descriptions.is_a?(Hash) || !parent_descriptions.is_a?(Hash)
-        return{
-            changed: changed_descriptions,
-            removed: removed_descriptions,
-            added: added_descriptions
-        }
-        end
 
-        # Iterate over each language in the current descriptions
-        (current_descriptions || {}).each do |lang, current_description|
-        parent_description = parent_descriptions[lang]
-    
-        if parent_description.nil?
-            added_descriptions << { lang: lang }
-        elsif current_description != parent_description
-            changed_descriptions << { lang: lang }
-        end
-        end
-    
-        # Iterate over each language in the parent descriptions to find removed descriptions
-        (parent_descriptions || {}).each do |lang, parent_description|
-        if current_descriptions[lang].nil?
-            removed_descriptions << { lang: lang }
-        end
+
+        # if parentid is 0, add all current description as added and return it
+        if parent_content.nil?
+            if !current_descriptions.empty?
+                current_descriptions.each do |lang, description|
+                    added_descriptions << { lang: lang }
+                end
+            end
+            return {
+                changed_descriptions: changed_descriptions,
+                removed_descriptions: removed_descriptions,
+                added_descriptions: added_descriptions
+            }
+        else
+            # Iterate over each language in the current descriptions
+            (current_descriptions).each do |lang, current_description|
+                # checking if the parent descriptions is empty
+                if parent_descriptions.empty?
+                    added_descriptions << { lang: lang }
+                elsif parent_descriptions[lang].nil?
+                    added_descriptions << { lang: lang }
+                elsif current_description != parent_descriptions[lang]
+                    changed_descriptions << { lang: lang }
+                end
+            end
+            
+                # Iterate over each language in the parent descriptions to find removed descriptions
+            (parent_descriptions).each do |lang, parent_description|
+                if current_descriptions.empty?
+                    removed_descriptions << { lang: lang }
+                end
+            end
         end
-    
         {
-        changed: changed_descriptions,
-        removed: removed_descriptions,
-        added: added_descriptions
+            changed_descriptions: changed_descriptions,
+            removed_descriptions: removed_descriptions,
+            added_descriptions: added_descriptions
         }
     end
 end

data/lib/wikidata/diff/form_analyzer.rb

@@ -0,0 +1,67 @@
+require_relative 'representation_analyzer'
+require_relative 'inside_claim_analyzer'
+class FormAnalyzer
+    def self.isolate_forms_differences(current_content, parent_content)
+        added_forms = []
+        removed_forms = []
+        changed_forms = []
+        added_representations = []
+        removed_representations = []
+        changed_representations = []
+        added_formclaims = []
+        removed_formclaims = []
+        changed_formclaims = []
+      
+        current_forms = current_content&.fetch("forms", []) || []
+        parent_forms = parent_content&.fetch("forms", []) || []
+      
+        current_forms.each_with_index do |current_form, index|
+          parent_form = parent_forms[index]
+      
+          if parent_form.nil?
+            # Claim was added
+            added_forms << { index: index }
+          elsif current_form
+            # Claim was changed
+            changed_forms << { index: index }
+          end
+      
+          representations = RepresentationAnalyzer.isolate_representation_differences(current_form, parent_form)
+          added_representations += representations[:added]
+          removed_representations += representations[:removed]
+          changed_representations += representations[:changed]
+      
+          formclaims = InsideClaimAnalyzer.isolate_inside_claim_differences(current_form, parent_form)
+          added_formclaims += formclaims[:added]
+          removed_formclaims += formclaims[:removed]
+          changed_formclaims += formclaims[:changed]
+        end
+      
+        parent_forms.each_with_index do |parent_form, index|
+          current_form = current_forms[index]
+      
+          if current_form.nil?
+            # Claim was removed
+            removed_forms << { index: index }
+      
+            representations = RepresentationAnalyzer.isolate_representation_differences(nil, parent_form)
+            removed_representations += representations[:removed]
+      
+            formclaims = InsideClaimAnalyzer.isolate_inside_claim_differences(nil, parent_form)
+            removed_formclaims += formclaims[:removed]
+          end
+        end
+      
+        {
+          added_forms: added_forms,
+          removed_forms: removed_forms,
+          changed_forms: changed_forms,
+          added_representations: added_representations,
+          removed_representations: removed_representations,
+          changed_representations: changed_representations,
+          added_formclaims: added_formclaims,
+          removed_formclaims: removed_formclaims,
+          changed_formclaims: changed_formclaims
+        }
+      end
+end

data/lib/wikidata/diff/gloss_analyzer.rb

@@ -0,0 +1,71 @@
+class GlossAnalyzer
+    def self.isolate_gloss_differences(current_content, parent_content)
+        return {
+        changed: [],
+        removed: [],
+        added: []
+        } if current_content.nil? && parent_content.nil?
+    
+        if current_content
+            current_glosses = current_content['glosses']
+            if current_glosses.nil? || current_glosses.is_a?(Array)
+                current_glosses = {}
+            end
+        else
+            current_glosses = {}
+        end
+
+        if parent_content
+            parent_glosses = parent_content['glosses'] 
+            if parent_glosses.nil? || parent_glosses.is_a?(Array)
+                parent_glosses = {}
+            end
+        else
+            parent_glosses = {}
+        end
+
+
+
+        changed = []
+        removed = []
+        added = []
+
+        # if parentid is 0, then add all labels as added and return it
+        if parent_content.nil?
+            current_glosses.each do |lang, label|
+                added << { lang: lang }
+            end
+            return {
+                changed: changed,
+                removed: removed,
+                added: added
+            }
+        else
+
+        
+            # Iterate over each language in the current labels
+            (current_glosses || {}).each do |lang, current_gloss|
+                parent_gloss = parent_glosses[lang]
+            
+                if parent_gloss.nil?
+                    added << { lang: lang }
+                elsif current_gloss != parent_gloss
+                    changed << { lang: lang }
+                end
+            end
+            
+                # Iterate over each language in the parent labels to find removed labels
+            (parent_glosses || {}).each do |lang, parent_gloss|
+                if current_glosses[lang].nil?
+                    removed << { lang: lang }
+                end
+            end
+        end
+    
+        {
+        changed: changed,
+        removed: removed,
+        added: added
+        }
+    end
+end