wikidata-diff-analyzer 0.1.1 → 2.0.0

data/lib/wikidata/diff/analyzer.rb

@@ -2,7 +2,6 @@
 require_relative 'large_batches_analyzer'
 require_relative 'revision_analyzer'
 require_relative 'total'
-require_relative 'mediawiki_login'
 
 module WikidataDiffAnalyzer
   class Error < StandardError; end
@@ -34,7 +33,37 @@ module WikidataDiffAnalyzer
       descriptions_changed: 0,
       sitelinks_added: 0,
       sitelinks_removed: 0,
-      sitelinks_changed: 0
+      sitelinks_changed: 0,
+      lemmas_added: 0,
+      lemmas_removed: 0,
+      lemmas_changed: 0,
+      forms_added: 0,
+      forms_removed: 0,
+      forms_changed: 0,
+      representations_added: 0,
+      representations_removed: 0,
+      representations_changed: 0,
+      formclaims_added: 0,
+      formclaims_removed: 0,
+      formclaims_changed: 0,
+      senses_added: 0,
+      senses_removed: 0,
+      senses_changed: 0,
+      glosses_added: 0,
+      glosses_removed: 0,
+      glosses_changed: 0,
+      senseclaims_added: 0,
+      senseclaims_removed: 0,
+      senseclaims_changed: 0,
+      merge_to: 0,
+      merge_from: 0,
+      redirect: 0,
+      undo: 0,
+      restore: 0,
+      clear_item: 0,
+      create_item: 0,
+      create_property: 0,
+      create_lexeme: 0
     }
 
     # if revision_ids has 0, then 0 can never be analyzed, so remove it and add in not analyzed
@@ -42,24 +71,13 @@ module WikidataDiffAnalyzer
       revision_ids.delete(0)
       diffs_not_analyzed << 0
     end
-    # # if mediawiki can be logged in call for 500, otherwise call for 50
-    # if MediawikiLogin.mediawiki_login
-    #   puts 'Logged in to mediawiki'
-    #   result = LargeBatchesAnalyzer.handle_large_batches(revision_ids, 500)
-    # else
-    #   puts 'Not logged in to mediawiki'
-    #   result = LargeBatchesAnalyzer.handle_large_batches(revision_ids, 50)
-    # end
 
     result = LargeBatchesAnalyzer.handle_large_batches(revision_ids, 50)
-    # result is a hash which has contents like this:
-    # result[revid] = { current_content: data[:content], parent_content: parent_content }
 
     result.each do |revision_id, revision_data|
       current_content = revision_data[:current_content]
-      parent_content = revision_data[:parent_content]
-      if current_content && parent_content
-        diff = RevisionAnalyzer.analyze_diff(current_content, parent_content)
+      if current_content
+        diff = RevisionAnalyzer.analyze_diff(revision_data)
         diffs[revision_id] = diff
         Total.accumulate_totals(diff, total)
         diffs_analyzed << revision_id
@@ -78,6 +96,3 @@ module WikidataDiffAnalyzer
     }
   end
 end
-
-
-

data/lib/wikidata/diff/api.rb

@@ -17,12 +17,11 @@ class Api
             prop: 'revisions',
             revids: revision_ids.join('|'),
             rvslots: 'main',
-            rvprop: 'content|ids',
+            rvprop: 'content|ids|comment',
             format: 'json'
         )
 
         if response.nil?
-            puts "No response received for revision IDs: #{revision_ids.join(', ')}"
             return {}
         end
 
@@ -30,31 +29,45 @@ class Api
 
         # checks if it has pages
         if response.data['pages'].nil?
-            puts "No pages found in the response for revision IDs: #{revision_ids.join(', ')}"
             return nil
         end
 
         response.data['pages'].keys.each do |page|
             page = response.data['pages'][page]
             revisions = page['revisions']
-
+          
             revisions.each do |revision|
-            if revision['slots']['main']['contentmodel'] != 'wikibase-item'
-                puts "Content model is not wikibase-item"
-                # include the revision ids that are not wikibase-item
-                puts "Revision ID #{revision['revid']} is a #{revision['slots']['main']['contentmodel']}"
-            else
-                content = revision['slots']['main']['*']
-                revid = revision['revid']
-                if revid == 0
-                    parsed_contents[revid] = {content: nil, parentid: 0}
+              content_model = revision['slots']['main']['contentmodel']
+              if content_model == 'wikibase-item' || content_model == 'wikibase-property' || content_model == 'wikibase-lexeme'
+                if revision.key?('texthidden')
+                  puts "Content has been hidden or deleted"
+                  revid = revision['revid']
+                  parentid = revision['parentid']
+                  parsed_contents[revid] = { content: nil, comment: nil, parentid: parentid, model: content_model }
+                # checking if comment has been deleted
+                elsif revision.key?('commenthidden')
+                  puts "Comment has been hidden or deleted"
+                  revid = revision['revid']
+                  content = revision['slots']['main']['*']
+                  parentid = revision['parentid']
+                  parsed_contents[revid] = { content: JSON.parse(content), comment: nil, parentid: parentid, model: content_model }
                 else
-                    parentid = revision['parentid']
-                    parsed_contents[revid] = {content: JSON.parse(content), parentid: parentid}
+                  content = revision['slots']['main']['*']
+                  revid = revision['revid']
+                  comment = revision['comment']
+                  parentid = revision['parentid']
+                  if revid == 0 || revid.nil?
+                    parsed_contents[revid] = { content: nil, comment: nil, parentid: nil, model: 'wikibase-item' }
+                  else
+                    parsed_contents[revid] = { content: JSON.parse(content), comment: comment, parentid: parentid, model: content_model}
+                  end
                 end
+              else
+                puts "Content model is #{content_model}"
+                puts "Revision id is #{revision['revid']}"
+              end
             end
-            end
-        end
+          end
         return parsed_contents
         rescue MediawikiApi::ApiError => e
         puts "Error retrieving revision content: #{e.message}"
@@ -64,4 +77,4 @@ class Api
         raise e
         end
     end
-end
+  end

data/lib/wikidata/diff/claim_analyzer.rb

@@ -1,5 +1,8 @@
+require_relative 'reference_analyzer'
+require_relative 'qualifier_analyzer'
+
 class ClaimAnalyzer
-    def self.isolate_claim_differences(current_content, parent_content)
+    def self.isolate_claims_differences(current_content, parent_content)
         # Initialize empty arrays to store the added, removed, and changed claims
         added_claims = []
         removed_claims = []
@@ -11,234 +14,114 @@ class ClaimAnalyzer
         removed_qualifiers = []
         changed_qualifiers = []
 
-        if !current_content["claims"].is_a?(Hash) || !parent_content["claims"].is_a?(Hash)
-        return {
-            added_claims: added_claims,
-            removed_claims: removed_claims,
-            changed_claims: changed_claims,
-            added_references: added_references,
-            removed_references: removed_references,
-            changed_references: changed_references,
-            added_qualifiers: added_qualifiers,
-            removed_qualifiers: removed_qualifiers,
-            changed_qualifiers: changed_qualifiers
-        }
-        end
-        # Iterate over each claim key in the current content
-        current_content["claims"].each do |claim_key, current_claims|
-        # Check if the claim key exists in the parent content
-        if parent_content["claims"].key?(claim_key)
-            parent_claims = parent_content["claims"][claim_key]
-            # Iterate over each claim in the current and parent content
-            current_claims.each_with_index do |current_claim, index|
-            parent_claim = parent_claims[index]
-            if parent_claim.nil?
-                # Claim was added
-                added_claims << { key: claim_key, index: index }
-                # check if there's any references or qualifiers in this claim
-                added_references = reference_updates(current_claim, added_references, claim_key, index)
-                added_qualifiers = qualifier_updates(current_claim, added_qualifiers, claim_key, index)
-
-            elsif current_claim != parent_claim
-                # Claim was changed
-                changed_claims << { key: claim_key, index: index }
-                # check if there's any references or qualifiers in this claim
-                changed = handle_changed_references(current_claim, parent_claim, changed_references, added_references, removed_references, claim_key, index)
-                added_references = changed[:added_references]
-                removed_references = changed[:removed_references]
-                changed_references = changed[:changed_references]
-                changed_qualifiers = handle_changed_qualifiers(current_claim, parent_claim, changed_qualifiers, added_qualifiers, removed_qualifiers, claim_key, index)
-                added_qualifiers = changed_qualifiers[:added_qualifiers]
-                removed_qualifiers = changed_qualifiers[:removed_qualifiers]
-                changed_qualifiers = changed_qualifiers[:changed_qualifiers]
-            end
-            end
-            # Check for removed claims
-            parent_claims.each_with_index do |parent_claim, index|
-            current_claim = current_claims[index]
-            if current_claim.nil?
-                # Claim was removed
-                removed_claims << { key: claim_key, index: index }
-
-                # check if there's any references or qualifiers in this claim
-                removed_references = reference_updates(parent_claim, removed_references, claim_key, index)
-                removed_qualifiers = qualifier_updates(parent_claim, removed_qualifiers, claim_key, index)
-            end
-            end
+        if current_content.nil?
+            current_content_claims = {}
         else
-            # All claims in current content with this key were added
-            current_claims.each_index do |index|
-            added_claims << { key: claim_key, index: index }
-            # check if there's any references or qualifiers in this claim
-            added_references = reference_updates(current_claims[index], added_references, claim_key, index)
-            added_qualifiers = qualifier_updates(current_claims[index], added_qualifiers, claim_key, index)
+            current_content_claims = current_content["claims"]
+            if !current_content_claims.is_a?(Hash)
+                current_content_claims = {}
             end
         end
-        end
+        
 
-        parent_content["claims"].each do |claim_key, parent_claims|
-        # current content[claims] can be nil
-        parent_claims.each_index do |index|
-            if current_content["claims"].nil? || !current_content["claims"].key?(claim_key)
-            removed_claims << { key: claim_key, index: index }
-            # check if there's any references or qualifiers in this claim
-            removed_references = reference_updates(parent_claims[index], removed_references, claim_key, index)
-            removed_qualifiers = qualifier_updates(parent_claims[index], removed_qualifiers, claim_key, index)
+        if parent_content.nil?
+            parent_content_claims = {}
+        else
+            parent_content_claims = parent_content["claims"]
+            if !parent_content_claims.is_a?(Hash)
+                parent_content_claims = {}
             end
         end
-        end
-    
-        # puts "Added claims: #{added_claims}"
-        # puts "Removed claims: #{removed_claims}"
-        # puts "Changed claims: #{changed_claims}"
-        # puts "Added references: #{added_references}"
-        # puts "Removed references: #{removed_references}"
-        # puts "Changed references: #{changed_references}"
-        # puts "Added qualifiers: #{added_qualifiers}"
-        # puts "Removed qualifiers: #{removed_qualifiers}"
-        # puts "Changed qualifiers: #{changed_qualifiers}"
-
-
-        {
-        added_claims: added_claims,
-        removed_claims: removed_claims,
-        changed_claims: changed_claims,
-        added_references: added_references,
-        removed_references: removed_references,
-        changed_references: changed_references,
-        added_qualifiers: added_qualifiers,
-        removed_qualifiers: removed_qualifiers,
-        changed_qualifiers: changed_qualifiers
-        }
-    end
-
-    # helper method for adding and removing references
-    def self.reference_updates(claim, updated_references, claim_key, claim_index)
-        if claim["references"]
-        claim["references"].each_with_index do |current_ref, ref_index|
-            updated_references << { claim_key: claim_key, claim_index: claim_index, reference_index: ref_index }
-        end
-        end
-        updated_references
-    end
-
-    # helper method for changed references
-    def self.handle_changed_references(current_claim, parent_claim, changed_references, added_references, removed_references, claim_key, claim_index)
-        current_references = current_claim["references"] ? current_claim["references"] : []
-        parent_references = parent_claim["references"] ? parent_claim["references"] : []
-    
-        current_references.each_with_index do |current_ref, ref_index|
-        if parent_references.empty?
-            added_references << { claim_key: claim_key, claim_index: claim_index, reference_index: ref_index }
-        elsif !parent_references.include?(current_ref)
-            added_references << { claim_key: claim_key, claim_index: claim_index, reference_index: ref_index }
-        elsif ref_modified?(current_ref, parent_references)
-            changed_references << { claim_key: claim_key, claim_index: claim_index, reference_index: ref_index }
-        end
-        end
-    
-        parent_references.each_with_index do |parent_ref, ref_index|
-        if !current_references.include?(parent_ref)
-            removed_references << { claim_key: claim_key, claim_index: claim_index, reference_index: ref_index }
-        end
-        end
+        
+        # if parentid is 0, add all current claims as added claims and return it
+        if parent_content.nil?
+            current_content_claims.each do |claim_key, current_claims|
+                current_claims.each_with_index do |current_claim, index|
+                    added_claims << { key: claim_key, index: index }
+                    # check if there's any references or qualifiers in this claim
+                    added_references = ReferenceAnalyzer.reference_updates(current_claim, added_references, claim_key, index)
+                    added_qualifiers = QualifierAnalyzer.qualifier_updates(current_claim, added_qualifiers, claim_key, index)
+                end
+            end
+        else
+            # Iterate over each claim key in the current content
+            current_content_claims.each do |claim_key, current_claims|
+            # Check if the claim key exists in the parent content
+            if parent_content_claims.key?(claim_key)
+                parent_claims = parent_content_claims[claim_key]
+                # Iterate over each claim in the current and parent content
+                current_claims.each_with_index do |current_claim, index|
+                    parent_claim = parent_claims[index]
+                    if parent_claim.nil?
+                        # Claim was added
+                        added_claims << { key: claim_key, index: index }
+                        # check if there's any references or qualifiers in this claim
+                        added_references = ReferenceAnalyzer.reference_updates(current_claim, added_references, claim_key, index)
+                        added_qualifiers = QualifierAnalyzer.qualifier_updates(current_claim, added_qualifiers, claim_key, index)
 
-        {
-        added_references: added_references,
-        removed_references: removed_references,
-        changed_references: changed_references
-        }
-    end
+                    elsif current_claim != parent_claim
+                        # Claim was changed
+                        changed_claims << { key: claim_key, index: index }
+                        # check if there's any references or qualifiers in this claim
+                        changed_references_hash = ReferenceAnalyzer.handle_changed_references(current_claim, parent_claim, changed_references, added_references, removed_references, claim_key, index)
 
-    # helper method for checking if a reference has been modified
-    def self.ref_modified?(current_reference, parent_references)
-        parent_references.each do |parent_reference|
-        if current_reference["snaks"] != parent_reference["snaks"]
-            return true
-        end
-        end
-        false
-    end
+                        added_references = changed_references_hash[:added_references]
+                        removed_references = changed_references_hash[:removed_references]
+                        changed_references = changed_references_hash[:changed_references]
 
-    # helper method for adding qualifiers
-    # handles added and removed qualifiers
-    def self.qualifier_updates(claim, updated_qualifiers, claim_key, claim_index)
-        if claim["qualifiers"]
-        qualifiers = claim["qualifiers"]
-        qualifiers.each do |qualifier_key, qualifier_values|
-            qualifier_values.each_with_index do |qualifier_value, qualifier_index|
-            updated_qualifiers << {
-            claim_key: claim_key,
-            claim_index: claim_index,
-            qualifier_key: qualifier_key,
-            qualifier_index: qualifier_index
-            }
-            end
-        end
-        end
-        updated_qualifiers
-    end
+                        changed_qualifiers_hash = QualifierAnalyzer.handle_changed_qualifiers(current_claim, parent_claim, changed_qualifiers, added_qualifiers, removed_qualifiers, claim_key, index)
 
-    # helper method for changed qualifiers
-    def self.handle_changed_qualifiers(current_claim, parent_claim, changed_qualifiers, added_qualifiers, removed_qualifiers, claim_key, claim_index)
-        current_qualifiers = current_claim["qualifiers"] ? current_claim["qualifiers"] : {}
-        parent_qualifiers = parent_claim["qualifiers"] ? parent_claim["qualifiers"] : {}
+                        added_qualifiers = changed_qualifiers_hash[:added_qualifiers]
+                        removed_qualifiers = changed_qualifiers_hash[:removed_qualifiers]
+                        changed_qualifiers = changed_qualifiers_hash[:changed_qualifiers]
+                    end
+                end
+                # Check for removed claims
+                parent_claims.each_with_index do |parent_claim, index|
+                    current_claim = current_claims[index]
+                    if current_claim.nil?
+                        # Claim was removed
+                        removed_claims << { key: claim_key, index: index }
 
-        current_qualifiers.each do |qualifier_key, qualifier_values|
-        qualifier_values.each_with_index do |qualifier_value, qualifier_index|
-            if parent_qualifiers.key?(qualifier_key)
-            parent = parent_qualifiers[qualifier_key]
-            end
-            # Check if the qualifier index exists in the parent content
-            if !parent.nil?
-            parent = parent[qualifier_index]
-            # check if the parent claim was changed by comparing the objects first
-            if parent != qualifier_value
-                # Claim was changed
-                changed_qualifiers << {
-                claim_key: claim_key,
-                claim_index: claim_index,
-                qualifier_key: qualifier_key,
-                qualifier_index: qualifier_index
-                }
-            end
+                        # check if there's any references or qualifiers in this claim
+                        removed_references = ReferenceAnalyzer.reference_updates(parent_claim, removed_references, claim_key, index)
+                        removed_qualifiers = QualifierAnalyzer.qualifier_updates(parent_claim, removed_qualifiers, claim_key, index)
+                    end
+                end
             else
-            # Claim was added
-            added_qualifiers << {
-                claim_key: claim_key,
-                claim_index: claim_index,
-                qualifier_key: qualifier_key,
-                qualifier_index: qualifier_index
-            }
-            end
-        end
-        end
-        # Check for removed claims
-        parent_qualifiers.each do |qualifier_key, qualifier_values|
-        qualifier_values.each_with_index do |qualifier_value, qualifier_index|
-            if current_qualifiers.key?(qualifier_key)
-            current = current_qualifiers[qualifier_key]
+                # All claims in current content with this key were added
+                current_claims.each_index do |index|
+                    added_claims << { key: claim_key, index: index }
+                    # check if there's any references or qualifiers in this claim
+                    added_references = ReferenceAnalyzer.reference_updates(current_claims[index], added_references, claim_key, index)
+                    added_qualifiers = QualifierAnalyzer.qualifier_updates(current_claims[index], added_qualifiers, claim_key, index)
+                end
             end
-            # Check if the qualifier index exists in the current content
-            if !current.nil?
-            current = current[qualifier_index]
             end
-            if current.nil?
-            # Claim was removed
-            removed_qualifiers << {
-                claim_key: claim_key,
-                claim_index: claim_index,
-                qualifier_key: qualifier_key,
-                qualifier_index: qualifier_index
-            }
+
+            parent_content_claims.each do |claim_key, parent_claims|
+                # current content[claims] can be nil
+                parent_claims.each_index do |index|
+                    if current_content_claims.nil? || !current_content_claims.key?(claim_key)
+                    removed_claims << { key: claim_key, index: index }
+                    # check if there's any references or qualifiers in this claim
+                    removed_references = ReferenceAnalyzer.reference_updates(parent_claims[index], removed_references, claim_key, index)
+                    removed_qualifiers = QualifierAnalyzer.qualifier_updates(parent_claims[index], removed_qualifiers, claim_key, index)
+                    end
+                end
             end
         end
-        end
 
         {
-        added_qualifiers: added_qualifiers,
-        removed_qualifiers: removed_qualifiers,
-        changed_qualifiers: changed_qualifiers
+            added_claims: added_claims,
+            removed_claims: removed_claims,
+            changed_claims: changed_claims,
+            added_references: added_references,
+            removed_references: removed_references,
+            changed_references: changed_references,
+            added_qualifiers: added_qualifiers,
+            removed_qualifiers: removed_qualifiers,
+            changed_qualifiers: changed_qualifiers
         }
     end
 end

data/lib/wikidata/diff/comment_analyzer.rb

@@ -0,0 +1,49 @@
+class CommentAnalyzer
+    def self.isolate_comment_differences(comment)
+    phrases = {
+        'merge_to': 0,
+        'merge_from': 0,
+        'redirect': 0,
+        'undo': 0,
+        'restore': 0,
+        'clear_item': 0,
+        'create_item': 0,
+    }
+
+    if comment.nil?
+        return phrases
+    end
+
+    if comment.include?('wbmergeitems-from')
+        phrases[:merge_from] = 1
+    end
+
+    if comment.include?('wbmergeitems-to')
+        phrases[:merge_to] = 1
+    end
+
+    if comment.include?('wbcreateredirect')
+        phrases[:redirect] = 1
+    end
+
+    if comment.include?('undo:')
+        phrases[:undo] = 1
+    end
+
+    if comment.include?('restore:')
+        phrases[:restore] = 1
+    end
+
+    if comment.include?('wbeditentity-override')
+        phrases[:clear_item] = 1
+    end
+
+    # create-property, create-item, create-lexeme all includes this phrase
+    # so based on content model in revision analyzer, it is decided which one it is
+    if comment.include?('wbeditentity-create')
+        phrases[:create_item] = 1
+    end
+
+    return phrases
+    end
+end

data/lib/wikidata/diff/description_analyzer.rb

@@ -1,49 +1,71 @@
-class DescriptionAnalyzer
+class DescriptionAnalyzer  
     def self.isolate_descriptions_differences(current_content, parent_content)
         return {
-        changed: [],
-        removed: [],
-        added: []
+        changed_descriptions: [],
+        removed_descriptions: [],
+        added_descriptions: []
         } if current_content.nil? && parent_content.nil?
-    
-        current_descriptions = current_content['descriptions'] || {}
-        parent_descriptions = parent_content['descriptions'] || {}
 
+        if current_content
+            current_descriptions = current_content['descriptions']
+            if current_descriptions.nil? || current_descriptions.is_a?(Array)
+                current_descriptions = {}
+            end
+        else
+            current_descriptions = {}
+        end
+    
+        
+        if parent_content
+            parent_descriptions = parent_content['descriptions']
+            if parent_descriptions.nil? || parent_descriptions.is_a?(Array)
+                parent_descriptions = {}
+            end
+        else
+            parent_descriptions = {}
+        end
     
         changed_descriptions = []    # Initialize as an array
         removed_descriptions = []    # Initialize as an array
         added_descriptions = []      # Initialize as an array
-    
-        if !current_descriptions.is_a?(Hash) || !parent_descriptions.is_a?(Hash)
-        return{
-            changed: changed_descriptions,
-            removed: removed_descriptions,
-            added: added_descriptions
-        }
-        end
 
-        # Iterate over each language in the current descriptions
-        (current_descriptions || {}).each do |lang, current_description|
-        parent_description = parent_descriptions[lang]
-    
-        if parent_description.nil?
-            added_descriptions << { lang: lang }
-        elsif current_description != parent_description
-            changed_descriptions << { lang: lang }
-        end
-        end
-    
-        # Iterate over each language in the parent descriptions to find removed descriptions
-        (parent_descriptions || {}).each do |lang, parent_description|
-        if current_descriptions[lang].nil?
-            removed_descriptions << { lang: lang }
-        end
+
+        # if parentid is 0, add all current description as added and return it
+        if parent_content.nil?
+            if !current_descriptions.empty?
+                current_descriptions.each do |lang, description|
+                    added_descriptions << { lang: lang }
+                end
+            end
+            return {
+                changed_descriptions: changed_descriptions,
+                removed_descriptions: removed_descriptions,
+                added_descriptions: added_descriptions
+            }
+        else
+            # Iterate over each language in the current descriptions
+            (current_descriptions).each do |lang, current_description|
+                # checking if the parent descriptions is empty
+                if parent_descriptions.empty?
+                    added_descriptions << { lang: lang }
+                elsif parent_descriptions[lang].nil?
+                    added_descriptions << { lang: lang }
+                elsif current_description != parent_descriptions[lang]
+                    changed_descriptions << { lang: lang }
+                end
+            end
+            
+                # Iterate over each language in the parent descriptions to find removed descriptions
+            (parent_descriptions).each do |lang, parent_description|
+                if current_descriptions.empty?
+                    removed_descriptions << { lang: lang }
+                end
+            end
         end
-    
         {
-        changed: changed_descriptions,
-        removed: removed_descriptions,
-        added: added_descriptions
+            changed_descriptions: changed_descriptions,
+            removed_descriptions: removed_descriptions,
+            added_descriptions: added_descriptions
         }
     end
 end