wikidata-diff-analyzer 0.1.0 → 2.0.0

data/lib/wikidata/diff/reference_analyzer.rb

@@ -0,0 +1,49 @@
+class ReferenceAnalyzer
+    # helper method for adding and removing references
+    def self.reference_updates(claim, updated_references, claim_key, claim_index)
+        if claim["references"]
+            claim["references"].each_with_index do |current_ref, ref_index|
+                updated_references << { claim_key: claim_key, claim_index: claim_index, reference_index: ref_index }
+            end
+        end
+        updated_references
+    end
+
+    # helper method for changed references
+    def self.handle_changed_references(current_claim, parent_claim, changed_references, added_references, removed_references, claim_key, claim_index)
+        current_references = current_claim["references"] ? current_claim["references"] : []
+        parent_references = parent_claim["references"] ? parent_claim["references"] : []
+    
+        current_references.each_with_index do |current_ref, ref_index|
+            if parent_references.empty?
+                added_references << { claim_key: claim_key, claim_index: claim_index, reference_index: ref_index }
+            elsif !parent_references.include?(current_ref)
+                added_references << { claim_key: claim_key, claim_index: claim_index, reference_index: ref_index }
+            elsif ref_modified?(current_ref, parent_references)
+                changed_references << { claim_key: claim_key, claim_index: claim_index, reference_index: ref_index }
+            end
+        end
+    
+        parent_references.each_with_index do |parent_ref, ref_index|
+            if !current_references.include?(parent_ref)
+                removed_references << { claim_key: claim_key, claim_index: claim_index, reference_index: ref_index }
+            end
+        end
+
+        {
+            added_references: added_references,
+            removed_references: removed_references,
+            changed_references: changed_references
+        }
+    end
+
+    # helper method for checking if a reference has been modified
+    def self.ref_modified?(current_reference, parent_references)
+        parent_references.each do |parent_reference|
+            if current_reference["snaks"] != parent_reference["snaks"]
+                return true
+            end
+        end
+        false
+    end
+end

data/lib/wikidata/diff/representation_analyzer.rb

@@ -0,0 +1,71 @@
+class RepresentationAnalyzer
+    def self.isolate_representation_differences(current_content, parent_content)
+        return {
+        changed: [],
+        removed: [],
+        added: []
+        } if current_content.nil? && parent_content.nil?
+    
+        if current_content
+            current_representations = current_content['representations']
+            if current_representations.nil? || current_representations.is_a?(Array)
+                current_representations = {}
+            end
+        else
+            current_representations = {}
+        end
+        
+        if parent_content
+            parent_representations = parent_content['representations'] 
+            if parent_representations.nil? || parent_representations.is_a?(Array)
+                parent_representations = {}
+            end
+        else
+            parent_representations = {}
+        end
+
+
+        changed = []
+        removed = []
+        added = []
+
+
+        # if parentid is 0, then add all labels as added and return it
+        if parent_content.nil?
+            current_representations.each do |lang, label|
+                added << { lang: lang }
+            end
+            return {
+                changed: changed,
+                removed: removed,
+                added: added
+            }
+        else
+
+        
+            # Iterate over each language in the current labels
+            (current_representations || {}).each do |lang, current_representation|
+                parent_representation =  parent_representations[lang]
+            
+                if parent_representation.nil?
+                    added << { lang: lang }
+                elsif current_representation != parent_representation
+                    changed << { lang: lang }
+                end
+                end
+            
+                # Iterate over each language in the parent labels to find removed labels
+            (parent_representations || {}).each do |lang, parent_representation|
+                if current_representations[lang].nil?
+                    removed << { lang: lang }
+                end
+            end
+        end
+    
+        {
+        changed: changed,
+        removed: removed,
+        added: added
+        }
+    end
+end

data/lib/wikidata/diff/revision_analyzer.rb

@@ -3,48 +3,164 @@ require_relative 'alias_analyzer'
 require_relative 'label_analyzer'
 require_relative 'description_analyzer'
 require_relative 'sitelink_analyzer'
+require_relative 'comment_analyzer'
+require_relative 'form_analyzer'  
+require_relative 'sense_analyzer'
+require_relative 'lemma_analyzer'
+
 
 class RevisionAnalyzer
+  CLAIM_TYPES = [:added_claims, :removed_claims, :changed_claims,:added_references, :removed_references, :changed_references,:added_qualifiers, :removed_qualifiers, :changed_qualifiers].freeze
+  ALIAS_TYPES = [:added_aliases, :removed_aliases, :changed_aliases].freeze
+  LABEL_TYPES = [:added_labels, :removed_labels, :changed_labels].freeze
+  DESCRIPTION_TYPES = [:added_descriptions, :removed_descriptions, :changed_descriptions].freeze
+  SITELINK_TYPES = [:added_sitelinks, :removed_sitelinks, :changed_sitelinks].freeze
+  COMMENT_TYPES = [:merge_to, :merge_from, :redirect, :undo, :restore, :clear_item].freeze
+  LEMMA_TYPES = [:added_lemmas, :removed_lemmas, :changed_lemmas].freeze
+  FORM_TYPES = [:added_forms, :removed_forms, :changed_forms, :added_representations, :removed_representations, :changed_representations, :added_formclaims, :removed_formclaims, :changed_formclaims].freeze
+  SENSE_TYPES = [:added_senses, :removed_senses, :changed_senses, :added_glosses, :removed_glosses, :changed_glosses, :added_senseclaims, :removed_senseclaims, :changed_senseclaims].freeze
+  NOT_IN_ITEM = [:create_lexeme, :create_property, :added_lemmas, :removed_lemmas, :changed_lemmas, :added_forms, :removed_forms, :changed_forms, :added_senses, :removed_senses, :changed_senses, :added_representations, :removed_representations, :changed_representations, :added_glosses, :removed_glosses, :changed_glosses, :added_formclaims, :removed_formclaims, :changed_formclaims, :added_senseclaims, :removed_senseclaims, :changed_senseclaims].freeze
+  NOT_IN_PROPERTY = [:create_lexeme, :create_item, :added_sitelinks, :removed_sitelinks, :changed_sitelinks, :added_lemmas, :removed_lemmas, :changed_lemmas, :added_forms, :removed_forms, :changed_forms, :added_senses, :removed_senses, :changed_senses, :added_representations, :removed_representations, :changed_representations, :added_glosses, :removed_glosses, :changed_glosses, :added_formclaims, :removed_formclaims, :changed_formclaims, :added_senseclaims, :removed_senseclaims, :changed_senseclaims].freeze
+  NOT_IN_LEXEME = [:create_item, :create_property, :added_sitelinks, :changed_sitelinks, :removed_sitelinks, :added_aliases, :changed_aliases, :removed_aliases, :added_labels, :changed_labels, :removed_labels, :added_descriptions, :changed_descriptions, :removed_descriptions].freeze
+  
   # This method takes two revisions as input and returns the differences between them.
-  def self.analyze_diff(current_content, parent_content)
+  def self.analyze_diff(revision_data)
+      model = revision_data[:model]
       diff = {}
-      # Calculate claim differences includes references and qualifiers
-      claim_diff = ClaimAnalyzer.isolate_claim_differences(current_content, parent_content)
-      diff[:added_claims] = claim_diff[:added_claims].length
-      diff[:removed_claims] = claim_diff[:removed_claims].length
-      diff[:changed_claims] = claim_diff[:changed_claims].length
-      diff[:added_references] = claim_diff[:added_references].length
-      diff[:removed_references] = claim_diff[:removed_references].length
-      diff[:changed_references] = claim_diff[:changed_references].length
-      diff[:added_qualifiers] = claim_diff[:added_qualifiers].length
-      diff[:removed_qualifiers] = claim_diff[:removed_qualifiers].length
-      diff[:changed_qualifiers] = claim_diff[:changed_qualifiers].length
+      if model == 'wikibase-item'
+        item(diff, revision_data)
+      elsif model == 'wikibase-property'
+        property(diff, revision_data)
+      elsif model == 'wikibase-lexeme'
+        lexeme(diff, revision_data)
+      end
+      diff
+  end
+
+  def self.item(diff, revision_data)
+    current_content = revision_data[:current_content]
+    parent_content = revision_data[:parent_content]
+    comment = revision_data[:comment]
+
+
+    claim_diff = ClaimAnalyzer.isolate_claims_differences(current_content, parent_content)
+    CLAIM_TYPES.each do |change_type|
+      diff[change_type] = claim_diff[change_type].length
+    end
+
+    alias_diff = AliasAnalyzer.isolate_aliases_differences(current_content, parent_content)
+    ALIAS_TYPES.each do |change_type|
+      diff[change_type] = alias_diff[change_type].length
+    end
+
+    # Calculate label differences
+    label_diff = LabelAnalyzer.isolate_labels_differences(current_content, parent_content)
+    LABEL_TYPES.each do |change_type|
+      diff[change_type] = label_diff[change_type].length
+    end
+
+    # Calculate description differences
+    description_diff = DescriptionAnalyzer.isolate_descriptions_differences(current_content, parent_content)
+    DESCRIPTION_TYPES.each do |change_type|
+      diff[change_type] = description_diff[change_type].length
+    end
+
+    # Calculate sitelink differences
+    sitelink_diff = SitelinkAnalyzer.isolate_sitelinks_differences(current_content, parent_content)
+    SITELINK_TYPES.each do |change_type|
+      diff[change_type] = sitelink_diff[change_type].length
+    end
+
+    phrases = CommentAnalyzer.isolate_comment_differences(comment)
+    COMMENT_TYPES.each do |change_type|
+      diff[change_type] = phrases[change_type]
+    end
+
+    NOT_IN_ITEM.each do |change_type|
+      diff[change_type] = 0
+    end
+
+    diff[:create_item] = phrases[:create_item]
+
+    diff
+  end
+
+  def self.property(diff, revision_data)
+    current_content = revision_data[:current_content]
+    parent_content = revision_data[:parent_content]
+    comment = revision_data[:comment]
+
+    claim_diff = ClaimAnalyzer.isolate_claims_differences(current_content, parent_content)
+    CLAIM_TYPES.each do |change_type|
+      diff[change_type] = claim_diff[change_type].length
+    end
+
+    alias_diff = AliasAnalyzer.isolate_aliases_differences(current_content, parent_content)
+    ALIAS_TYPES.each do |change_type|
+      diff[change_type] = alias_diff[change_type].length
+    end
+
+    # Calculate label differences
+    label_diff = LabelAnalyzer.isolate_labels_differences(current_content, parent_content)
+
+    LABEL_TYPES.each do |change_type|
+      diff[change_type] = label_diff[change_type].length
+    end
+
+    # Calculate description differences
+    description_diff = DescriptionAnalyzer.isolate_descriptions_differences(current_content, parent_content)
+    DESCRIPTION_TYPES.each do |change_type|
+      diff[change_type] = description_diff[change_type].length
+    end
+  
+
+    phrases = CommentAnalyzer.isolate_comment_differences(comment)
+    COMMENT_TYPES.each do |change_type|
+      diff[change_type] = phrases[change_type]
+    end
+
+    diff[:create_property] = phrases[:create_item]
+
+    NOT_IN_PROPERTY.each do |change_type|
+      diff[change_type] = 0
+    end
+  end
+
+  def self.lexeme(diff, revision_data)
+    current_content = revision_data[:current_content]
+    parent_content = revision_data[:parent_content]
+    comment = revision_data[:comment]
     
-      # Calculate alias differences
-      alias_diff = AliasAnalyzer.isolate_aliases_differences(current_content, parent_content)
-      diff[:added_aliases] = alias_diff[:added].length
-      diff[:removed_aliases] = alias_diff[:removed].length
-      diff[:changed_aliases] = alias_diff[:changed].length
-
-
-      # Calculate label differences
-      label_diff = LabelAnalyzer.isolate_labels_differences(current_content, parent_content)
-      diff[:added_labels] = label_diff[:added].length
-      diff[:removed_labels] = label_diff[:removed].length
-      diff[:changed_labels] = label_diff[:changed].length
-
-      # Calculate description differences
-      description_diff = DescriptionAnalyzer.isolate_descriptions_differences(current_content, parent_content)
-      diff[:added_descriptions] = description_diff[:added].length
-      diff[:removed_descriptions] = description_diff[:removed].length
-      diff[:changed_descriptions] = description_diff[:changed].length
-
-      # Calculate sitelink differences
-      sitelink_diff = SitelinkAnalyzer.isolate_sitelinks_differences(current_content, parent_content)
-      diff[:added_sitelinks] = sitelink_diff[:added].length
-      diff[:removed_sitelinks] = sitelink_diff[:removed].length
-      diff[:changed_sitelinks] = sitelink_diff[:changed].length
+    claim_diff = ClaimAnalyzer.isolate_claims_differences(current_content, parent_content)
+    CLAIM_TYPES.each do |change_type|
+      diff[change_type] = claim_diff[change_type].length
+    end
 
-      diff
+    forms_diff = FormAnalyzer.isolate_forms_differences(current_content, parent_content)
+    FORM_TYPES.each do |change_type|
+      diff[change_type] = forms_diff[change_type].length
+    end
+
+    # Calculate label differences
+    lemmas_diff = LemmaAnalyzer.isolate_lemmas_differences(current_content, parent_content)
+    LEMMA_TYPES.each do |change_type|
+      diff[change_type] = lemmas_diff[change_type].length
+    end
+
+    # Calculate description differences
+    senses_diff = SenseAnalyzer.isolate_senses_differences(current_content, parent_content)
+    SENSE_TYPES.each do |change_type|
+      diff[change_type] = senses_diff[change_type].length
+    end
+
+    phrases = CommentAnalyzer.isolate_comment_differences(comment)
+    COMMENT_TYPES.each do |change_type|
+      diff[change_type] = phrases[change_type]
+    end
+      
+    NOT_IN_LEXEME.each do |change_type|
+      diff[change_type] = 0
+    end
+    diff[:create_lexeme] = phrases[:create_item]
   end
 end

data/lib/wikidata/diff/sense_analyzer.rb

@@ -0,0 +1,106 @@
+require_relative 'gloss_analyzer'
+require_relative 'inside_claim_analyzer'
+class SenseAnalyzer
+    def self.isolate_senses_differences(current_content, parent_content)
+         # Initialize empty arrays to store the added, removed, and changed claims
+         added_senses = []
+         removed_senses = []
+         changed_senses = []
+         added_glosses = []
+         removed_glosses = []
+         changed_glosses = []
+         added_senseclaims = []
+         removed_senseclaims = []
+         changed_senseclaims = []
+ 
+         current_content_senses = current_content["senses"] if current_content
+         parent_content_senses = parent_content["senses"] if parent_content
+ 
+         if !current_content_senses.is_a?(Array) || !parent_content_senses.is_a?(Array)
+         return {
+            added_senses: added_senses,
+            removed_senses: removed_senses,
+            changed_senses: changed_senses,
+            added_glosses: added_glosses,
+            removed_glosses: removed_glosses,
+            changed_glosses: changed_glosses,
+            added_senseclaims: added_senseclaims,
+            removed_senseclaims: removed_senseclaims,
+            changed_senseclaims: changed_senseclaims
+         }
+         end
+ 
+         current_content_senses = current_content["senses"] || []
+         parent_content_senses = parent_content["senses"] || []
+         
+         # if parentid is 0, add all current claims as added claims and return it
+         if parent_content.nil?
+            current_content_senses.each_with_index do |current_claim, index|
+                 added_senses << { index: index }
+                 glosses = GlossAnalyzer.isolate_gloss_differences(current_claim, nil)
+                 added_glosses += glosses[:added]
+                 removed_glosses += glosses[:removed]
+                 changed_glosses += glosses[:changed]
+                 senseclaims = InsideClaimAnalyzer.isolate_inside_claim_differences(current_claim, nil)
+                 added_senseclaims += senseclaims[:added]
+                 removed_senseclaims += senseclaims[:removed]
+                 changed_senseclaims += senseclaims[:changed]
+            end
+         else
+            current_content_senses.each_with_index do |current_claim, index|
+                 parent_claim = parent_content_senses[index]
+                 if parent_claim.nil?
+                     # Claim was added
+                    added_senses << { index: index }
+                    glosses = GlossAnalyzer.isolate_gloss_differences(current_claim, parent_claim)
+                    added_glosses += glosses[:added]
+                    removed_glosses += glosses[:removed]
+                    changed_glosses += glosses[:changed]
+                    senseclaims = InsideClaimAnalyzer.isolate_inside_claim_differences(current_claim, nil)
+                    added_senseclaims += senseclaims[:added]
+                    removed_senseclaims += senseclaims[:removed]
+                    changed_senseclaims += senseclaims[:changed]
+                 elsif current_claim != parent_claim
+                     # Claim was changed
+                     changed_senses << { index: index }
+                     glosses = GlossAnalyzer.isolate_gloss_differences(current_claim, parent_claim)
+                    added_glosses += glosses[:added]
+                    removed_glosses += glosses[:removed]
+                    changed_glosses += glosses[:changed]
+                    senseclaims = InsideClaimAnalyzer.isolate_inside_claim_differences(current_claim, nil)
+                    added_senseclaims += senseclaims[:added]
+                    removed_senseclaims += senseclaims[:removed]
+                    changed_senseclaims += senseclaims[:changed]
+                 end
+             end
+         end
+ 
+         # Iterate over each claim key in the parent content
+         parent_content_senses.each_with_index do |parent_claim, index|
+             current_claim = current_content_senses[index]
+             if current_claim.nil?
+                # Claim was removed
+                removed_senses << { index: index }
+                glosses = GlossAnalyzer.isolate_gloss_differences(current_claim, parent_claim)
+                added_glosses += glosses[:added]
+                removed_glosses += glosses[:removed]
+                changed_glosses += glosses[:changed]
+                senseclaims = InsideClaimAnalyzer.isolate_inside_claim_differences(current_claim, nil)
+                added_senseclaims += senseclaims[:added]
+                removed_senseclaims += senseclaims[:removed]
+                changed_senseclaims += senseclaims[:changed]
+             end
+         end
+         {
+            added_senses: added_senses,
+            removed_senses: removed_senses,
+            changed_senses: changed_senses,
+            added_glosses: added_glosses,
+            removed_glosses: removed_glosses,
+            changed_glosses: changed_glosses,
+            added_senseclaims: added_senseclaims,
+            removed_senseclaims: removed_senseclaims,
+            changed_senseclaims: changed_senseclaims
+         }
+    end
+end

data/lib/wikidata/diff/sitelink_analyzer.rb

@@ -47,15 +47,11 @@ class SitelinkAnalyzer
           # All sitelinks are removed if current content is nil
           removed_sitelinks = parent_content['sitelinks']
         end
-      
-        # puts "Added sitelinks: #{added_sitelinks}"
-        # puts "Removed sitelinks: #{removed_sitelinks}"
-        # puts "Changed sitelinks: #{changed_sitelinks}"
     
         {
-          added: added_sitelinks,
-          removed: removed_sitelinks,
-          changed: changed_sitelinks
+          added_sitelinks: added_sitelinks,
+          removed_sitelinks: removed_sitelinks,
+          changed_sitelinks: changed_sitelinks
         }
     end
 end 

data/lib/wikidata/diff/total.rb

@@ -21,5 +21,36 @@ class Total
         total[:sitelinks_added] += diff_data[:added_sitelinks]
         total[:sitelinks_removed] += diff_data[:removed_sitelinks]
         total[:sitelinks_changed] += diff_data[:changed_sitelinks]
+        total[:lemmas_added] += diff_data[:added_lemmas]
+        total[:lemmas_removed] += diff_data[:removed_lemmas]
+        total[:lemmas_changed] += diff_data[:changed_lemmas]
+        total[:forms_added] += diff_data[:added_forms]
+        total[:forms_removed] += diff_data[:removed_forms]
+        total[:forms_changed] += diff_data[:changed_forms]
+        total[:representations_added] += diff_data[:added_representations]
+        total[:representations_removed] += diff_data[:removed_representations]
+        total[:representations_changed] += diff_data[:changed_representations]
+        total[:formclaims_added] += diff_data[:added_formclaims]
+        total[:formclaims_removed] += diff_data[:removed_formclaims]
+        total[:formclaims_changed] += diff_data[:changed_formclaims]
+        total[:senses_added] += diff_data[:added_senses]
+        total[:senses_removed] += diff_data[:removed_senses]
+        total[:senses_changed] += diff_data[:changed_senses]
+        total[:glosses_added] += diff_data[:added_glosses]
+        total[:glosses_removed] += diff_data[:removed_glosses]
+        total[:glosses_changed] += diff_data[:changed_glosses]
+        total[:senseclaims_added] += diff_data[:added_senseclaims]
+        total[:senseclaims_removed] += diff_data[:removed_senseclaims]
+        total[:senseclaims_changed] += diff_data[:changed_senseclaims]
+        total[:merge_from] += diff_data[:merge_from]
+        total[:merge_to] += diff_data[:merge_to]
+        total[:undo] += diff_data[:undo]
+        total[:restore] += diff_data[:restore]
+        total[:clear_item] += diff_data[:clear_item]
+        total[:create_item] += diff_data[:create_item]
+        total[:create_property] += diff_data[:create_property]
+        total[:create_lexeme] += diff_data[:create_lexeme]
+        total[:redirect] += diff_data[:redirect]
+
     end
 end

data/wikidata-diff-analyzer.gemspec

@@ -30,6 +30,7 @@ Gem::Specification.new do |spec|
   spec.bindir = "exe"
   spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
+  spec.files.reject! { |f| File.extname(f) == '.gem' }
 
   # Adding the dependencies
   spec.add_dependency 'json', '~> 2.1'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: wikidata-diff-analyzer
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 2.0.0
 platform: ruby
 authors:
 - Sulagna Saha
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2023-07-05 00:00:00.000000000 Z
+date: 2023-07-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json
@@ -34,6 +34,7 @@ extra_rdoc_files: []
 files:
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
+- CONTRIBUTING.md
 - Gemfile
 - Gemfile.lock
 - LICENSE
@@ -44,11 +45,19 @@ files:
 - lib/wikidata/diff/analyzer/version.rb
 - lib/wikidata/diff/api.rb
 - lib/wikidata/diff/claim_analyzer.rb
+- lib/wikidata/diff/comment_analyzer.rb
 - lib/wikidata/diff/description_analyzer.rb
+- lib/wikidata/diff/form_analyzer.rb
+- lib/wikidata/diff/gloss_analyzer.rb
+- lib/wikidata/diff/inside_claim_analyzer.rb
 - lib/wikidata/diff/label_analyzer.rb
 - lib/wikidata/diff/large_batches_analyzer.rb
-- lib/wikidata/diff/mediawiki_login.rb
+- lib/wikidata/diff/lemma_analyzer.rb
+- lib/wikidata/diff/qualifier_analyzer.rb
+- lib/wikidata/diff/reference_analyzer.rb
+- lib/wikidata/diff/representation_analyzer.rb
 - lib/wikidata/diff/revision_analyzer.rb
+- lib/wikidata/diff/sense_analyzer.rb
 - lib/wikidata/diff/sitelink_analyzer.rb
 - lib/wikidata/diff/total.rb
 - sig/wikidata/diff/analyzer.rbs

data/lib/wikidata/diff/mediawiki_login.rb

@@ -1,12 +0,0 @@
-require 'mediawiki_api'
-# to load env variable
-require 'dotenv/load'
-
-# THIS IS NOT WORKING YET
-class MediawikiLogin
-    def self.mediawiki_login
-        client = MediawikiApi::Client.new('https://www.mediawiki.org/w/api.php')
-        client.log_in(ENV['MEDIAWIKI_USERNAME'], ENV['MEDIAWIKI_PASSWORD'])
-        client.logged_in?  # Return whether login was successful
-    end
-end