wikian 0.1.12 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dac3dedbc9c20098849e966beda97855a9a133d8b9334ad26d5a2a651ce18fa2
-  data.tar.gz: 347b7ead2d39dee453f122757f1cfb758559eb86e30ec924e55a6aca48101e4f
+  metadata.gz: 34b650b2c286774ef79f660715de782f54cd5b86d7690f6055dd662cdda8e1d0
+  data.tar.gz: 4b11d22fc6556d696b75dcd1d0373573e02a8eda58a02d661090be8faf03ec8d
 SHA512:
-  metadata.gz: 518a100775e95f60f37925142dafea7614298c654ff5a87f5358966dc7e40c25a3551f06c547dac971d81bf3589100967815104ee4fc3d6e239788a87553b8b0
-  data.tar.gz: a0230996611e118bb272dfdc83c8ef424bf180fb7b5fd51057f6ceee8a24e3ffe931493ce1b11ee77668d38c131346020d7e8342cd544c64147782d8e41d1c9d
+  metadata.gz: 9688be1c774d25208822ccba31d0381f512d9fb161c829631b22d70a824d841ed78b70303a32161f3826649fc1ec34a1af819724a1c34d6c0471d87cdf2ef92f
+  data.tar.gz: 5d3cb888c767d07f3ead8c9dbc56d962042dcb1a719274dd9e28493e8e146ed5807e646af17cf08219737b6e85f3ebe76c60d48854d73f4e1fa0438d2685b490

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    wikian (0.1.12)
+    wikian (0.2.0)
 
 GEM
   remote: https://rubygems.org/

data/lib/wikian.rb

@@ -13,6 +13,7 @@ require 'fileutils'
 require 'json'
 require 'net/http'
 require 'open-uri'
+require 'tempfile'
 require 'yaml'
 
 class Wikian
@@ -43,6 +44,7 @@ class Wikian
       api = Wikian::Get.new(args)
       api.doit
       api.extract_wikitext
+      api.save_metadata
     elsif subcommand[0] == 's'
       api = Wikian::Search.new(args)
       api.doit
@@ -58,6 +60,15 @@ class Wikian
     puts "#{e.class} #{e.message} in #{__FILE__}"
   end
 
+  def self.meta_dir
+    '.wikian'
+  end
+
+  # file to store metadata of fetched articles
+  def self.meta_file
+    File.join(meta_dir, 'meta.yml')
+  end
+
   def help
     puts <<~eos
       Usage:
@@ -76,7 +87,7 @@ class Wikian
         -v, --version              print version number
 
       Subcommands:
-        c, contributions [N]    get user last N contributions. N defaults to 20
+        c, contributions [N]    get user last N contributions (defaults to #{Contributions::DEFAULT_MAX_CONTRIBUTIONS})
         g, get                  get wikitext from a Wikipedia article
         p, post                 post wikitext to a Wikipedia article
         s, search               search wikitext in Wikipedia

data/lib/wikian/contributions.rb

@@ -11,15 +11,15 @@ class Wikian
 
       max_contributions = args.find(&:numeric?) || DEFAULT_MAX_CONTRIBUTIONS
 
-      raise(BadUrlError, "Try passing the '-t' option") unless yaml['meta']['site']
+      raise(BadUrlError, "Try passing the '-t' option") unless config['meta']['site']
 
-      @output_file = 'User:' +  ENV['WIKI_USER'] + '.contributions.' + yaml['meta']['site']
+      @output_file = 'User:' +  ENV['WIKI_USER'] + '.contributions.' + config['meta']['site']
 
       @params.merge!('ucuser' => ENV['WIKI_USER'], 'uclimit' => max_contributions, 'format' => Wikian::RESPONSE_FORMAT)
 
       @query = @params.to_query
 
-      @api_url = URI("https://#{yaml['meta']['site']}/w/api.php?#{query}")
+      @api_url = URI("https://#{config['meta']['site']}/w/api.php?#{query}")
     rescue => e
       puts "#{e.class} in #{__FILE__}. #{e.message}"
       exit

data/lib/wikian/get.rb

@@ -1,17 +1,16 @@
 class Wikian
   class WikianGetError < StandardError; end
-  class ExtractWikiError < WikianGetError; end
   class ArgumentRequiredError < WikianGetError; end
 
   class Get < Subcommand
-    attr_accessor :title
+    attr_accessor :title, :url, :latest_revision
 
     def initialize(args)
       raise ArgumentRequiredError if args.empty?
 
       super
 
-      url = URI(args.find{|arg| arg =~ URI.regexp})
+      @url = URI(args.find{|arg| arg =~ URI.regexp})
 
       raise BadUrlError unless url.path
 
@@ -33,46 +32,43 @@ class Wikian
     #
     # return: nil
     def extract_wikitext
-      if !res['content-type'].match?('json') || !(pages = JSON.parse(res.body).dig('query','pages'))
-        raise ExtractWikiError, 'JSON response has no pages'
-      end
+      pages = JSON.parse(res_body).dig('query','pages')
 
-      create_wiki = -> (title, revisions) do
-        revisions.each do |revision|
-          wiki_file= File.basename(response_file, File.extname(response_file)) + '.wiki'
-          if revision['revid'].nil? && revisions.size > 1
-            STDERR.puts "Warning: you should specify 'revid' in #{Wikian::CONFIG_FILE} to prevent overriding different revisions"
-          end
-          File.open(wiki_file,'w') do |f|
-            content = revision.dig('slots', 'main', 'content') ||
-                      revision.dig('slots', '*') ||
-                      revision.dig('*')
-            STDERR.puts "Warning: nil 'content' in #{Wikian::CONFIG_FILE}" unless content
-            STDERR.puts "Writing to #{wiki_file}"
-            f.puts content
-          end
-        end
-      end
+      # Wikipedia is inconsistent in their value for 'pages', it's sometimes a hash, sometimes an array
+      @latest_revision = (pages.respond_to?(:keys) ? pages.values.first : pages.first)['revisions'].first
+
+      content = latest_revision.dig('slots', 'main', 'content') ||
+                latest_revision.dig('slots', '*') ||
+                latest_revision.dig('*')
 
-      # this is ugly, but Wikipedia is inconsistent in their JSON value for 'pages'. Sometimes it's a hash, sometimes it's an array.
-      if pages.respond_to? :keys
-        create_wiki.call(pages.values.first['title'], pages.values.first['revisions'])
-      else
-        pages.each do |page|
-          create_wiki.call(page['title'], page['revisions'])
-        end
+      wiki_file= File.basename(response_file, File.extname(response_file)) + '.wiki'
+
+      File.open(wiki_file,'w') do |f|
+        STDERR.puts "Warning: nil 'content' in #{Wikian::CONFIG_FILE}" unless content
+        STDERR.puts "Writing to #{wiki_file}"
+        f.puts content
       end
 
     rescue => e
-      puts "An error occurred while extracting the wikitext",
-           "Try using a new config file by pasing the '-t' option.",
-           "Or pass the '-d' option for debugging"
+      puts "An error occurred while extracting the wikitext"
       exit
     end
 
+    # save article metadata
+    #
+    # metadata like article timestamp is used to solve edit conflicts
+    def save_metadata
+      FileUtils.mkdir_p(Wikian.meta_dir)
+
+      metadata = File.exist?(Wikian.meta_file) ? YAML.load(File.open(Wikian.meta_file)) : {}
+      metadata['meta'].merge!(title => {'timestamp' => latest_revision['timestamp']})
+
+      File.write(Wikian.meta_file, YAML.dump(metadata))
+    end
+
     def template
       <<~eos
-        # for a list of parameters to use here see: https://www.mediawiki.org/wiki/API:Get_the_contents_of_a_page
+        # for a list of parameters to use here see: https://www.mediawiki.org/wiki/API:Revisions
         meta:
           headers:
             user-agent: Wikian
@@ -83,9 +79,10 @@ class Wikian
             - revisions
           rvprop:
             - content
-          #rvsection: # get specific sections
-          #  - 0
-          #  - 2
+            - timestamp
+         #rvsection: # get specific sections
+         #  - 0
+         #  - 2
           rvslots:
             - main
           formatversion:

data/lib/wikian/monkeypatches.rb

@@ -9,7 +9,7 @@ end
 class Hash
   # return a query string representation of a hash
   def to_query
-    URI.decode(URI.encode_www_form(self))
+    URI::DEFAULT_PARSER.unescape(URI.encode_www_form(self))
   end
 end
 

data/lib/wikian/post.rb

@@ -1,19 +1,24 @@
 class Wikian
   class WikianPostError < StandardError; end
   class WikiFileError < WikianPostError; end
+  class WikiFileNameError < WikianPostError; end
+  class WikiMergeError < WikianPostError; end
 
   class Post
     attr_accessor :args, :baseurl, :header, :input_file, :debug, :login_token,
-                  :login_cookie, :csrf_token, :csrf_cookie, :query, :body_text, :username
+                  :login_cookie, :csrf_token, :csrf_cookie, :query, :body_text,
+                  :username, :params, :latest_revision, :latest_content, :metadata
 
     def initialize(args)
       @args = args
 
+      long_to_short_options
+
       # input wikitext file
-      @input_file = args.find{|f| File.exist? f}
-      raise WikiFileError unless input_file
+      raise WikiFileError unless @input_file = args.find{|f| File.exist? f}
 
-      site = input_file.match(/\.(.*)\.wiki/)[1]
+      site = input_file.match(/\.(.*)\.wiki/)&.[](1)
+      raise(WikiFileNameError, "Use the Input file name convention <article_name>.<site>.wiki") unless site
 
       @baseurl = "https://#{site}/w/api.php"
 
@@ -21,15 +26,20 @@ class Wikian
 
       @username = ENV['WIKI_USER']
 
-      @debug = (args & %w(-d --debug)).length > 0 ? true : false
+      @debug = (args & %w(-d)).length > 0 ? true : false
     rescue => e
       puts "#{e.class} in #{__FILE__}. #{e.message}"
       exit
     end
 
+    # transform long options like '--message' to short options like '-m'
+    def long_to_short_options
+      args.map! {|opt| opt[0,2] == '--' ? opt[1,2] : opt}
+    end
+
     def post
       # remove expired cookie
-      if expired_cookie? || args.have?(%w(-r --remove-cookie))
+      if expired_cookie? || args.have?(%w(-r))
         FileUtils.rm_f(csrf_cookie_file)
       end
 
@@ -41,10 +51,17 @@ class Wikian
 
         get_csrf_cookie
       end
-      get_csrf_token
-
       build_query_string
 
+      get_latest_revision
+
+      if @body_text &&  Time.parse(params['starttimestamp']) < Time.parse(params['basetimestamp'])
+        puts "\e[31mEdit conflict detected, merging with latest version\e[m"
+        merge_versions
+      end
+
+      get_csrf_token
+
       upload_article
     end
 
@@ -84,9 +101,19 @@ class Wikian
       puts(res.body) if debug
     end
 
+    def get_latest_revision
+      res = URI.open("#{baseurl}?action=query&prop=revisions&titles=#{params['title']}&rvslots=main&rvprop=content|timestamp&format=json")
+      @latest_revision = JSON.parse(res.read).dig('query', 'pages').values.first.dig('revisions').first
+      params['basetimestamp'] = latest_revision['timestamp']
+      @latest_content = latest_revision.dig('slots', 'main', 'content') ||
+                        latest_revision.dig('slots', 'main', '*') ||
+                        latest_revision.dig('slots', '*') ||
+                        latest_revision.dig('*')
+    end
+
     def get_csrf_token
       puts("\nGetting csrf token using csrf cookies") if debug
-      url = URI("#{baseurl}?action=query&meta=tokens&format=json&type=csrf")
+      url = URI("#{baseurl}?action=query&meta=tokens&format=json&prop=info|revisions&rvprop=timestamp")
       res = URI.open(url, header.merge('cookie' => csrf_cookie))
       json = JSON.parse(res.read)
       @csrf_token = json.dig('query','tokens','csrftoken')
@@ -94,32 +121,51 @@ class Wikian
     end
 
     def build_query_string
-      params={}
+      @params={}
       params['action'] = 'edit'
       params['format'] = Wikian::RESPONSE_FORMAT
       params['title'] = input_file.sub(/\..*/,'')
+      @metadata =  File.exist?(Wikian.meta_file) ? YAML.load(File.open(Wikian.meta_file)) : {}
+      params['starttimestamp'] =
+        if timestamp = metadata.dig('meta', params['title'], 'timestamp')
+          timestamp
+        else
+          FileUtils.mkdir_p(Wikian.meta_dir)
+          metadata = {'meta' => {'title'  => {'timestamp' => File.ctime(input_file).utc.iso8601}}}
+          File.write(Wikian.meta_file, YAML.dump(metadata))
+        end
       wikitext = File.read(input_file)
-      if args.have?(%w(-a --append))
+      if args.have?(%w(-a))
         params['appendtext'] = wikitext
-      elsif args.have?(%w(-p --prepend))
+      elsif args.have?(%w(-p))
         params['prependtext'] = wikitext
       else
         # pass the wikitext in request body
         @body_text = wikitext
       end
-      if args.have?(%w(-c --captcha))
+      if args.have?(%w(-c))
         params['captchaid'], params['captchaword'] = args[args.index('-c')+1].split(':')
       end
-      if args.have?(%w(-m --message))
+      if args.have?(%w(-m))
         params['summary'] = args[args.index('-m')+1]
       end
-      if args.have?(%w(-s --section))
+      if args.have?(%w(-s))
         params['section'] = args[args.index('-s')+1]
       end
-      @query = URI.encode_www_form(params)
+    end
+
+    def merge_versions
+      tmp_local = Tempfile.open {|f| f.write @body_text; f}
+      tmp_latest = Tempfile.open {|f| f.write latest_content; f}
+      @body_text = %x(diff --line-format %L #{tmp_local.path} #{tmp_latest.path})
+      metadata['meta'].merge!(params['title'] => {'timestamp' => Time.now.utc.iso8601})
+    rescue => e
+      puts "WikiMergeError in #{__FILE__}"
+      exit
     end
 
     def upload_article
+      @query = URI.encode_www_form(params)
       puts("\nUploading the wiki article using csrf token #{csrf_token}") if debug
       url = URI("#{baseurl}?#{query}")
       req = Net::HTTP::Post.new(url, header.merge('cookie' => csrf_cookie, 'content-type' => 'application/x-www-form-urlencoded'))

data/lib/wikian/search.rb

@@ -6,13 +6,13 @@ class Wikian
     def initialize(args)
       super
 
-      @output_file = yaml['api']['srsearch'].first
+      @output_file = config['api']['srsearch'].first
 
       @params.merge!('format' => Wikian::RESPONSE_FORMAT)
 
       @query = @params.to_query
 
-      @api_url = URI("https://#{yaml['meta']['site']}/w/api.php?#{query}")
+      @api_url = URI("https://#{config['meta']['site']}/w/api.php?#{query}")
     rescue => e
       puts "#{e.class} in #{__FILE__}. #{e.message}"
       exit

data/lib/wikian/subcommand.rb

@@ -1,12 +1,11 @@
-#!/usr/bin/env -S ruby -W0
 class Wikian
   class WikianSubcommandError < StandardError; end
-  class MissingConfigFileError < WikianSubcommandError; end
   class BadUrlError < WikianSubcommandError; end
 
   # class to be inherited by other Wikian classes
   class Subcommand
-    attr_accessor :args, :res, :yaml, :query, :title, :api_url, :debug, :output_file
+    attr_accessor :args, :res, :config, :query, :title, :api_url,
+                  :debug, :output_file, :res_body
 
     def initialize(args)
       @args = args
@@ -19,11 +18,15 @@ class Wikian
 
       @debug = (args & %w(-d --debug)).length > 0 ? true : false
 
-      raise MissingConfigFileError unless File.exist?(Wikian::CONFIG_FILE)
-      @yaml=YAML.load(File.open(Wikian::CONFIG_FILE))
+      @config =
+        if File.exist?(Wikian::CONFIG_FILE)
+          YAML.load(File.open(Wikian::CONFIG_FILE))
+        else
+          YAML.load(template)
+        end
 
       # some params like 'titles' can contain multiple entries joined by '|'. More info in Wikipedia API docs
-      @params = Hash[yaml['api'].keys.zip(yaml['api'].values.map{|arr| arr.join("|")})]
+      @params = Hash[config['api'].keys.zip(config['api'].values.map{|arr| arr.join("|")})]
     rescue MissingConfigFileError => e
       puts "#{e.class} try passing the '-t' option to generate #{Wikian::CONFIG_FILE} in #{__FILE__}"
       exit
@@ -42,27 +45,23 @@ class Wikian
 
     # HTTP response file name. Its extension depends on the 'content-type' header
     def response_file
-      output_file + '.' + res['content-type'].split('/').last.sub(/;.*/,'')
+      output_file + '.' + res.meta['content-type'].split('/').last.sub(/;.*/,'')
     end
 
     # write response in to `response_file`
     def write_response
       STDERR.puts "Writing to #{response_file}"
       File.open(response_file, 'w') do |f|
-        f.puts prettify(res.body)
+        f.puts prettify(res_body)
       end
     end
 
     def doit
       puts api_url if debug
 
-      req = Net::HTTP::Get.new(api_url, yaml['meta']['headers'])
+      @res=URI.open(api_url, config['meta']['headers'])
 
-      http = Net::HTTP.new(api_url.host, api_url.port)
-
-      http.use_ssl = true
-
-      @res=http.request(req)
+      @res_body = res.read
 
       write_response
     rescue => e
@@ -74,7 +73,7 @@ class Wikian
 
     # if response is JSON prettify it, otherwise return it unchanged
     def prettify(str)
-      res['content-type'].match?('json') ? JSON.pretty_generate(JSON.parse(str)) : str
+      res.meta['content-type'].match?('json') ? JSON.pretty_generate(JSON.parse(str)) : str
     end
   end
 end

data/lib/wikian/version.rb

@@ -1,3 +1,3 @@
 class Wikian
-  VERSION = "0.1.12"
+  VERSION = "0.2.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: wikian
 version: !ruby/object:Gem::Version
-  version: 0.1.12
+  version: 0.2.0
 platform: ruby
 authors:
 - sergioro
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-09-13 00:00:00.000000000 Z
+date: 2020-09-15 00:00:00.000000000 Z
 dependencies: []
 description: Get and update Wikipedia articles
 email: