wikian 0.1.10 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9b9d29a2c2542c0fab6ff8a49f2f3803250b497956b62b1ae4985e6edfb1aead
-  data.tar.gz: 524ab9f53e433589678b3751bac059fcec6293b2620f89523285dee5d5db75a0
+  metadata.gz: 66726501f7fd096888803fc73c97612ef35b26e3246c54c4f9fbb7ea887c9d78
+  data.tar.gz: 9cda60a1184cb39a79cec8a93cb36fba2cde382bb7d1d69ca7d1beb79f36dc4e
 SHA512:
-  metadata.gz: 12844ea63af9aba3181ace13c9f1c36dfb7448df719925f21bc93a4e15ec105e1846bfbbb47cb8101432682674bfb895baaefca22df5e1d45d6d76dbc821002e
-  data.tar.gz: 2e4eba98c8ba630050bf650b496f3221e388449ae963ab78c49cd0acf8708d76293905909e7bccd613a56eb5bb94bd5c9c5d239cb3fe3f4658bdafae6f872361
+  metadata.gz: 5370603bfe66ffe85be969cbb5eadcd4f96dedcc59c594effb81f76c1b7538ba9588f822e7ef50c52c04a46e54217e086d93cd64cda742607018d1e5c9d81dc9
+  data.tar.gz: b9039ccf6311700d85ce6b80f62a8a9238e08ba3727a684e82764ace39778975bff5f543932e91b84749f1e094e89cef68a671d24607026fab75300366d3a85c

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    wikian (0.1.10)
+    wikian (0.2.2)
 
 GEM
   remote: https://rubygems.org/

data/lib/wikian.rb

@@ -13,6 +13,7 @@ require 'fileutils'
 require 'json'
 require 'net/http'
 require 'open-uri'
+require 'tempfile'
 require 'yaml'
 
 class Wikian
@@ -43,6 +44,7 @@ class Wikian
       api = Wikian::Get.new(args)
       api.doit
       api.extract_wikitext
+      api.save_metadata
     elsif subcommand[0] == 's'
       api = Wikian::Search.new(args)
       api.doit
@@ -58,6 +60,15 @@ class Wikian
     puts "#{e.class} #{e.message} in #{__FILE__}"
   end
 
+  def self.meta_dir
+    '.wikian'
+  end
+
+  # file to store metadata of fetched articles
+  def self.meta_file
+    File.join(meta_dir, 'meta.yml')
+  end
+
   def help
     puts <<~eos
       Usage:
@@ -66,46 +77,49 @@ class Wikian
       Options:
         -a, --append               append the input file
         -c, --captcha ID:MESSAGE   captcha info
-        -d, --debug                debug
+        -d, --debug                print debugging messages
+        -f, --force                force post (this ignores edit conflicts)
+        -h, --help                 print this help message
         -m, --message MESSAGE      add a commit message (HIGHLY recommended)
         -p, --prepend              prepend the input file
         -r, --remove-cookie        remove API cookie
         -s, --section NUMBER       section to edit
         -t, --template             create template configuration file
-        -v, --version
+        -v, --version              print version number
 
       Subcommands:
-        c, contributions [N]    get user last N contributions. N defaults to 20
-        g, get                  get wikitext file from a wikipedia article
-        p, post                 post wikitext file to a wikipedia article
-        s, search               search wikitext file to a wikipedia article
+        c, contributions [N]    get user last N contributions (defaults to #{Contributions::DEFAULT_MAX_CONTRIBUTIONS})
+        g, get                  get wikitext from a Wikipedia article
+        p, post                 post wikitext to a Wikipedia article
+        s, search               search wikitext in Wikipedia
 
       Examples:
         # create wiki.yml template
-        wiki -t
+        wi g -t
 
-        # download article and create response and wikitext files
-        wiki get https://en.wikipedia.org/wiki/Spider-Man
+        # download article
+        wi get -t
+        wi get https://en.wikipedia.org/wiki/Wikipedia:Sandbox
 
-        # upload file to English Wikipedia
-        wiki post Spider-Man.en.wikipedia.org.wiki
+        # upload file
+        wi post Wikipedia:Sandbox.en.wikipedia.org.wiki
 
         # upload file to Spanish Wikipedia
-        wiki post Spider-Man.es.wikipedia.org.wiki
+        wi post Wikipedia:Sandbox.es.wikipedia.org.wiki
 
         # upload file to English Wiktionary
-        wiki file to Spider-Man.es.wiktionary.org.wiki
+        wi post Wikipedia:Sandbox.en.wiktionary.org.wiki
 
-        # append new section to article
-        wiki post -a Spider-Man-new-section.wiki
+        # append wikitext to section 2 of the article
+        wi post -a -s 2 Wikipedia:Sandbox.en.wikipedia.org.wiki
 
         # heavy use of the API may require cache validation
-        wiki post -c 1234:someMessage spider-Man.wiki
+        wi post -c 1234:someMessage spider-Man.wiki
 
       Comments:
         Posted files must follow the convention:
-          <article_name>.<host>.wiki
-        where <host> is a wikimedia site.
+          <article_name>.<site>.wiki
+        where <site> is a wikimedia site.
         More info at: https://meta.wikimedia.org/wiki/Our_projects
     eos
     exit

data/lib/wikian/contributions.rb

@@ -11,17 +11,17 @@ class Wikian
 
       max_contributions = args.find(&:numeric?) || DEFAULT_MAX_CONTRIBUTIONS
 
-      raise(BadUrlError, "Try passing the '-t' option") unless yaml['meta']['site']
+      raise(BadUrlError, "Try passing the '-t' option") unless config['meta']['site']
 
-      @output_file = 'User:' +  ENV['WIKI_USER'] + '.contributions.' + yaml['meta']['site']
+      @output_file = 'User:' +  ENV['WIKI_USER'] + '.contributions.' + config['meta']['site']
 
       @params.merge!('ucuser' => ENV['WIKI_USER'], 'uclimit' => max_contributions, 'format' => Wikian::RESPONSE_FORMAT)
 
       @query = @params.to_query
 
-      @api_url = URI("https://#{yaml['meta']['site']}/w/api.php?#{query}")
+      @api_url = URI("https://#{config['meta']['site']}/w/api.php?#{query}")
     rescue => e
-      puts "#{e.class} in #{__FILE__}. #{e.message}"
+      puts "#{e.class} in #{__FILE__}", e.message
       exit
     end
 
@@ -29,11 +29,9 @@ class Wikian
       <<~eos
         # for a list of parameters to use here see: https://www.mediawiki.org/wiki/API:Usercontribs
         meta:
-          http_method: get
           site: en.wikipedia.org
           headers:
-            #accept-encoding: gzip
-            user-agent:  Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:60.0) Gecko/20100101 Firefox/60.0
+            user-agent: Wikian
         api:
           action:
             - query

data/lib/wikian/get.rb

@@ -1,17 +1,16 @@
 class Wikian
   class WikianGetError < StandardError; end
-  class ExtractWikiError < WikianGetError; end
   class ArgumentRequiredError < WikianGetError; end
 
   class Get < Subcommand
-    attr_accessor :title
+    attr_accessor :title, :url, :latest_revision
 
     def initialize(args)
       raise ArgumentRequiredError if args.empty?
 
       super
 
-      url = URI(args.find{|arg| arg =~ URI.regexp})
+      @url = URI(args.find{|arg| arg =~ URI.regexp})
 
       raise BadUrlError unless url.path
 
@@ -25,7 +24,7 @@ class Wikian
 
       @api_url = URI("https://#{url.host}/w/api.php?#{query}")
     rescue => e
-      puts "#{e.class} in #{__FILE__}. #{e.message}"
+      puts "#{e.class} in #{__FILE__}", e.message
       exit
     end
 
@@ -33,46 +32,43 @@ class Wikian
     #
     # return: nil
     def extract_wikitext
-      if !res['content-type'].match?('json') || !(pages = JSON.parse(res.body).dig('query','pages'))
-        raise ExtractWikiError, 'JSON response has no pages'
-      end
+      pages = JSON.parse(res_body).dig('query','pages')
 
-      create_wiki = -> (title, revisions) do
-        revisions.each do |revision|
-          wiki_file= File.basename(response_file, File.extname(response_file)) + '.wiki'
-          if revision['revid'].nil? && revisions.size > 1
-            STDERR.puts "Warning: you should specify 'revid' in #{Wikian::CONFIG_FILE} to prevent overriding different revisions"
-          end
-          File.open(wiki_file,'w') do |f|
-            content = revision.dig('slots', 'main', 'content') ||
-                      revision.dig('slots', '*') ||
-                      revision.dig('*')
-            STDERR.puts "Warning: nil 'content' in #{Wikian::CONFIG_FILE}" unless content
-            STDERR.puts "Writing to #{wiki_file}"
-            f.puts content
-          end
-        end
-      end
+      # Wikipedia is inconsistent in their value for 'pages', it's sometimes a hash, sometimes an array
+      @latest_revision = (pages.respond_to?(:keys) ? pages.values.first : pages.first)['revisions'].first
+
+      content = latest_revision.dig('slots', 'main', 'content') ||
+                latest_revision.dig('slots', '*') ||
+                latest_revision.dig('*')
 
-      # this is ugly, but Wikipedia is inconsistent in their JSON value for 'pages'. Sometimes it's a hash, sometimes it's an array.
-      if pages.respond_to? :keys
-        create_wiki.call(pages.values.first['title'], pages.values.first['revisions'])
-      else
-        pages.each do |page|
-          create_wiki.call(page['title'], page['revisions'])
-        end
+      wiki_file= File.basename(response_file, File.extname(response_file)) + '.wiki'
+
+      File.open(wiki_file,'w') do |f|
+        STDERR.puts "Warning: nil 'content' in #{Wikian::CONFIG_FILE}" unless content
+        STDERR.puts "Writing to #{wiki_file}"
+        f.puts content
       end
 
     rescue => e
-      puts "An error occurred while extracting the wikitext",
-           "Try using a new config file by pasing the '-t' option.",
-           "Or pass the '-d' option for debugging"
+      puts "An error occurred while extracting the wikitext"
       exit
     end
 
+    # save article metadata
+    #
+    # metadata like article timestamp is used to solve edit conflicts
+    def save_metadata
+      FileUtils.mkdir_p(Wikian.meta_dir)
+
+      metadata = File.exist?(Wikian.meta_file) ? YAML.load(File.open(Wikian.meta_file)) : {'meta' => {}}
+      metadata['meta'].merge!(title => {'timestamp' => latest_revision['timestamp']})
+
+      File.write(Wikian.meta_file, YAML.dump(metadata))
+    end
+
     def template
       <<~eos
-        # for a list of parameters to use here see: https://www.mediawiki.org/wiki/API:Get_the_contents_of_a_page
+        # for a list of parameters to use here see: https://www.mediawiki.org/wiki/API:Revisions
         meta:
           headers:
             user-agent: Wikian
@@ -83,9 +79,10 @@ class Wikian
             - revisions
           rvprop:
             - content
-          #rvsection: # get specific sections
-          #  - 0
-          #  - 2
+            - timestamp
+         #rvsection: # get specific sections
+         #  - 0
+         #  - 2
           rvslots:
             - main
           formatversion:

data/lib/wikian/monkeypatches.rb

@@ -9,7 +9,7 @@ end
 class Hash
   # return a query string representation of a hash
   def to_query
-    URI.decode(URI.encode_www_form(self))
+    URI::DEFAULT_PARSER.unescape(URI.encode_www_form(self))
   end
 end
 

data/lib/wikian/post.rb

@@ -1,19 +1,24 @@
 class Wikian
   class WikianPostError < StandardError; end
   class WikiFileError < WikianPostError; end
+  class WikiFileNameError < WikianPostError; end
+  class WikiMergeError < WikianPostError; end
 
   class Post
     attr_accessor :args, :baseurl, :header, :input_file, :debug, :login_token,
-                  :login_cookie, :csrf_token, :csrf_cookie, :query, :body_text, :username
+                  :login_cookie, :csrf_token, :csrf_cookie, :query, :body_text,
+                  :username, :params, :latest_revision, :latest_content, :metadata
 
     def initialize(args)
       @args = args
 
+      long_to_short_options
+
       # input wikitext file
-      @input_file = args.find{|f| File.exist? f}
-      raise WikiFileError unless input_file
+      raise WikiFileError unless @input_file = args.find{|f| File.exist? f}
 
-      site = input_file.match(/\.(.*)\.wiki/)[1]
+      site = input_file.match(/\.(.*)\.wiki/)&.[](1)
+      raise(WikiFileNameError, "Use the Input file name convention <article_name>.<site>.wiki") unless site
 
       @baseurl = "https://#{site}/w/api.php"
 
@@ -21,15 +26,20 @@ class Wikian
 
       @username = ENV['WIKI_USER']
 
-      @debug = (args & %w(-d --debug)).length > 0 ? true : false
+      @debug = (args & %w(-d)).length > 0 ? true : false
     rescue => e
-      puts "#{e.class} in #{__FILE__}. #{e.message}"
+      puts "#{e.class} in #{__FILE__}", e.message
       exit
     end
 
+    # transform long options like '--message' to short options like '-m'
+    def long_to_short_options
+      args.map! {|opt| opt[0,2] == '--' ? opt[1,2] : opt}
+    end
+
     def post
       # remove expired cookie
-      if expired_cookie? || args.have?(%w(-r --remove-cookie))
+      if expired_cookie? || args.have?(%w(-r))
         FileUtils.rm_f(csrf_cookie_file)
       end
 
@@ -41,10 +51,17 @@ class Wikian
 
         get_csrf_cookie
       end
-      get_csrf_token
-
       build_query_string
 
+      get_latest_revision
+
+      if !args.include?('-f') && @body_text &&  Time.parse(params['starttimestamp']) < Time.parse(params['basetimestamp'])
+        puts "Edit conflict detected"
+        merge_versions(@body_text, latest_content)
+      end
+
+      get_csrf_token
+
       upload_article
     end
 
@@ -84,9 +101,19 @@ class Wikian
       puts(res.body) if debug
     end
 
+    def get_latest_revision
+      res = URI.open("#{baseurl}?action=query&prop=revisions&titles=#{params['title']}&rvslots=main&rvprop=content|timestamp&format=json")
+      @latest_revision = JSON.parse(res.read).dig('query', 'pages').values.first.dig('revisions').first
+      params['basetimestamp'] = latest_revision['timestamp']
+      @latest_content = latest_revision.dig('slots', 'main', 'content') ||
+                        latest_revision.dig('slots', 'main', '*') ||
+                        latest_revision.dig('slots', '*') ||
+                        latest_revision.dig('*')
+    end
+
     def get_csrf_token
       puts("\nGetting csrf token using csrf cookies") if debug
-      url = URI("#{baseurl}?action=query&meta=tokens&format=json&type=csrf")
+      url = URI("#{baseurl}?action=query&meta=tokens&format=json&prop=info|revisions&rvprop=timestamp")
       res = URI.open(url, header.merge('cookie' => csrf_cookie))
       json = JSON.parse(res.read)
       @csrf_token = json.dig('query','tokens','csrftoken')
@@ -94,32 +121,65 @@ class Wikian
     end
 
     def build_query_string
-      params={}
+      @params={}
       params['action'] = 'edit'
       params['format'] = Wikian::RESPONSE_FORMAT
       params['title'] = input_file.sub(/\..*/,'')
+      @metadata = File.exist?(Wikian.meta_file) ? YAML.load(File.open(Wikian.meta_file)) : {}
+      params['starttimestamp'] =
+        if timestamp = metadata.dig('meta', params['title'], 'timestamp')
+          timestamp
+        else
+          FileUtils.mkdir_p(Wikian.meta_dir)
+          metadata = {'meta' => {'title'  => {'timestamp' => File.mtime(input_file).utc.iso8601}}}
+          File.write(Wikian.meta_file, YAML.dump(metadata))
+        end
       wikitext = File.read(input_file)
-      if args.have?(%w(-a --append))
+      if args.have?(%w(-a))
         params['appendtext'] = wikitext
-      elsif args.have?(%w(-p --prepend))
+      elsif args.have?(%w(-p))
         params['prependtext'] = wikitext
       else
         # pass the wikitext in request body
         @body_text = wikitext
       end
-      if args.have?(%w(-c --captcha))
+      if args.have?(%w(-c))
         params['captchaid'], params['captchaword'] = args[args.index('-c')+1].split(':')
       end
-      if args.have?(%w(-m --message))
+      if args.have?(%w(-m))
         params['summary'] = args[args.index('-m')+1]
       end
-      if args.have?(%w(-s --section))
+      if args.have?(%w(-s))
         params['section'] = args[args.index('-s')+1]
       end
-      @query = URI.encode_www_form(params)
+    end
+
+    # merge two version with diff.
+    # TODO the merge command is ADDING differences, but it should MERGE differences.
+    def merge_versions(content_one, content_two)
+      tmp_local = Tempfile.open {|f| f.write content_one; f}
+      tmp_latest = Tempfile.open {|f| f.write content_two; f}
+
+      diff_cmd = "diff #{tmp_local.path} #{tmp_latest.path}"
+      system("#{diff_cmd}")
+
+      # raise error until the above TODO is solved
+      raise WikiMergeError, "Please merge with latest version and try again"
+
+      merge_cmd = "diff --line-format %L #{tmp_local.path} #{tmp_latest.path}"
+      @body_text = %x(#{merge_cmd})
+    rescue => e
+      puts "#{e.class} in #{__FILE__}", e.message
+      exit
+    end
+
+    def update_metadata
+      metadata['meta'].merge!(params['title'] => {'timestamp' => Time.now.utc.iso8601})
+      File.write(Wikian.meta_file, YAML.dump(metadata))
     end
 
     def upload_article
+      @query = URI.encode_www_form(params)
       puts("\nUploading the wiki article using csrf token #{csrf_token}") if debug
       url = URI("#{baseurl}?#{query}")
       req = Net::HTTP::Post.new(url, header.merge('cookie' => csrf_cookie, 'content-type' => 'application/x-www-form-urlencoded'))
@@ -136,6 +196,7 @@ class Wikian
              "Try pasing the '-r' option to remove '#{csrf_cookie_file}'",
              "Or pass '-d' option for debugging"
       else
+        update_metadata
         puts "Article uploaded"
       end
     end

data/lib/wikian/search.rb

@@ -6,15 +6,15 @@ class Wikian
     def initialize(args)
       super
 
-      @output_file = yaml['api']['srsearch'].first
+      @output_file = config['api']['srsearch'].first
 
       @params.merge!('format' => Wikian::RESPONSE_FORMAT)
 
       @query = @params.to_query
 
-      @api_url = URI("https://#{yaml['meta']['site']}/w/api.php?#{query}")
+      @api_url = URI("https://#{config['meta']['site']}/w/api.php?#{query}")
     rescue => e
-      puts "#{e.class} in #{__FILE__}. #{e.message}"
+      puts "#{e.class} in #{__FILE__}", e.message
       exit
     end
 

data/lib/wikian/subcommand.rb

@@ -1,12 +1,11 @@
-#!/usr/bin/env -S ruby -W0
 class Wikian
   class WikianSubcommandError < StandardError; end
-  class MissingConfigFileError < WikianSubcommandError; end
   class BadUrlError < WikianSubcommandError; end
 
   # class to be inherited by other Wikian classes
   class Subcommand
-    attr_accessor :args, :res, :yaml, :query, :title, :api_url, :debug, :output_file
+    attr_accessor :args, :res, :config, :query, :title, :api_url,
+                  :debug, :output_file, :res_body
 
     def initialize(args)
       @args = args
@@ -19,11 +18,15 @@ class Wikian
 
       @debug = (args & %w(-d --debug)).length > 0 ? true : false
 
-      raise MissingConfigFileError unless File.exist?(Wikian::CONFIG_FILE)
-      @yaml=YAML.load(File.open(Wikian::CONFIG_FILE))
+      @config =
+        if File.exist?(Wikian::CONFIG_FILE)
+          YAML.load(File.open(Wikian::CONFIG_FILE))
+        else
+          YAML.load(template)
+        end
 
       # some params like 'titles' can contain multiple entries joined by '|'. More info in Wikipedia API docs
-      @params = Hash[yaml['api'].keys.zip(yaml['api'].values.map{|arr| arr.join("|")})]
+      @params = Hash[config['api'].keys.zip(config['api'].values.map{|arr| arr.join("|")})]
     rescue MissingConfigFileError => e
       puts "#{e.class} try passing the '-t' option to generate #{Wikian::CONFIG_FILE} in #{__FILE__}"
       exit
@@ -42,31 +45,27 @@ class Wikian
 
     # HTTP response file name. Its extension depends on the 'content-type' header
     def response_file
-      output_file + '.' + res['content-type'].split('/').last.sub(/;.*/,'')
+      output_file + '.' + res.meta['content-type'].split('/').last.sub(/;.*/,'')
     end
 
     # write response in to `response_file`
     def write_response
       STDERR.puts "Writing to #{response_file}"
       File.open(response_file, 'w') do |f|
-        f.puts prettify(res.body)
+        f.puts prettify(res_body)
       end
     end
 
     def doit
       puts api_url if debug
 
-      req = Net::HTTP::Get.new(api_url, yaml['meta']['headers'])
+      @res=URI.open(api_url, config['meta']['headers'])
 
-      http = Net::HTTP.new(api_url.host, api_url.port)
-
-      http.use_ssl = true
-
-      @res=http.request(req)
+      @res_body = res.read
 
       write_response
     rescue => e
-      puts "#{e.class} in #{__FILE__}. #{e.message}"
+      puts "#{e.class} in #{__FILE__}", e.message
       exit
     end
 
@@ -74,7 +73,7 @@ class Wikian
 
     # if response is JSON prettify it, otherwise return it unchanged
     def prettify(str)
-      res['content-type'].match?('json') ? JSON.pretty_generate(JSON.parse(str)) : str
+      res.meta['content-type'].match?('json') ? JSON.pretty_generate(JSON.parse(str)) : str
     end
   end
 end

data/lib/wikian/version.rb

@@ -1,3 +1,3 @@
 class Wikian
-  VERSION = "0.1.10"
+  VERSION = "0.2.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: wikian
 version: !ruby/object:Gem::Version
-  version: 0.1.10
+  version: 0.2.2
 platform: ruby
 authors:
 - sergioro
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-09-13 00:00:00.000000000 Z
+date: 2020-09-15 00:00:00.000000000 Z
 dependencies: []
 description: Get and update Wikipedia articles
 email: