wikian 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1b0fc9564425d05bbd600b386508385cbbd089f2a9174fcebee5020933aa8e2f
-  data.tar.gz: d56b498ab07f9fa12a754248560963cf7dfda45874c7540e6840f832bad605df
+  metadata.gz: 91aba0c780899759805ebdd5d68297d87ecce58afaac94f99d2314883c760343
+  data.tar.gz: 4b586d366edfcddc22db22a4e56255845e2da3545aee038a888b69db123f7dc4
 SHA512:
-  metadata.gz: 5eafc016cab2eae0f4ebb8288d41ba5e445562a5d51c9ec923be48a1b332988558ea1874a2f5e7a08a4cd7443e214257f0a1f593b74b8a426373dd26bcfc9895
-  data.tar.gz: d98f2043ada681f9fc2a88433c7e852908b2af9835efa3da518aa05619b8bb746442c247eda6611991d8209e5833f150b026a578abbcbdc7ebc910272f2d0ada
+  metadata.gz: 513a2b007c95ba82da1c701409a3cfe3323f406a97a13215db6edf4ea72693d1f11d43198a0f0c51dd7f82ff9d428f4b91cc21fed3dd46dcd63727a9936304ba
+  data.tar.gz: 58d5b4a8873584183b7efe189f56a8f3f762ff0326989cc38610fdc793ad8bb89310b1567536362c039bd0e87f7dc653b614ffe6c5cbd3f807a4d59927a341b6

data/Gemfile.lock

@@ -0,0 +1,21 @@
+PATH
+  remote: .
+  specs:
+    wikian (0.1.1)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    minitest (5.14.2)
+    rake (12.3.3)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  minitest (~> 5.0)
+  rake (~> 12.0)
+  wikian!
+
+BUNDLED WITH
+   2.1.4

data/README.md

@@ -1,40 +1,62 @@
 # Wikian
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/wikian`. To experiment with that code, run `bin/console` for an interactive prompt.
+Want to be happier while editing wiki files?
+Try [Wikian](https://rubygems.org/gems/wikian):
 
-TODO: Delete this and the text above, and describe your gem
-
-## Installation
+```
+$ gem install wikian
+```
 
-Add this line to your application's Gemfile:
+To use it, first create a wikipedia account and get a user name and a password.
+Then define and export the `WIKI_USER` and `SECRET_WIKI_PASS` variables in your `.bashrc`:
 
-```ruby
-gem 'wikian'
+```bash
+export WIKI_USER='Example_user'
+export SECRET_WIKI_PASS='example_password'
 ```
 
-And then execute:
+Wikian works across [Wikimedia sites](https://meta.wikimedia.org/wiki/Our_projects).
+Create a file following the convention:
 
-    $ bundle install
+```
+<article_name>.<site>.wiki
+```
 
-Or install it yourself as:
+## Examples
 
-    $ gem install wikian
+To append some text to your Wiktionary user profile:
 
-## Usage
+```bash
+$ cat User:$WIKI_USER.en.wikitionary.org.wiki
 
-TODO: Write usage instructions here
+== Last section==
+testing
 
-## Development
+$ wi post -p User:$WIKI_USER.en.wikitionary.org.wiki
+Article uploaded
+```
 
-After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+To edit an article first you need to get its wikitext.
+Generate a template `wiki.yml` config:
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+```
+wi g -t
+```
 
-## Contributing
+Then get an article to edit
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/wikian.
+```bash
+$ wi get https://en.wikipedia.org/wiki/Wikipedia:Sandbox
+Writing to Wikipedia:Sandbox.en.wikipedia.org.json
+Writing to Wikipedia:Sandbox.en.wikipedia.org.wiki
+```
 
+Edits the wiki file in Vim, \*ahem\*, in your favorite text editor.
+When ready, uploaded it:
 
-## License
+```bash
+$ wi post Wikipedia:Sandbox.en.wikipedia.org.wiki
+Article uploaded
+```
 
-The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+Vim users should try out [mediawiki.vim](https://en.wikipedia.org/wiki/Help:Text_editor_support#Vim), it's awesome.

data/exe/wi

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift File.expand_path('../lib', __dir__)
+
+require "wikian"
+
+Wikian.new(nil).help if ARGV.empty?
+
+Wikian.new(*ARGV).run

data/exe/wikian

@@ -1,5 +1,4 @@
 #!/usr/bin/env ruby
 
-$LOAD_PATH.unshift File.expand_path('../lib', __dir__)
-
-require "wikian"
+# fall back to `wi` executable
+system("wi #{ARGV.join(' ')}")

data/lib/wikian.rb

@@ -1,8 +1,122 @@
 $LOAD_PATH.unshift __dir__
 
-require "wikian/version"
+require 'wikian/subcommand'
+require 'wikian/get'
+require 'wikian/post'
+require 'wikian/search'
+require 'wikian/version'
 
-module Wikian
-  class Error < StandardError; end
-  # Your code goes here...
+# stdlib
+require 'fileutils'
+require 'json'
+require 'net/http'
+require 'open-uri'
+require 'yaml'
+
+# external libraries
+require 'byebug'
+
+class Array
+  # return true if `self` and `elms` have any element in common
+  def have?(elms)
+    (self & elms).length > 0 ? true : false
+  end
+  alias_method :has?, :have?
+end
+
+class Hash
+  # return a query string representation of a hash
+  def to_query
+    URI.decode(URI.encode_www_form(self))
+  end
+end
+
+class Wikian
+  class WikianError < StandardError; end
+  class UnknownSubcommandError < WikianError; end
+
+  attr_accessor :subcommand, :args
+
+  CONFIG_FILE = 'wiki.yml'
+  RESPONSE_FORMAT = 'json'
+
+  def initialize(*args)
+    @args = args
+  end
+
+  def run
+    if args.have?(%w(-h --help))
+      help
+    end
+
+    @subcommand = args.shift
+
+    raise(UnknownSubcommandError, "Unkown Subcommand") unless %w(g p s get post search).include?(subcommand)
+
+    if subcommand[0] == 'g'
+      api = Wikian::Get.new(args)
+      api.doit
+      api.extract_wikitext
+    elsif subcommand[0] == 's'
+      api = Wikian::Search.new(args)
+      api.doit
+    else
+      api = Wikian::Post.new(args)
+      api.post
+    end
+
+  rescue UnknownSubcommandError => e
+    puts "#{e.class} #{e.message}"
+  end
+
+  def help
+    puts <<~eos
+      Usage:
+        wiki [options] <subcommand>  [url|file]
+
+      Options:
+        -a, --append               append the input file
+        -c, --captcha ID:MESSAGE   captcha info
+        -d, --debug                debug
+        -m, --message MESSAGE      add a commit message (HIGHLY recommended)
+        -p, --prepend              prepend the input file
+        -r, --remove-cookie        remove API cookie
+        -t, --template             create template configuration file
+        -v, --version
+
+      Subcommands:
+        g, get           get wikitext file from a wikipedia article
+        p, post          post wikitext file to a wikipedia article
+        s, search        search wikitext file to a wikipedia article
+
+      Examples:
+        # create wiki.yml template
+        wiki -t
+
+        # download article and create response and wikitext files
+        wiki get https://en.wikipedia.org/wiki/Spider-Man
+
+        # upload file to English Wikipedia
+        wiki post Spider-Man.en.wikipedia.org.wiki
+
+        # upload file to Spanish Wikipedia
+        wiki post Spider-Man.es.wikipedia.org.wiki
+
+        # upload file to English Wiktionary
+        wiki file to Spider-Man.es.wiktionary.org.wiki
+
+        # append new section to article
+        wiki post -a Spider-Man-new-section.wiki
+
+        # heavy use of the API may require cache validation
+        wiki post -c 1234:someMessage spider-Man.wiki
+
+      Comments:
+        Posted files must follow the convention:
+          <article_name>.<host>.wiki
+        where <host> is a wikimedia site.
+        More info at: https://meta.wikimedia.org/wiki/Our_projects
+    eos
+    exit
+  end
 end

data/lib/wikian/get.rb

@@ -0,0 +1,96 @@
+class Wikian
+  class WikianGetError < StandardError; end
+  class ExtractWikiError < WikianGetError; end
+  class ArgumentRequiredError < WikianGetError; end
+
+  class Get < Subcommand
+    attr_accessor :title
+
+    def initialize(args)
+      raise ArgumentRequiredError if args.empty?
+
+      super
+
+      url = URI(args.find{|arg| arg =~ URI.regexp})
+
+      @title = File.basename(url.path)
+
+      @output_file = title + '.' + url.host
+
+      @params.merge!('titles' => title, 'format' => Wikian::RESPONSE_FORMAT)
+
+      @query = @params.to_query
+
+      @api_url = URI("https://#{url.host}/w/api.php?#{query}")
+    rescue => e
+      puts "#{e.class} #{e.message} in #{__FILE__}"
+      exit
+    end
+
+    # extract wikitext from the response file and save it into a `.wiki` file
+    #
+    # return: nil
+    def extract_wikitext
+      if !res['content-type'].match?('json') || !(pages = JSON.parse(res.body).dig('query','pages'))
+        raise ExtractWikiError, 'JSON response has no pages'
+      end
+
+      create_wiki = -> (title, revisions) do
+        revisions.each do |revision|
+          wiki_file= File.basename(response_file, File.extname(response_file)) + '.wiki'
+          if revision['revid'].nil? && revisions.size > 1
+            STDERR.puts "Warning: you should specify 'revid' in #{Wikian::CONFIG_FILE} to prevent overriding different revisions"
+          end
+          File.open(wiki_file,'w') do |f|
+            content = revision.dig('slots', 'main', 'content') ||
+                      revision.dig('slots', '*') ||
+                      revision.dig('*')
+            STDERR.puts "Warning: nil 'content' in #{Wikian::CONFIG_FILE}" unless content
+            STDERR.puts "Writing to #{wiki_file}"
+            f.puts content
+          end
+        end
+      end
+
+      # this is ugly, but Wikipedia is inconsistent in their JSON value for 'pages'. Sometimes it's a hash, sometimes it's an array.
+      if pages.respond_to? :keys
+        byebug
+        create_wiki.call(pages.values.first['title'], pages.values.first['revisions'])
+      else
+        pages.each do |page|
+          create_wiki.call(page['title'], page['revisions'])
+        end
+      end
+
+    rescue => e
+      puts "An error occurred while extracting the wikitext",
+           "Try using a new config file by pasing the '-t' option.",
+           "Or pass '-d' option for debugging"
+      exit
+    end
+
+    def template
+      <<~eos
+        meta:
+          headers:
+            user-agent: Wikian
+        api:
+          action:
+            - query
+          prop:
+            - revisions
+          rvprop:
+            - content
+              #rvsection:
+              #  - 0
+              #  - 2
+          rvslots:
+            - main
+          formatversion:
+            - 2
+          format:
+            - json
+      eos
+    end
+  end
+end

data/lib/wikian/post.rb

@@ -0,0 +1,140 @@
+class Wikian
+  class WikianPostError < StandardError; end
+  class WikiFileError < WikianPostError; end
+
+  class Post
+    attr_accessor :args, :baseurl, :header, :input_file, :debug, :login_token,
+                  :login_cookie, :csrf_token, :csrf_cookie, :query, :body_text, :username
+
+    def initialize(args)
+      @args = args
+
+      # input wikitext file
+      @input_file = args.find{|f| File.exist? f}
+      raise WikiFileError unless input_file
+
+      site = input_file.match(/\.(.*)\.wiki/)[1]
+
+      @baseurl = "https://#{site}/w/api.php"
+
+      @header = {}
+
+      @username = ENV['WIKI_USER']
+
+      @debug = (args & %w(-d --debug)).length > 0 ? true : false
+    rescue => e
+      puts "#{e.class} in #{__FILE__}"
+      exit
+    end
+
+    def post
+      # remove expired cookie
+      if expired_cookie? || args.have?(%w(-r --remove-cookie))
+        FileUtils.rm_f(csrf_cookie_file)
+      end
+
+      # csrf_cookie can be reused among multiple requests. But csrf_token must be updated on each request
+      if File.exist?(csrf_cookie_file)
+        @csrf_cookie = File.read(csrf_cookie_file)
+      else
+        get_login_token
+
+        get_csrf_cookie
+      end
+      get_csrf_token
+
+      build_query_string
+
+      upload_article
+    end
+
+    # check if the cookie is expired (older than an hour)
+    def expired_cookie?
+      File.exist?(csrf_cookie_file) && ((Time.now - File.open(csrf_cookie_file).stat.ctime)/3600 > 1)
+    end
+
+    def csrf_cookie_file
+      'csrf_cookie'
+    end
+
+    def remove_cookie_metadata(cookie)
+      cookie.gsub(/secure;|path=.*?[,;]|httponly[;,]|samesite=.*?[;,]|expires=.....*?[,;]|domain=.*?;|max-age=.*?[;,]/i,'').squeeze(' ')
+    end
+
+    def get_login_token
+      puts("Getting login token/cookie") if debug
+      url = URI("#{baseurl}?action=query&meta=tokens&format=json&type=login")
+      res = URI.open(url)
+      json = JSON.parse(res.read)
+      @login_token = json.dig('query','tokens','logintoken')
+      @login_cookie = remove_cookie_metadata(res.meta['set-cookie'])
+      puts(json) if debug
+    end
+
+    def get_csrf_cookie
+      puts("\nGetting csrf cookie using token #{login_token}") if debug
+      url = URI("#{baseurl}?action=login&lgname=#{username}&format=json")
+      req = Net::HTTP::Post.new(url, header.merge('cookie' => login_cookie, 'content-type' => 'application/x-www-form-urlencoded'))
+      req.set_form_data('lgpassword' => ENV['SECRET_WIKI_PASS'], 'lgtoken' => login_token)
+      http = Net::HTTP.new(url.host, url.port)
+      http.use_ssl = true
+      res=http.request(req)
+      @csrf_cookie = remove_cookie_metadata(res['set-cookie'])
+      File.write(csrf_cookie_file, @csrf_cookie)
+      puts(res.body) if debug
+    end
+
+    def get_csrf_token
+      puts("\nGetting csrf token using csrf cookies") if debug
+      url = URI("#{baseurl}?action=query&meta=tokens&format=json&type=csrf")
+      res = URI.open(url, header.merge('cookie' => csrf_cookie))
+      json = JSON.parse(res.read)
+      @csrf_token = json.dig('query','tokens','csrftoken')
+      puts(json) if debug
+    end
+
+    def build_query_string
+      params={}
+      params['action'] = 'edit'
+      params['format'] = Wikian::RESPONSE_FORMAT
+      params['title'] = input_file.sub(/\..*/,'')
+      wikitext = File.read(input_file)
+      if args.have?(%w(-a --append))
+        params['appendtext'] = wikitext
+      elsif args.have?(%w(-p --prepend))
+        params['prependtext'] = wikitext
+      else
+        # pass the wikitext in request body
+        @body_text = wikitext
+      end
+      if args.include?('-c')
+        params['captchaid'], params['captchaword'] = args[args.index('-c')+1].split(':')
+      end
+      if args.include?('-m')
+        params['summary'] = args[args.index('-m')+1]
+      end
+      @query = URI.encode_www_form(params)
+    end
+
+    def upload_article
+      puts("\nUploading the wiki article using csrf token #{csrf_token}") if debug
+      url = URI("#{baseurl}?#{query}")
+      req = Net::HTTP::Post.new(url, header.merge('cookie' => csrf_cookie, 'content-type' => 'application/x-www-form-urlencoded'))
+      http = Net::HTTP.new(url.host, url.port)
+      req_body = body_text.nil? ? {token: csrf_token} : {token: csrf_token, text: body_text}
+      req.set_form_data(req_body)
+      http = Net::HTTP.new(url.host, url.port)
+      http.use_ssl = true
+      res=http.request(req)
+      json = JSON.parse(res.body)
+      puts(json) if debug
+      if json.dig('error')
+        puts "An error occurred while uploding the file",
+             "Try pasing the '-r' option to remove '#{csrf_cookie_file}'",
+             "Or pass '-d' option for debugging"
+      else
+        puts "Article uploaded"
+      end
+    end
+  end
+end

data/lib/wikian/search.rb

@@ -0,0 +1,40 @@
+class Wikian
+  class WikianGetError < StandardError; end
+  class ArgumentRequiredError < WikianGetError; end
+
+  class Search < Subcommand
+    def initialize(args)
+      super
+
+      @output_file = yaml['api']['srsearch'].first
+
+      @params.merge!('format' => Wikian::RESPONSE_FORMAT)
+
+      @query = @params.to_query
+
+      @api_url = URI("https://#{yaml['meta']['site']}/w/api.php?#{query}")
+    rescue => e
+      puts "#{e.class} #{e.message} in #{__FILE__}"
+      exit
+    end
+
+    def template
+      <<~eos
+        # Get last 5 revisions of the Main Page.
+        meta:
+          site: en.wikipedia.org
+          headers:
+            user-agent: Wikian
+        api:
+          action:
+            - query
+          list:
+            - search
+          srsearch:
+            - Craig Noone
+          format:
+            - json
+      eos
+    end
+  end
+end

data/lib/wikian/subcommand.rb

@@ -0,0 +1,74 @@
+#!/usr/bin/env -S ruby -W0
+class Wikian
+  # class to be inherited by other Wikian classes
+  class Subcommand
+    attr_accessor :args, :res, :yaml, :query, :title, :api_url, :debug, :output_file
+
+    def initialize(args)
+      @args = args
+
+      if args.have?(%w(-t --template))
+        puts "Creating template #{Wikian::CONFIG_FILE}"
+        make_template
+        exit
+      end
+
+      @debug = (args & %w(-d --debug)).length > 0 ? true : false
+
+      @yaml=YAML.load(File.open(Wikian::CONFIG_FILE))
+
+      # some params like 'titles' can contain multiple entries joined by '|'. More info in Wikipedia API docs
+      @params = Hash[yaml['api'].keys.zip(yaml['api'].values.map{|arr| arr.join("|")})]
+    end
+
+    def make_template
+      if File.exist?(CONFIG_FILE)
+        puts "Overwrite existing '#{CONFIG_FILE}'? [yn]"
+        answer = STDIN.gets.chomp
+        (puts 'Bye'; exit) if answer != 'y'
+      end
+
+      File.open(CONFIG_FILE, 'w') do |f|
+        f.write template
+      end
+      exit
+    end
+
+    # HTTP response file name. Its extension depends on the 'content-type' header
+    def response_file
+      output_file + '.' + res['content-type'].split('/').last.sub(/;.*/,'')
+    end
+
+    # write response in to `response_file`
+    def write_response
+      STDERR.puts "Writing to #{response_file}"
+      File.open(response_file, 'w') do |f|
+        f.puts prettify(res.body)
+      end
+    end
+
+    def doit
+      puts api_url if debug
+
+      req = Net::HTTP::Get.new(api_url, yaml['meta']['headers'])
+
+      http = Net::HTTP.new(api_url.host, api_url.port)
+
+      http.use_ssl = true
+
+      @res=http.request(req)
+
+      write_response
+    rescue => e
+      puts "#{e.class} #{e.message} in #{__FILE__}"
+      exit
+    end
+
+    private
+
+    # if response is JSON prettify it, otherwise return it unchanged
+    def prettify(str)
+      res['content-type'].match?('json') ? JSON.pretty_generate(JSON.parse(str)) : str
+    end
+  end
+end

data/lib/wikian/version.rb

@@ -1,3 +1,3 @@
-module Wikian
-  VERSION = "0.1.0"
+class Wikian
+  VERSION = "0.1.1"
 end

metadata

@@ -1,19 +1,20 @@
 --- !ruby/object:Gem::Specification
 name: wikian
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - sergioro
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-09-03 00:00:00.000000000 Z
+date: 2020-09-11 00:00:00.000000000 Z
 dependencies: []
 description: Get and edit wikipedia articles
 email:
 - yo@sergioro.com
 executables:
+- wi
 - wikian
 extensions: []
 extra_rdoc_files: []
@@ -21,13 +22,20 @@ files:
 - ".gitignore"
 - ".travis.yml"
 - Gemfile
+- Gemfile.lock
 - LICENSE.txt
 - README.md
 - Rakefile
 - bin/console
 - bin/setup
+- exe/wi
 - exe/wikian
+- lib/.gitignore
 - lib/wikian.rb
+- lib/wikian/get.rb
+- lib/wikian/post.rb
+- lib/wikian/search.rb
+- lib/wikian/subcommand.rb
 - lib/wikian/version.rb
 - wikian.gemspec
 homepage: