enwrite 0.2.0

data/lib/enwrite.rb

@@ -0,0 +1,299 @@
+#!/usr/bin/env ruby
+
+#
+# enwrite - power a web site using Evernote
+#
+# Diego Zamboni, March 2015
+# Time-stamp: <2015-04-30 13:39:37 diego>
+
+require 'rubygems'
+require 'bundler/setup'
+
+require "digest/md5"
+require 'evernote-thrift'
+require 'evernote-utils'
+require "optparse"
+require "ostruct"
+require 'util'
+require 'yaml'
+require 'deep_merge'
+
+class Enwrite
+  PLUGINS = %w[hugo]
+  module Version
+    MAJOR = 0
+    MINOR = 2
+    PATCH = 0
+
+    STRING = [MAJOR, MINOR, PATCH].compact.join('.')
+  end
+  
+  def self.run
+
+    options = OpenStruct.new
+    options.removetags = []
+    options.verbose = false
+    options.debug = false
+    options.outputplugin = 'hugo'
+    options.configtag = '_enwrite_config'
+    options.filestagprefix = '_enwrite_files'
+
+    opts = OptionParser.new do |opts|
+      def opts.version_string
+        "Enwrite v#{Enwrite::Version::STRING}"
+      end
+      
+      opts.banner = "#{opts.version_string}\n\nUsage: #{$0} [options] (at least one of -n or -s has to be specified)"
+
+      def opts.show_usage
+        puts self
+        exit
+      end
+
+      def opts.show_version
+        puts version_string
+        exit
+      end
+
+      opts.separator "\nSearch options:"
+      opts.on("-n", "--notebook NOTEBOOK",
+              "Process notes from specified notebook.") do |notebook|
+        options.notebook = notebook
+      end
+      opts.on("-t", "--tag TAG",
+              "Process only notes that have this tag",
+              "within the given notebook.") do |tag|
+        options.tag = tag
+      end
+      opts.on("--remove-tags [t1,t2,t3]", Array,
+              "List of tags to remove from output posts.",
+              "If no argument given, defaults to --tag.") do |removetags|
+        options.removetags = removetags || [options.tag]
+      end
+      opts.on("-s", "--search SEARCHEXP",
+              "Process notes that match given search",
+              "expression. If specified, --notebook",
+              "and --tag are ignored.") do |searchexp|
+        options.searchexp = searchexp
+        options.tag = nil
+        options.notebook = nil
+      end
+      opts.separator 'Output options:'
+      opts.on("-p", "--output-plugin PLUGIN", PLUGINS,
+              "Output plugin to use (Valid values: #{PLUGINS.join(', ')})") do |plugin|
+        options.outputplugin = plugin
+      end
+      opts.on("-o", "--output-dir OUTDIR",
+              "Base dir of hugo output installation") do |outdir|
+        options.outdir = outdir
+      end
+      opts.on("--rebuild-all",
+              "Process all notes that match the given",
+              "conditions (normally only updated notes",
+              "are processed)") { options.rebuild_all = true }
+      opts.separator 'Other options:'
+      opts.on("--auth [TOKEN]",
+              "Force Evernote reauthentication (will",
+              "happen automatically if needed). Use",
+              "TOKEN if given, otherwise get one",
+              "interactively.") do |forceauth|
+        options.forceauth = true
+        options.authtoken = forceauth
+      end
+      opts.on("--config-tag TAG",
+              "Specify tag to determine config notes",
+              "(default: #{options.configtag})") { |conftag|
+        options.configtag = conftag
+      }
+      opts.on_tail("-v", "--verbose", "Verbose mode") { options.verbose=true }
+      opts.on_tail("-d", "--debug", "Debug output mode") {
+        options.debug=true
+        options.verbose=true
+      }
+      opts.on_tail("--version", "Show version") { opts.show_version }
+      opts.on_tail("-h", "--help", "Shows this help message") { opts.show_usage }
+    end
+
+    opts.parse!
+
+    begin
+      eval "require 'output/#{options.outputplugin}'"
+    rescue LoadError
+      error "There was an error loading output module '#{plugin}': #{e.to_s}"
+      exit 1
+    end
+
+    $enwrite_verbose = options.verbose
+    $enwrite_debug = options.debug
+
+    verbose("Options: " + options.to_s)
+
+    if not (options.notebook or options.searchexp or options.forceauth)
+      error "You have to specify at least one of --notebook, --search or --auth"
+      exit(1)
+    end
+    exps = [ options.searchexp ? options.searchexp : nil,
+             options.notebook ? "notebook:#{options.notebook}" : nil,
+             options.tag ? "tag:#{options.tag}" : nil,
+           ].reject(&:nil?)
+    searchexp = exps.join(' ')
+
+    verbose "Output dir: #{options.outdir}"
+    verbose "Search expression: #{searchexp}"
+
+    begin
+      
+      # Initialize Evernote access
+      Evernote_utils.init(options.forceauth, options.authtoken)
+
+      if not searchexp # Only --auth was specified
+        exit 0
+      end
+      
+      updatecount_index = "updatecount_#{searchexp}"
+      latestUpdateCount = config(updatecount_index, 0)
+      if options.rebuild_all
+        msg "Processing ALL notes (--rebuild-all)."
+        latestUpdateCount = 0
+      end
+
+      verbose "Latest stored update count for #{searchexp}: #{latestUpdateCount}"
+
+      currentState = Evernote_utils.noteStore.getSyncState(Evernote_utils.authToken)
+      currentUpdateCount = currentState.updateCount
+
+      verbose "Current update count for the account: #{currentUpdateCount}"
+
+      if (currentUpdateCount > latestUpdateCount)
+        msg "Reading #{options.rebuild_all ? 'all' : 'updated'} notes that match #{searchexp}"
+
+        filter = Evernote::EDAM::NoteStore::NoteFilter.new
+        filter.words = searchexp
+        filter.order = Evernote::EDAM::Type::NoteSortOrder::UPDATE_SEQUENCE_NUMBER
+        filter.ascending = false
+
+        spec = Evernote::EDAM::NoteStore::NotesMetadataResultSpec.new
+        spec.includeTitle = true
+        spec.includeCreated = true
+        spec.includeTagGuids = true
+        spec.includeContentLength = true
+        spec.includeUpdateSequenceNum = true
+        spec.includeDeleted = true
+
+        results = Evernote_utils.noteStore.findNotesMetadata(Evernote_utils.authToken,
+                                                             filter,
+                                                             0,
+                                                             Evernote::EDAM::Limits::EDAM_USER_NOTES_MAX,
+                                                             spec)
+
+        # Get also deleted notes so we can remove from the blog
+        filter.inactive = true
+        delresults = Evernote_utils.noteStore.findNotesMetadata(Evernote_utils.authToken,
+                                                                filter,
+                                                                0,
+                                                                Evernote::EDAM::Limits::EDAM_USER_NOTES_MAX,
+                                                                spec)
+
+        # Go through the list looking for config notes, parse them and remove
+        # them from the list
+        enwriteconfig = { 'hugo' => {
+                            'base_dir' => options.outdir,
+                            'rebuild_all' => options.rebuild_all,
+                          },
+                        }
+
+        if Evernote_utils.tags.include?(options.configtag)
+          config_tag_guid = Evernote_utils.tags[options.configtag].guid
+          results.notes.select { |note|
+            note.tagGuids.include?(config_tag_guid)
+          }.each { |confignotemd|
+            msg "Found config note '#{confignotemd.title}'"
+            confignote = Evernote_utils.getWholeNote(confignotemd)
+            enml = ENML_utils.new(confignote.content)
+            configtext = enml.to_text
+            debug "   Config note text: '#{configtext}'"
+            configyaml = YAML.load(configtext)
+            debug "   Config note YAML: #{configyaml}"
+            enwriteconfig.deep_merge!(configyaml)
+            debug "   enwriteconfig = #{enwriteconfig}"
+            results.notes.delete(confignotemd)
+            results.totalNotes -= 1
+          }
+        end
+        verbose "Final enwrite config: #{enwriteconfig}"
+
+        files_tag = "#{options.filestagprefix}_#{options.outputplugin}"
+
+        if Evernote_utils.tags.include?(files_tag)
+          files_tag_guid = Evernote_utils.tags[files_tag].guid
+          results.notes.select { |note|
+            note.tagGuids.include?(files_tag_guid) &&
+              note.updateSequenceNum > latestUpdateCount
+          }.each { |filesnotemd|
+            msg "Found files note '#{filesnotemd.title}'"
+            filesnote = Evernote_utils.getWholeNote(filesnotemd)
+            enml = ENML_utils.new(filesnote.content, filesnote.resources)
+            files = enml.resource_files
+            Dir.chdir("#{options.outdir}")
+            files.each do |file|
+              case
+              when file[:basename] =~ /\.tar.gz$/
+                f = Tempfile.new('enwrite')
+                begin
+                  verbose "   Saving file #{file[:basename]} to #{f.path}"
+                  f.write(file[:data])
+                  f.close
+                  verbose "   Unpacking file #{f.path} with tar"
+                  ok = system("tar zxf #{f.path}")
+                  unless ok
+                    error "   An error occurred when unpacking #{f.path}"
+                  end
+                ensure
+                  f.close
+                  f.unlink
+                end
+              else
+                open("#{options.outdir}/#{file[:basename]}", "w") do |f|
+                  verbose "   Saving file #{f.path}"
+                  f.write(file[:data])
+                end
+              end
+            end
+            results.notes.delete(filesnotemd)
+            results.totalNotes -= 1
+          }
+        end
+
+        debug "Evaluating: #{options.outputplugin.capitalize}.new(enwriteconfig[options.outputplugin])"
+        writer = eval "#{options.outputplugin.capitalize}.new(enwriteconfig[options.outputplugin])"
+        
+        (results.notes + delresults.notes).select {
+          |note| note.updateSequenceNum > latestUpdateCount
+        }.sort_by {
+          |note| note.updateSequenceNum
+        }.each do |metadata|
+          verbose "######################################################################"
+          note = Evernote_utils.getWholeNote(metadata)
+          note.tagNames = note.tagNames - options.removetags
+          # This either creates or deletes posts as appropriate
+          writer.output_note(note)
+        end
+        # Persist the latest updatecount for next time
+        setconfig(updatecount_index, currentUpdateCount)
+
+        exit 0
+      else
+        msg "No updated notes that match #{searchexp}"
+        exit 1
+      end
+    rescue Evernote::EDAM::Error::EDAMUserException => e
+      #the exceptions that come back from Evernote are hard to read, but really important to keep track of
+      msg = "Caught an exception from Evernote trying to create a note.  #{Evernote_utils.translate_error(e)}"
+      raise msg
+    rescue Evernote::EDAM::Error::EDAMSystemException => e
+      #the exceptions that come back from Evernote are hard to read, but really important to keep track of
+      msg = "Caught an exception from Evernote trying to create a note.  #{Evernote_utils.translate_error(e)}"
+      raise msg
+    end
+  end
+end

data/lib/evernote-utils.rb

@@ -0,0 +1,234 @@
+#
+# Evernote access utilities
+#
+# Diego Zamboni, March 2015
+# Time-stamp: <2015-04-16 23:09:37 diego>
+
+# Load libraries required by the Evernote OAuth
+require 'oauth'
+require 'oauth/consumer'
+
+# Load Thrift & Evernote Ruby libraries
+require "evernote_oauth"
+
+class Evernote_utils
+
+  # Client credentials for enwrite
+  OAUTH_CONSUMER_KEY = "zzamboni-2648"
+  OAUTH_CONSUMER_SECRET = "05f988c37b5e8c68"
+  
+  # Connect to Sandbox server?
+  SANDBOX = false
+
+  # Environment variable in which to look for the token
+  ENVVAR = 'ENWRITE_AUTH_TOKEN'
+  
+  @@authToken = nil
+  @@userStore = nil
+  @@noteStore = nil
+  @@notebooks = nil
+  @@tags      = nil
+  @@forceAuth = false
+
+  def self.interactiveGetToken
+    callback_url = "http://zzamboni.org/enwrite-callback.html"
+    client = EvernoteOAuth::Client.new(token: nil, consumer_key: OAUTH_CONSUMER_KEY, consumer_secret: OAUTH_CONSUMER_SECRET, sandbox: SANDBOX)
+    request_token = client.request_token(:oauth_callback => callback_url)
+    authorize_url = request_token.authorize_url
+
+    puts("Welcome to enwrite's Evernote authentication.
+
+Please open the following URL:
+#{authorize_url}
+
+Once you authenticate you will be redirected to
+a page in the zzamboni.org domain that will show you an authentication verifier
+token. Please enter that token now.")
+    print("> ")
+    $stdout.flush
+    oauth_verifier = gets.chomp
+
+    access_token = request_token.get_access_token(:oauth_verifier => oauth_verifier)
+
+    puts("Thank you! Your access token is the following string:
+#{access_token.token}
+
+I can store the token for you in the config file (#{config_file}),
+then enwrite will use it automatically in the future.
+")
+    print "Would you like me to do that for you now (Y/n)? "
+    $stdout.flush
+    yesno = gets.chomp
+    if yesno =~ /^[yY]/
+      setconfig(:evernote_auth_token, access_token.token)
+      puts "Token stored."
+    else
+      puts "OK, I won't store the token, just use it for now.
+
+You can also store it in the ENWRITE_AUTH_TOKEN environment variable."
+    end
+    # Cancel force mode after we've gotten the token
+    @@forceAuth = false
+    
+    return access_token.token
+  end
+  
+  def self.getToken
+    if @@forceAuth
+      return self.interactiveGetToken
+    elsif not ENV[ENVVAR].nil?
+      return ENV[ENVVAR]
+    elsif not config(:evernote_auth_token).nil?
+      return config(:evernote_auth_token)
+    else
+      return self.interactiveGetToken
+    end
+  end
+  
+  def self.authToken
+    if @@authToken == nil || @@forceAuth
+      @@authToken = self.getToken
+    end
+    return @@authToken
+  end
+
+  def self.userStore
+    if @@userStore == nil
+      # Initial development is performed on our sandbox server. To use the production
+      # service, change "sandbox.evernote.com" to "www.evernote.com" and replace your
+      # developer token above with a token from
+      # https://www.evernote.com/api/DeveloperToken.action
+      evernoteHost = SANDBOX ? "sandbox.evernote.com" : "www.evernote.com"
+      userStoreUrl = "https://#{evernoteHost}/edam/user"
+
+      userStoreTransport = Thrift::HTTPClientTransport.new(userStoreUrl)
+      userStoreProtocol = Thrift::BinaryProtocol.new(userStoreTransport)
+      @@userStore = Evernote::EDAM::UserStore::UserStore::Client.new(userStoreProtocol)
+    end
+    return @@userStore
+  end
+
+  def self.checkVersion
+    versionOK = self.userStore.checkVersion("enwrite",
+				            Evernote::EDAM::UserStore::EDAM_VERSION_MAJOR,
+				            Evernote::EDAM::UserStore::EDAM_VERSION_MINOR)
+    verbose "Is my Evernote API version up to date?  #{versionOK}"
+    unless versionOK
+      error "Please update the Evernote Ruby libraries - they are not up to date."
+      exit(1)
+    end
+  end
+
+  def self.noteStore
+    if @@noteStore == nil
+      # Get the URL used to interact with the contents of the user's account
+      # When your application authenticates using OAuth, the NoteStore URL will
+      # be returned along with the auth token in the final OAuth request.
+      # In that case, you don't need to make this call.
+      noteStoreUrl = self.userStore.getNoteStoreUrl(self.authToken)
+
+      noteStoreTransport = Thrift::HTTPClientTransport.new(noteStoreUrl)
+      noteStoreProtocol = Thrift::BinaryProtocol.new(noteStoreTransport)
+      @@noteStore = Evernote::EDAM::NoteStore::NoteStore::Client.new(noteStoreProtocol)
+    end
+    return @@noteStore
+  end
+  
+  def self.notebooks(force=false)
+    if (@@notebooks == nil) or force
+      # List all of the notebooks in the user's account
+      @@notebooks = self.noteStore.listNotebooks(self.authToken)
+      verbose "Found #{notebooks.size} notebooks:"
+      defaultNotebook = notebooks.first
+      notebooks.each do |notebook|
+        verbose "  * #{notebook.name}"
+      end
+    end
+    return @@notebooks
+  end
+
+  def self.tags(force=false)
+    if (@@tags == nil) or force
+      verbose "Reading all tags:"
+      
+      # Get list of all tags, cache it for future use
+      taglist = self.noteStore.listTags(self.authToken)
+      # Create a hash for easier reference
+      @@tags = {}
+      tagstr = ""
+      for t in taglist
+        @@tags[t.guid] = t
+        @@tags[t.name] = t
+        tagstr += "#{t.name} "
+      end
+      verbose tagstr
+    end
+    return @@tags
+  end
+  
+  def self.init(force=false, token=nil)
+    @@forceAuth = force
+    if not token.nil?
+      @@forceAuth = false
+      @@authToken = token
+    end
+    self.authToken
+    self.userStore
+    self.checkVersion
+    self.noteStore
+    
+    self.notebooks
+    self.tags
+  end
+
+  def self.getWholeNote(metadata)
+    note = self.noteStore.getNote(self.authToken, metadata.guid, true, true, false, false)
+    note.tagNames = []
+    if metadata.tagGuids != nil
+      tags = Evernote_utils.tags
+      note.tagNames = metadata.tagGuids.map { |guid| tags[guid].name }
+    end
+    verbose "Tags: #{note.tagNames}"
+    return note
+  end
+
+  # From http://pollen.io/2012/12/creating-a-note-in-evernote-from-ruby/
+  # With changes to handle RATE_LIMIT_REACHED
+  def self.translate_error(e)
+    error_name = "unknown"
+    case e.errorCode
+    when Evernote::EDAM::Error::EDAMErrorCode::AUTH_EXPIRED
+      error_name = "AUTH_EXPIRED"
+    when Evernote::EDAM::Error::EDAMErrorCode::BAD_DATA_FORMAT
+      error_name = "BAD_DATA_FORMAT"
+    when Evernote::EDAM::Error::EDAMErrorCode::DATA_CONFLICT
+      error_name = "DATA_CONFLICT"
+    when Evernote::EDAM::Error::EDAMErrorCode::DATA_REQUIRED
+      error_name = "DATA_REQUIRED"
+    when Evernote::EDAM::Error::EDAMErrorCode::ENML_VALIDATION
+      error_name = "ENML_VALIDATION"
+    when Evernote::EDAM::Error::EDAMErrorCode::INTERNAL_ERROR
+      error_name = "INTERNAL_ERROR"
+    when Evernote::EDAM::Error::EDAMErrorCode::INVALID_AUTH
+      error_name = "INVALID_AUTH"
+    when Evernote::EDAM::Error::EDAMErrorCode::LIMIT_REACHED
+      error_name = "LIMIT_REACHED"
+    when Evernote::EDAM::Error::EDAMErrorCode::PERMISSION_DENIED
+      error_name = "PERMISSION_DENIED"
+    when Evernote::EDAM::Error::EDAMErrorCode::QUOTA_REACHED
+      error_name = "QUOTA_REACHED"
+    when Evernote::EDAM::Error::EDAMErrorCode::SHARD_UNAVAILABLE
+      error_name = "SHARD_UNAVAILABLE"
+    when Evernote::EDAM::Error::EDAMErrorCode::UNKNOWN
+      error_name = "UNKNOWN"
+    when Evernote::EDAM::Error::EDAMErrorCode::VALID_VALUES
+      error_name = "VALID_VALUES"
+    when Evernote::EDAM::Error::EDAMErrorCode::VALUE_MAP
+      error_name = "VALUE_MAP"
+    when Evernote::EDAM::Error::EDAMErrorCode::RATE_LIMIT_REACHED
+      error_name = "RATE_LIMIT_REACHED"
+      e.message = "Rate limit reached. Please retry in #{e.rateLimitDuration} seconds"
+    end
+    rv = "Error code was: #{error_name}[#{e.errorCode}] and parameter: [#{e.message}]"
+  end
+end