enwrite 0.2.0

data/lib/filters.rb

@@ -0,0 +1,64 @@
+# coding: utf-8
+require 'htmlentities'
+
+module Filters
+
+  def run_filters(text)
+    newtext = text
+    text.scan(/(\[(\w+)([^\]]*)\])/) do |m|
+      match = m[0]
+      debug "match = #{match}"
+      filter = m[1]
+      args = m[2]
+      if Filters.method_defined?("filter_#{filter}")
+        fn=Filters.method("filter_#{filter}")
+        arg = {}
+        args = HTMLEntities.new.decode(args)
+        debug "args = #{args}"
+        args.scan(/\b(\w+)=["“]([^"]*)["”]/) { |a|
+          arg[a[0]] = a[1]
+        }
+        verbose "Calling filter_#{filter} with args #{arg}"
+        result = fn.(arg)
+        if not result.nil?
+          newtext = newtext.gsub(match, result)
+        end
+      else
+        # We only produce this message in verbose mode because it gets triggered
+        # every time [some text in brackets] is used
+        warn("Warning: nonexistent filter #{filter} used, leaving text as is") if $enwrite_verbose
+      end
+    end
+    debug "After running filters:"
+    debug newtext
+    return newtext
+  end
+  
+  def filter_youtube(args)
+    if args.include?('url')
+      args['src'] = args['url']
+      args.delete('url')
+    elsif args.include?('id')
+      args['src'] = "https://www.youtube.com/embed/#{args['id']}"
+    end
+    if args['src'].nil?
+      return nil
+    end
+    args['src'].gsub!(/\/watch\?v=/, "/embed/")
+    return "<iframe "+args.each.map{ |k,v| "#{k}=\"#{v}\"" }.join(" ")+"></iframe>"
+  end
+
+  def filter_gist(args)
+    if args.include?('url')
+      args['src'] = args['url']
+      args.delete('url')
+    end
+    if args['src'].nil?
+      return nil
+    end
+    if not args['src'] =~ /\.js$/
+      args['src'] += ".js"
+    end
+    return "<script "+args.each.map{ |k,v| "#{k}=\"#{v}\"" }.join(" ")+"></script>"
+  end
+end

data/lib/output.rb

@@ -0,0 +1,9 @@
+# Base class to output pages
+#
+# Diego Zamboni, March 2015
+# Time-stamp: <2015-03-29 00:51:04 diego>
+
+class Output
+  def output_note(metadata, note)
+  end
+end

data/lib/output/hugo.rb

@@ -0,0 +1,236 @@
+#
+# Output class for Hugo
+#
+# Diego Zamboni, March 2015
+# Time-stamp: <2015-04-30 12:40:31 diego>
+
+require 'output'
+require 'filters'
+require 'enml-utils'
+require 'fileutils'
+require 'yaml/store'
+require 'digest'
+
+include Filters
+
+class Hugo < Output
+  def initialize(opts = {})
+    @opts = opts
+    @base_dir = opts['base_dir']
+    unless @base_dir
+      error "The 'base_dir' option of the Hugo plugin must be set!"
+    end
+    @use_filters = opts['use_filters'] || true
+    @rebuild_all = opts['rebuild_all'] || false
+
+    # Persistent store for this base_dir
+    datadir = "#{@base_dir}/data"
+    FileUtils.mkdir_p datadir
+    @config_store = YAML::Store.new("#{datadir}/enwrite_data.yaml")
+
+    # Initialize GUID-to-filename map if needed
+    @config_store.transaction { @config_store[:note_files] = {} unless @config_store[:note_files] }
+    
+    # These are [ realpath, urlpath ]
+    @static_dir = [ "#{@base_dir}/#{opts['static_subdir'] || 'static' }", opts['static_url'] || "" ]
+    @static_subdirs = { 'image' => opts['image_subdir'] || 'img',
+                        'audio' => opts['audio_subdir'] || 'audio',
+                        'video' => opts['video_subdir'] || 'video',
+                        'files' => opts['files_subdir'] || 'files',
+                      }
+
+    # Tag-to-type map
+    @tag_to_type = opts['tag_to_type'] || { "default" => "post/",
+                                            "post" => "post/",
+                                            "page" => "" }
+    @tag_to_type_order = opts['tag_to_type_order'] || [ "post", "page", "default" ]
+
+    @tag_to_type_order.each { |type|
+      @tag_to_type[type] = "" unless @tag_to_type.include?(type)
+      @tag_to_type[type] = "" if @tag_to_type[type].nil?
+    }
+    
+    # Markdown tag
+    @markdown_tag = opts['markdown_tag'] || "markdown"
+
+    # Command to run hugo
+    @hugo_cmd = opts['hugo_cmd'] || "hugo"
+  end
+
+  def set_static_dirs(note)
+    @static_dirs = {}
+    @static_dirs['note'] = [ "#{@static_dir[0]}/note/#{note.guid}",
+                             "#{@static_dir[1]}/note/#{note.guid}" ]
+    ['image', 'audio', 'video', 'files']. each do |type|
+      @static_dirs[type] = [ "#{@static_dirs['note'][0]}/#{@static_subdirs[type]}",  # full path
+                             "#{@static_dirs['note'][1]}/#{@static_subdirs[type]}" ]; # url path
+    end
+  end
+  
+  def delete_note(note, fname)
+    set_static_dirs(note)
+
+    if File.exist?(fname)
+      msg "   This note has been deleted from Evernote, deleting its file #{fname}"
+      File.delete(fname)
+    end
+    if Dir.exist?(@static_dirs['note'][0])
+      msg "   Deleting static files for deleted note #{@static_dirs['note'][0]}"
+      FileUtils.rmtree(@static_dirs['note'][0])
+    end
+  end
+
+  def output_note(note)
+    set_static_dirs(note)
+
+    msg "Found note '#{note.title}'"
+    verbose "Created: #{Time.at(note.created/1000)}" if note.created
+    verbose "Deleted: #{Time.at(note.deleted/1000)}" if note.deleted
+    verbose "Content length: #{note.contentLength}"  if note.contentLength
+    verbose "Clipped from: #{note.attributes.sourceURL}" if note.attributes.sourceURL
+
+    markdown = note.tagNames.include?(@markdown_tag)
+    if markdown
+      msg "   It has the '#{ @markdown_tag }' tag, so I will interpret it as markdown"
+      note.tagNames -= [ @markdown_tag ]
+    end
+
+    type = nil
+    # Detect the type of post according to its tags
+    @tag_to_type_order.each do |tag|
+      if note.tagNames.include?(tag) or tag == "default"
+        type = @tag_to_type[tag]
+        break
+      end
+    end
+    if type.nil?
+      error "   ### I couldn't determine the type for this post - skipping it"
+      return
+    end
+
+    # Determine if we should include the page in the main menu
+    inmainmenu = note.tagNames.include?('_mainmenu')
+    if inmainmenu
+      note.tagNames -= [ '_mainmenu' ]
+    end
+
+    # Determine if we should use a custom slug
+    slug = nil
+    note.tagNames.grep(/^_slug=(\S+)/) do |slugtag|
+      slug = $1
+      note.tagNames -= [ slugtag ]
+      verbose "   Will use custom slug for this post: #{slug}"
+    end
+
+    # Get our note GUID-to-filename map
+    note_files = config(:note_files, {}, @config_store)
+    
+    # Determine the name I would assign to this note when published to Hugo
+    date = Time.at(note.created/1000).strftime('%F')
+    post_filename = "#{type}#{date}-#{note.title}.#{markdown ? 'md' : 'html'}"
+    # Do we already have a post for this note (by GUID)? If so, we remove the
+    # old file since it will be regenerated anyway, which also takes care of the
+    # case when the note was renamed and the filename will change, to avoid
+    # post duplication. If the note has been deleted, we just delete the
+    # old filename and stop here.
+    oldfile = note_files[note.guid]
+    if oldfile
+      verbose "   I already had a file for note #{note.guid}, removing #{oldfile}"
+      File.delete(oldfile) if File.exist?(oldfile)
+      note_files.delete(note.guid)
+      setconfig(:note_files, note_files, @config_store)
+      if note.deleted
+        delete_note(note, oldfile)
+        return
+      end
+    end
+    
+    # Run hugo to create the file, then read it back it to update the front matter
+    # with our tags.
+    # We run "hugo new" also for deleted notes so that hugo gives us the filename
+    # to delete.
+    fname = nil
+    frontmatter = nil
+    Dir.chdir(@base_dir) do
+      # Force -f yaml because it's so much easier to process
+      while true
+        post_filename.gsub!(/"/, '\"')
+        cmd = %Q(#{@hugo_cmd} new -f yaml "#{post_filename}" 2>&1)
+        debug "Executing: #{cmd}"
+        output = %x(#{cmd})
+        if output =~ /^(.+) created$/
+          # Get the full filename as reported by Hugo
+          fname = $1
+          if note.deleted
+            delete_note(note, fname)
+            return
+          end
+          # Load the frontmatter
+          frontmatter = YAML.load_file(fname)
+          # Update title because Hugo gets it wrong sometimes depending on the characters in the title, and to get rid of the date we put in the filename
+          frontmatter['title'] = note.title
+          # Fix the date to the date when the note was created
+          frontmatter['date'] = date
+          # Update tags, for now set categories to the same
+          frontmatter['tags'] = note.tagNames
+          frontmatter['categories'] = note.tagNames
+          # Set slug to work around https://github.com/spf13/hugo/issues/1017
+          frontmatter['slug'] = slug ? slug : note.title.downcase.gsub(/\W+/, "-").gsub(/^-+/, "").gsub(/-+$/, "")
+          # Set main menu tag if needed
+          frontmatter['menu'] = 'main' if inmainmenu
+          break
+        elsif output =~ /ERROR: \S+ (.+) already exists/
+          # Get the full filename as reported by Hugo
+          fname = $1
+          # If the file existed already, remove it and regenerate it
+          File.delete(fname)
+          if note.deleted
+            delete_note(note, fname)
+            return
+          end
+          # This shouldn't happen due to the index check above
+          unless @rebuild_all
+            error "   I found a file that should not be there (#{fname}). This might indicate"
+            error "   an inconsistency in my internal note-to-file map. Please re-run with"
+            error "   --rebuild-all to regenerate it. I am deleting the file and continuing"
+            error "   for now, but please review the results carefully."
+          end
+          redo
+        else
+          error "   Hugo returned unknown output when trying to create this post - skipping it: #{output}"
+          return
+        end
+      end
+    end
+
+    debug "Updated frontmatter: #{frontmatter.to_s}"
+    
+    File.open(fname, "w") do |f|
+      f.write(frontmatter.to_yaml)
+      f.puts("---")
+      f.puts
+      enml = ENML_utils.new(note.content, note.resources, @static_dirs, note.guid)
+      output = markdown ? enml.to_text : enml.to_html
+      if @use_filters
+        verbose "Running filters on text"
+        output = run_filters(output)
+      end
+      if note.attributes.sourceURL
+        f.puts(%(<p class="clip-attribute">via <a href="#{note.attributes.sourceURL}">#{note.attributes.sourceURL}</a></p>))
+      end
+      f.puts(output)
+      enml.resource_files.each do |resfile|
+        FileUtils.mkdir_p File.dirname(resfile[:fname])
+        File.open(resfile[:fname], "w") do |r|
+          r.write(resfile[:data])
+        end
+        verbose "Wrote file #{resfile[:fname]}"
+      end
+    end
+
+    verbose "Wrote file #{fname}"
+    note_files[note.guid] = fname
+    setconfig(:note_files, note_files, @config_store)
+    
+  end
+end

data/lib/util.rb

@@ -0,0 +1,44 @@
+require 'yaml/store'
+require 'colorize'
+
+# Message output
+
+def verbose(msg)
+  puts ("Enwrite [VERBOSE]: " + msg).blue if $enwrite_verbose
+end
+
+def debug(msg)
+  puts ("Enwrite [DEBUG]: " + msg) if $enwrite_debug
+end
+
+def error(msg)
+  $stderr.puts ("Enwrite [ERROR]: " + msg).red
+end
+
+def msg(msg)
+  puts ("Enwrite [INFO]: " + msg).green
+end
+
+def warn(msg)
+  $stderr.puts ("Enwrite [WARN]: " + msg).light_yellow
+end
+
+# Config file storage
+
+def config_file
+  return "#{ENV['HOME']}/.enwrite.config"
+end
+
+def config_store
+  return YAML::Store.new(config_file())
+end
+
+# Get a persistent config value
+def config(key, defval=nil, store=config_store)
+  return store.transaction { store.fetch(key, defval) }
+end
+
+# Set a persistent config value
+def setconfig(key, val, store=config_store)
+  return store.transaction { store[key] = val }
+end  

data/test/helper.rb

@@ -0,0 +1,34 @@
+require 'simplecov'
+
+module SimpleCov::Configuration
+  def clean_filters
+    @filters = []
+  end
+end
+
+SimpleCov.configure do
+  clean_filters
+  load_adapter 'test_frameworks'
+end
+
+ENV["COVERAGE"] && SimpleCov.start do
+  add_filter "/.rvm/"
+end
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+require 'shoulda'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'enwrite'
+
+class Test::Unit::TestCase
+end

data/test/test_enwrite.rb

@@ -0,0 +1,7 @@
+require 'helper'
+
+class TestEnwrite < Test::Unit::TestCase
+  should "probably rename this file and start testing for real" do
+    flunk "hey buddy, you should probably rename this file and start testing for real"
+  end
+end

metadata

@@ -0,0 +1,179 @@
+--- !ruby/object:Gem::Specification
+name: enwrite
+version: !ruby/object:Gem::Version
+  version: 0.2.0
+platform: ruby
+authors:
+- Diego Zamboni
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-04-30 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: colorize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+- !ruby/object:Gem::Dependency
+  name: deep_merge
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: evernote-thrift
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.25'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.25'
+- !ruby/object:Gem::Dependency
+  name: evernote_oauth
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+- !ruby/object:Gem::Dependency
+  name: htmlentities
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.3'
+- !ruby/object:Gem::Dependency
+  name: rdoc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.12'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: jeweler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+description: |-
+  Enwrite allows you to generate a website from contents stored in Evernote.
+  At the moment only Hugo (http://gohugo.io) is supported as an output format,
+  but others can be added through plugins.
+email: diego@zzamboni.org
+executables:
+- enwrite
+extensions: []
+extra_rdoc_files:
+- LICENSE
+- LICENSE.txt
+- README.md
+files:
+- ".document"
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/enwrite
+- lib/enml-utils.rb
+- lib/enwrite.rb
+- lib/evernote-utils.rb
+- lib/filters.rb
+- lib/output.rb
+- lib/output/hugo.rb
+- lib/util.rb
+- test/helper.rb
+- test/test_enwrite.rb
+homepage: http://github.com/zzamboni/enwrite
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: 'Enwrite: Power a web site using Evernote'
+test_files: []