sitefs 1.0.0.beta1

data/lib/sitefs/command_config.rb

@@ -0,0 +1,79 @@
+class Sitefs::CommandConfig
+  attr_accessor :index_format, :port, :ignore_patterns
+  attr_reader :root_path, :manifest
+  attr_writer :quiet, :force
+
+  def initialize seed
+    @root_path = '.'
+    @aws_region = 'aws_region'
+    @index_format = 'all-index'
+    @ignore_patterns = []
+
+    seed.each do |k, v|
+      instance_variable_set "@#{k}", v
+    end
+
+    self.root_path = @root_path
+
+    @manifest = Manifest.new root_path
+  end
+
+  def quiet?; @quiet; end
+  def force?; @force; end
+
+  def root_path= path
+    @root_path = File.join(File.expand_path(@root_path), '')
+  end
+
+  def aws_creds
+    @aws_creds ||= Aws::Credentials.new(@aws_access_key_id, @aws_secret_access_key)
+  end
+
+  def s3
+    @s3 ||= Aws::S3::Client.new credentials: aws_creds, region: @aws_region
+  end
+
+  def s3_bucket
+    @bucket ||= s3.bucket(@aws_s3_bucket)
+  end
+
+  def s3_pages
+    s3.list_objects(bucket: @aws_s3_bucket)
+  end
+
+  def put_s3_object path, content_type: nil, **args
+    if content_type
+      args[:content_type] = content_type
+    end
+
+    s3.put_object(bucket: @aws_s3_bucket, key: path, **args)
+  end
+
+  def s3_object_contents path
+    resp = s3.get_object(bucket: @aws_s3_bucket, key: path)
+
+    resp && resp.body.read
+  end
+
+  def s3_object path
+    return @s3_object[path.to_s] if @s3_object
+
+    @s3_object = {}
+
+    s3_pages.each do |request|
+      request.contents.each do |obj|
+        @s3_object[obj.key.to_s] = obj
+      end
+    end
+
+    @s3_object[path.to_s]
+  end
+
+  class << self
+    attr_reader :current
+
+    def from_file filename
+      @current = new YAML.load_file(filename)
+    end
+  end
+end

data/lib/sitefs/dsl_context.rb

@@ -0,0 +1,49 @@
+# To make sure that inside of the DSLs you can import yaml files
+require 'yaml'
+
+class Sitefs::DslContext
+  attr_reader :_pages_to_render, :path_helper
+  def initialize path_helper
+    @path_helper = path_helper
+    @_pages_to_render = []
+  end
+
+  def source_file
+    @path_helper.source_file
+  end
+
+  def path_for_href href
+    @path_helper.pathname_for href
+  end
+
+  def root_path
+    @path_helper.root_path
+  end
+
+  def _eval
+    instance_eval(File.read(source_file), source_file)
+    @_pages_to_render
+  end
+
+  def current_dir
+    File.dirname(source_file)
+  end
+
+  def files_like matcher
+    if matcher.is_a?(Regexp)
+      Dir[File.join(current_dir,'**/*')].grep(matcher)
+    else
+      Dir.glob(File.join(current_dir, matcher), File::FNM_CASEFOLD)
+    end
+  end
+
+  def new_page
+    Page.new path_helper
+  end
+
+  def render page, with:, **args
+    page._rendering_template = @path_helper.full_path_for(with)
+    @_pages_to_render << page
+    page
+  end
+end

data/lib/sitefs/file_action.rb

@@ -0,0 +1,155 @@
+require 'digest/md5'
+require 'fileutils'
+require 'diffy'
+require 'mime/types'
+
+Diffy::Diff.default_format = :color
+
+MAP_MIME_TYPE = MIME::Type.new('application/json+sourcemap') {|type| type.add_extensions('map')}
+MIME::Types.add(MAP_MIME_TYPE)
+
+class Sitefs::FileAction
+  attr_accessor :render_error, :content_type, :path
+
+  POSSIBLE_ACTIONS = %i{upload write}
+
+  def initialize possible_actions: POSSIBLE_ACTIONS, path:, content: nil, content_type: nil, &blk
+    @content_type = content_type || content_type_for(path)
+    @possible_actions = possible_actions
+    @path = path
+
+    @content = blk || content
+  end
+
+  def content_type_for path
+    unless mtype = MIME::Types.type_for(path).first
+      return
+    end
+
+    type = mtype.to_s
+    case mtype
+    when 'text/html', 'application/javascript'
+      type += '; charset=utf-8'
+    when MAP_MIME_TYPE
+      type = 'application/json'
+    end
+
+    type
+  end
+
+  def content
+    if @content.respond_to?(:call)
+      @content = @content.call
+    end
+
+    @content
+  end
+
+  def content_hash
+    @content_hash ||= Digest::MD5.hexdigest(content)
+  end
+
+  def destination_hash config, action, path
+    case action
+    when :write
+      if File.exist? path
+        Digest::MD5.hexdigest(File.read(path))
+      end
+    when :upload
+      if manifest = config.manifest.path_uploaded_hash(path)
+        manifest
+      elsif obj = config.s3_object(path)
+        obj.etag[1...-1]
+      end
+    end
+  end
+
+  # TODO: make this return a reason instead of bool
+  def should_write? config, action, path
+    return true if config.force?
+
+    d_hash = destination_hash(config, action, path)
+
+    result = !d_hash || (content_hash != d_hash)
+    if result && false
+      puts "should_write? #{path}"
+      puts "  dest hash: #{d_hash}"
+      puts "  cont hash: #{content_hash}"
+
+      remote_content = config.s3_object_contents(path)
+
+      diff = Diffy::Diff.new(remote_content, content)
+      puts "  #{diff.to_s}"
+    end
+
+    result
+  end
+
+  def write_path_for path, config
+    return path unless path =~ /\.html$/
+
+    case config.index_format
+    when 'github', 'standard'
+      path
+    when 'all-index'
+      if path =~ /index\.html$/i
+        path
+      else
+        path.sub(/\.html$/, '/index.html')
+      end
+    when 'aws'
+      if path =~ /index\.html$/i
+        path
+      else
+        path.sub(/\.html$/, '')
+      end
+    end
+  end
+
+  def perform_action config, action
+    unless POSSIBLE_ACTIONS.include? action
+      raise ArgumentError.new("invalid action: #{action}")
+    end
+
+    unless @possible_actions.include? action
+      return
+    end
+
+    path = write_path_for @path, config
+
+    dirname = File.dirname path
+
+    config.manifest.set_path_content_type path, content_type if content_type
+
+    case action
+    when :write
+      if should_write? config, action, path
+        log :generate, path
+        FileUtils.mkdir_p dirname
+        File.open(path, 'w') { |file| file.write content }
+
+        config.manifest.generated_path path
+      else
+        log :identical, path
+      end
+    when :upload
+      if should_write? config, action, path
+        log :upload, path
+
+        config.put_s3_object path, content_type: content_type, body: content
+        config.manifest.uploaded_path path
+        config.manifest.set_path_uploaded_hash path, content_hash
+      else
+        log :identical, path
+      end
+    end
+  end
+
+  def log type, path
+    puts "  #{type} - #{path}"
+  end
+
+  def to_s
+    inspect
+  end
+end

data/lib/sitefs/file_action_set.rb

@@ -0,0 +1,51 @@
+class Sitefs::FileActionSet
+  include Enumerable
+
+  def initialize(file_actions)
+    @file_actions = file_actions
+    @errored_on = []
+  end
+
+  def each
+    @file_actions.each do |file_action|
+      yield file_action
+    end
+  end
+
+  def [] i
+    @file_actions[i]
+  end
+
+  def has_errors?
+    !@errored_on.empty?
+  end
+
+  def output_paths
+    map(&:path)
+  end
+
+  def includes_path? path
+    path = File.expand_path(path)
+
+    output_paths.include? path
+  end
+
+  def call config, action
+
+    each do |file_action|
+      begin
+        file_action.perform_action config, action
+      rescue Exception => e
+        STDERR.puts "Error: file(#{file_action.inspect})"
+        STDERR.puts e
+
+        raise e
+
+        file_action.render_error = e
+        @errored_on << file_action
+      end
+    end
+
+    self
+  end
+end

data/lib/sitefs/file_registry.rb

@@ -0,0 +1,61 @@
+class Sitefs::FileRegistry
+
+  def initialize
+    @handlers = []
+  end
+
+  def << handler
+    @pages = nil
+
+    if handler.respond_to? :registry=
+      handler.registry = self
+    end
+
+    @handlers << handler
+  end
+
+  def handlers
+    @handlers.select {|h| !h.delay_generation }
+  end
+
+  def delayed_handlers
+    @handlers.select {|h| h.delay_generation }
+  end
+
+   def pages
+    @pages ||= handlers.flat_map do |h|
+      h.pages
+    end.compact
+  end
+
+  def pages_tagged tag_str, all: false
+    # If it hasnt been published always sort by the same
+    default_time = Time.now
+
+    pages.select do |page|
+      is_tagged = page.tags.include? tag_str
+
+      if all
+        is_tagged
+      else
+        is_tagged && page.published_at
+      end
+    end.sort_by do |page|
+      [page.published_at || default_time, page.title]
+    end
+  end
+
+  def public_tags
+    pages.flat_map(&:public_tags).uniq
+  end
+
+  def gather_actions
+    all_handlers = handlers + delayed_handlers
+
+    actions = all_handlers.flat_map do |handler|
+      handler.file_actions self
+    end
+
+    FileActionSet.new(actions)
+  end
+end

data/lib/sitefs/handler.rb

@@ -0,0 +1,46 @@
+class Sitefs::Handler
+  attr_reader :root_path, :source_file
+
+  def initialize root_path, source_file
+    @root_path = root_path
+    @source_file = source_file
+  end
+
+  def should_generate? config
+    true
+  end
+
+  def delay_generation
+    false
+  end
+
+  def pages
+    []
+  end
+
+  def path_helper
+    @path_helper ||= PathHelper.new(root_path, source_file)
+  end
+
+  def file_actions registry
+    []
+  end
+
+  def tags
+    []
+  end
+
+  def html_pipeline_result
+    @html_pipeline_result ||= {}
+  end
+
+  def attributes
+    @attributes ||= AttributeSet.new.tap do |attrs|
+      attrs['tags'] ||= tags
+      attrs['title'] ||= html_pipeline_result[:title]
+      attrs['subtitle'] ||= html_pipeline_result[:subtitle]
+      attrs['description'] ||= html_pipeline_result[:subtitle]
+    end
+  end
+
+end

data/lib/sitefs/handlers/markdown.rb

@@ -0,0 +1,102 @@
+class Sitefs::Handlers::Markdown < Sitefs::Handler
+  def output_path
+    @source_file.sub(/\.page.*$/, '.html').sub(root_path, '')
+  end
+
+  def path
+    File.join('', @source_file.gsub(/\.page.*$/, '').sub(@root_path, ''))
+  end
+
+  def config_strings
+    _read unless @config_strings
+    @config_strings
+  end
+
+  def markdown
+    _read unless @markdown
+    @markdown.join
+  end
+
+  MARKER_REG = /^\-{3,}$/
+
+  def _read
+    @config_strings = []
+    @markdown = []
+
+    in_config = false
+
+    File.open(@source_file).each_with_index do |line, i|
+      if i === 0 && line =~ MARKER_REG
+        in_config = true
+        next
+      end
+
+      if in_config
+        if line =~ MARKER_REG
+          in_config = false
+          next
+        else
+          @config_strings << line
+        end
+      else
+        @markdown << line
+      end
+    end
+  end
+
+  def attributes
+    @attributes ||= AttributeSet.from_yaml(config_strings.join).tap do |attrs|
+      attrs['tags'] ||= []
+      attrs['title'] ||= html_pipeline_result[:title]
+      attrs['subtitle'] ||= html_pipeline_result[:subtitle]
+      attrs['description'] ||= html_pipeline_result[:subtitle]
+     end
+  end
+
+  def pages
+    @pages ||= begin
+      page = Page.new(path_helper, attributes)
+      page.path = path
+      [page]
+    end
+  end
+
+  def html_pipeline_result
+    @html_pipeline_result ||= HtmlPipelines.markdown.call(markdown)
+  end
+
+  def xml
+    html_pipeline_result[:output]
+  end
+
+  def output
+    xml.to_s
+  end
+
+  def markdown_pipeline
+    @markdown_pipeline ||= RendererPipeline.for(
+      root_path: root_path,
+      source_file: source_file,
+      content_text: output,
+      layout_name: '_layout.md.html.erb',
+    )
+  end
+
+  def render_pipleine
+    @render_pipleine ||= RendererPipeline.for(root_path: root_path, source_file: source_file, content_text: markdown_pipeline)
+  end
+
+  def _render context=nil
+    render_pipleine.render(context)
+  end
+
+  def file_actions registry
+    return [] unless page = pages.first
+
+    FileAction.new path: output_path do
+      context = RenderContext.new(registry, current_page: page)
+      _render(context)
+    end
+  end
+
+end

data/lib/sitefs/handlers/noop.rb

@@ -0,0 +1,15 @@
+class Sitefs::Handlers::Noop < Sitefs::Handler
+  def output_path
+    @source_file.sub(root_path, '')
+  end
+
+  def should_generate? config
+    (File.basename(source_file)[0] != '_') && !config.manifest.generated?(output_path)
+  end
+
+  def file_actions registry
+    FileAction.new(path: output_path, possible_actions: %i{upload}) do
+      File.read(@source_file)
+    end
+  end
+end

data/lib/sitefs/handlers/ruby_gen.rb

@@ -0,0 +1,33 @@
+class Sitefs::Handlers::RubyGen < Sitefs::Handler
+  def dsl
+    @dsl ||= DslContext.new path_helper
+  end
+
+  def pages
+    @pages ||= begin
+      dsl._eval
+    end
+  end
+
+  def renderer_for page
+    content_path = page._rendering_template
+    content_text = File.read(content_path)
+
+    RendererPipeline.for(root_path: root_path, source_file: source_file, content_text: content_text)
+  end
+
+  def file_actions registry
+    pages.map do |page|
+      output_path = page.expanded_path + '.html'
+      output_path.sub!(root_path, '')
+
+      FileAction.new path: output_path do
+        renderer = renderer_for page
+        context = RenderContext.new(registry, current_page: page)
+
+        renderer.render context
+      end
+    end
+  end
+
+end

data/lib/sitefs/handlers/scss.rb

@@ -0,0 +1,63 @@
+require 'sass'
+
+class Sitefs::Handlers::Scss < Sitefs::Handler
+
+  def content
+    engine.render
+  end
+
+  def engine_options
+    {
+      cache: true,
+      syntax: :scss,
+      style: :compressed,
+      filename: source_file,
+      sourcemap: :inline,
+      load_paths: [root_path],
+    }
+  end
+
+  def engine
+    @engine ||= Sass::Engine.new(File.read(source_file), engine_options)
+  end
+
+  def render_with_sourcemap
+    @render_with_sourcemap ||= engine.render_with_sourcemap(sourcemap_output_pathname)
+  end
+
+  # def delay_generation
+  #   true
+  # end
+
+  def should_generate? config
+    File.basename(source_file)[0] != '_'
+  end
+
+  def output_path
+    source_file.sub(/\.scss$/, '.css').sub(root_path, '')
+  end
+
+  def output_pathname
+    File.join('', output_path)
+  end
+
+  def sourcemap_output_pathname
+    output_pathname + '.map'
+  end
+
+  def sourcemap_output_path
+    output_path + '.map'
+  end
+
+  def file_actions registry
+    sourcemap_options = {
+      css_path: output_path,
+      sourcemap_path: sourcemap_output_path,
+    }
+
+    [
+      FileAction.new(path: output_path) { render_with_sourcemap[0] },
+      FileAction.new(path: sourcemap_output_path) { render_with_sourcemap[1].to_json(sourcemap_options) },
+    ]
+  end
+end

data/lib/sitefs/handlers/single_erb.rb

@@ -0,0 +1,49 @@
+class Sitefs::Handlers::SingleErb < Sitefs::Handler
+  def basename
+    File.basename(source_file)
+  end
+
+  def extension
+    if match = basename.match(/\.page(\..*)?\.erb$/)
+      match[1]
+    else
+      '.html'
+    end
+  end
+
+  def output_path
+    source_file.sub(/\.page(\..*)?\.erb$/,'\1').sub(root_path, '')
+  end
+
+  def content_text
+    @content_text ||= File.read(source_file)
+  end
+
+  def renderer
+    RendererPipeline.for(root_path: root_path, source_file: source_file, content_text: content_text)
+  end
+
+  def pages
+    @pages ||= begin
+      page = Page.new path_helper, attributes
+      page.published_at = File.birthtime(source_file)
+      [page]
+    end
+  end
+
+  def page
+    pages.first
+  end
+
+  def html_pipeline_result
+    @html_pipeline_result ||= HtmlPipelines.content.call(content_text)
+  end
+
+  def file_actions registry
+    FileAction.new path: output_path do
+      context = RenderContext.new(registry, current_page: page)
+
+      renderer.render context
+    end
+  end
+end