bookwatch 1.0.0

data/lib/bookwatch/html_document_manipulator.rb

@@ -0,0 +1,21 @@
+require 'nokogiri'
+
+module Bookwatch
+  class HtmlDocumentManipulator
+    def set_attribute(document: nil,
+                      selector: nil,
+                      attribute: nil,
+                      value: nil)
+      doc = Nokogiri::HTML.fragment(document)
+      node_set = doc.css(selector)
+      node_set.attr(attribute, value)
+      doc.to_html
+    end
+
+    def read_html_in_tag(document: nil, tag: nil)
+      doc = Nokogiri::HTML(document)
+      doc.css(tag).inner_html
+    end
+  end
+end
+

data/lib/bookwatch/ingest/cloner_factory.rb

@@ -0,0 +1,26 @@
+require_relative 'git_cloner'
+require_relative 'local_filesystem_cloner'
+
+module Bookwatch
+  module Ingest
+    class ClonerFactory
+      def initialize(streams, filesystem, version_control_system)
+        @streams = streams
+        @filesystem = filesystem
+        @version_control_system = version_control_system
+      end
+
+      def produce(user_repo_dir)
+        if user_repo_dir
+          LocalFilesystemCloner.new(streams, filesystem, user_repo_dir)
+        else
+          GitCloner.new(version_control_system)
+        end
+      end
+
+      private
+
+      attr_reader :streams, :filesystem, :version_control_system
+    end
+  end
+end

data/lib/bookwatch/ingest/destination_directory.rb

@@ -0,0 +1,21 @@
+module Bookwatch
+  module Ingest
+    DestinationDirectory = Struct.new(:full_repo_name, :desired_destination_dir_name) do
+      def to_str
+        if desired_destination_dir_name
+          desired_destination_dir_name.to_s
+        elsif full_repo_name
+          full_repo_name.split('/').last
+        else
+          ""
+        end
+      end
+
+      alias :to_s :to_str
+
+      def ==(other)
+        to_str == other
+      end
+    end
+  end
+end

data/lib/bookwatch/ingest/git_accessor.rb

@@ -0,0 +1,102 @@
+require 'git'
+require_relative '../directory_helpers'
+require_relative '../local_filesystem_accessor'
+require_relative 'update_failure'
+require_relative 'update_success'
+
+module Bookwatch
+  module Ingest
+    class GitAccessor
+      TagExists = Class.new(RuntimeError)
+      InvalidTagRef = Class.new(RuntimeError)
+      include DirectoryHelperMethods
+
+      def clone(url, name, path: nil, checkout: 'master')
+        cached_clone(url, name, Pathname(path)).tap do |git|
+          git.checkout(checkout)
+        end
+      end
+
+      def update(cloned_path)
+        Git.open(cloned_path).pull
+        Ingest::UpdateSuccess.new
+      rescue ArgumentError, Git::GitExecuteError => e
+        case e.message
+        when /overwritten by merge/
+          Ingest::UpdateFailure.new('merge error')
+        when /path does not exist/
+          Ingest::UpdateFailure.new('not found')
+        else
+          raise
+        end
+      end
+
+      def remote_tag(url, tagname, commit_or_object)
+        Dir.mktmpdir do |dir|
+          path = Pathname(dir)
+          git = cached_clone(url, temp_name("tag"), path)
+          git.config('user.name', 'Bookwatch')
+          git.config('user.email', 'bookwatch@cloudfoundry.org')
+          begin
+            git.add_tag(tagname, "origin/#{commit_or_object}",
+              message: 'Tagged by Bookwatch')
+          rescue Git::GitExecuteError => e
+            case e.message
+              when /already exists/
+                raise TagExists
+              when /as a valid ref/
+                raise InvalidTagRef
+              else
+                raise
+            end
+          end
+          git.push("origin", "master", tags: true)
+        end
+      end
+
+      def author_date(path, exclusion_flag: '[exclude]', dita: false)
+        fs = LocalFilesystemAccessor.new
+
+        if dita
+          source_dir = 'preprocessing'
+          path_to_file = path.sub(/\.html(.md)?(.erb)?/, '.xml')
+        else
+          source_dir = source_dir_name
+          path_to_file = path
+        end
+
+
+        Pathname(path).dirname.ascend do |current_dir|
+          if (
+              current_dir.to_s.include?(source_dir) &&
+              current_dir.entries.include?(Pathname(".git")) &&
+              fs.source_file_exists?(Pathname(path).dirname, path_to_file)
+            )
+
+            git = Git.open(current_dir)
+            logs = git.gblob(path_to_file).log
+
+            last_non_excluded_commit = logs.detect { |log| !log.message.include?(exclusion_flag) }
+
+            return last_non_excluded_commit.author.date if last_non_excluded_commit
+          end
+        end
+      end
+
+      private
+
+      def temp_name(purpose)
+        "bookwatch-git-accessor-#{purpose}"
+      end
+
+      def cached_clone(url, name, path)
+        dest_dir = path.join(name)
+        if dest_dir.exist?
+          Git.open(dest_dir)
+        else
+          Git.clone(url, name, path: path, recursive: true)
+        end
+      end
+    end
+  end
+end

data/lib/bookwatch/ingest/git_cloner.rb

@@ -0,0 +1,36 @@
+require_relative 'destination_directory'
+require_relative 'repo_identifier'
+require_relative 'working_copy'
+
+module Bookwatch
+  module Ingest
+    class GitCloner
+      def initialize(version_control_system)
+        @version_control_system = version_control_system
+      end
+
+      def call(source_repo_name: nil,
+               source_ref: "master",
+               destination_parent_dir: nil,
+               destination_dir_name: nil)
+        dest_dir = DestinationDirectory.new(source_repo_name, destination_dir_name)
+        copied_to = Pathname(destination_parent_dir).join(dest_dir)
+        version_control_system.clone(
+          RepoIdentifier.new(source_repo_name),
+          dest_dir,
+          path: destination_parent_dir,
+          checkout: source_ref
+        )
+        WorkingCopy.new(
+          copied_to: copied_to,
+          full_name: source_repo_name,
+          ref: source_ref
+        )
+      end
+
+      private
+
+      attr_reader :version_control_system
+    end
+  end
+end

data/lib/bookwatch/ingest/local_filesystem_cloner.rb

@@ -0,0 +1,66 @@
+require_relative 'destination_directory'
+require_relative 'working_copy'
+require_relative 'missing_working_copy'
+
+module Bookwatch
+  module Ingest
+    class LocalFilesystemCloner
+      def initialize(streams, filesystem, user_repo_dir)
+        @streams = streams
+        @user_repo_dir = user_repo_dir
+        @filesystem = filesystem
+      end
+
+      def call(source_repo_name: nil,
+               source_ref: nil,
+               destination_parent_dir: nil,
+               destination_dir_name: nil)
+        link!(
+          source_repo_name,
+          Pathname(user_repo_dir).join(source_repo_name.split('/').last),
+          Pathname(destination_parent_dir).join(DestinationDirectory.new(source_repo_name, destination_dir_name)),
+          source_ref,
+          source_repo_name.split('/').first
+        )
+      end
+
+      private
+
+      attr_reader :streams, :filesystem, :user_repo_dir
+
+      def link!(source_repo_name, source_dir, dest_dir, source_ref, source_org)
+        source_exists = filesystem.file_exist?(source_dir)
+        unless source_exists
+          source_dir_with_ref = "#{source_dir}-#{source_ref}"
+          source_exists = filesystem.file_exist?(source_dir_with_ref)
+
+          if source_exists
+            source_dir = source_dir_with_ref
+          else
+            source_dir_with_org_and_ref = "#{source_dir}-#{source_org}-#{source_ref}"
+            source_exists = filesystem.file_exist?(source_dir_with_org_and_ref)
+
+            source_dir = source_dir_with_org_and_ref if source_exists
+          end
+        end
+
+        if !source_exists
+          streams[:out].puts "  skipping (not found) #{source_dir}"
+          MissingWorkingCopy.new(source_repo_name, source_dir)
+        else
+          streams[:out].puts "  copying #{source_dir}"
+
+          unless filesystem.file_exist?(dest_dir)
+            filesystem.link_creating_intermediate_dirs(source_dir, dest_dir)
+          end
+
+          WorkingCopy.new(
+            copied_to: dest_dir,
+            full_name: source_repo_name,
+            ref: source_ref
+          )
+        end
+      end
+    end
+  end
+end

data/lib/bookwatch/ingest/missing_working_copy.rb

@@ -0,0 +1,27 @@
+module Bookwatch
+  module Ingest
+    class MissingWorkingCopy
+      def initialize(source_repo_name, source_dir, source_ref=nil)
+        @source_repo_name = source_repo_name
+        @source_dir = source_dir
+        @source_ref = source_ref
+      end
+
+      def full_name
+        @source_repo_name
+      end
+
+      def path
+        Pathname(@source_dir)
+      end
+
+      def ref
+        @source_ref
+      end
+
+      def available?
+        false
+      end
+    end
+  end
+end

data/lib/bookwatch/ingest/repo_identifier.rb

@@ -0,0 +1,45 @@
+module Bookwatch
+  module Ingest
+    class RepoIdentifier
+      DEFAULT_VCS_PREFIX = 'git@github.com:'
+
+      def initialize(input_identifier)
+        @input_identifier = input_identifier
+      end
+
+      def to_str
+        if input_identifier.nil?
+          ""
+        elsif input_identifier.include?(':') || input_identifier.match(/^\//)
+          input_identifier
+        else
+          "#{DEFAULT_VCS_PREFIX}#{input_identifier}"
+        end
+      end
+
+      alias :to_s :to_str
+
+      def inspect
+        %Q("#{to_s}")
+      end
+
+      def split(*args)
+        input_identifier.split(*args)
+      end
+
+      def hash
+        to_str.hash
+      end
+
+      def ==(other)
+        to_str == other
+      end
+
+      alias :eql? :==
+
+      private
+
+      attr_reader :input_identifier
+    end
+  end
+end

data/lib/bookwatch/ingest/section_repository.rb

@@ -0,0 +1,49 @@
+require_relative '../values/section'
+require_relative '../values/product_info'
+
+module Bookwatch
+  module Ingest
+    class SectionRepository
+      def fetch(configured_sections: [],
+                destination_dir: nil,
+                ref_override: nil,
+                cloner: nil,
+                streams: nil)
+        configured_sections.map do |section_config|
+          streams[:success].puts("Gathering #{section_config.repo_name}")
+
+          working_copy = cloner.call(source_repo_name: section_config.repo_name,
+                                     source_ref: ref_override || section_config.repo_ref,
+                                     destination_parent_dir: destination_dir,
+                                     destination_dir_name: section_config.desired_directory_name)
+
+          if section_config.dependent_sections.any?
+            section_config.dependent_sections.map do |dependent_config|
+              unless dependent_config.no_docs?
+                streams[:success].puts("Gathering #{dependent_config.repo_name}")
+                cloner.call(source_repo_name: dependent_config.repo_name,
+                            source_ref: ref_override || dependent_config.repo_ref,
+                            destination_parent_dir: "#{destination_dir}/#{section_config.desired_directory_name}",
+                            destination_dir_name: dependent_config.desired_directory_name)
+              end
+            end
+          end
+
+          Section.new(
+            working_copy.path,
+            working_copy.full_name,
+            section_config.desired_directory_name,
+            section_config.subnav_template,
+            section_config.subnav_name,
+            section_config.preprocessor_config,
+            section_config.at_repo_path,
+            section_config.repo_name,
+            working_copy.ref,
+            section_config.pdf_output_filename,
+            section_config.product_info
+          )
+        end
+      end
+    end
+  end
+end

data/lib/bookwatch/ingest/update_failure.rb

@@ -0,0 +1,15 @@
+module Bookwatch
+  module Ingest
+    class UpdateFailure
+      attr_reader :reason
+
+      def initialize(reason)
+        @reason = reason
+      end
+
+      def success?
+        false
+      end
+    end
+  end
+end

data/lib/bookwatch/ingest/update_success.rb

@@ -0,0 +1,12 @@
+module Bookwatch
+  module Ingest
+    class UpdateSuccess
+      def reason
+      end
+
+      def success?
+        true
+      end
+    end
+  end
+end

data/lib/bookwatch/ingest/working_copy.rb

@@ -0,0 +1,36 @@
+require_relative '../errors/programmer_mistake'
+
+module Bookwatch
+  module Ingest
+    class WorkingCopy
+      attr_reader :full_name, :ref
+
+      def initialize(copied_to: nil,
+                     full_name: nil,
+                     ref: nil)
+        if [copied_to, full_name].none?
+          raise Errors::ProgrammerMistake.new("Must provide copied_to and/or full_name to WorkingCopy.new")
+        else
+          @copied_to = copied_to
+          @full_name = full_name
+          @ref = ref
+        end
+      end
+
+      def available?
+        !! @copied_to
+      end
+
+      def path
+        Pathname(@copied_to)
+      end
+
+      def ==(other)
+        [@copied_to, @full_name, @ref] ==
+          [other.instance_variable_get(:@copied_to),
+           other.instance_variable_get(:@full_name),
+           other.instance_variable_get(:@ref)]
+      end
+    end
+  end
+end