collavre_github 0.4.1 → 0.5.1

data/app/jobs/collavre_github/initial_markdown_sync_job.rb

@@ -0,0 +1,19 @@
+module CollavreGithub
+  class InitialMarkdownSyncJob < ApplicationJob
+    queue_as :default
+    retry_on StandardError, wait: :polynomially_longer, attempts: 3
+
+    def perform(repository_link_id)
+      link = CollavreGithub::RepositoryLink.find_by(id: repository_link_id)
+      return unless link&.markdown_sync_enabled?
+
+      user = link.creative.user
+      return unless user
+
+      CollavreGithub::MarkdownSync::InitialImportService.new(
+        repository_link: link,
+        user: user
+      ).call
+    end
+  end
+end

data/app/jobs/collavre_github/markdown_sync_job.rb

@@ -0,0 +1,16 @@
+module CollavreGithub
+  class MarkdownSyncJob < ApplicationJob
+    queue_as :default
+    retry_on StandardError, wait: :polynomially_longer, attempts: 3
+
+    def perform(repository_link_id, push_payload)
+      link = CollavreGithub::RepositoryLink.find_by(id: repository_link_id)
+      return unless link&.markdown_sync_enabled?
+
+      CollavreGithub::MarkdownSync::IncrementalSyncService.new(
+        repository_link: link,
+        push_payload: push_payload
+      ).call
+    end
+  end
+end

data/app/models/collavre_github/repository_link.rb

@@ -4,12 +4,19 @@ module CollavreGithub
 
     belongs_to :creative, class_name: "Collavre::Creative"
     belongs_to :github_account, class_name: "CollavreGithub::Account"
+    belongs_to :markdown_root_creative, class_name: "Collavre::Creative", optional: true
 
     validates :repository_full_name, presence: true
     validates :webhook_secret, presence: true
 
     before_validation :ensure_webhook_secret
 
+    scope :markdown_sync, -> { where(markdown_sync_enabled: true) }
+
+    def markdown_sync_branch
+      sync_branch.presence || "main"
+    end
+
     private
 
     def ensure_webhook_secret

data/app/services/collavre_github/client.rb

@@ -85,6 +85,9 @@ module CollavreGithub
           active: true
         }
       )
+    rescue Octokit::Error, Faraday::Error => e
+      Rails.logger.warn("GitHub create webhook failed for #{repo_full_name}: #{e.message}")
+      nil
     end
 
     def update_repository_webhook(repo_full_name, hook_id, url:, secret:, events:, content_type: "json")
@@ -102,10 +105,57 @@ module CollavreGithub
           active: true
         }
       )
+    rescue Octokit::Error, Faraday::Error => e
+      Rails.logger.warn("GitHub update webhook failed for #{repo_full_name}: #{e.message}")
+      nil
     end
 
     def delete_repository_webhook(repo_full_name, hook_id)
       client.remove_hook(repo_full_name, hook_id)
+    rescue Octokit::Error, Faraday::Error => e
+      Rails.logger.warn("GitHub delete webhook failed for #{repo_full_name}: #{e.message}")
+      nil
+    end
+
+    # Fetch the default branch name for a repository
+    def default_branch(repo_full_name)
+      repo = client.repository(repo_full_name)
+      repo.default_branch
+    rescue Octokit::Error, Faraday::Error => e
+      Rails.logger.warn("GitHub default branch fetch failed: #{e.message}")
+      "main"
+    end
+
+    # Fetch the full recursive tree for a given branch/sha
+    def tree(repo_full_name, branch)
+      sha = client.ref(repo_full_name, "heads/#{branch}").object.sha
+      result = client.tree(repo_full_name, sha, recursive: true)
+      result.tree
+    rescue Octokit::Error, Faraday::Error => e
+      Rails.logger.warn("GitHub tree fetch failed: #{e.message}")
+      []
+    end
+
+    # Fetch file content (decoded) for a given path and ref
+    def file_content(repo_full_name, path, ref: nil)
+      opts = ref ? { ref: ref } : {}
+      content = client.contents(repo_full_name, path: path, **opts)
+      if content.encoding == "base64"
+        Base64.decode64(content.content).force_encoding("UTF-8")
+      else
+        content.content
+      end
+    rescue Octokit::Error, Faraday::Error => e
+      Rails.logger.warn("GitHub file content fetch failed for #{path}: #{e.message}")
+      nil
+    end
+
+    # Compare two commits and return changed files
+    def compare(repo_full_name, base, head)
+      client.compare(repo_full_name, base, head)
+    rescue Octokit::Error, Faraday::Error => e
+      Rails.logger.warn("GitHub compare failed: #{e.message}")
+      nil
     end
 
     private

data/app/services/collavre_github/markdown_sync/content_processor.rb

@@ -0,0 +1,113 @@
+module CollavreGithub
+  module MarkdownSync
+    class ContentProcessor
+      IMAGE_EXTENSIONS = %w[.png .jpg .jpeg .gif .svg .webp .ico .bmp .tiff .avif].freeze
+
+      def initialize(client:, repo:, branch:, path_to_creative_map:)
+        @client = client
+        @repo = repo
+        @branch = branch
+        @path_to_creative_map = path_to_creative_map
+        @image_cache = {}
+      end
+
+      # Returns [processed_content, array_of_blobs]
+      def process(markdown_content, file_path)
+        content = markdown_content.dup
+        blobs = []
+        dir = File.dirname(file_path)
+
+        content = content.gsub(/(!?)\[([^\]]*)\]\(([^)]+)\)/) do |match|
+          prefix = Regexp.last_match(1)
+          label = Regexp.last_match(2)
+          url = Regexp.last_match(3)
+
+          next match if absolute_url?(url) || anchor_only?(url)
+
+          if prefix == "!"
+            process_image(match, label, url, dir, blobs)
+          else
+            process_link(match, label, url, dir)
+          end
+        end
+
+        [ content, blobs ]
+      end
+
+      private
+
+      def process_image(match, alt, url, dir, blobs)
+        url_path = strip_fragment_and_query(url)
+        resolved = resolve_path(dir, url_path)
+
+        blob = download_and_upload(resolved)
+        return match unless blob
+
+        blobs << blob
+        blob_url = Rails.application.routes.url_helpers.rails_blob_path(blob, only_path: true)
+        "![#{alt}](#{blob_url})"
+      end
+
+      def process_link(match, label, url, dir)
+        url_path, fragment = url.split("#", 2)
+        url_path = strip_query(url_path)
+        resolved = resolve_path(dir, url_path)
+
+        creative = @path_to_creative_map[resolved]
+        return match unless creative
+
+        link = "/creatives/#{creative.id}"
+        link += "##{fragment}" if fragment.present?
+        "[#{label}](#{link})"
+      end
+
+      def download_and_upload(resolved_path)
+        return @image_cache[resolved_path] if @image_cache.key?(resolved_path)
+
+        raw = @client.file_content(@repo, resolved_path, ref: @branch)
+        unless raw.present?
+          @image_cache[resolved_path] = nil
+          return nil
+        end
+
+        filename = File.basename(resolved_path)
+        ext = File.extname(filename).downcase
+        content_type = Rack::Mime.mime_type(ext, "application/octet-stream")
+
+        blob = ActiveStorage::Blob.create_and_upload!(
+          io: StringIO.new(raw),
+          filename: filename,
+          content_type: content_type
+        )
+        @image_cache[resolved_path] = blob
+        blob
+      rescue StandardError => e
+        Rails.logger.warn("[MarkdownSync] Image download failed for #{resolved_path}: #{e.message}")
+        @image_cache[resolved_path] = nil
+        nil
+      end
+
+      def resolve_path(dir, relative)
+        clean = relative.to_s.strip
+        return clean if clean.empty?
+        Pathname.new(File.join(dir, clean)).cleanpath.to_s
+      end
+
+      def absolute_url?(url)
+        url.match?(%r{\A(https?://|//|mailto:)})
+      end
+
+      def anchor_only?(url)
+        url.start_with?("#")
+      end
+
+      def strip_fragment_and_query(url)
+        url.split("#").first.split("?").first
+      end
+
+      def strip_query(url)
+        url.split("?").first
+      end
+    end
+  end
+end

data/app/services/collavre_github/markdown_sync/incremental_sync_service.rb

@@ -0,0 +1,231 @@
+module CollavreGithub
+  module MarkdownSync
+    class IncrementalSyncService
+      def initialize(repository_link:, push_payload:)
+        @link = repository_link
+        @payload = push_payload
+        @client = CollavreGithub::Client.new(repository_link.github_account)
+        @repo = repository_link.repository_full_name
+        @user = repository_link.creative.user
+      end
+
+      def call
+        return unless @link.markdown_sync_enabled?
+        return unless target_branch_push?
+
+        root = @link.markdown_root_creative
+        return unless root
+
+        commits = @payload["commits"] || []
+        return if commits.empty?
+
+        added_paths = []
+        modified_paths = []
+        removed_paths = []
+
+        commits.each do |commit|
+          added_paths.concat(Array(commit["added"]))
+          modified_paths.concat(Array(commit["modified"]))
+          removed_paths.concat(Array(commit["removed"]))
+        end
+
+        # Only care about .md files
+        added_paths = added_paths.uniq.select { |p| p.end_with?(".md") }
+        modified_paths = modified_paths.uniq.select { |p| p.end_with?(".md") }
+        removed_paths = removed_paths.uniq.select { |p| p.end_with?(".md") }
+
+        # Remove from modified if also in added (new file)
+        modified_paths -= added_paths
+
+        return if added_paths.empty? && modified_paths.empty? && removed_paths.empty?
+
+        branch = @link.markdown_sync_branch
+
+        # Pre-fetch tree once for SHA lookups (avoid per-file API calls)
+        @tree_cache = @client.tree(@repo, branch).each_with_object({}) do |entry, h|
+          h[entry.path] = entry.sha
+        end
+
+        @synced_creatives = load_synced_creatives
+        processor = ContentProcessor.new(
+          client: @client, repo: @repo, branch: branch,
+          path_to_creative_map: @synced_creatives
+        )
+
+        created = []
+
+        removed_paths.each do |path|
+          creative = @synced_creatives[path]
+          next unless creative
+          creative.archive! if creative.respond_to?(:archive!)
+          @synced_creatives.delete(path)
+        end
+
+        modified_paths.each do |path|
+          creative = @synced_creatives[path]
+          next unless creative
+
+          content = @client.file_content(@repo, path, ref: branch)
+          next if content.blank?
+
+          source = creative.data["source"].merge(
+            "markdown" => content,
+            "sha" => @tree_cache[path]
+          )
+          source.delete("rendered_html")
+          creative.skip_github_validation = true
+          creative.update!(data: creative.data.merge("source" => source))
+
+          processed, blobs = processor.process(content, path)
+          update_content_comment(creative, processed, blobs)
+        end
+
+        added_paths.each do |path|
+          next if @synced_creatives[path] # idempotency: skip if already synced (redelivery/retry)
+
+          parts = path.split("/")
+          parts.pop
+
+          parent, new_dirs = ensure_parent_directories(parts, root)
+          created.concat(new_dirs)
+          content = @client.file_content(@repo, path, ref: branch)
+          next if content.blank?
+
+          filename = path.split("/").last
+          creative = Collavre::Creative.new(
+            description: filename,
+            parent: parent,
+            user: @user,
+            data: {
+              "source" => {
+                "type" => "github_markdown",
+                "repo" => @repo,
+                "path" => path,
+                "sha" => @tree_cache[path],
+                "markdown" => content,
+                "repository_link_id" => @link.id
+              }
+            }
+          )
+          creative.skip_github_validation = true
+          creative.save!
+          @synced_creatives[path] = creative
+
+          processed, blobs = processor.process(content, path)
+          comment = create_content_comment(creative, processed)
+          attach_blobs(comment, blobs) if blobs.any?
+          created << creative
+        end
+
+        resequence_affected_parents(created)
+        @link.update!(last_synced_at: Time.current)
+        Collavre::Creative::RealtimeBroadcastable.broadcast_batch_created(created) if created.any?
+      end
+
+      private
+
+      def target_branch_push?
+        ref = @payload["ref"]
+        branch = ref&.sub("refs/heads/", "")
+        branch == @link.markdown_sync_branch
+      end
+
+      # Load all synced creatives for this repository link, indexed by path
+      def load_synced_creatives
+        scope = Collavre::Creative.where(archived_at: nil)
+
+        if scope.connection.adapter_name == "PostgreSQL"
+          scope = scope.where("(data->'source'->>'repository_link_id')::integer = ?", @link.id)
+        else
+          scope = scope.where("json_extract(data, '$.source.repository_link_id') = ?", @link.id)
+        end
+
+        scope.each_with_object({}) { |c, h| h[c.data.dig("source", "path")] = c }
+      end
+
+      def create_content_comment(creative, markdown_content)
+        topic = creative.content_topic(fallback_user: @user)
+        creative.comments.create!(
+          content: markdown_content,
+          topic: topic,
+          user: @user,
+          skip_dispatch: true
+        )
+      end
+
+      def update_content_comment(creative, markdown_content, blobs = [])
+        topic = creative.content_topic(fallback_user: @user)
+        comment = creative.comments.where(topic: topic).order(:created_at).first
+        if comment
+          comment.images.purge if comment.images.attached?
+          comment.update!(content: markdown_content)
+          attach_blobs(comment, blobs) if blobs.any?
+        else
+          comment = create_content_comment(creative, markdown_content)
+          attach_blobs(comment, blobs) if blobs.any?
+        end
+      end
+
+      def attach_blobs(comment, blobs)
+        blobs.each { |blob| comment.images.attach(blob) }
+      end
+
+      def resequence_affected_parents(new_creatives)
+        parent_ids = new_creatives.map(&:parent_id).compact.uniq
+        Collavre::Creative.transaction do
+          parent_ids.each do |pid|
+            siblings = Collavre::Creative.where(parent_id: pid, archived_at: nil)
+              .select(:id, :data, :description, :sequence)
+            sorted = siblings.sort_by do |c|
+              path = c.data&.dig("source", "path") || ""
+              is_dir = path.end_with?("/") || path == ""
+              [ is_dir ? 0 : 1, c.description.to_s.downcase ]
+            end
+            sorted.each_with_index do |c, idx|
+              c.update_column(:sequence, idx) if c.sequence != idx
+            end
+          end
+        end
+      end
+
+      # Returns [parent_creative, newly_created_directories]
+      def ensure_parent_directories(parts, root)
+        parent = root
+        current_path = ""
+        new_dirs = []
+
+        parts.each do |part|
+          current_path = current_path.empty? ? part : "#{current_path}/#{part}"
+          dir_path = "#{current_path}/"
+
+          existing = @synced_creatives[dir_path]
+
+          if existing
+            parent = existing
+          else
+            dir_creative = Collavre::Creative.new(
+              description: part,
+              parent: parent,
+              user: @user,
+              data: {
+                "source" => {
+                  "type" => "github_markdown",
+                  "repo" => @repo,
+                  "path" => dir_path,
+                  "repository_link_id" => @link.id
+                }
+              }
+            )
+            dir_creative.skip_github_validation = true
+            dir_creative.save!
+            @synced_creatives[dir_path] = dir_creative
+            new_dirs << dir_creative
+            parent = dir_creative
+          end
+        end
+
+        [ parent, new_dirs ]
+      end
+    end
+  end
+end

data/app/services/collavre_github/markdown_sync/initial_import_service.rb

@@ -0,0 +1,190 @@
+module CollavreGithub
+  module MarkdownSync
+    class InitialImportService
+      MAX_FILES = 200
+
+      def initialize(repository_link:, user:)
+        @link = repository_link
+        @user = user
+        @client = CollavreGithub::Client.new(repository_link.github_account)
+        @repo = repository_link.repository_full_name
+      end
+
+      def call
+        branch = resolve_branch
+        tree_entries = @client.tree(@repo, branch)
+        md_entries = tree_entries.select { |e| e.type == "blob" && e.path.end_with?(".md") }
+
+        if md_entries.size > MAX_FILES
+          Rails.logger.warn("[MarkdownSync] #{@repo}: #{md_entries.size} .md files found, limiting to #{MAX_FILES}")
+          md_entries = md_entries.first(MAX_FILES)
+        end
+
+        # Archive existing root tree before reimport (pause/resume flow)
+        root = @link.markdown_root_creative
+        if root && root.archived_at.nil?
+          root.archive!
+        end
+
+        parent_creative = @link.creative
+        root_creative = create_root_creative(parent_creative)
+        @link.update!(markdown_root_creative_id: root_creative.id, last_synced_at: Time.current)
+
+        dir_map = { "" => root_creative }
+        created = [ root_creative ]
+        file_creatives = []
+
+        if md_entries.empty?
+          Collavre::Creative::RealtimeBroadcastable.broadcast_batch_created(created) if created.size > 1
+          return created
+        end
+
+        md_entries.sort_by(&:path).each do |entry|
+          parts = entry.path.split("/")
+          filename = parts.pop
+
+          parent = ensure_directories(parts, dir_map, root_creative, created)
+
+          content = @client.file_content(@repo, entry.path, ref: branch)
+          next if content.blank?
+
+          creative = Collavre::Creative.new(
+            description: filename,
+            parent: parent,
+            user: @user,
+            data: {
+              "source" => {
+                "type" => "github_markdown",
+                "repo" => @repo,
+                "path" => entry.path,
+                "sha" => entry.sha,
+                "markdown" => content,
+                "repository_link_id" => @link.id
+              }
+            }
+          )
+          creative.skip_github_validation = true
+          creative.save!
+          file_creatives << creative
+          created << creative
+        end
+
+        # Second pass: resolve relative links and images now that all creatives exist
+        path_map = build_path_map(created)
+        processor = ContentProcessor.new(client: @client, repo: @repo, branch: branch, path_to_creative_map: path_map)
+
+        file_creatives.each do |creative|
+          raw_md = creative.data.dig("source", "markdown")
+          next if raw_md.blank?
+
+          processed, blobs = processor.process(raw_md, creative.data.dig("source", "path"))
+          comment = create_content_comment(creative, processed)
+          attach_blobs(comment, blobs) if blobs.any?
+        end
+
+        resequence_directories_first(created)
+        Collavre::Creative::RealtimeBroadcastable.broadcast_batch_created(created) if created.size > 1
+        created
+      end
+
+      private
+
+      def resolve_branch
+        if @link.sync_branch.present?
+          @link.sync_branch
+        else
+          branch = @client.default_branch(@repo)
+          @link.update_column(:sync_branch, branch)
+          branch
+        end
+      end
+
+      def create_root_creative(parent)
+        repo_name = @repo.split("/").last
+        creative = Collavre::Creative.new(
+          description: repo_name,
+          parent: parent,
+          user: @user,
+          data: {
+            "source" => {
+              "type" => "github_markdown",
+              "repo" => @repo,
+              "path" => "",
+              "repository_link_id" => @link.id
+            }
+          }
+        )
+        creative.skip_github_validation = true
+        creative.save!
+        creative
+      end
+
+      def build_path_map(creatives)
+        creatives.each_with_object({}) do |c, map|
+          path = c.data&.dig("source", "path")
+          map[path] = c if path.present?
+        end
+      end
+
+      def create_content_comment(creative, markdown_content)
+        topic = creative.content_topic(fallback_user: @user)
+        creative.comments.create!(
+          content: markdown_content,
+          topic: topic,
+          user: @user,
+          skip_dispatch: true
+        )
+      end
+
+      def attach_blobs(comment, blobs)
+        blobs.each { |blob| comment.images.attach(blob) }
+      end
+
+      def resequence_directories_first(creatives)
+        Collavre::Creative.transaction do
+          creatives.group_by(&:parent_id).each do |_parent_id, siblings|
+            sorted = siblings.sort_by do |c|
+              path = c.data&.dig("source", "path") || ""
+              is_dir = path.end_with?("/") || path == ""
+              [ is_dir ? 0 : 1, c.description.to_s.downcase ]
+            end
+            sorted.each_with_index do |c, idx|
+              c.update_column(:sequence, idx)
+            end
+          end
+        end
+      end
+
+      def ensure_directories(parts, dir_map, root, created)
+        current_path = ""
+        parent = root
+
+        parts.each do |part|
+          current_path = current_path.empty? ? part : "#{current_path}/#{part}"
+          unless dir_map[current_path]
+            dir_creative = Collavre::Creative.new(
+              description: part,
+              parent: parent,
+              user: @user,
+              data: {
+                "source" => {
+                  "type" => "github_markdown",
+                  "repo" => @repo,
+                  "path" => "#{current_path}/",
+                  "repository_link_id" => @link.id
+                }
+              }
+            )
+            dir_creative.skip_github_validation = true
+            dir_creative.save!
+            dir_map[current_path] = dir_creative
+            created << dir_creative
+          end
+          parent = dir_map[current_path]
+        end
+
+        parent
+      end
+    end
+  end
+end

data/app/services/collavre_github/tools/concerns/github_client_finder.rb

@@ -30,6 +30,23 @@ module CollavreGithub
 
           CollavreGithub::Client.new(link.github_account)
         end
+
+        # Find client or return error hash. Yields the client to the block.
+        # Wraps the block with a StandardError rescue that returns { error: ... }.
+        #
+        # @param creative_id [Integer]
+        # @param repo [String]
+        # @param error_context [String] human-readable label for error messages
+        # @yield [client] the GitHub client
+        # @return [Hash] the block's result or an error hash
+        def with_github_client(creative_id:, repo:, error_context:, &block)
+          client = find_github_client(creative_id, repo)
+          return { error: "GitHub account not found for this creative and repository" } unless client
+
+          yield client
+        rescue StandardError => e
+          { error: "Failed to #{error_context}: #{e.message}" }
+        end
       end
     end
   end