collavre_github 0.4.1 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8c11fdb398224bc3a5dfeeb326b2f5ddd1575c1a371907ce0fcb9d5592e2f9cf
-  data.tar.gz: '058cfbcf2a41677dd39ec6da17f3b8f31471875db56e158855f587985e26005b'
+  metadata.gz: d8f3ffe59f1a7e85a1a42fd49cd5fa3ee1ad40667996a69e405fef0bb8415318
+  data.tar.gz: 5a9c05bf70a341ca56253bfa2d6a2849acc335038e42c1b9919747be2d564f7f
 SHA512:
-  metadata.gz: c9063bc158175ca780be591a837e7b185eb2e6fc7595422c4f52b17ae91c5f459246521ccaa0eac5d6dbb2de2b6fda1331f0df59cbab16730f7112b06eb780ff
-  data.tar.gz: 1fbcc7a46e22b45146a812335524dcd3a08ec10b2cc21d6f7588978a4a3960fad36b7fd6f680a29c30b632f44cf3367f230c78cbde588bc0342ade2c69a908c1
+  metadata.gz: ee53fe7588353302c64f84decc488e62b1d8f9fb9d65f8332439d0bd8d2f353fc8777ff59724c9c9d82390fd5094ab513f796ab5b1666fd456e382fd0d13099b
+  data.tar.gz: aa439ab76e282b9841abcaea1a84ad0e3ccfaf06214cb8380af7e24661ea93848de8197fd8f69b4108b5a4b609ed2cc8caaae69cc5c59e48f833328a6829b2e6

data/app/controllers/collavre_github/creatives/integrations_controller.rb

@@ -1,12 +1,13 @@
 module CollavreGithub
   module Creatives
     class IntegrationsController < ApplicationController
+      include Collavre::IntegrationSetup
       include Collavre::IntegrationPermission
 
       before_action :set_creative
       before_action :set_origin
       before_action :ensure_read_permission
-      before_action :ensure_admin_permission, only: [ :show, :update ]
+      before_action :ensure_admin_permission, only: [ :show, :update, :resync ]
 
       def show
         account = Current.user.github_account
@@ -25,7 +26,14 @@ module CollavreGithub
           },
           selected_repositories: links.map(&:repository_full_name),
           all_repositories: all_repositories,
-          webhooks: serialize_webhooks(links)
+          webhooks: serialize_webhooks(links),
+          markdown_sync: links.each_with_object({}) { |l, h|
+            h[l.repository_full_name] = {
+              enabled: l.markdown_sync_enabled?,
+              last_synced_at: l.last_synced_at,
+              sync_branch: l.markdown_sync_branch
+            }
+          }
         }
       end
 
@@ -56,6 +64,24 @@ module CollavreGithub
           links = linked_repository_links(account).to_a
         end
 
+        # Handle markdown sync toggle BEFORE webhook provisioning
+        # so events_for() sees the updated markdown_sync_enabled state
+        markdown_sync = integration_attributes[:markdown_sync]
+        if markdown_sync.is_a?(ActionController::Parameters) || markdown_sync.is_a?(Hash)
+          links.each do |link|
+            repo = link.repository_full_name
+            next unless markdown_sync.key?(repo)
+
+            enabled = ActiveModel::Type::Boolean.new.cast(markdown_sync[repo])
+            was_enabled = link.markdown_sync_enabled?
+            link.update!(markdown_sync_enabled: enabled)
+
+            if enabled && !was_enabled
+              CollavreGithub::InitialMarkdownSyncJob.perform_later(link.id)
+            end
+          end
+        end
+
         CollavreGithub::WebhookProvisioner.ensure_for_links(
           account: account,
           links: links,
@@ -65,12 +91,43 @@ module CollavreGithub
         render json: {
           success: true,
           selected_repositories: links.map(&:repository_full_name),
-          webhooks: serialize_webhooks(links)
+          webhooks: serialize_webhooks(links),
+          markdown_sync: links.each_with_object({}) { |l, h|
+            h[l.repository_full_name] = {
+              enabled: l.markdown_sync_enabled?,
+              last_synced_at: l.last_synced_at,
+              sync_branch: l.markdown_sync_branch
+            }
+          }
         }
       rescue ActiveRecord::RecordInvalid => e
         render json: { error: e.message }, status: :unprocessable_entity
       end
 
+      def resync
+        account = Current.user.github_account
+        unless account
+          render json: { error: I18n.t("collavre_github.errors.not_connected") }, status: :unprocessable_entity
+          return
+        end
+
+        repo = params[:repository]
+        link = linked_repository_links(account).find_by(repository_full_name: repo)
+        unless link&.markdown_sync_enabled?
+          render json: { error: I18n.t("collavre_github.markdown_sync.not_enabled") }, status: :unprocessable_entity
+          return
+        end
+
+        # Archive existing synced tree and re-import
+        if link.markdown_root_creative
+          link.markdown_root_creative.archive! if link.markdown_root_creative.respond_to?(:archive!)
+          link.update!(markdown_root_creative_id: nil)
+        end
+
+        CollavreGithub::InitialMarkdownSyncJob.perform_later(link.id)
+        render json: { success: true, message: I18n.t("collavre_github.markdown_sync.resync_started") }
+      end
+
       def destroy
         unless @creative.has_permission?(Current.user, :write)
           render json: { error: integration_forbidden_message }, status: :forbidden
@@ -139,14 +196,6 @@ module CollavreGithub
 
       private
 
-      def set_creative
-        @creative = Collavre::Creative.find(params[:creative_id])
-      end
-
-      def set_origin
-        @origin = @creative.effective_origin
-      end
-
       def integration_forbidden_message
         I18n.t("collavre_github.errors.forbidden")
       end
@@ -158,7 +207,7 @@ module CollavreGithub
       end
 
       def integration_params
-        params.permit(repositories: [])
+        params.permit(repositories: [], markdown_sync: {})
       end
 
       def serialize_webhooks(links)

data/app/controllers/collavre_github/webhooks_controller.rb

@@ -13,7 +13,13 @@ module CollavreGithub
       return head :unauthorized unless valid_signature?(raw_body)
 
       payload = payload.presence || {}
-      create_system_comment(event, payload) if @repository_link&.creative
+
+      # Process all links for this repo (same repo can be linked to multiple creatives)
+      all_links = all_repository_links_for(payload)
+      all_links.each do |link|
+        create_system_comment_for(link, event, payload) if link.creative
+        trigger_markdown_sync_for(link, event, payload) if link.markdown_sync_enabled?
+      end
 
       head :ok
     rescue JSON::ParserError
@@ -22,8 +28,8 @@ module CollavreGithub
 
     private
 
-    def create_system_comment(event, payload)
-      creative = @repository_link.creative&.effective_origin
+    def create_system_comment_for(link, event, payload)
+      creative = link.creative&.effective_origin
       return unless creative
 
       content = format_github_event(event, payload)
@@ -33,7 +39,6 @@ module CollavreGithub
         content: content,
         private: false
       )
-      # Dispatch handled by Comment#after_create_commit (skipped for system messages with user: nil)
     end
 
     def format_github_event(event, payload)
@@ -202,6 +207,22 @@ module CollavreGithub
       I18n.t("collavre_github.webhooks.#{key}", **options)
     end
 
+    def trigger_markdown_sync_for(link, event, payload)
+      return unless event == "push"
+
+      CollavreGithub::MarkdownSyncJob.perform_later(
+        link.id,
+        payload.as_json
+      )
+    end
+
+    def all_repository_links_for(payload)
+      repo = payload&.dig("repository", "full_name") || payload&.dig(:repository, :full_name)
+      return [ @repository_link ].compact if repo.blank?
+
+      CollavreGithub::RepositoryLink.where(repository_full_name: repo).to_a
+    end
+
     def find_repository_link(payload)
       if payload.blank?
         Rails.logger.warn("[GitHub Webhook] Payload is blank")

data/app/jobs/collavre_github/initial_markdown_sync_job.rb

@@ -0,0 +1,19 @@
+module CollavreGithub
+  class InitialMarkdownSyncJob < ApplicationJob
+    queue_as :default
+    retry_on StandardError, wait: :polynomially_longer, attempts: 3
+
+    def perform(repository_link_id)
+      link = CollavreGithub::RepositoryLink.find_by(id: repository_link_id)
+      return unless link&.markdown_sync_enabled?
+
+      user = link.creative.user
+      return unless user
+
+      CollavreGithub::MarkdownSync::InitialImportService.new(
+        repository_link: link,
+        user: user
+      ).call
+    end
+  end
+end

data/app/jobs/collavre_github/markdown_sync_job.rb

@@ -0,0 +1,16 @@
+module CollavreGithub
+  class MarkdownSyncJob < ApplicationJob
+    queue_as :default
+    retry_on StandardError, wait: :polynomially_longer, attempts: 3
+
+    def perform(repository_link_id, push_payload)
+      link = CollavreGithub::RepositoryLink.find_by(id: repository_link_id)
+      return unless link&.markdown_sync_enabled?
+
+      CollavreGithub::MarkdownSync::IncrementalSyncService.new(
+        repository_link: link,
+        push_payload: push_payload
+      ).call
+    end
+  end
+end

data/app/models/collavre_github/repository_link.rb

@@ -4,12 +4,19 @@ module CollavreGithub
 
     belongs_to :creative, class_name: "Collavre::Creative"
     belongs_to :github_account, class_name: "CollavreGithub::Account"
+    belongs_to :markdown_root_creative, class_name: "Collavre::Creative", optional: true
 
     validates :repository_full_name, presence: true
     validates :webhook_secret, presence: true
 
     before_validation :ensure_webhook_secret
 
+    scope :markdown_sync, -> { where(markdown_sync_enabled: true) }
+
+    def markdown_sync_branch
+      sync_branch.presence || "main"
+    end
+
     private
 
     def ensure_webhook_secret

data/app/services/collavre_github/client.rb

@@ -85,6 +85,9 @@ module CollavreGithub
           active: true
         }
       )
+    rescue Octokit::Error, Faraday::Error => e
+      Rails.logger.warn("GitHub create webhook failed for #{repo_full_name}: #{e.message}")
+      nil
     end
 
     def update_repository_webhook(repo_full_name, hook_id, url:, secret:, events:, content_type: "json")
@@ -102,10 +105,57 @@ module CollavreGithub
           active: true
         }
       )
+    rescue Octokit::Error, Faraday::Error => e
+      Rails.logger.warn("GitHub update webhook failed for #{repo_full_name}: #{e.message}")
+      nil
     end
 
     def delete_repository_webhook(repo_full_name, hook_id)
       client.remove_hook(repo_full_name, hook_id)
+    rescue Octokit::Error, Faraday::Error => e
+      Rails.logger.warn("GitHub delete webhook failed for #{repo_full_name}: #{e.message}")
+      nil
+    end
+
+    # Fetch the default branch name for a repository
+    def default_branch(repo_full_name)
+      repo = client.repository(repo_full_name)
+      repo.default_branch
+    rescue Octokit::Error, Faraday::Error => e
+      Rails.logger.warn("GitHub default branch fetch failed: #{e.message}")
+      "main"
+    end
+
+    # Fetch the full recursive tree for a given branch/sha
+    def tree(repo_full_name, branch)
+      sha = client.ref(repo_full_name, "heads/#{branch}").object.sha
+      result = client.tree(repo_full_name, sha, recursive: true)
+      result.tree
+    rescue Octokit::Error, Faraday::Error => e
+      Rails.logger.warn("GitHub tree fetch failed: #{e.message}")
+      []
+    end
+
+    # Fetch file content (decoded) for a given path and ref
+    def file_content(repo_full_name, path, ref: nil)
+      opts = ref ? { ref: ref } : {}
+      content = client.contents(repo_full_name, path: path, **opts)
+      if content.encoding == "base64"
+        Base64.decode64(content.content).force_encoding("UTF-8")
+      else
+        content.content
+      end
+    rescue Octokit::Error, Faraday::Error => e
+      Rails.logger.warn("GitHub file content fetch failed for #{path}: #{e.message}")
+      nil
+    end
+
+    # Compare two commits and return changed files
+    def compare(repo_full_name, base, head)
+      client.compare(repo_full_name, base, head)
+    rescue Octokit::Error, Faraday::Error => e
+      Rails.logger.warn("GitHub compare failed: #{e.message}")
+      nil
     end
 
     private

data/app/services/collavre_github/markdown_sync/content_processor.rb

@@ -0,0 +1,113 @@
+module CollavreGithub
+  module MarkdownSync
+    class ContentProcessor
+      IMAGE_EXTENSIONS = %w[.png .jpg .jpeg .gif .svg .webp .ico .bmp .tiff .avif].freeze
+
+      def initialize(client:, repo:, branch:, path_to_creative_map:)
+        @client = client
+        @repo = repo
+        @branch = branch
+        @path_to_creative_map = path_to_creative_map
+        @image_cache = {}
+      end
+
+      # Returns [processed_content, array_of_blobs]
+      def process(markdown_content, file_path)
+        content = markdown_content.dup
+        blobs = []
+        dir = File.dirname(file_path)
+
+        content = content.gsub(/(!?)\[([^\]]*)\]\(([^)]+)\)/) do |match|
+          prefix = Regexp.last_match(1)
+          label = Regexp.last_match(2)
+          url = Regexp.last_match(3)
+
+          next match if absolute_url?(url) || anchor_only?(url)
+
+          if prefix == "!"
+            process_image(match, label, url, dir, blobs)
+          else
+            process_link(match, label, url, dir)
+          end
+        end
+
+        [ content, blobs ]
+      end
+
+      private
+
+      def process_image(match, alt, url, dir, blobs)
+        url_path = strip_fragment_and_query(url)
+        resolved = resolve_path(dir, url_path)
+
+        blob = download_and_upload(resolved)
+        return match unless blob
+
+        blobs << blob
+        blob_url = Rails.application.routes.url_helpers.rails_blob_path(blob, only_path: true)
+        "![#{alt}](#{blob_url})"
+      end
+
+      def process_link(match, label, url, dir)
+        url_path, fragment = url.split("#", 2)
+        url_path = strip_query(url_path)
+        resolved = resolve_path(dir, url_path)
+
+        creative = @path_to_creative_map[resolved]
+        return match unless creative
+
+        link = "/creatives/#{creative.id}"
+        link += "##{fragment}" if fragment.present?
+        "[#{label}](#{link})"
+      end
+
+      def download_and_upload(resolved_path)
+        return @image_cache[resolved_path] if @image_cache.key?(resolved_path)
+
+        raw = @client.file_content(@repo, resolved_path, ref: @branch)
+        unless raw.present?
+          @image_cache[resolved_path] = nil
+          return nil
+        end
+
+        filename = File.basename(resolved_path)
+        ext = File.extname(filename).downcase
+        content_type = Rack::Mime.mime_type(ext, "application/octet-stream")
+
+        blob = ActiveStorage::Blob.create_and_upload!(
+          io: StringIO.new(raw),
+          filename: filename,
+          content_type: content_type
+        )
+        @image_cache[resolved_path] = blob
+        blob
+      rescue StandardError => e
+        Rails.logger.warn("[MarkdownSync] Image download failed for #{resolved_path}: #{e.message}")
+        @image_cache[resolved_path] = nil
+        nil
+      end
+
+      def resolve_path(dir, relative)
+        clean = relative.to_s.strip
+        return clean if clean.empty?
+        Pathname.new(File.join(dir, clean)).cleanpath.to_s
+      end
+
+      def absolute_url?(url)
+        url.match?(%r{\A(https?://|//|mailto:)})
+      end
+
+      def anchor_only?(url)
+        url.start_with?("#")
+      end
+
+      def strip_fragment_and_query(url)
+        url.split("#").first.split("?").first
+      end
+
+      def strip_query(url)
+        url.split("?").first
+      end
+    end
+  end
+end

data/app/services/collavre_github/markdown_sync/incremental_sync_service.rb

@@ -0,0 +1,231 @@
+module CollavreGithub
+  module MarkdownSync
+    class IncrementalSyncService
+      def initialize(repository_link:, push_payload:)
+        @link = repository_link
+        @payload = push_payload
+        @client = CollavreGithub::Client.new(repository_link.github_account)
+        @repo = repository_link.repository_full_name
+        @user = repository_link.creative.user
+      end
+
+      def call
+        return unless @link.markdown_sync_enabled?
+        return unless target_branch_push?
+
+        root = @link.markdown_root_creative
+        return unless root
+
+        commits = @payload["commits"] || []
+        return if commits.empty?
+
+        added_paths = []
+        modified_paths = []
+        removed_paths = []
+
+        commits.each do |commit|
+          added_paths.concat(Array(commit["added"]))
+          modified_paths.concat(Array(commit["modified"]))
+          removed_paths.concat(Array(commit["removed"]))
+        end
+
+        # Only care about .md files
+        added_paths = added_paths.uniq.select { |p| p.end_with?(".md") }
+        modified_paths = modified_paths.uniq.select { |p| p.end_with?(".md") }
+        removed_paths = removed_paths.uniq.select { |p| p.end_with?(".md") }
+
+        # Remove from modified if also in added (new file)
+        modified_paths -= added_paths
+
+        return if added_paths.empty? && modified_paths.empty? && removed_paths.empty?
+
+        branch = @link.markdown_sync_branch
+
+        # Pre-fetch tree once for SHA lookups (avoid per-file API calls)
+        @tree_cache = @client.tree(@repo, branch).each_with_object({}) do |entry, h|
+          h[entry.path] = entry.sha
+        end
+
+        @synced_creatives = load_synced_creatives
+        processor = ContentProcessor.new(
+          client: @client, repo: @repo, branch: branch,
+          path_to_creative_map: @synced_creatives
+        )
+
+        created = []
+
+        removed_paths.each do |path|
+          creative = @synced_creatives[path]
+          next unless creative
+          creative.archive! if creative.respond_to?(:archive!)
+          @synced_creatives.delete(path)
+        end
+
+        modified_paths.each do |path|
+          creative = @synced_creatives[path]
+          next unless creative
+
+          content = @client.file_content(@repo, path, ref: branch)
+          next if content.blank?
+
+          source = creative.data["source"].merge(
+            "markdown" => content,
+            "sha" => @tree_cache[path]
+          )
+          source.delete("rendered_html")
+          creative.skip_github_validation = true
+          creative.update!(data: creative.data.merge("source" => source))
+
+          processed, blobs = processor.process(content, path)
+          update_content_comment(creative, processed, blobs)
+        end
+
+        added_paths.each do |path|
+          next if @synced_creatives[path] # idempotency: skip if already synced (redelivery/retry)
+
+          parts = path.split("/")
+          parts.pop
+
+          parent, new_dirs = ensure_parent_directories(parts, root)
+          created.concat(new_dirs)
+          content = @client.file_content(@repo, path, ref: branch)
+          next if content.blank?
+
+          filename = path.split("/").last
+          creative = Collavre::Creative.new(
+            description: filename,
+            parent: parent,
+            user: @user,
+            data: {
+              "source" => {
+                "type" => "github_markdown",
+                "repo" => @repo,
+                "path" => path,
+                "sha" => @tree_cache[path],
+                "markdown" => content,
+                "repository_link_id" => @link.id
+              }
+            }
+          )
+          creative.skip_github_validation = true
+          creative.save!
+          @synced_creatives[path] = creative
+
+          processed, blobs = processor.process(content, path)
+          comment = create_content_comment(creative, processed)
+          attach_blobs(comment, blobs) if blobs.any?
+          created << creative
+        end
+
+        resequence_affected_parents(created)
+        @link.update!(last_synced_at: Time.current)
+        Collavre::Creative::RealtimeBroadcastable.broadcast_batch_created(created) if created.any?
+      end
+
+      private
+
+      def target_branch_push?
+        ref = @payload["ref"]
+        branch = ref&.sub("refs/heads/", "")
+        branch == @link.markdown_sync_branch
+      end
+
+      # Load all synced creatives for this repository link, indexed by path
+      def load_synced_creatives
+        scope = Collavre::Creative.where(archived_at: nil)
+
+        if scope.connection.adapter_name == "PostgreSQL"
+          scope = scope.where("(data->'source'->>'repository_link_id')::integer = ?", @link.id)
+        else
+          scope = scope.where("json_extract(data, '$.source.repository_link_id') = ?", @link.id)
+        end
+
+        scope.each_with_object({}) { |c, h| h[c.data.dig("source", "path")] = c }
+      end
+
+      def create_content_comment(creative, markdown_content)
+        topic = creative.content_topic(fallback_user: @user)
+        creative.comments.create!(
+          content: markdown_content,
+          topic: topic,
+          user: @user,
+          skip_dispatch: true
+        )
+      end
+
+      def update_content_comment(creative, markdown_content, blobs = [])
+        topic = creative.content_topic(fallback_user: @user)
+        comment = creative.comments.where(topic: topic).order(:created_at).first
+        if comment
+          comment.images.purge if comment.images.attached?
+          comment.update!(content: markdown_content)
+          attach_blobs(comment, blobs) if blobs.any?
+        else
+          comment = create_content_comment(creative, markdown_content)
+          attach_blobs(comment, blobs) if blobs.any?
+        end
+      end
+
+      def attach_blobs(comment, blobs)
+        blobs.each { |blob| comment.images.attach(blob) }
+      end
+
+      def resequence_affected_parents(new_creatives)
+        parent_ids = new_creatives.map(&:parent_id).compact.uniq
+        Collavre::Creative.transaction do
+          parent_ids.each do |pid|
+            siblings = Collavre::Creative.where(parent_id: pid, archived_at: nil)
+              .select(:id, :data, :description, :sequence)
+            sorted = siblings.sort_by do |c|
+              path = c.data&.dig("source", "path") || ""
+              is_dir = path.end_with?("/") || path == ""
+              [ is_dir ? 0 : 1, c.description.to_s.downcase ]
+            end
+            sorted.each_with_index do |c, idx|
+              c.update_column(:sequence, idx) if c.sequence != idx
+            end
+          end
+        end
+      end
+
+      # Returns [parent_creative, newly_created_directories]
+      def ensure_parent_directories(parts, root)
+        parent = root
+        current_path = ""
+        new_dirs = []
+
+        parts.each do |part|
+          current_path = current_path.empty? ? part : "#{current_path}/#{part}"
+          dir_path = "#{current_path}/"
+
+          existing = @synced_creatives[dir_path]
+
+          if existing
+            parent = existing
+          else
+            dir_creative = Collavre::Creative.new(
+              description: part,
+              parent: parent,
+              user: @user,
+              data: {
+                "source" => {
+                  "type" => "github_markdown",
+                  "repo" => @repo,
+                  "path" => dir_path,
+                  "repository_link_id" => @link.id
+                }
+              }
+            )
+            dir_creative.skip_github_validation = true
+            dir_creative.save!
+            @synced_creatives[dir_path] = dir_creative
+            new_dirs << dir_creative
+            parent = dir_creative
+          end
+        end
+
+        [ parent, new_dirs ]
+      end
+    end
+  end
+end