star-dlp 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 25b1201d34bb3a3e4d219f9faadde2b8d34aba5747c1ceb926482699107b44e2
-  data.tar.gz: f431fbe097ef52988772208ac1021d4e1dbd11f59fe16c4b8ab512e84ad19906
+  metadata.gz: 49aecf46afd8779a951f317d8412ae41d157bcb50d6df163db0eec69f556881b
+  data.tar.gz: 4f3b4809beb3fddc5508f2f2e55cd62012829ff6f6b142b28034eee4e2aaedc0
 SHA512:
-  metadata.gz: a21e8d101a58153efd911c4974e25683291d9b1ae52dc1f519b8b01f93dbb679c23f0d442eaad8dbf5c1b941e9db6dfa9abea42bc917c1f2a6d65984590dd195
-  data.tar.gz: 6783258316126bcdd95f5ebedd283d9ffa2ce38c6f24acfdd124e7e4ff584549cf1dc1ed9caa0d3118a1bd210ba2f5de64b51ac456777d6225e61d979480a4fc
+  metadata.gz: 824c986da6d7c0e30f058bec67b254d289024da4f7effdfbf1975af7d2a5414671e551d561df110331d5cdcff2a3c8f3427028600ade35d86fe3958264df370a
+  data.tar.gz: 18278facb4fd629b173af6f78f9a3c0b10c8ec2cd035590031f4a7fbe30d9a0d016d39fc2c6c2cfc3d285a5b52ac2cd41f6ea250a65a5e0e2589e3b54e213c1a

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    star-dlp (0.1.1)
+    star-dlp (0.1.2)
       fileutils (~> 1.6)
       github_api (~> 0.19.0)
       json (~> 2.6)

data/lib/star/dlp/downloader.rb

@@ -2,10 +2,12 @@
 
 require "github_api"
 require "json"
+require "tempfile"
 require "fileutils"
 require "time"
 require "base64"
 require "thread"
+require "open3"
 
 module Star
   module Dlp
@@ -18,6 +20,33 @@ module Star
       DEFAULT_RETRY_COUNT = 5
       DEFAULT_RETRY_DELAY = 1 # seconds
       
+      # Supported README formats in order of preference
+      README_FORMATS = [
+        "README.md",
+        "README.markdown",
+        "readme.md",
+        "README.org",
+        "README.rst",
+        "README.txt",
+        "README.rdoc",
+        "README.adoc",
+        "README",
+        "readme.org",
+        "readme.rst",
+        "readme.txt",
+        "readme.rdoc",
+        "readme.adoc",
+        "readme"
+      ]
+      
+      # Formats that need conversion to markdown
+      FORMATS_NEEDING_CONVERSION = {
+        ".org" => "org",
+        ".rst" => "rst",
+        ".txt" => "txt",
+        "" => "txt"  # For files without extension
+      }
+      
       def initialize(config, username, thread_count: DEFAULT_THREAD_COUNT, skip_readme: false, retry_count: DEFAULT_RETRY_COUNT, retry_delay: DEFAULT_RETRY_DELAY)
         @config = config
         @username = username
@@ -209,9 +238,9 @@ module Star
           ->(repo) { repo }, # Item name is the repo name itself
           ->(repo_full_name) {
             # Try to download README
-            readme_content = fetch_readme(repo_full_name)
+            readme_result = fetch_readme(repo_full_name)
             
-            if readme_content
+            if readme_result && readme_result[:content]
               # Get starred_at date if available, or use current date as fallback
               date = nil
               if repo_dates.key?(repo_full_name) && repo_dates[repo_full_name]
@@ -232,7 +261,9 @@ module Star
                 if File.exist?(md_filepath)
                   # Append README content to existing file
                   File.open(md_filepath, 'a') do |file|
-                    file.puts "\n\n## README\n\n#{readme_content}\n"
+                    file.puts "\n\n## README"
+                    file.puts "\n*Format: #{readme_result[:format]}*\n" if readme_result[:format] != "markdown"
+                    file.puts "\n#{readme_result[:content]}\n"
                   end
                 else
                   # Create new file with repository information and README
@@ -245,10 +276,14 @@ module Star
                     [View on GitHub](https://github.com/#{repo_full_name})
                     
                     ## README
-                    
-                    #{readme_content}
                   MARKDOWN
                   
+                  # Add format note if not markdown
+                  content += "\n*Format: #{readme_result[:format]}*\n" if readme_result[:format] != "markdown"
+                  
+                  # Add README content
+                  content += "\n#{readme_result[:content]}\n"
+                  
                   File.write(md_filepath, content)
                 end
                 
@@ -282,63 +317,158 @@ module Star
         }
       end
       
-      # Fetch README.md content from GitHub
+      # Fetch README content from GitHub
+      # Returns a hash with :content and :format keys, or nil if not found
       def fetch_readme(repo_full_name)
-        begin
-          # Get README content using GitHub API
-          response = github.repos.contents.get(
-            user: repo_full_name.split('/').first,
-            repo: repo_full_name.split('/').last,
-            path: 'README.md'
-          )
-          
-          # Decode content from Base64
-          if response.content && response.encoding == 'base64'
-            return Base64.decode64(response.content).force_encoding('UTF-8')
-          end
-        rescue Github::Error::NotFound
-          # Try README.markdown if README.md not found
+        # Try each README format in order
+        README_FORMATS.each do |readme_path|
           begin
+            # Get README content using GitHub API
             response = github.repos.contents.get(
               user: repo_full_name.split('/').first,
               repo: repo_full_name.split('/').last,
-              path: 'README.markdown'
+              path: readme_path
             )
             
+            # Decode content from Base64
             if response.content && response.encoding == 'base64'
-              return Base64.decode64(response.content).force_encoding('UTF-8')
+              content = Base64.decode64(response.content).force_encoding('UTF-8')
+              
+              # Get file extension
+              ext = File.extname(readme_path).downcase
+              
+              # Check if we need to convert the content
+              if FORMATS_NEEDING_CONVERSION.key?(ext)
+                format = FORMATS_NEEDING_CONVERSION[ext]
+                puts "Converting #{readme_path} from #{format} to markdown for #{repo_full_name}"
+                
+                # Create a temporary file with the content
+                temp_file = Tempfile.new(['readme', ".#{format}"])
+                begin
+                  temp_file.write(content)
+                  temp_file.close
+                  
+                  # Use pandoc to convert to markdown
+                  markdown_content, status = convert_to_markdown(temp_file.path, format)
+                  
+                  if status.success?
+                    return { content: markdown_content, format: format }
+                  else
+                    puts "Pandoc conversion failed for #{repo_full_name}, using original content"
+                    return { content: content, format: format }
+                  end
+                ensure
+                  temp_file.unlink
+                end
+              else
+                # Already markdown, no conversion needed
+                return { content: content, format: "markdown" }
+              end
             end
           rescue Github::Error::NotFound
-            # Try readme.md (lowercase) if previous attempts failed
-            begin
-              response = github.repos.contents.get(
-                user: repo_full_name.split('/').first,
-                repo: repo_full_name.split('/').last,
-                path: 'readme.md'
-              )
+            # Try next format
+            next
+          rescue => e
+            puts "Error fetching #{readme_path} for #{repo_full_name}: #{e.message}"
+            next
+          end
+        end
+        
+        # No README found in predefined formats, check for any readme-like file in the root directory
+        begin
+          # Get repository contents
+          contents = github.repos.contents.get(
+            user: repo_full_name.split('/').first,
+            repo: repo_full_name.split('/').last,
+            path: ""  # Root directory
+          )
+          
+          # Look for any file with name matching /readme/i
+          readme_file = contents.find { |item| item.type == "file" && item.name =~ /readme/i }
+          
+          if readme_file
+            puts "Found alternative README file: #{readme_file.name} for #{repo_full_name}"
+            
+            # Get README content
+            readme_content = github.repos.contents.get(
+              user: repo_full_name.split('/').first,
+              repo: repo_full_name.split('/').last,
+              path: readme_file.name
+            )
+            
+            # Decode content from Base64
+            if readme_content.content && readme_content.encoding == 'base64'
+              content = Base64.decode64(readme_content.content).force_encoding('UTF-8')
+              
+              # Get file extension
+              ext = File.extname(readme_file.name).downcase
               
-              if response.content && response.encoding == 'base64'
-                return Base64.decode64(response.content).force_encoding('UTF-8')
+              # Check if we need to convert the content
+              if FORMATS_NEEDING_CONVERSION.key?(ext)
+                format = FORMATS_NEEDING_CONVERSION[ext]
+                puts "Converting #{readme_file.name} from #{format} to markdown for #{repo_full_name}"
+                
+                # Create a temporary file with the content
+                temp_file = Tempfile.new(['readme', ".#{format}"])
+                begin
+                  temp_file.write(content)
+                  temp_file.close
+                  
+                  # Use pandoc to convert to markdown
+                  markdown_content, status = convert_to_markdown(temp_file.path, format)
+                  
+                  if status.success?
+                    return { content: markdown_content, format: format }
+                  else
+                    puts "Pandoc conversion failed for #{repo_full_name}, using original content"
+                    return { content: content, format: format }
+                  end
+                ensure
+                  temp_file.unlink
+                end
+              else
+                # Determine format based on extension or default to txt
+                format = ext.empty? ? "txt" : ext[1..]
+                # Use markdown format if extension suggests it's already markdown
+                format = "markdown" if [".md", ".markdown"].include?(ext)
+                
+                return { content: content, format: format }
               end
-            rescue Github::Error::NotFound
-              # README not found
-              return nil
-            rescue => e
-              puts "Error fetching lowercase readme.md for #{repo_full_name}: #{e.message}"
-              raise e
             end
-          rescue => e
-            puts "Error fetching README.markdown for #{repo_full_name}: #{e.message}"
-            raise e
           end
         rescue => e
-          puts "Error fetching README.md for #{repo_full_name}: #{e.message}"
-          raise e
+          puts "Error checking root directory for README-like files for #{repo_full_name}: #{e.message}"
         end
         
+        # No README found in any format
         nil
       end
       
+      # Convert content from a given format to markdown using pandoc
+      def convert_to_markdown(file_path, format)
+        begin
+          # Check if pandoc is installed
+          version_output, status = Open3.capture2e("pandoc --version")
+          unless status.success?
+            puts "Warning: pandoc is not installed or not in PATH. Cannot convert non-markdown formats."
+            return [File.read(file_path), status]
+          end
+          
+          # Use pandoc to convert to markdown
+          output, status = Open3.capture2e("pandoc", "-f", format, "-t", "markdown", file_path)
+          
+          if status.success?
+            return [output, status]
+          else
+            puts "Pandoc conversion failed: #{output}"
+            return [File.read(file_path), status]
+          end
+        rescue => e
+          puts "Error during conversion: #{e.message}"
+          return [File.read(file_path), OpenStruct.new(success?: false)]
+        end
+      end
+      
       private
       
       # Process a list of items using multiple threads
@@ -513,9 +643,12 @@ module Star
         
         # Try to fetch README.md content if not skipped
         unless @skip_readme
-          readme_content = fetch_readme(repo_full_name)
-          if readme_content
-            content += "\n\n## README\n\n#{readme_content}\n"
+          readme_result = fetch_readme(repo_full_name)
+          if readme_result && readme_result[:content]
+            content += "\n\n## README"
+            # Add format note if not markdown
+            content += "\n*Format: #{readme_result[:format]}*\n" if readme_result[:format] != "markdown"
+            content += "\n#{readme_result[:content]}\n"
           else
             content += "\n\n## Description\n\n#{get_description(star)}\n"
           end

data/lib/star/dlp/version.rb

@@ -2,6 +2,6 @@
 
 module Star
   module Dlp
-    VERSION = "0.1.1"
+    VERSION = "0.1.2"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: star-dlp
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Liu Xiang
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-03-16 00:00:00.000000000 Z
+date: 2025-03-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: github_api