star-dlp 0.1.1 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 25b1201d34bb3a3e4d219f9faadde2b8d34aba5747c1ceb926482699107b44e2
-  data.tar.gz: f431fbe097ef52988772208ac1021d4e1dbd11f59fe16c4b8ab512e84ad19906
+  metadata.gz: 89dbd739d276a3c9015fbd3305a97a546bcc9973f789305aadd12f7cd1e4ff98
+  data.tar.gz: 1a1fdf75575b6c805653fe83e0415f234ca70212e6550e48b31147a92976c93b
 SHA512:
-  metadata.gz: a21e8d101a58153efd911c4974e25683291d9b1ae52dc1f519b8b01f93dbb679c23f0d442eaad8dbf5c1b941e9db6dfa9abea42bc917c1f2a6d65984590dd195
-  data.tar.gz: 6783258316126bcdd95f5ebedd283d9ffa2ce38c6f24acfdd124e7e4ff584549cf1dc1ed9caa0d3118a1bd210ba2f5de64b51ac456777d6225e61d979480a4fc
+  metadata.gz: 160c8340faff0de94cca2214b54e00d094545fc9a9e1966e8231c4f36579f5b638bc0328084524562fd4546d3899bfde0d017759edca7a8b4e4d754c786b8c0c
+  data.tar.gz: 9e69361912bc9b4ca02cfa2bfa07dd0001c1b308a40c2550c7d39095254d8c7533a3d51e5b7543984c36c3b568bdda4c4c2a0e1c0022c4410cde9b1b966a1b87

data/Gemfile.lock

@@ -1,10 +1,13 @@
 PATH
   remote: .
   specs:
-    star-dlp (0.1.1)
+    star-dlp (0.1.3)
       fileutils (~> 1.6)
       github_api (~> 0.19.0)
-      json (~> 2.6)
+      json (>= 2.6)
+      logger
+      multipart-post (< 2.4.0)
+      ostruct
       thor (~> 1.2)
 
 GEM
@@ -17,7 +20,7 @@ GEM
     descendants_tracker (0.0.4)
       thread_safe (~> 0.3, >= 0.3.1)
     diff-lcs (1.5.0)
-    faraday (1.10.4)
+    faraday (1.10.6)
       faraday-em_http (~> 1.0)
       faraday-em_synchrony (~> 1.0)
       faraday-excon (~> 1.1)
@@ -33,13 +36,13 @@ GEM
     faraday-em_synchrony (1.0.0)
     faraday-excon (1.1.0)
     faraday-httpclient (1.0.1)
-    faraday-multipart (1.1.0)
+    faraday-multipart (1.2.0)
       multipart-post (~> 2.0)
     faraday-net_http (1.0.2)
     faraday-net_http_persistent (1.2.0)
     faraday-patron (1.0.0)
     faraday-rack (1.0.0)
-    faraday-retry (1.0.3)
+    faraday-retry (1.0.4)
     fileutils (1.7.0)
     github_api (0.19.0)
       addressable (~> 2.4)
@@ -48,19 +51,21 @@ GEM
       hashie (~> 3.5, >= 3.5.2)
       oauth2 (~> 1.0)
     hashie (3.6.0)
-    json (2.6.3)
+    json (2.21.2)
     jwt (2.10.1)
       base64
+    logger (1.7.0)
     multi_json (1.15.0)
     multi_xml (0.7.1)
       bigdecimal (~> 3.1)
-    multipart-post (2.4.1)
+    multipart-post (2.3.0)
     oauth2 (1.4.11)
       faraday (>= 0.17.3, < 3.0)
       jwt (>= 1.0, < 3.0)
       multi_json (~> 1.3)
       multi_xml (~> 0.5)
       rack (>= 1.2, < 4)
+    ostruct (0.6.3)
     public_suffix (6.0.1)
     rack (3.1.12)
     rake (13.2.1)
@@ -83,6 +88,7 @@ GEM
 
 PLATFORMS
   arm64-darwin-23
+  arm64-darwin-25
 
 DEPENDENCIES
   github_api (~> 0.19.0)
@@ -91,4 +97,4 @@ DEPENDENCIES
   star-dlp!
 
 BUNDLED WITH
-   2.4.12
+  4.0.18

data/lib/star/dlp/downloader.rb

@@ -2,10 +2,12 @@
 
 require "github_api"
 require "json"
+require "tempfile"
 require "fileutils"
 require "time"
 require "base64"
 require "thread"
+require "open3"
 
 module Star
   module Dlp
@@ -18,6 +20,33 @@ module Star
       DEFAULT_RETRY_COUNT = 5
       DEFAULT_RETRY_DELAY = 1 # seconds
       
+      # Supported README formats in order of preference
+      README_FORMATS = [
+        "README.md",
+        "README.markdown",
+        "readme.md",
+        "README.org",
+        "README.rst",
+        "README.txt",
+        "README.rdoc",
+        "README.adoc",
+        "README",
+        "readme.org",
+        "readme.rst",
+        "readme.txt",
+        "readme.rdoc",
+        "readme.adoc",
+        "readme"
+      ]
+      
+      # Formats that need conversion to markdown
+      FORMATS_NEEDING_CONVERSION = {
+        ".org" => "org",
+        ".rst" => "rst",
+        ".txt" => "txt",
+        "" => "txt"  # For files without extension
+      }
+      
       def initialize(config, username, thread_count: DEFAULT_THREAD_COUNT, skip_readme: false, retry_count: DEFAULT_RETRY_COUNT, retry_delay: DEFAULT_RETRY_DELAY)
         @config = config
         @username = username
@@ -209,9 +238,9 @@ module Star
           ->(repo) { repo }, # Item name is the repo name itself
           ->(repo_full_name) {
             # Try to download README
-            readme_content = fetch_readme(repo_full_name)
+            readme_result = fetch_readme(repo_full_name)
             
-            if readme_content
+            if readme_result && readme_result[:content]
               # Get starred_at date if available, or use current date as fallback
               date = nil
               if repo_dates.key?(repo_full_name) && repo_dates[repo_full_name]
@@ -232,7 +261,9 @@ module Star
                 if File.exist?(md_filepath)
                   # Append README content to existing file
                   File.open(md_filepath, 'a') do |file|
-                    file.puts "\n\n## README\n\n#{readme_content}\n"
+                    file.puts "\n\n## README"
+                    file.puts "\n*Format: #{readme_result[:format]}*\n" if readme_result[:format] != "markdown"
+                    file.puts "\n#{readme_result[:content]}\n"
                   end
                 else
                   # Create new file with repository information and README
@@ -245,10 +276,14 @@ module Star
                     [View on GitHub](https://github.com/#{repo_full_name})
                     
                     ## README
-                    
-                    #{readme_content}
                   MARKDOWN
                   
+                  # Add format note if not markdown
+                  content += "\n*Format: #{readme_result[:format]}*\n" if readme_result[:format] != "markdown"
+                  
+                  # Add README content
+                  content += "\n#{readme_result[:content]}\n"
+                  
                   File.write(md_filepath, content)
                 end
                 
@@ -282,63 +317,158 @@ module Star
         }
       end
       
-      # Fetch README.md content from GitHub
+      # Fetch README content from GitHub
+      # Returns a hash with :content and :format keys, or nil if not found
       def fetch_readme(repo_full_name)
-        begin
-          # Get README content using GitHub API
-          response = github.repos.contents.get(
-            user: repo_full_name.split('/').first,
-            repo: repo_full_name.split('/').last,
-            path: 'README.md'
-          )
-          
-          # Decode content from Base64
-          if response.content && response.encoding == 'base64'
-            return Base64.decode64(response.content).force_encoding('UTF-8')
-          end
-        rescue Github::Error::NotFound
-          # Try README.markdown if README.md not found
+        # Try each README format in order
+        README_FORMATS.each do |readme_path|
           begin
+            # Get README content using GitHub API
             response = github.repos.contents.get(
               user: repo_full_name.split('/').first,
               repo: repo_full_name.split('/').last,
-              path: 'README.markdown'
+              path: readme_path
             )
             
+            # Decode content from Base64
             if response.content && response.encoding == 'base64'
-              return Base64.decode64(response.content).force_encoding('UTF-8')
+              content = Base64.decode64(response.content).force_encoding('UTF-8')
+              
+              # Get file extension
+              ext = File.extname(readme_path).downcase
+              
+              # Check if we need to convert the content
+              if FORMATS_NEEDING_CONVERSION.key?(ext)
+                format = FORMATS_NEEDING_CONVERSION[ext]
+                puts "Converting #{readme_path} from #{format} to markdown for #{repo_full_name}"
+                
+                # Create a temporary file with the content
+                temp_file = Tempfile.new(['readme', ".#{format}"])
+                begin
+                  temp_file.write(content)
+                  temp_file.close
+                  
+                  # Use pandoc to convert to markdown
+                  markdown_content, status = convert_to_markdown(temp_file.path, format)
+                  
+                  if status.success?
+                    return { content: markdown_content, format: format }
+                  else
+                    puts "Pandoc conversion failed for #{repo_full_name}, using original content"
+                    return { content: content, format: format }
+                  end
+                ensure
+                  temp_file.unlink
+                end
+              else
+                # Already markdown, no conversion needed
+                return { content: content, format: "markdown" }
+              end
             end
           rescue Github::Error::NotFound
-            # Try readme.md (lowercase) if previous attempts failed
-            begin
-              response = github.repos.contents.get(
-                user: repo_full_name.split('/').first,
-                repo: repo_full_name.split('/').last,
-                path: 'readme.md'
-              )
+            # Try next format
+            next
+          rescue => e
+            puts "Error fetching #{readme_path} for #{repo_full_name}: #{e.message}"
+            next
+          end
+        end
+        
+        # No README found in predefined formats, check for any readme-like file in the root directory
+        begin
+          # Get repository contents
+          contents = github.repos.contents.get(
+            user: repo_full_name.split('/').first,
+            repo: repo_full_name.split('/').last,
+            path: ""  # Root directory
+          )
+          
+          # Look for any file with name matching /readme/i
+          readme_file = contents.find { |item| item.type == "file" && item.name =~ /readme/i }
+          
+          if readme_file
+            puts "Found alternative README file: #{readme_file.name} for #{repo_full_name}"
+            
+            # Get README content
+            readme_content = github.repos.contents.get(
+              user: repo_full_name.split('/').first,
+              repo: repo_full_name.split('/').last,
+              path: readme_file.name
+            )
+            
+            # Decode content from Base64
+            if readme_content.content && readme_content.encoding == 'base64'
+              content = Base64.decode64(readme_content.content).force_encoding('UTF-8')
+              
+              # Get file extension
+              ext = File.extname(readme_file.name).downcase
               
-              if response.content && response.encoding == 'base64'
-                return Base64.decode64(response.content).force_encoding('UTF-8')
+              # Check if we need to convert the content
+              if FORMATS_NEEDING_CONVERSION.key?(ext)
+                format = FORMATS_NEEDING_CONVERSION[ext]
+                puts "Converting #{readme_file.name} from #{format} to markdown for #{repo_full_name}"
+                
+                # Create a temporary file with the content
+                temp_file = Tempfile.new(['readme', ".#{format}"])
+                begin
+                  temp_file.write(content)
+                  temp_file.close
+                  
+                  # Use pandoc to convert to markdown
+                  markdown_content, status = convert_to_markdown(temp_file.path, format)
+                  
+                  if status.success?
+                    return { content: markdown_content, format: format }
+                  else
+                    puts "Pandoc conversion failed for #{repo_full_name}, using original content"
+                    return { content: content, format: format }
+                  end
+                ensure
+                  temp_file.unlink
+                end
+              else
+                # Determine format based on extension or default to txt
+                format = ext.empty? ? "txt" : ext[1..]
+                # Use markdown format if extension suggests it's already markdown
+                format = "markdown" if [".md", ".markdown"].include?(ext)
+                
+                return { content: content, format: format }
               end
-            rescue Github::Error::NotFound
-              # README not found
-              return nil
-            rescue => e
-              puts "Error fetching lowercase readme.md for #{repo_full_name}: #{e.message}"
-              raise e
             end
-          rescue => e
-            puts "Error fetching README.markdown for #{repo_full_name}: #{e.message}"
-            raise e
           end
         rescue => e
-          puts "Error fetching README.md for #{repo_full_name}: #{e.message}"
-          raise e
+          puts "Error checking root directory for README-like files for #{repo_full_name}: #{e.message}"
         end
         
+        # No README found in any format
         nil
       end
       
+      # Convert content from a given format to markdown using pandoc
+      def convert_to_markdown(file_path, format)
+        begin
+          # Check if pandoc is installed
+          version_output, status = Open3.capture2e("pandoc --version")
+          unless status.success?
+            puts "Warning: pandoc is not installed or not in PATH. Cannot convert non-markdown formats."
+            return [File.read(file_path), status]
+          end
+          
+          # Use pandoc to convert to markdown
+          output, status = Open3.capture2e("pandoc", "-f", format, "-t", "markdown", file_path)
+          
+          if status.success?
+            return [output, status]
+          else
+            puts "Pandoc conversion failed: #{output}"
+            return [File.read(file_path), status]
+          end
+        rescue => e
+          puts "Error during conversion: #{e.message}"
+          return [File.read(file_path), OpenStruct.new(success?: false)]
+        end
+      end
+      
       private
       
       # Process a list of items using multiple threads
@@ -513,9 +643,12 @@ module Star
         
         # Try to fetch README.md content if not skipped
         unless @skip_readme
-          readme_content = fetch_readme(repo_full_name)
-          if readme_content
-            content += "\n\n## README\n\n#{readme_content}\n"
+          readme_result = fetch_readme(repo_full_name)
+          if readme_result && readme_result[:content]
+            content += "\n\n## README"
+            # Add format note if not markdown
+            content += "\n*Format: #{readme_result[:format]}*\n" if readme_result[:format] != "markdown"
+            content += "\n#{readme_result[:content]}\n"
           else
             content += "\n\n## Description\n\n#{get_description(star)}\n"
           end

data/lib/star/dlp/version.rb

@@ -2,6 +2,6 @@
 
 module Star
   module Dlp
-    VERSION = "0.1.1"
+    VERSION = "0.1.3"
   end
 end

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: star-dlp
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.3
 platform: ruby
 authors:
 - Liu Xiang
-autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-03-16 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: github_api
@@ -24,6 +23,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.19.0
+- !ruby/object:Gem::Dependency
+  name: multipart-post
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 2.4.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 2.4.0
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
@@ -56,16 +69,44 @@ dependencies:
   name: json
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '2.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '2.6'
+- !ruby/object:Gem::Dependency
+  name: ostruct
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: logger
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: star-dlp is a tool that helps you download, organize, and manage repositories
   you've starred on GitHub
 email:
@@ -101,7 +142,6 @@ metadata:
   homepage_uri: https://github.com/lululau/star-dlp
   source_code_uri: https://github.com/lululau/star-dlp
   changelog_uri: https://github.com/lululau/star-dlp/blob/main/CHANGELOG.md
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -116,8 +156,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.1
-signing_key:
+rubygems_version: 4.0.3
 specification_version: 4
 summary: A Ruby gem for downloading and managing your GitHub stars
 test_files: []