spurline-test 0.3.0

data/lib/spurline/cartographer/analyzers/manifests.rb

@@ -0,0 +1,217 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Spurline
+  module Cartographer
+    module Analyzers
+      class Manifests < Analyzer
+        WEB_FRAMEWORKS = {
+          rails: "rails",
+          sinatra: "sinatra",
+          express: "express",
+          django: "django",
+        }.freeze
+
+        TEST_FRAMEWORKS = {
+          rspec: %w[rspec rspec-rails],
+          minitest: %w[minitest],
+          jest: %w[jest],
+          pytest: %w[pytest],
+        }.freeze
+
+        LINTERS = {
+          rubocop: "rubocop",
+          eslint: "eslint",
+          prettier: "prettier",
+        }.freeze
+
+        def analyze
+          gemfile = read_file("Gemfile")
+          gemfile_lock = read_file("Gemfile.lock")
+          package = parse_json_file("package.json")
+          pyproject = read_file("pyproject.toml")
+
+          frameworks = {}
+
+          web_name, web_version = detect_web_framework(
+            gemfile: gemfile,
+            gemfile_lock: gemfile_lock,
+            package: package,
+            pyproject: pyproject
+          )
+          frameworks[:web] = { name: web_name, version: web_version } if web_name
+
+          test_framework = detect_test_framework(
+            gemfile: gemfile,
+            gemfile_lock: gemfile_lock,
+            package: package,
+            pyproject: pyproject
+          )
+          frameworks[:test] = test_framework if test_framework
+
+          linters = detect_linters(
+            gemfile: gemfile,
+            gemfile_lock: gemfile_lock,
+            package: package,
+            pyproject: pyproject
+          )
+          frameworks[:linter] = linters.length == 1 ? linters.first : linters unless linters.empty?
+
+          ruby_version = detect_ruby_version(gemfile)
+          node_version = detect_node_version(package)
+
+          parsed_files = []
+          parsed_files << "Gemfile" if gemfile
+          parsed_files << "Gemfile.lock" if gemfile_lock
+          parsed_files << "package.json" if package
+          parsed_files << "pyproject.toml" if pyproject
+          parsed_files << ".ruby-version" if file_exists?(".ruby-version")
+          parsed_files << ".node-version" if file_exists?(".node-version")
+          parsed_files << ".python-version" if file_exists?(".python-version")
+
+          result = {
+            frameworks: frameworks,
+            metadata: {
+              manifests: {
+                parsed_files: parsed_files,
+              },
+            },
+          }
+          result[:ruby_version] = ruby_version if ruby_version
+          result[:node_version] = node_version if node_version
+
+          @findings = result
+        end
+
+        def confidence
+          parsed_count = findings.dig(:metadata, :manifests, :parsed_files)&.length.to_i
+          parsed_count.zero? ? 0.6 : 0.95
+        end
+
+        private
+
+        def detect_web_framework(gemfile:, gemfile_lock:, package:, pyproject:)
+          return [:rails, gem_version("rails", gemfile, gemfile_lock, package)] if dependency_present?("rails", gemfile, gemfile_lock, package, pyproject)
+          return [:sinatra, gem_version("sinatra", gemfile, gemfile_lock, package)] if dependency_present?("sinatra", gemfile, gemfile_lock, package, pyproject)
+          return [:express, package_version("express", package)] if dependency_present?("express", gemfile, gemfile_lock, package, pyproject)
+          return [:django, python_version("django", pyproject)] if dependency_present?("django", gemfile, gemfile_lock, package, pyproject)
+
+          [nil, nil]
+        end
+
+        def detect_test_framework(gemfile:, gemfile_lock:, package:, pyproject:)
+          TEST_FRAMEWORKS.each do |key, package_names|
+            return key if package_names.any? { |name| dependency_present?(name, gemfile, gemfile_lock, package, pyproject) }
+          end
+
+          nil
+        end
+
+        def detect_linters(gemfile:, gemfile_lock:, package:, pyproject:)
+          LINTERS.each_with_object([]) do |(key, package_name), list|
+            list << key if dependency_present?(package_name, gemfile, gemfile_lock, package, pyproject)
+          end
+        end
+
+        def detect_ruby_version(gemfile)
+          version_file = read_file(".ruby-version")&.strip
+          return version_file unless version_file.to_s.empty?
+
+          gemfile&.match(/^\s*ruby\s+["']([^"']+)["']/)&.captures&.first
+        end
+
+        def detect_node_version(package)
+          version_file = read_file(".node-version")&.strip
+          return version_file unless version_file.to_s.empty?
+
+          nvmrc_version = read_file(".nvmrc")&.strip
+          return nvmrc_version unless nvmrc_version.to_s.empty?
+
+          package&.dig("engines", "node")
+        end
+
+        def dependency_present?(name, gemfile, gemfile_lock, package, pyproject)
+          gem_declared?(name, gemfile) ||
+            gem_locked?(name, gemfile_lock) ||
+            package_dependency?(name, package) ||
+            python_dependency?(name, pyproject)
+        end
+
+        def gem_declared?(name, gemfile)
+          return false unless gemfile
+
+          gemfile.match?(/^\s*gem\s+["']#{Regexp.escape(name)}["']/)
+        end
+
+        def gem_locked?(name, gemfile_lock)
+          return false unless gemfile_lock
+
+          gemfile_lock.match?(/^\s{4}#{Regexp.escape(name)}\s+\(/)
+        end
+
+        def gem_version(name, gemfile, gemfile_lock, package)
+          version = gemfile_lock&.match(/^\s{4}#{Regexp.escape(name)}\s+\(([^)]+)\)/)&.captures&.first
+          return version if version
+
+          declared = gem_declared_version(name, gemfile)
+          return declared if declared
+
+          package_version(name, package)
+        end
+
+        def gem_declared_version(name, gemfile)
+          return nil unless gemfile
+
+          line = gemfile.each_line.find { |text| text.match?(/^\s*gem\s+["']#{Regexp.escape(name)}["']/) }
+          return nil unless line
+
+          match = line.match(/^\s*gem\s+["']#{Regexp.escape(name)}["']\s*,\s*["']([^"']+)["']/)
+          return nil unless match
+
+          normalize_version_requirement(match[1])
+        end
+
+        def normalize_version_requirement(value)
+          token = value.to_s.split(",").first.to_s.strip
+          token = token.sub(/\A[~><=\s]*/, "")
+          token.empty? ? nil : token
+        end
+
+        def package_dependency?(name, package)
+          return false unless package.is_a?(Hash)
+
+          package.key?("dependencies") && package["dependencies"].is_a?(Hash) && package["dependencies"].key?(name) ||
+            package.key?("devDependencies") && package["devDependencies"].is_a?(Hash) && package["devDependencies"].key?(name)
+        end
+
+        def package_version(name, package)
+          return nil unless package.is_a?(Hash)
+
+          package.dig("dependencies", name) || package.dig("devDependencies", name)
+        end
+
+        def python_dependency?(name, pyproject)
+          return false unless pyproject
+
+          pyproject.match?(/#{Regexp.escape(name)}\s*(?:[<>=~!]|$)/i)
+        end
+
+        def python_version(name, pyproject)
+          return nil unless pyproject
+
+          pyproject.match(/#{Regexp.escape(name)}\s*([<>=~!].+?)?(?:\n|$)/i)&.captures&.first&.strip
+        end
+
+        def parse_json_file(relative_path)
+          content = read_file(relative_path)
+          return nil unless content
+
+          JSON.parse(content)
+        rescue JSON::ParserError
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/spurline/cartographer/analyzers/security_scan.rb

@@ -0,0 +1,223 @@
+# frozen_string_literal: true
+
+require "find"
+require "json"
+
+module Spurline
+  module Cartographer
+    module Analyzers
+      class SecurityScan < Analyzer
+        SECRET_PATTERNS = {
+          openai_key: /\bsk-[A-Za-z0-9]{20,}\b/,
+          aws_access_key_id: /\bAKIA[0-9A-Z]{16}\b/,
+          github_token: /\bghp_[A-Za-z0-9]{36}\b/,
+        }.freeze
+        GENERIC_ASSIGNMENT_PATTERN = /\b(api[_-]?key|token|password|secret)\b\s*[:=]\s*["']([^"']{16,})["']/i.freeze
+
+        CURATED_SUSPICIOUS_DEPENDENCIES = %w[
+          pymafka
+          requests-darwin
+          osxroot
+          colourfool
+        ].freeze
+
+        MAX_SCAN_BYTES = 512_000
+
+        def analyze
+          findings = []
+          findings.concat(scan_sensitive_filenames)
+          findings.concat(scan_secret_patterns)
+          findings.concat(scan_suspicious_dependencies)
+
+          @findings = {
+            security_findings: findings,
+          }
+        end
+
+        def confidence
+          0.9
+        end
+
+        private
+
+        def scan_sensitive_filenames
+          each_candidate_file.each_with_object([]) do |(path, rel_path), findings|
+            basename = File.basename(path)
+
+            if rel_path == ".env" || rel_path.end_with?("/.env")
+              findings << finding(
+                type: :sensitive_file,
+                severity: :high,
+                file: rel_path,
+                detail: "Environment file is committed."
+              )
+            end
+
+            if rel_path == "config/credentials.yml" || rel_path == "credentials.yml"
+              findings << finding(
+                type: :sensitive_file,
+                severity: :high,
+                file: rel_path,
+                detail: "Credentials file is committed."
+              )
+            end
+
+            if basename.match?(/\.(pem|key)\z/i)
+              findings << finding(
+                type: :sensitive_file,
+                severity: :high,
+                file: rel_path,
+                detail: "Private key material appears committed (#{basename})."
+              )
+            end
+          end
+        end
+
+        def scan_secret_patterns
+          each_candidate_file.each_with_object([]) do |(path, rel_path), findings|
+            next if binary_file?(path)
+
+            content = read_limited(path)
+            next if content.nil? || content.empty?
+
+            SECRET_PATTERNS.each do |name, pattern|
+              next unless content.match?(pattern)
+
+              findings << finding(
+                type: :hardcoded_secret,
+                severity: :high,
+                file: rel_path,
+                detail: "Matched #{name} pattern."
+              )
+            end
+
+            generic_secret_assignment_details(content).each do |detail|
+              findings << finding(
+                type: :hardcoded_secret,
+                severity: :high,
+                file: rel_path,
+                detail: detail
+              )
+            end
+          end
+        end
+
+        def scan_suspicious_dependencies
+          findings = []
+
+          package = parse_json_file("package.json")
+          if package
+            deps = [package["dependencies"], package["devDependencies"]].compact
+                      .select { |value| value.is_a?(Hash) }
+                      .flat_map(&:keys)
+                      .uniq
+
+            deps.each do |dependency|
+              next unless CURATED_SUSPICIOUS_DEPENDENCIES.include?(dependency)
+
+              findings << finding(
+                type: :suspicious_dependency,
+                severity: :medium,
+                file: "package.json",
+                detail: "Dependency '#{dependency}' is in the curated suspicious list."
+              )
+            end
+          end
+
+          gemfile = read_file("Gemfile")
+          if gemfile
+            gemfile.scan(/^\s*gem\s+["']([^"']+)["']/).flatten.each do |dependency|
+              next unless CURATED_SUSPICIOUS_DEPENDENCIES.include?(dependency)
+
+              findings << finding(
+                type: :suspicious_dependency,
+                severity: :medium,
+                file: "Gemfile",
+                detail: "Gem '#{dependency}' is in the curated suspicious list."
+              )
+            end
+          end
+
+          findings
+        end
+
+        def each_candidate_file
+          files = []
+
+          Find.find(repo_path) do |path|
+            rel_path = relative_path(path)
+            rel_path = "." if rel_path.empty?
+
+            if File.directory?(path)
+              if rel_path != "." && excluded_relative_path?(rel_path)
+                Find.prune
+              else
+                next
+              end
+            end
+
+            next if excluded_relative_path?(rel_path)
+            next unless File.file?(path)
+
+            files << [path, rel_path]
+          end
+
+          files
+        end
+
+        def parse_json_file(relative_path)
+          content = read_file(relative_path)
+          return nil unless content
+
+          JSON.parse(content)
+        rescue JSON::ParserError
+          nil
+        end
+
+        def read_limited(path)
+          File.open(path, "rb") { |file| file.read(MAX_SCAN_BYTES) }
+              &.force_encoding(Encoding::UTF_8)
+              &.scrub
+        rescue Errno::ENOENT, ArgumentError
+          nil
+        end
+
+        def binary_file?(path)
+          sample = File.open(path, "rb") { |file| file.read(1024) }
+          return false unless sample
+
+          sample.include?("\x00")
+        rescue Errno::ENOENT
+          false
+        end
+
+        def finding(type:, severity:, file:, detail:)
+          {
+            type: type,
+            severity: severity,
+            file: file,
+            detail: detail,
+          }
+        end
+
+        def generic_secret_assignment_details(content)
+          content.scan(GENERIC_ASSIGNMENT_PATTERN).filter_map do |(name, value)|
+            next if benign_secret_value?(value)
+
+            "Matched generic_secret_assignment pattern for #{name.downcase}."
+          end.uniq
+        end
+
+        def benign_secret_value?(value)
+          normalized = value.to_s.strip
+          return true if normalized.empty?
+          return true if normalized.match?(/\A\[REDACTED/i)
+          return true if normalized.match?(/\A(redacted|test|example|dummy|fake|placeholder|changeme|replace_me)/i)
+          return true if normalized.include?("user@example.com")
+
+          false
+        end
+      end
+    end
+  end
+end

data/lib/spurline/cartographer/repo_profile.rb

@@ -0,0 +1,140 @@
+# frozen_string_literal: true
+
+require "json"
+require "time"
+
+module Spurline
+  module Cartographer
+    # Immutable, serializable analysis output for a repository.
+    class RepoProfile
+      CURRENT_VERSION = "1.0"
+
+      attr_reader :version, :analyzed_at, :repo_path,
+                  :languages, :frameworks, :ruby_version, :node_version,
+                  :ci, :entry_points, :environment_vars_required,
+                  :security_findings, :confidence, :metadata
+
+      def initialize(**attrs)
+        @version = CURRENT_VERSION
+        @analyzed_at = normalize_time(attrs.fetch(:analyzed_at, Time.now.utc.iso8601))
+        @repo_path = attrs.fetch(:repo_path)
+        @languages = deep_copy(attrs.fetch(:languages, {}))
+        @frameworks = deep_copy(attrs.fetch(:frameworks, {}))
+        @ruby_version = attrs.fetch(:ruby_version, nil)
+        @node_version = attrs.fetch(:node_version, nil)
+        @ci = deep_copy(attrs.fetch(:ci, {}))
+        @entry_points = deep_copy(attrs.fetch(:entry_points, {}))
+        @environment_vars_required = deep_copy(attrs.fetch(:environment_vars_required, []))
+        @security_findings = deep_copy(attrs.fetch(:security_findings, []))
+        @confidence = deep_copy(attrs.fetch(:confidence, {}))
+        @metadata = deep_copy(attrs.fetch(:metadata, {}))
+
+        deep_freeze(@languages)
+        deep_freeze(@frameworks)
+        deep_freeze(@ci)
+        deep_freeze(@entry_points)
+        deep_freeze(@environment_vars_required)
+        deep_freeze(@security_findings)
+        deep_freeze(@confidence)
+        deep_freeze(@metadata)
+        freeze
+      end
+
+      def to_h
+        {
+          version: version,
+          analyzed_at: analyzed_at,
+          repo_path: repo_path,
+          languages: deep_copy(languages),
+          frameworks: deep_copy(frameworks),
+          ruby_version: ruby_version,
+          node_version: node_version,
+          ci: deep_copy(ci),
+          entry_points: deep_copy(entry_points),
+          environment_vars_required: deep_copy(environment_vars_required),
+          security_findings: deep_copy(security_findings),
+          confidence: deep_copy(confidence),
+          metadata: deep_copy(metadata),
+        }
+      end
+
+      def self.from_h(hash)
+        data = deep_symbolize(hash || {})
+        new(
+          analyzed_at: data[:analyzed_at],
+          repo_path: data.fetch(:repo_path),
+          languages: data[:languages] || {},
+          frameworks: data[:frameworks] || {},
+          ruby_version: data[:ruby_version],
+          node_version: data[:node_version],
+          ci: data[:ci] || {},
+          entry_points: data[:entry_points] || {},
+          environment_vars_required: data[:environment_vars_required] || [],
+          security_findings: data[:security_findings] || [],
+          confidence: data[:confidence] || {},
+          metadata: data[:metadata] || {}
+        )
+      end
+
+      def to_json(*)
+        JSON.generate(to_h)
+      end
+
+      def secure?
+        security_findings.empty?
+      end
+
+      private
+
+      def normalize_time(value)
+        return value.utc.iso8601 if value.respond_to?(:utc) && value.respond_to?(:iso8601)
+
+        value.to_s
+      end
+
+      def deep_copy(value)
+        case value
+        when Hash
+          value.each_with_object({}) do |(key, item), copy|
+            copy[key] = deep_copy(item)
+          end
+        when Array
+          value.map { |item| deep_copy(item) }
+        else
+          value
+        end
+      end
+
+      def deep_freeze(value)
+        case value
+        when Hash
+          value.each do |key, item|
+            deep_freeze(key)
+            deep_freeze(item)
+          end
+        when Array
+          value.each { |item| deep_freeze(item) }
+        end
+
+        value.freeze
+      end
+
+      class << self
+        private
+
+        def deep_symbolize(value)
+          case value
+          when Hash
+            value.each_with_object({}) do |(key, item), hash|
+              hash[key.to_sym] = deep_symbolize(item)
+            end
+          when Array
+            value.map { |item| deep_symbolize(item) }
+          else
+            value
+          end
+        end
+      end
+    end
+  end
+end

data/lib/spurline/cartographer/runner.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+module Spurline
+  module Cartographer
+    class Runner
+      ANALYZERS = [
+        Analyzers::FileSignatures,
+        Analyzers::Manifests,
+        Analyzers::CIConfig,
+        Analyzers::Dotfiles,
+        Analyzers::EntryPoints,
+        Analyzers::SecurityScan,
+      ].freeze
+
+      # ASYNC-READY:
+      def analyze(repo_path:, scheduler: Spurline::Adapters::Scheduler::Sync.new)
+        expanded_path = File.expand_path(repo_path)
+        validate_path!(expanded_path)
+
+        results = {}
+        confidences = {}
+        active_scheduler = scheduler.is_a?(Class) ? scheduler.new : scheduler
+
+        ANALYZERS.each do |klass|
+          analyzer = klass.new(repo_path: expanded_path)
+          layer_result = active_scheduler.run { analyzer.analyze }
+
+          unless layer_result.is_a?(Hash)
+            raise Spurline::AnalyzerError,
+              "#{klass.name} returned #{layer_result.class} instead of Hash"
+          end
+
+          results = deep_merge(results, layer_result)
+          confidences[analyzer_key(klass)] = analyzer.confidence
+        rescue StandardError => e
+          confidences[analyzer_key(klass)] = 0.0
+          results[:metadata] ||= {}
+          (results[:metadata][:analyzer_errors] ||= []) << {
+            analyzer: klass.name,
+            error: e.message,
+          }
+        end
+
+        results = deep_merge(results, confidence: build_confidence(confidences))
+
+        RepoProfile.new(repo_path: expanded_path, **results)
+      end
+
+      private
+
+      def validate_path!(path)
+        return if File.directory?(path)
+
+        raise Spurline::CartographerAccessError,
+          "Repository path '#{path}' does not exist or is not a directory. " \
+          "Provide an absolute path to a valid repository."
+      end
+
+      def analyzer_key(klass)
+        name = klass.name || "AnonymousAnalyzer#{klass.object_id}"
+        name = name.split("::").last
+        name = name.gsub(/([A-Z]+)([A-Z][a-z])/, "\\1_\\2")
+        name = name.gsub(/([a-z\\d])([A-Z])/, "\\1_\\2")
+        name.downcase.to_sym
+      end
+
+      def build_confidence(layer_confidences)
+        scores = layer_confidences.values
+        {
+          overall: scores.empty? ? 0.0 : (scores.sum / scores.size).round(2),
+          per_layer: layer_confidences,
+        }
+      end
+
+      def deep_merge(left, right)
+        left.merge(right) do |_key, left_value, right_value|
+          if left_value.is_a?(Hash) && right_value.is_a?(Hash)
+            deep_merge(left_value, right_value)
+          elsif left_value.is_a?(Array) && right_value.is_a?(Array)
+            (left_value + right_value).uniq
+          else
+            right_value
+          end
+        end
+      end
+    end
+  end
+end

data/lib/spurline/cartographer.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module Spurline
+  module Cartographer
+  end
+end