capsium 0.1.1 → 0.2.0

data/lib/capsium/package.rb

@@ -1,39 +1,51 @@
 # frozen_string_literal: true
 
-# lib/capsium/package.rb
 require "fileutils"
-require "json"
-require "yaml"
-require "csv"
-require "sqlite3"
-require "zip"
-require_relative "package/manifest"
-require_relative "package/metadata"
-require_relative "package/routes"
-require_relative "package/dataset"
-require_relative "package/storage"
-require_relative "packager"
+require "pathname"
+require "tmpdir"
 
 module Capsium
   class Package
-    attr_reader :name, :path, :manifest, :metadata, :routes, :datasets, :storage
+    autoload :Dataset, "capsium/package/dataset"
+    autoload :DatasetConfig, "capsium/package/storage_config"
+    autoload :DigitalSignatures, "capsium/package/security_config"
+    autoload :IntegrityChecks, "capsium/package/security_config"
+    autoload :Manifest, "capsium/package/manifest"
+    autoload :ManifestConfig, "capsium/package/manifest_config"
+    autoload :Metadata, "capsium/package/metadata"
+    autoload :MetadataData, "capsium/package/metadata_config"
+    autoload :Repository, "capsium/package/metadata_config"
+    autoload :Resource, "capsium/package/manifest_config"
+    autoload :Route, "capsium/package/routes_config"
+    autoload :Routes, "capsium/package/routes"
+    autoload :RoutesConfig, "capsium/package/routes_config"
+    autoload :Security, "capsium/package/security"
+    autoload :SecurityConfig, "capsium/package/security_config"
+    autoload :SecurityData, "capsium/package/security_config"
+    autoload :Storage, "capsium/package/storage"
+    autoload :StorageConfig, "capsium/package/storage_config"
+    autoload :StorageData, "capsium/package/storage_config"
+    autoload :Validator, "capsium/package/validator"
+
+    attr_reader :name, :path, :manifest, :metadata, :routes, :storage,
+                :security, :load_type
 
     MANIFEST_FILE = "manifest.json"
     METADATA_FILE = "metadata.json"
-    PACKAGING_FILE = "packaging.json"
-    SIGNATURE_FILE = "signature.json"
     STORAGE_FILE = "storage.json"
     ROUTES_FILE = "routes.json"
+    SECURITY_FILE = "security.json"
     CONTENT_DIR = "content"
     DATA_DIR = "data"
-    ENCRYPTED_PACKAGING_FILE = "package.enc"
 
-    def initialize(path)
+    def initialize(path, load_type: nil)
       @original_path = Pathname.new(path).expand_path
-      @path = prepare_package(@original_path)
+      @path = prepare_package(@original_path).to_s
+      @load_type = load_type || determine_load_type(path)
       create_package_structure
       load_package
       @name = metadata.name
+      verify_integrity!
     end
 
     def prepare_package(path)
@@ -42,47 +54,23 @@ module Capsium
       if File.file?(path)
         return decompress_cap_file(path) if File.extname(path) == ".cap"
 
-        raise "Error: The package must have a .cap extension"
-
+        raise Error, "The package must have a .cap extension"
       end
 
-      raise "Invalid package path: #{path}"
+      raise Error, "Invalid package path: #{path}"
     end
 
     def solidify
       @manifest.save_to_file
       @metadata.save_to_file
       @routes.save_to_file
-      @storage.save_to_file
+      @storage.save_to_file unless @storage.empty?
     end
 
     def decompress_cap_file(file_path)
-      temp_dir = Dir.mktmpdir
-      metadata_path = File.join(temp_dir, METADATA_FILE)
-
-      # Extract metadata.json first
-      Zip::File.open(file_path) do |zip_file|
-        if entry = zip_file.find_entry(METADATA_FILE)
-          entry.extract(metadata_path)
-        end
-      end
-
-      metadata = Metadata.new(metadata_path)
-      package_name = metadata.name
-      package_version = metadata.version
-      package_dependencies = metadata.dependencies
-
-      package_path = File.join(temp_dir, "#{package_name}-#{package_version}")
+      package_path = File.join(Dir.mktmpdir, package_stem(file_path))
       FileUtils.mkdir_p(package_path)
-
-      Zip::File.open(file_path) do |zip_file|
-        zip_file.each do |entry|
-          entry_path = File.join(package_path, entry.name)
-          FileUtils.mkdir_p(File.dirname(entry_path))
-          entry.extract(entry_path)
-        end
-      end
-
+      Packager.new.unpack(file_path, package_path)
       package_path
     end
 
@@ -92,51 +80,77 @@ module Capsium
 
       # Optional
       @manifest = Manifest.new(manifest_path)
-      @routes = Routes.new(routes_path, @manifest)
       @storage = Storage.new(storage_path)
-      # @datasets = load_datasets
+      @routes = Routes.new(routes_path, @manifest, @storage)
+      @security = Security.new(security_path)
     end
 
     def cleanup
-      return unless @path != @original_path && File.directory?(@path)
+      return unless @path != @original_path.to_s && File.directory?(@path)
 
       FileUtils.remove_entry(@path)
     end
 
-    def package_files
-      @packager.package_files
+    def datasets
+      storage.datasets
     end
 
-    private
-
-    def create_package_structure
-      FileUtils.mkdir_p(@path)
-      FileUtils.mkdir_p(content_path)
-      FileUtils.mkdir_p(data_path)
+    # The .cap file this package was loaded from, or nil when loaded
+    # from a directory.
+    def cap_file_path
+      @original_path.to_s if load_type == :cap_file
     end
 
-    def content_path
-      File.join(@path, CONTENT_DIR)
+    def content_files
+      Dir.glob(File.join(content_path, "**", "*")).select do |file|
+        File.file?(file)
+      end
     end
 
-    def data_path
-      File.join(@path, DATA_DIR)
+    def determine_load_type(path)
+      return :directory if File.directory?(path)
+      return :cap_file if File.extname(path) == ".cap"
+
+      :unsaved
     end
 
-    def routes_path
-      File.join(@path, ROUTES_FILE)
+    # Verifies the package against security.json (ARCHITECTURE.md section
+    # 6). Returns a list of typed errors; empty when no security.json is
+    # present or all checksums match.
+    def verify_integrity
+      return [] unless @security.present?
+
+      @security.verify(@path)
     end
 
-    def storage_path
-      File.join(@path, STORAGE_FILE)
+    def verify_integrity!
+      @security.verify!(@path) if @security.present?
     end
 
-    def metadata_path
-      File.join(@path, METADATA_FILE)
+    private
+
+    def package_stem(file_path)
+      File.basename(file_path, ".cap")
     end
 
-    def manifest_path
-      File.join(@path, MANIFEST_FILE)
+    def create_package_structure
+      FileUtils.mkdir_p(@path)
+      FileUtils.mkdir_p(content_path)
+      FileUtils.mkdir_p(data_path)
     end
+
+    def content_path = File.join(@path, CONTENT_DIR)
+
+    def data_path = File.join(@path, DATA_DIR)
+
+    def routes_path = File.join(@path, ROUTES_FILE)
+
+    def storage_path = File.join(@path, STORAGE_FILE)
+
+    def metadata_path = File.join(@path, METADATA_FILE)
+
+    def manifest_path = File.join(@path, MANIFEST_FILE)
+
+    def security_path = File.join(@path, SECURITY_FILE)
   end
 end

data/lib/capsium/packager.rb

@@ -1,100 +1,98 @@
 # frozen_string_literal: true
 
-# lib/capsium/packager.rb
-require "json"
 require "fileutils"
-require "rubygems"
+require "pathname"
+require "tmpdir"
 require "zip"
 
 module Capsium
   class Packager
     class FileAlreadyExistsError < StandardError; end
 
-    def pack(package, options = {})
-      package_name = package.metadata.name
-      package_version = package.metadata.version
-      cap_file_path = File.expand_path(
-        File.join(package.path, "..", "#{package_name}-#{package_version}.cap")
-      )
+    # Raised when a .cap archive contains an entry whose name would be
+    # written outside the destination directory (absolute paths, drive
+    # letters, ".." segments) — zip-slip protection, on par with the Lua
+    # reactor's extractor.
+    class UnsafeEntryError < Capsium::Error; end
 
-      if File.exist?(cap_file_path)
-        unless options[:force]
-          raise FileAlreadyExistsError,
-                "Package target already exists, aborting: `#{relative_path_current(cap_file_path)}`"
-        end
+    DRIVE_LETTER_PATTERN = %r{\A[A-Za-z]:[/\\]}
 
+    def pack(package, options = {})
+      directory = package.path
+      output_file_name = "#{package.metadata.name}-#{package.metadata.version}.cap"
+      output_directory = File.dirname(directory)
+      cap_file_path = File.join(output_directory, output_file_name)
+
+      if File.exist?(cap_file_path) && !options[:force]
+        raise FileAlreadyExistsError,
+              "Package target already exists, aborting: `#{relative_path_current(cap_file_path)}`"
+      elsif File.exist?(cap_file_path)
         puts "Package target already exists, overwriting: `#{relative_path_current(cap_file_path)}`"
         FileUtils.rm_f(cap_file_path)
       end
 
       Dir.mktmpdir do |dir|
-        FileUtils.cp_r("#{package.path}/.", dir)
+        FileUtils.cp_r("#{directory}/.", dir)
+        FileUtils.rm_f(File.join(dir, Package::SECURITY_FILE))
         new_package = Package.new(dir)
         new_package.solidify
-        compress_package(new_package, cap_file_path)
+        generate_security(new_package)
+        new_cap_file_path = File.join(dir, output_file_name)
+        compress_package(new_package, new_cap_file_path)
+        puts "Package built at: #{new_cap_file_path}"
+        FileUtils.mv(new_cap_file_path, cap_file_path)
         puts "Package created: #{relative_path_current(cap_file_path)}"
+        return cap_file_path
+      end
+    end
+
+    def unpack(cap_file_path, destination)
+      destination = File.expand_path(destination)
+      Zip::File.open(cap_file_path) do |zip_file|
+        zip_file.each do |entry|
+          entry_path = safe_entry_path(destination, entry.name)
+          FileUtils.mkdir_p(File.dirname(entry_path))
+          entry.extract(entry_path)
+        end
       end
     end
 
     def compress_package(package, cap_file)
-      Zip::File.open(cap_file, Zip::File::CREATE) do |zipfile|
-        Dir[File.join(package.path, "**", "**")].each do |file|
+      entries = Dir[File.join(package.path, "**", "**")].reject do |file|
+        File.expand_path(file) == File.expand_path(cap_file)
+      end
+      Zip::File.open(cap_file, create: true) do |zipfile|
+        entries.each do |file|
           zipfile.add(file.sub("#{package.path}/", ""), file)
         end
       end
     end
 
-    # def package_files
-    #   create_metadata_file
-    #   create_manifest_file
-    #   create_packaging_file
-
-    #   compressor = Compressor.new(@package, @package.metadata[:compression])
-    #   compressor.compress
-
-    #   protector = Protector.new(@package, @package.metadata[:encryption], @package.metadata[:signature])
-    #   protector.apply_encryption_and_sign
-    # end
-
-    # private
+    def relative_path_current(absolute_path)
+      Pathname.new(absolute_path).relative_path_from(Dir.pwd).to_s
+    end
 
-    # def create_metadata_file
-    #   metadata_path = File.join(@package.path, Package::METADATA_FILE)
-    #   metadata_content = @package.metadata.to_h
-    #   write_json_file(metadata_path, metadata_content)
-    # end
+    private
 
-    # def create_manifest_file
-    #   manifest_path = File.join(@package.path, Package::MANIFEST_FILE)
-    #   manifest_content = {
-    #     files: Dir[File.join(@package.path, '**', '*')].reject { |f| File.directory?(f) }
-    #   }
-    #   write_json_file(manifest_path, manifest_content)
-    # end
+    # Resolves an entry name against the destination and returns the
+    # absolute target path, raising UnsafeEntryError when the entry would
+    # escape the destination (zip-slip).
+    def safe_entry_path(destination, entry_name)
+      entry_path = File.expand_path(entry_name, destination)
+      return entry_path if contained?(entry_path, destination) &&
+                           !entry_name.match?(DRIVE_LETTER_PATTERN)
 
-    def create_packaging_file
-      packaging_path = File.join(@package.path, Package::PACKAGING_FILE)
-      packaging_content = {
-        name: @package.name,
-        content_path: relative_path_package(@package.content_path),
-        data_path: relative_path_package(@package.data_path),
-        datasets: @package.datasets.map(&:to_h)
-      }
-      write_json_file(packaging_path, packaging_content)
+      raise UnsafeEntryError,
+            "Refusing to extract unsafe zip entry: #{entry_name}"
     end
 
-    def write_json_file(path, content)
-      File.open(path, "w") do |file|
-        file.write(JSON.pretty_generate(content))
-      end
+    def contained?(entry_path, destination)
+      entry_path.start_with?("#{destination}#{File::SEPARATOR}")
     end
 
-    def relative_path_package(absolute_path)
-      Pathname.new(absolute_path).relative_path_from(Pathname.new(@package.path)).to_s
-    end
-
-    def relative_path_current(absolute_path)
-      Pathname.new(absolute_path).relative_path_from(Dir.pwd).to_s
+    def generate_security(package)
+      security = Package::Security.generate(package.path)
+      security.save_to_file
     end
   end
 end

data/lib/capsium/reactor/introspection.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+require "digest"
+require "json"
+require "time"
+
+module Capsium
+  class Reactor
+    # Monitoring HTTP API reports for the package this reactor serves
+    # (ARCHITECTURE.md section 7). Each report wraps its single entry in
+    # the list shape all reactors converge on.
+    class Introspection
+      METADATA_PATH = "/api/v1/introspect/metadata"
+      ROUTES_PATH = "/api/v1/introspect/routes"
+      CONTENT_HASHES_PATH = "/api/v1/introspect/content-hashes"
+      CONTENT_VALIDITY_PATH = "/api/v1/introspect/content-validity"
+      PATHS = [METADATA_PATH, ROUTES_PATH, CONTENT_HASHES_PATH,
+               CONTENT_VALIDITY_PATH].freeze
+
+      attr_reader :package
+
+      def initialize(package)
+        @package = package
+      end
+
+      def endpoint?(path)
+        PATHS.include?(path)
+      end
+
+      # The report body for an introspection endpoint, or nil when the
+      # path is not an introspection endpoint.
+      def report_for(path)
+        case path
+        when METADATA_PATH then metadata_report
+        when ROUTES_PATH then routes_report
+        when CONTENT_HASHES_PATH then content_hashes_report
+        when CONTENT_VALIDITY_PATH then content_validity_report
+        end
+      end
+
+      def metadata_report
+        metadata = package.metadata
+        { packages: [{
+          name: metadata.name,
+          version: metadata.version,
+          author: metadata.author,
+          description: metadata.description
+        }] }
+      end
+
+      def routes_report
+        entries = package.routes.config.routes.map do |route|
+          { method: route.http_method || "GET", path: route.path }
+        end
+        { routes: [{ package: package.name, routes: entries }] }
+      end
+
+      def content_hashes_report
+        { contentHashes: [{ package: package.name, hash: content_hash }] }
+      end
+
+      def content_validity_report
+        errors = package.verify_integrity
+        entry = {
+          package: package.name,
+          valid: errors.empty?,
+          lastChecked: Time.now.utc.iso8601
+        }
+        entry[:reason] = errors.map(&:message).join("; ") unless errors.empty?
+        { contentValidity: [entry] }
+      end
+
+      private
+
+      # SHA-256 of the .cap blob when the package was loaded from one.
+      # For directory sources there is no blob, so the hash covers a
+      # canonical (sorted-key) JSON serialization of the package content
+      # checksums — the same data security.json integrityChecks carry.
+      def content_hash
+        cap_file = package.cap_file_path
+        return Digest::SHA256.file(cap_file).hexdigest if cap_file
+
+        checksums = Package::Security.checksums_for(package.path)
+        Digest::SHA256.hexdigest(JSON.generate(checksums))
+      end
+    end
+  end
+end

data/lib/capsium/reactor.rb

@@ -1,69 +1,78 @@
 # frozen_string_literal: true
 
-# lib/capsium/reactor.rb
-require "webrick"
 require "json"
 require "listen"
-require "capsium/package"
+require "webrick"
 
 module Capsium
   class Reactor
+    autoload :Introspection, "capsium/reactor/introspection"
+
     DEFAULT_PORT = 8864
-    attr_accessor :package, :package_path, :routes, :port, :server, :server_thread
+    DEFAULT_CACHE_CONTROL = "public, max-age=31536000"
 
-    def initialize(package, port = DEFAULT_PORT)
-      @package = package
-      @package_path = package.path
+    attr_reader :package, :package_path, :routes, :port, :cache_control,
+                :server, :server_thread, :introspection
+
+    def initialize(package:, port: DEFAULT_PORT,
+                   cache_control: DEFAULT_CACHE_CONTROL, do_not_listen: false)
+      @package = package.is_a?(String) ? Package.new(package) : package
+      @package_path = @package.path
       @port = port
-      @server = WEBrick::HTTPServer.new(Port: @port)
-      load_routes
+      @cache_control = cache_control
+      setup_server(do_not_listen)
+      load_state
       mount_routes
     end
 
     def serve
+      trap("INT") { shutdown_server }
       @server_thread = start_server
       start_listener
     end
 
-    private
+    def handle_request(request, response)
+      return serve_introspection(request, response) if @introspection.endpoint?(request.path)
 
-    def load_routes
-      @routes = @package.routes.data.routes
+      route = @routes.resolve(request.path)
+      route ? serve_route(route, response) : respond_not_found(response)
     end
 
     def mount_routes
-      @routes.each do |route|
-        path = route.path
-        target = route.target
-        content_path = target.fs_path(@package.manifest)
-        puts "mounting route: #{path} => #{content_path}"
-        @server.mount_proc(path.to_s) do |_req, res|
-          res.body = File.read(content_path)
-          res.content_type = target.mime(@package.manifest)
-        end
+      paths = @routes.config.routes.map(&:path) + Introspection::PATHS
+      paths.each do |path|
+        @server.mount_proc(path.to_s) { |req, res| handle_request(req, res) }
       end
     end
 
-    def start_server
-      @server_thread = Thread.new do
-        trap("INT") do
-          puts "\nShutting down server..."
-          @server.shutdown
-          exit
-        end
+    def restart_server
+      @server.shutdown
+      @server_thread&.join
+      load_package
+      setup_server(false)
+      mount_routes
+      @server_thread = start_server
+    end
+
+    private
 
+    def setup_server(do_not_listen)
+      server_options = { Port: @port }
+      server_options[:DoNotListen] = true if do_not_listen
+      @server = WEBrick::HTTPServer.new(server_options)
+    end
+
+    def start_server
+      Thread.new do
         puts "Starting server on http://localhost:#{@port}"
         @server.start
       end
     end
 
-    def restart_server
+    def shutdown_server
+      puts "\nShutting down server..."
       @server.shutdown
-      # @server_thread.kill
-      load_package
-      @server = WEBrick::HTTPServer.new(Port: @port)
-      mount_routes
-      start_server
+      exit
     end
 
     def start_listener
@@ -74,9 +83,72 @@ module Capsium
 
       listener.start
       puts "Listening for changes in #{@package_path}..."
-
-      # Wait for the server thread to finish
       @server_thread.join
     end
+
+    def load_package
+      @package = Package.new(@package_path)
+      load_state
+    end
+
+    def load_state
+      @routes = @package.routes
+      @introspection = Introspection.new(@package)
+    end
+
+    def serve_introspection(request, response)
+      return respond_method_not_allowed(response) unless request.request_method == "GET"
+
+      response.status = 200
+      response["Content-Type"] = "application/json"
+      response.body = JSON.generate(@introspection.report_for(request.path))
+    end
+
+    def serve_route(route, response)
+      case route.kind
+      when :dataset then serve_dataset(route.dataset, response)
+      when :resource then serve_file(route, response)
+      else respond_not_implemented(response)
+      end
+    end
+
+    def serve_file(route, response)
+      content_path = route.fs_path(@package.path)
+      if content_path && File.exist?(content_path)
+        response.status = 200
+        response["Content-Type"] = route.mime(@package.manifest) || "application/octet-stream"
+        headers_for(route).each { |name, value| response[name] = value }
+        response.body = File.read(content_path)
+      else
+        respond_not_found(response)
+      end
+    end
+
+    def headers_for(route)
+      route.headers || (@cache_control ? { "Cache-Control" => @cache_control } : {})
+    end
+
+    def serve_dataset(dataset_name, response)
+      dataset = @package.storage.dataset(dataset_name)
+      if dataset
+        response.status = 200
+        response["Content-Type"] = "application/json"
+        response.body = JSON.generate(dataset.data)
+      else
+        respond_not_found(response)
+      end
+    end
+
+    def respond_not_found(response) = respond_text(response, 404, "Not Found")
+
+    def respond_not_implemented(response) = respond_text(response, 501, "Not Implemented")
+
+    def respond_method_not_allowed(response) = respond_text(response, 405, "Method Not Allowed")
+
+    def respond_text(response, status, body)
+      response.status = status
+      response["Content-Type"] = "text/plain"
+      response.body = body
+    end
   end
 end

data/lib/capsium/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Capsium
-  VERSION = "0.1.1"
+  VERSION = "0.2.0"
 end