capsium 0.1.2 → 0.2.0

data/lib/capsium/package/storage.rb

@@ -1,76 +1,51 @@
 # frozen_string_literal: true
 
-# lib/capsium/package/storage.rb
-require "json"
-require "shale"
-require_relative "dataset"
-require_relative "storage_config"
+require "forwardable"
 
 module Capsium
   class Package
     class Storage
       extend Forwardable
-      attr_reader :config
 
-      def_delegators :@config, :to_json
+      attr_reader :path, :config, :datasets
 
-      attr_reader :datasets
+      def_delegators :@config, :to_json, :to_hash
 
       def initialize(path)
         @path = path
         @dir = File.dirname(path)
-        @datasets_path = File.join(@dir, DATA_DIR)
         @config = if File.exist?(path)
                     StorageConfig.from_json(File.read(path))
                   else
-                    StorageConfig.new(datasets: generate_datasets)
+                    StorageConfig.new
                   end
-        @datasets = load_datasets || generate_datasets
+        @datasets = load_datasets
+      end
+
+      def data_sets
+        @config.data_sets
       end
 
       def load_datasets
-        if File.exist?(@path)
-          storage_data = StorageConfig.from_json(File.read(@path))
-          storage_data.datasets.map do |dataset_config|
-            dataset_config.to_dataset(@datasets_path)
-          end
+        data_sets.map do |name, dataset_config|
+          dataset_config.to_dataset(name, @dir)
         end
       end
 
-      def save_to_file(output_path = @path)
-        storage_config = StorageConfig.new(datasets: @datasets.map do |dataset|
-                                                       DatasetConfig.from_dataset(dataset)
-                                                     end)
-        File.write(output_path, storage_config.to_json)
+      def dataset(name)
+        @datasets.find { |dataset| dataset.name == name }
       end
 
-      def generate_datasets
-        paths = File.join(@datasets_path, "*.{yaml,yml,json,csv,tsv,sqlite,db}")
-        Dir.glob(paths).map do |file_path|
-          Dataset.new(config: DatasetConfig.new(
-            name: File.basename(file_path,
-                                ".*"), source: file_path, format: detect_format(file_path)
-          ))
-        end
+      def dataset_names
+        data_sets.keys.sort
       end
 
-      private
+      def empty?
+        data_sets.empty?
+      end
 
-      def detect_format(file_path)
-        case File.extname(file_path).downcase
-        when ".yaml", ".yml"
-          "yaml"
-        when ".json"
-          "json"
-        when ".csv"
-          "csv"
-        when ".tsv"
-          "tsv"
-        when ".sqlite", ".db"
-          "sqlite"
-        else
-          raise "Unsupported data file type: #{File.extname(file_path)}"
-        end
+      def save_to_file(output_path = @path)
+        File.write(output_path, to_json)
       end
     end
   end

data/lib/capsium/package/storage_config.rb

@@ -1,9 +1,106 @@
-require "shale"
+# frozen_string_literal: true
+
+require "json"
+require "lutaml/model"
 
 module Capsium
   class Package
-    class StorageConfig < Shale::Mapper
-      attribute :datasets, DatasetConfig, collection: true
+    # A single dataset entry (ARCHITECTURE.md section 5). Paths are
+    # package-relative POSIX paths. Kinds: schema-backed file (via
+    # "source") or SQLite ("databaseFile" + "table").
+    class DatasetConfig < Lutaml::Model::Serializable
+      FORMATS = {
+        ".yaml" => "yaml", ".yml" => "yaml", ".json" => "json",
+        ".csv" => "csv", ".tsv" => "tsv",
+        ".sqlite" => "sqlite", ".db" => "sqlite"
+      }.freeze
+      SCHEMA_TYPES = %w[json-schema].freeze
+
+      attribute :source, :string
+      attribute :schema_file, :string
+      attribute :schema_type, :string, values: SCHEMA_TYPES
+      attribute :database_file, :string
+      attribute :table, :string
+
+      json do
+        map :source, to: :source
+        map "schemaFile", to: :schema_file
+        map "schemaType", to: :schema_type
+        map "databaseFile", to: :database_file
+        map :table, to: :table
+      end
+
+      def format
+        FORMATS.fetch(File.extname(backing_file).downcase) do
+          raise Error, "Unsupported data file type: #{File.extname(backing_file)}"
+        end
+      end
+
+      def sqlite?
+        !database_file.nil?
+      end
+
+      def backing_file
+        database_file || source.to_s
+      end
+
+      def to_dataset(name, package_path)
+        Dataset.new(name: name, config: self, package_path: package_path)
+      end
+    end
+
+    # The "storage" object holding the dataSets map.
+    class StorageData < Lutaml::Model::Serializable
+      attribute :data_sets, :hash, default: {}
+
+      json do
+        map "dataSets", with: { from: :data_sets_from_json, to: :data_sets_to_json }
+      end
+
+      def data_sets_from_json(model, value)
+        model.data_sets = (value || {}).to_h do |name, attributes|
+          [name, DatasetConfig.from_json(JSON.generate(attributes))]
+        end
+      end
+
+      def data_sets_to_json(model, doc)
+        doc["dataSets"] = model.data_sets.sort.to_h do |name, dataset|
+          [name, JSON.parse(dataset.to_json)]
+        end
+      end
+    end
+
+    # Canonical storage.json model. The legacy gem form
+    # ({"datasets": [{name, source, format, schema}]}) is accepted on read
+    # and normalized; writers emit only the canonical form.
+    class StorageConfig < Lutaml::Model::Serializable
+      attribute :storage, StorageData
+
+      json do
+        map :storage, to: :storage
+      end
+
+      def self.from_json(json)
+        doc = JSON.parse(json)
+        doc["storage"] ||= { "dataSets" => legacy_data_sets(doc.delete("datasets")) }
+        super(JSON.generate(doc))
+      end
+
+      def self.legacy_data_sets(datasets)
+        (datasets || []).to_h do |item|
+          entry = { "source" => item["source"] }
+          if item["schema"]
+            entry["schemaFile"] = item["schema"]
+            entry["schemaType"] = "json-schema"
+          end
+          [item["name"], entry]
+        end
+      end
+      private_class_method :legacy_data_sets
+
+      def data_sets
+        storage ? storage.data_sets : {}
+      end
     end
   end
 end

data/lib/capsium/package/validator.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+require "json"
+require "marcel"
+require "pathname"
+require "tmpdir"
+
+module Capsium
+  class Package
+    # Validates a package directory or .cap file against the canonical
+    # schemas (ARCHITECTURE.md sections 2-6) and reports one result per
+    # check. Used by `capsium package validate`.
+    class Validator
+      CheckResult = Data.define(:name, :ok, :messages) do
+        def ok?
+          ok
+        end
+      end
+
+      EXTERNAL_REFERENCE_PATTERN = %r{(?:src|href)\s*=\s*["']https?://}in
+      TEXT_CONTENT_MIME = %r{\Atext/|application/(json|javascript|xml)|image/svg\+xml}
+
+      def initialize(package_path)
+        @given_path = package_path
+      end
+
+      def run
+        Dir.mktmpdir do |dir|
+          @package_path = prepare(dir)
+          [metadata_check, manifest_check, routes_check,
+           storage_check, security_check, content_check]
+        end
+      end
+
+      def valid?
+        run.all?(&:ok?)
+      end
+
+      private
+
+      def prepare(dir)
+        return @given_path if File.directory?(@given_path)
+
+        destination = File.join(dir, "package")
+        Packager.new.unpack(@given_path, destination)
+        destination
+      end
+
+      def metadata_check
+        path = metadata_path(METADATA_FILE)
+        return failure("metadata", ["metadata.json is missing"]) unless File.exist?(path)
+
+        result("metadata", Metadata.new(path).config.format_errors)
+      rescue StandardError => e
+        failure("metadata", ["metadata.json is not valid: #{e.message}"])
+      end
+
+      def manifest_check
+        manifest = Manifest.new(metadata_path(MANIFEST_FILE))
+        missing = manifest.resources.keys.reject do |path|
+          File.file?(File.join(@package_path, path))
+        end
+        result("manifest", missing.map { |path| "resource missing on disk: #{path}" })
+      rescue StandardError => e
+        failure("manifest", ["manifest.json is not valid: #{e.message}"])
+      end
+
+      def routes_check
+        manifest = Manifest.new(metadata_path(MANIFEST_FILE))
+        storage = Storage.new(metadata_path(STORAGE_FILE))
+        routes = Routes.new(metadata_path(ROUTES_FILE), manifest, storage)
+        problems = route_problems(routes, storage) + index_problems(routes)
+        result("routes", problems)
+      rescue StandardError => e
+        failure("routes", ["routes.json is not valid: #{e.message}"])
+      end
+
+      def route_problems(routes, storage)
+        routes.config.routes.flat_map do |route|
+          route_target_problems(route, storage)
+        end
+      end
+
+      def route_target_problems(route, storage)
+        problems = []
+        if route.dataset_route? && !route.path.start_with?(Route::DATASET_PATH_PREFIX)
+          problems << "dataset route #{route.path} not under #{Route::DATASET_PATH_PREFIX}"
+        end
+        route.validate_target(@package_path, storage)
+        problems
+      rescue Error => e
+        problems + [e.message]
+      end
+
+      def index_problems(routes)
+        index = routes.config.index
+        return ["index route is missing"] if index.nil?
+        unless File.file?(File.join(@package_path, index))
+          return ["index missing on disk: #{index}"]
+        end
+        return [] if File.extname(index).downcase == ".html"
+
+        ["index is not an HTML file: #{index}"]
+      end
+
+      def storage_check
+        storage = Storage.new(metadata_path(STORAGE_FILE))
+        problems = storage.datasets.flat_map(&:validation_errors)
+        result("storage", problems)
+      rescue StandardError => e
+        failure("storage", ["storage.json is not valid: #{e.message}"])
+      end
+
+      def security_check
+        security = Security.new(metadata_path(SECURITY_FILE))
+        return result("security", []) unless security.present?
+
+        result("security", security.verify(@package_path).map(&:message))
+      end
+
+      def content_check
+        offenders = Dir.glob(File.join(@package_path, CONTENT_DIR, "**", "*")).select do |file|
+          File.file?(file) && text_file?(file) &&
+            File.binread(file).match?(EXTERNAL_REFERENCE_PATTERN)
+        end
+        result("content", offenders.map do |file|
+          "external reference in #{file.delete_prefix("#{@package_path}/")}"
+        end)
+      end
+
+      def text_file?(path)
+        mime = Marcel::MimeType.for(Pathname.new(path), name: File.basename(path))
+        mime.match?(TEXT_CONTENT_MIME)
+      end
+
+      def metadata_path(file_name)
+        File.join(@package_path, file_name)
+      end
+
+      def result(name, problems)
+        CheckResult.new(name: name, ok: problems.empty?, messages: problems)
+      end
+
+      def failure(name, messages)
+        CheckResult.new(name: name, ok: false, messages: messages)
+      end
+    end
+  end
+end

data/lib/capsium/package.rb

@@ -1,41 +1,51 @@
 # frozen_string_literal: true
 
-# lib/capsium/package.rb
 require "fileutils"
-require "json"
-require "yaml"
-require "csv"
-require "sqlite3"
-require "zip"
-require_relative "package/manifest"
-require_relative "package/metadata"
-require_relative "package/routes"
-require_relative "package/dataset"
-require_relative "package/storage"
-require_relative "packager"
+require "pathname"
+require "tmpdir"
 
 module Capsium
   class Package
-    attr_reader :name, :path, :manifest, :metadata, :routes, :datasets,
-                :storage, :load_type
+    autoload :Dataset, "capsium/package/dataset"
+    autoload :DatasetConfig, "capsium/package/storage_config"
+    autoload :DigitalSignatures, "capsium/package/security_config"
+    autoload :IntegrityChecks, "capsium/package/security_config"
+    autoload :Manifest, "capsium/package/manifest"
+    autoload :ManifestConfig, "capsium/package/manifest_config"
+    autoload :Metadata, "capsium/package/metadata"
+    autoload :MetadataData, "capsium/package/metadata_config"
+    autoload :Repository, "capsium/package/metadata_config"
+    autoload :Resource, "capsium/package/manifest_config"
+    autoload :Route, "capsium/package/routes_config"
+    autoload :Routes, "capsium/package/routes"
+    autoload :RoutesConfig, "capsium/package/routes_config"
+    autoload :Security, "capsium/package/security"
+    autoload :SecurityConfig, "capsium/package/security_config"
+    autoload :SecurityData, "capsium/package/security_config"
+    autoload :Storage, "capsium/package/storage"
+    autoload :StorageConfig, "capsium/package/storage_config"
+    autoload :StorageData, "capsium/package/storage_config"
+    autoload :Validator, "capsium/package/validator"
+
+    attr_reader :name, :path, :manifest, :metadata, :routes, :storage,
+                :security, :load_type
 
     MANIFEST_FILE = "manifest.json"
     METADATA_FILE = "metadata.json"
-    PACKAGING_FILE = "packaging.json"
-    SIGNATURE_FILE = "signature.json"
     STORAGE_FILE = "storage.json"
     ROUTES_FILE = "routes.json"
+    SECURITY_FILE = "security.json"
     CONTENT_DIR = "content"
     DATA_DIR = "data"
-    ENCRYPTED_PACKAGING_FILE = "package.enc"
 
     def initialize(path, load_type: nil)
       @original_path = Pathname.new(path).expand_path
-      @path = prepare_package(@original_path)
+      @path = prepare_package(@original_path).to_s
       @load_type = load_type || determine_load_type(path)
       create_package_structure
       load_package
       @name = metadata.name
+      verify_integrity!
     end
 
     def prepare_package(path)
@@ -44,45 +54,23 @@ module Capsium
       if File.file?(path)
         return decompress_cap_file(path) if File.extname(path) == ".cap"
 
-        raise "Error: The package must have a .cap extension"
+        raise Error, "The package must have a .cap extension"
       end
 
-      raise "Invalid package path: #{path}"
+      raise Error, "Invalid package path: #{path}"
     end
 
     def solidify
       @manifest.save_to_file
       @metadata.save_to_file
       @routes.save_to_file
-      @storage.save_to_file
+      @storage.save_to_file unless @storage.empty?
     end
 
     def decompress_cap_file(file_path)
-      temp_dir = Dir.mktmpdir
-      metadata_path = File.join(temp_dir, METADATA_FILE)
-
-      # Extract metadata.json first
-      Zip::File.open(file_path) do |zip_file|
-        if entry = zip_file.find_entry(METADATA_FILE)
-          entry.extract(metadata_path)
-        end
-      end
-
-      metadata = Metadata.new(metadata_path)
-      package_name = metadata.name
-      package_version = metadata.version
-
-      package_path = File.join(temp_dir, "#{package_name}-#{package_version}")
+      package_path = File.join(Dir.mktmpdir, package_stem(file_path))
       FileUtils.mkdir_p(package_path)
-
-      Zip::File.open(file_path) do |zip_file|
-        zip_file.each do |entry|
-          entry_path = File.join(package_path, entry.name)
-          FileUtils.mkdir_p(File.dirname(entry_path))
-          entry.extract(entry_path)
-        end
-      end
-
+      Packager.new.unpack(file_path, package_path)
       package_path
     end
 
@@ -94,16 +82,23 @@ module Capsium
       @manifest = Manifest.new(manifest_path)
       @storage = Storage.new(storage_path)
       @routes = Routes.new(routes_path, @manifest, @storage)
+      @security = Security.new(security_path)
     end
 
     def cleanup
-      return unless @path != @original_path && File.directory?(@path)
+      return unless @path != @original_path.to_s && File.directory?(@path)
 
       FileUtils.remove_entry(@path)
     end
 
-    def package_files
-      @packager.package_files
+    def datasets
+      storage.datasets
+    end
+
+    # The .cap file this package was loaded from, or nil when loaded
+    # from a directory.
+    def cap_file_path
+      @original_path.to_s if load_type == :cap_file
     end
 
     def content_files
@@ -119,36 +114,43 @@ module Capsium
       :unsaved
     end
 
+    # Verifies the package against security.json (ARCHITECTURE.md section
+    # 6). Returns a list of typed errors; empty when no security.json is
+    # present or all checksums match.
+    def verify_integrity
+      return [] unless @security.present?
+
+      @security.verify(@path)
+    end
+
+    def verify_integrity!
+      @security.verify!(@path) if @security.present?
+    end
+
     private
 
+    def package_stem(file_path)
+      File.basename(file_path, ".cap")
+    end
+
     def create_package_structure
       FileUtils.mkdir_p(@path)
       FileUtils.mkdir_p(content_path)
       FileUtils.mkdir_p(data_path)
     end
 
-    def content_path
-      File.join(@path, CONTENT_DIR)
-    end
+    def content_path = File.join(@path, CONTENT_DIR)
 
-    def data_path
-      File.join(@path, DATA_DIR)
-    end
+    def data_path = File.join(@path, DATA_DIR)
 
-    def routes_path
-      File.join(@path, ROUTES_FILE)
-    end
+    def routes_path = File.join(@path, ROUTES_FILE)
 
-    def storage_path
-      File.join(@path, STORAGE_FILE)
-    end
+    def storage_path = File.join(@path, STORAGE_FILE)
 
-    def metadata_path
-      File.join(@path, METADATA_FILE)
-    end
+    def metadata_path = File.join(@path, METADATA_FILE)
 
-    def manifest_path
-      File.join(@path, MANIFEST_FILE)
-    end
+    def manifest_path = File.join(@path, MANIFEST_FILE)
+
+    def security_path = File.join(@path, SECURITY_FILE)
   end
 end

data/lib/capsium/packager.rb

@@ -1,15 +1,22 @@
 # frozen_string_literal: true
 
-# lib/capsium/packager.rb
-require "json"
 require "fileutils"
-require "rubygems"
+require "pathname"
+require "tmpdir"
 require "zip"
 
 module Capsium
   class Packager
     class FileAlreadyExistsError < StandardError; end
 
+    # Raised when a .cap archive contains an entry whose name would be
+    # written outside the destination directory (absolute paths, drive
+    # letters, ".." segments) — zip-slip protection, on par with the Lua
+    # reactor's extractor.
+    class UnsafeEntryError < Capsium::Error; end
+
+    DRIVE_LETTER_PATTERN = %r{\A[A-Za-z]:[/\\]}
+
     def pack(package, options = {})
       directory = package.path
       output_file_name = "#{package.metadata.name}-#{package.metadata.version}.cap"
@@ -26,8 +33,10 @@ module Capsium
 
       Dir.mktmpdir do |dir|
         FileUtils.cp_r("#{directory}/.", dir)
+        FileUtils.rm_f(File.join(dir, Package::SECURITY_FILE))
         new_package = Package.new(dir)
         new_package.solidify
+        generate_security(new_package)
         new_cap_file_path = File.join(dir, output_file_name)
         compress_package(new_package, new_cap_file_path)
         puts "Package built at: #{new_cap_file_path}"
@@ -38,9 +47,10 @@ module Capsium
     end
 
     def unpack(cap_file_path, destination)
+      destination = File.expand_path(destination)
       Zip::File.open(cap_file_path) do |zip_file|
         zip_file.each do |entry|
-          entry_path = File.join(destination, entry.name)
+          entry_path = safe_entry_path(destination, entry.name)
           FileUtils.mkdir_p(File.dirname(entry_path))
           entry.extract(entry_path)
         end
@@ -48,8 +58,11 @@ module Capsium
     end
 
     def compress_package(package, cap_file)
-      Zip::File.open(cap_file, Zip::File::CREATE) do |zipfile|
-        Dir[File.join(package.path, "**", "**")].each do |file|
+      entries = Dir[File.join(package.path, "**", "**")].reject do |file|
+        File.expand_path(file) == File.expand_path(cap_file)
+      end
+      Zip::File.open(cap_file, create: true) do |zipfile|
+        entries.each do |file|
           zipfile.add(file.sub("#{package.path}/", ""), file)
         end
       end
@@ -58,5 +71,28 @@ module Capsium
     def relative_path_current(absolute_path)
       Pathname.new(absolute_path).relative_path_from(Dir.pwd).to_s
     end
+
+    private
+
+    # Resolves an entry name against the destination and returns the
+    # absolute target path, raising UnsafeEntryError when the entry would
+    # escape the destination (zip-slip).
+    def safe_entry_path(destination, entry_name)
+      entry_path = File.expand_path(entry_name, destination)
+      return entry_path if contained?(entry_path, destination) &&
+                           !entry_name.match?(DRIVE_LETTER_PATTERN)
+
+      raise UnsafeEntryError,
+            "Refusing to extract unsafe zip entry: #{entry_name}"
+    end
+
+    def contained?(entry_path, destination)
+      entry_path.start_with?("#{destination}#{File::SEPARATOR}")
+    end
+
+    def generate_security(package)
+      security = Package::Security.generate(package.path)
+      security.save_to_file
+    end
   end
 end