edgeway 0.1.0 → 0.3.0

data/lib/edgeway/cli/build/app_files.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Edgeway
+  module CLI
+    class Build
+      class AppFiles
+        def initialize(root)
+          @root = root
+        end
+
+        def require_lines(views: nil)
+          lines = []
+          app = File.join(@root, "app")
+          dirs = Dir.exist?(app) ? Dir.children(app).sort.select { |d| File.directory?(File.join(app, d)) } : []
+          dirs.each do |d|
+            next if %w[models controllers].include?(d)
+
+            lines.concat(requires("app/#{d}"))
+          end
+          lines.concat(requires("app/models"))
+          lines.concat(requires("app/controllers"))
+          lines << "require_relative #{views.dump}" if views
+          lines << 'require_relative "../config/routes"'
+          lines
+        end
+
+        # require_relative lines for the .rb files under dir, application_*.rb first.
+        def requires(dir)
+          full = File.join(@root, dir)
+          return [] unless Dir.exist?(full)
+
+          files = Dir.glob("**/*.rb", base: full).map { |f| File.join(dir, f) }
+          first, rest = files.partition { |f| File.basename(f).start_with?("application_") }
+          # String#dump quotes a path with spaces, quotes or "#{" safely.
+          (first.sort + rest.sort).map { |f| "require_relative #{"../#{f.delete_suffix(".rb")}".dump}" }
+        end
+      end
+    end
+  end
+end

data/lib/edgeway/cli/build/cache_config.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+require_relative "../../cache/limits"
+
+module Edgeway
+  module CLI
+    class Build
+      class CacheConfig
+        STORES = {
+          "kv" => "edgeway/service/cloudflare/kv",
+          "memory" => nil,
+          "null" => nil
+        }.freeze
+        KEYS = %w[store binding namespace expires_in].freeze
+
+        attr_reader :store, :binding_name, :namespace, :expires_in
+
+        def self.load(root, env)
+          path = File.join(root, "config", "cache.yml")
+          return new("null") unless File.exist?(path)
+
+          sections = YAML.safe_load_file(path) || {}
+          raise Error, "config/cache.yml must be a Hash of environments" unless sections.is_a?(Hash)
+
+          section = sections[env]
+          return new("null") if section.nil?
+          raise Error, "config/cache.yml: #{env} must be a Hash" unless section.is_a?(Hash)
+
+          from_section(section, env)
+        end
+
+        def self.from_section(section, env)
+          unknown = section.keys.map(&:to_s) - KEYS
+          raise Error, "config/cache.yml: unknown #{unknown.join(", ")} in #{env} (one of #{KEYS.join(", ")})" unless unknown.empty?
+
+          new(store_name(section), binding_name: section["binding"], namespace: section["namespace"],
+                               expires_in: section["expires_in"], env: env)
+        end
+
+        def self.store_name(section)
+          section.key?("store") && section["store"].nil? ? "null" : section["store"]
+        end
+
+        def initialize(store, binding_name: nil, namespace: nil, expires_in: nil, env: "development")
+          @env = env
+          @store = checked_store(store)
+          @binding_name = checked_binding(binding_name)
+          @namespace = checked_namespace(namespace)
+          @expires_in = checked_expires_in(expires_in)
+        end
+
+        def backend_file = STORES.fetch(@store)
+
+        def store_expression
+          case @store
+          when "kv" then kv_expression
+          when "memory" then "Edgeway::Cache::MemoryStore.new(#{store_options})"
+          else "Edgeway::Cache::NullStore.new"
+          end
+        end
+
+        private
+
+        def store_options = "namespace: #{@namespace.inspect}, expires_in: #{@expires_in.inspect}"
+
+        def kv_expression
+          "Edgeway::Cloudflare::KVStore.new(binding: #{@binding_name.inspect}, #{store_options}, " \
+            "transport: Edgeway::Cloudflare::KVImportTransport.new)"
+        end
+
+        def checked_store(store)
+          return store if STORES.key?(store)
+
+          raise Error, "config/cache.yml: store in #{@env} must be one of #{STORES.keys.join(", ")} (got #{store.inspect})"
+        end
+
+        def checked_binding(binding_name)
+          return checked_unused_binding(binding_name) unless @store == "kv"
+          raise Error, "config/cache.yml: binding: in #{@env} must be a String" unless binding_name.nil? || binding_name.is_a?(String)
+          return binding_name unless binding_name.nil? || binding_name.empty?
+
+          raise Error, "config/cache.yml: #{@env} has store: kv but no binding: (the name of a [[kv_namespaces]] binding in wrangler.toml)"
+        end
+
+        def checked_unused_binding(binding_name)
+          raise Error, "config/cache.yml: binding: is only for store: kv (in #{@env})" unless binding_name.nil?
+
+          nil
+        end
+
+        def checked_namespace(namespace)
+          return nil if namespace.nil?
+          raise Error, "config/cache.yml: namespace in #{@env} must be a non-empty String" unless namespace.is_a?(String) && !namespace.empty?
+          return namespace if namespace.valid_encoding? && !namespace.include?("\0")
+
+          raise Error, "config/cache.yml: namespace in #{@env} must be valid UTF-8 without a NUL byte"
+        end
+
+        def checked_expires_in(seconds)
+          return nil if seconds.nil?
+          return seconds if seconds.is_a?(Integer) && seconds.between?(1, Edgeway::Cache::MAX_EXPIRES_IN)
+
+          raise Error, "config/cache.yml: expires_in in #{@env} must be a whole number of seconds, 1 to #{Edgeway::Cache::MAX_EXPIRES_IN}"
+        end
+      end
+    end
+  end
+end

data/lib/edgeway/cli/build/database_config.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+module Edgeway
+  module CLI
+    class Build
+      class DatabaseConfig
+        DATABASE_NAME = /\A[a-z_][a-z0-9_]*\z/
+
+        def initialize(root)
+          @root = root
+        end
+
+        def section(env)
+          section = sections[env]
+          raise Error, "config/database.yml has no #{env} section" unless section.is_a?(Hash)
+
+          section
+        end
+
+        def section?(env)
+          File.exist?(path) && sections[env].is_a?(Hash)
+        end
+
+        def databases_in(section, env)
+          entries = section["databases"] || {}
+          raise Error, "config/database.yml: databases in #{env} must be a Hash" unless entries.is_a?(Hash)
+
+          entries.to_h do |name, entry|
+            name = name.to_s
+            [name, database_binding(name, entry, env)]
+          end
+        end
+
+        def primary_binding(section) = (section["binding"] || "DB").to_s
+
+        def session_bindings(section, env)
+          flags = {}
+          record_sessions(flags, "the #{env} section", primary_binding(section), section, "the #{env} section")
+          (section["databases"] || {}).each do |name, entry|
+            record_sessions(flags, "databases.#{name}", entry["binding"].to_s, entry, "databases.#{name} in #{env}")
+          end
+          flags.select { |_binding, (_label, enabled)| enabled }.keys
+        end
+
+        def names
+          names = []
+          return names unless File.exist?(path)
+
+          sections.each do |env, section|
+            next unless section.is_a?(Hash)
+            databases_in(section, env).each_key { |name| names << name unless names.include?(name) }
+          end
+          names
+        end
+
+        private
+
+        def path = File.join(@root, "config", "database.yml")
+
+        def sections
+          raise Error, "no config/database.yml in #{@root} (is this an Edgeway app?)" unless File.exist?(path)
+
+          YAML.safe_load_file(path) || {}
+        end
+
+        def record_sessions(flags, label, binding_name, entry, where)
+          enabled = sessions?(entry, where)
+          known_label, known = flags[binding_name]
+          if known_label && known != enabled
+            raise Error, "config/database.yml: #{known_label} and #{label} share binding #{binding_name} but differ in sessions"
+          end
+
+          flags[binding_name] = [label, enabled]
+        end
+
+        def sessions?(entry, where)
+          return false unless entry.is_a?(Hash) && entry.key?("sessions")
+          return entry["sessions"] if [true, false].include?(entry["sessions"])
+
+          raise Error, "config/database.yml: sessions in #{where} must be true or false"
+        end
+
+        def database_binding(name, entry, env)
+          unless name.match?(DATABASE_NAME) && name != "primary"
+            raise Error, "config/database.yml: bad database name #{name.inspect} in #{env} (lowercase letters, digits and _, and not primary)"
+          end
+
+          binding_name = entry.is_a?(Hash) ? entry["binding"].to_s : ""
+          raise Error, "config/database.yml: databases.#{name} in #{env} has no binding:" if binding_name.empty?
+
+          binding_name
+        end
+      end
+    end
+  end
+end

data/lib/edgeway/cli/build/js_files.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require "json"
+require "fileutils"
+
+module Edgeway
+  module CLI
+    class Build
+      module JsFiles
+        TEST_ONLY = /-test(-[^\/]*)?\.js\z/
+
+        def self.copy(destination, config_js, tests: false)
+          FileUtils.mkdir_p(destination)
+          paths(tests).each { |file| FileUtils.cp(file, destination) }
+          File.write(File.join(destination, "config.js"), config_js)
+        end
+
+        def self.paths(tests)
+          files = Dir.glob(File.join(CLI.gem_dir, "js/*.js"))
+          tests ? files : files.grep_v(TEST_ONLY)
+        end
+
+        def self.config_js(values, command)
+          "// Generated by `edgeway #{command}` from config/database.yml; do not edit.\n" \
+            "export default #{JSON.generate(values)};\n"
+        end
+      end
+    end
+  end
+end

data/lib/edgeway/cli/build/view_compiler.rb

@@ -0,0 +1,191 @@
+# frozen_string_literal: true
+
+require_relative "../../inflector"
+
+module Edgeway
+  module CLI
+    class Build
+      # app/views/**/*.html.erb -> Ruby that reopens the controllers.
+      #
+      # Spinel has no eval and no binding, so a template can't be read at
+      # runtime: each one becomes an instance method of its controller (it sees
+      # the action's @variables, params and helpers as any method of it does),
+      # and `edgeway build` / `edgeway test` require the result after the
+      # controllers.
+      #
+      #   app/views/todos/index.html.erb   TodosController#_view_todos_index
+      #   app/views/layouts/application.html.erb
+      #                                    ApplicationController#_render_layout
+      #
+      # A controller gets `_render_view(action)` too: a case over its templates,
+      # not a send, since with a computed name Spinel's send misses the methods
+      # of a subclass (matz/spinel#8592).
+      #
+      # The ERB it reads is a small subset (stdlib ERB is not used):
+      #   <%= expr %>    expr, HTML-escaped
+      #   <%== expr %>   expr, as it is
+      #   <% code %>     code
+      #   <%# text %>    nothing
+      #   <%%            a literal <%
+      #   <%-            starts a tag and drops the indentation before it
+      #   -%>            ends a tag and drops the newline after it
+      # and `<%= yield %>` (or `<%== yield %>`) in the layout is the page, as
+      # it is.
+      class ViewCompiler
+        EXT = ".html.erb"
+        LAYOUT = "layouts/application#{EXT}"
+        NAME = /\A[a-z_][a-z0-9_]*\z/
+
+        def initialize(root)
+          @root = root
+        end
+
+        # The Ruby for app/views, nil when the app has none.
+        def compile
+          views = File.join(@root, "app/views")
+          return nil unless Dir.exist?(views)
+
+          files = Dir.glob("**/*#{EXT}", base: views).sort
+          layout = files.delete(LAYOUT)
+          files.reject! { |f| f.start_with?("layouts/") }
+          by_dir = {}
+          files.each do |f|
+            dir = File.dirname(f)
+            raise Error, "app/views/#{f}: a template goes in a controller's directory (app/views/todos/index#{EXT})" if dir == "."
+
+            (by_dir[dir] ||= []) << f
+          end
+
+          out = ["# Generated by `edgeway build` from app/views; do not edit."]
+          out.concat(layout_source(layout)) if layout
+          by_dir.each { |dir, fs| out.concat(controller_source(dir, fs)) }
+          out.join("\n") + "\n"
+        end
+
+        def template_lines(text, path, layout: false)
+          lines = []
+          pos = 0
+          literal = +""
+          # the line of text[counted], advanced as the scan goes
+          counted = 0
+          line = 1
+          while (open = text.index("<%", pos))
+            if text[open + 2] == "%"
+              literal << text[pos...open] << "<%"
+              pos = open + 3
+              next
+            end
+            line += text[counted...open].count("\n")
+            counted = open
+            close = text.index("%>", open + 2) or raise Error, "#{path}:#{line}: unclosed <%"
+            literal << text[pos...open]
+            body = text[(open + 2)...close]
+            if body.start_with?("-")
+              body = body[1..]
+              # <%- also drops the indentation before it on its line
+              line_start = open > 0 ? (text.rindex("\n", open - 1) || -1) + 1 : 0
+              indent = open - line_start
+              literal = literal[0...(literal.length - indent)] if indent > 0 && text[line_start...open].strip.empty?
+            end
+            trim = body.end_with?("-")
+            body = body.chomp("-") if trim
+            after = close + 2
+            if trim && text[after] == "\n"
+              after += 1
+            elsif trim && text[after, 2] == "\r\n"
+              after += 2
+            end
+            flush(lines, literal)
+            code = body.strip
+            if body.start_with?("#")
+            elsif body.start_with?("==")
+              expr = code[2..].strip
+              lines << "# #{path}:#{line}" << output(expr, "(", ").to_s", layout, path, line)
+            elsif body.start_with?("=")
+              expr = code[1..].strip
+              lines << "# #{path}:#{line}" << output(expr, "Edgeway::View.escape(", ")", layout, path, line)
+            else
+              lines << "# #{path}:#{line}" << code
+            end
+            pos = after
+          end
+          literal << text[pos..]
+          flush(lines, literal)
+          lines
+        end
+
+        private
+
+        # An output tag; the closing paren goes on its own line so that a
+        # trailing `# comment` in the tag doesn't swallow it. `yield` is the
+        # page in the layout.
+        def output(expr, head, tail, layout, path, line)
+          if expr == "yield"
+            raise Error, "#{path}:#{line}: yield is only for the layout" unless layout
+
+            return "out << content"
+          end
+          "out << #{head}#{expr}\n#{tail}"
+        end
+
+        def flush(lines, literal)
+          return if literal.empty?
+
+          lines << "out << #{literal.dump}"
+          literal.clear
+        end
+
+        def layout_source(file)
+          path = "app/views/#{file}"
+          controller_file!("application", path)
+          body = template_lines(File.read(File.join(@root, path)), path, layout: true)
+          wrap("ApplicationController", method_source("_render_layout(content)", body, path))
+        end
+
+        def controller_source(dir, files)
+          controller_file!(dir, "app/views/#{dir}/")
+          methods = []
+          actions = []
+          files.each do |f|
+            path = "app/views/#{f}"
+            action = File.basename(f).delete_suffix(EXT)
+            raise Error, "#{path}: #{action.inspect} is not a valid action name" unless NAME.match?(action)
+            dir.split("/").each { |part| raise Error, "#{path}: #{part.inspect} is not a valid controller path" unless NAME.match?(part) }
+
+            name = "_view_#{dir.tr("/", "_")}_#{action}"
+            methods.concat(method_source(name, template_lines(File.read(File.join(@root, path)), path), path))
+            methods << ""
+            actions << [action, name]
+          end
+          dispatch = ["def _render_view(action)", "  case action"]
+          actions.each { |action, name| dispatch << "  when #{action.dump} then #{name}" }
+          dispatch << "  else super" << "  end" << "end"
+          wrap(Inflector.controller_class_name(dir), methods + dispatch)
+        end
+
+        def method_source(signature, body, path)
+          ["def #{signature}", "  # #{path}", "  out = +\"\""] +
+            body.map { |l| "  #{l}" } + ["  out", "end"]
+        end
+
+        def wrap(klass, body)
+          *modules, name = klass.split("::")
+          depth = modules.size
+          lines = [""]
+          modules.each_with_index { |m, i| lines << "#{"  " * i}module #{m}" }
+          lines << "#{"  " * depth}class #{name}"
+          body.each { |l| lines << (l.empty? ? "" : "#{"  " * (depth + 1)}#{l}") }
+          lines << "#{"  " * depth}end"
+          modules.size.times { |i| lines << "#{"  " * (modules.size - 1 - i)}end" }
+          lines
+        end
+
+        def controller_file!(dir, what)
+          return if File.exist?(File.join(@root, "app/controllers/#{dir}_controller.rb"))
+
+          raise Error, "#{what} has no controller: app/controllers/#{dir}_controller.rb does not exist"
+        end
+      end
+    end
+  end
+end

data/lib/edgeway/cli/build.rb

@@ -2,36 +2,48 @@
 
 require "optparse"
 require "json"
+require_relative "build/app_config"
+require_relative "build/app_files"
+require_relative "build/cache_config"
+require_relative "build/js_files"
+require_relative "build/view_compiler"
 
 module Edgeway
   module CLI
     # edgeway build: app/, config/ -> build/main.rb -> build/app.wasm, and the
-    # gem's js/ -> build/edgeway/.
+    # gem's js/ -> build/edgeway/. Spinel compiles with --incremental (cached C
+    # units, fast rebuilds, no inlining across units) unless --release, which
+    # builds as one unit for deploying.
     #
     # Spinel resolves require at compile time and has no autoloading, so
     # build/main.rb requires every file, in an order where a class body finds
     # what it names (a superclass, an extended module):
-    #   1. edgeway, the database adapter config/database.yml names, and the JWT
-    #      verifier
+    #   1. edgeway, the database adapter config/database.yml names, the JWT
+    #      verifier, the HTTP client, the rate limiter and the cache store
+    #      config/cache.yml names (when it needs a file)
     #   2. app/*/ other than models/ and controllers/ (services, ...)
     #   3. app/models/
     #   4. app/controllers/
-    #   5. config/routes.rb
+    #   5. build/views.rb, the Ruby compiled from app/views (when there is one)
+    #   6. config/routes.rb
     # Within each, application_*.rb come first (the base classes), then the
     # rest by path (byte order).
     class Build
       def self.run(args)
         env = "development"
+        release = false
         OptionParser.new do |o|
-          o.banner = "Usage: edgeway build [--env ENV]"
+          o.banner = "Usage: edgeway build [--env ENV] [--release]"
           o.on("--env ENV", "environment (default: development)") { |v| env = v }
+          o.on("--release", "build as one unit, without --incremental (for deploying)") { release = true }
         end.parse!(args)
-        new(Dir.pwd, env).run
+        new(Dir.pwd, env, release:).run
       end
 
-      def initialize(root, env)
+      def initialize(root, env, release: false)
         @root = root
         @env = env
+        @release = release
       end
 
       def run
@@ -48,40 +60,60 @@ module Edgeway
       private
 
       def write_main(config)
+        cache = CacheConfig.load(@root, @env)
         lines = ["# Generated by `edgeway build` from app/ and config/; do not edit.", 'require "edgeway"']
         lines << "require #{config.adapter_file.dump}"
-        # Always: js/cloudflare.js supplies the edgeway_jwt import to every app,
-        # and the verifier is only called by an app that authenticates.
+        # Always: js/cloudflare.js supplies the edgeway_jwt, edgeway_fetch and
+        # edgeway_ratelimit imports to every app, and each is only called by an
+        # app that authenticates, calls out or rate limits.
         lines << 'require "edgeway/service/cloudflare/jwt"'
+        lines << 'require "edgeway/service/fetch"'
+        lines << 'require "edgeway/service/cloudflare/rate_limit"'
+        lines << "require #{cache.backend_file.dump}" if cache.backend_file
         lines << ""
         lines << connection_line(config)
+        config.databases.each { |name, binding_name| lines << connect_line(config, name, binding_name) }
         lines << "Edgeway::Auth.verifier = Edgeway::Cloudflare::JWTVerifier.new"
+        lines << "Edgeway::HTTP.client = Edgeway::FetchTransport.new"
+        lines << "Edgeway.cache = #{cache.store_expression}"
+        lines << "Edgeway::Controller.rate_limiter = Edgeway::Cloudflare::RateLimiter.new(transport: Edgeway::Cloudflare::RateLimitImportTransport.new)"
         lines << ""
-        lines.concat(AppFiles.new(@root).require_lines)
+        views = ViewCompiler.new(@root).compile
+        File.write(File.join(@root, "build/views.rb"), views) if views
+        lines.concat(AppFiles.new(@root).require_lines(views: views && "views"))
         lines << ""
         lines << "Edgeway::Server.run"
         File.write(File.join(@root, "build/main.rb"), lines.join("\n") + "\n")
       end
 
       def connection_line(config)
+        "Edgeway::Model.connection = #{adapter_expression(config, config.binding_name)}"
+      end
+
+      def connect_line(config, name, binding_name)
+        "Edgeway::Model.connect(#{name.inspect}, #{adapter_expression(config, binding_name)})"
+      end
+
+      def adapter_expression(config, binding_name)
         case config.adapter
-        when "d1" then %(Edgeway::Model.connection = Edgeway::Cloudflare::D1Adapter.new(binding: #{config.binding_name.inspect}))
-        else "Edgeway::Model.connection = Edgeway::DB::MemoryAdapter.new"
+        when "d1" then "Edgeway::Cloudflare::D1Adapter.new(binding: #{binding_name.inspect})"
+        else "Edgeway::DB::MemoryAdapter.new"
         end
       end
 
       def copy_js(config)
-        dest = File.join(@root, "build/edgeway")
-        FileUtils.mkdir_p(dest)
-        Dir.glob(File.join(CLI.gem_dir, "js/*.js")).each { |f| FileUtils.cp(f, dest) }
-        js = "// Generated by `edgeway build` from config/database.yml; do not edit.\n" \
-             "export default #{JSON.generate({ "env" => config.env, "d1Binding" => config.binding_name })};\n"
-        File.write(File.join(dest, "config.js"), js)
+        JsFiles.copy(File.join(@root, "build/edgeway"), config_js(config))
+      end
+
+      def config_js(config)
+        values = { "env" => config.env, "d1Binding" => config.binding_name, "sessions" => config.sessions }
+        JsFiles.config_js(values, "build")
       end
 
       def compile
         spinel = Toolchain.spinel || raise(Error, "no Spinel compiler found (install it with mise, or set SPINEL=/path/to/spinel)")
         cmd = [spinel, "-I", File.join(CLI.gem_dir, "lib"), "--target=wasm32-wasi", "build/main.rb", "-o", "build/app.wasm"]
+        cmd << "--incremental" unless @release
         puts cmd.join(" ")
         system(*cmd, chdir: @root) || raise(Error, "spinel failed")
       end

data/lib/edgeway/cli/cloudflare_test/wasm_test_cli.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Edgeway
+  module CLI
+    class CloudflareTest
+      class WasmTestCLI < SpinelTest::Unit::CLI
+        def initialize(argv, root:, launcher:)
+          super(argv, root: root)
+          @launcher = launcher
+        end
+
+        private
+
+        def compile(spinel, main, bin, entries)
+          wasm = File.join(@root, "tmp/test_app.wasm")
+          command = [spinel, "--target=wasm32-wasi", "--incremental"] +
+                    include_dirs(entries).flat_map { |directory| ["-I", directory] } + [main, "-o", wasm]
+          output, status = Open3.capture2e(*command)
+          unless status.success?
+            warn output unless output.empty?
+            warn "edgeway test: compilation failed (#{command.join(" ")})"
+            return false
+          end
+          write_launcher(bin, wasm)
+        end
+
+        def write_launcher(bin, wasm)
+          File.write(bin, "#!/bin/sh\nexec #{Shellwords.join(@launcher + [wasm])} \"$@\"\n")
+          File.chmod(0o755, bin)
+          true
+        end
+      end
+    end
+  end
+end