esp-modkit 0.1.0

data/lib/esp/mw/reference_index.rb

@@ -0,0 +1,182 @@
+require 'json'
+require 'sqlite3'
+
+module Esp
+  module Mw
+    # SQLite index over the unpacked vanilla ESM JSONs. Replaces "grep over
+    # 200MB of JSON" with a millisecond keyed lookup.
+    #
+    # Schema is intentionally narrow: just enough to answer "where is this
+    # record defined?" and "what's its full JSON?". Full-record retrieval
+    # falls back to reading the source ESM at the stored array index.
+    #
+    # Storage location (step 23.5 slice 2): vanilla data is *not* per-project
+    # — it's identical for every user, and ~150 MB of it. It lives at
+    # `$ESP_REFERENCES_DIR` (explicit override), else
+    # `$ESP_DATA_DIR/references/`, else `~/.config/esp/references/`. One
+    # index across every project. The defaults resolve at call time, not at
+    # require, so an env-var change between invocations takes effect.
+    class ReferenceIndex
+      class << self
+        # Where vanilla `<Master>.esm.json` files live. Explicit override
+        # via $ESP_REFERENCES_DIR wins; otherwise fall under the per-user
+        # data dir (Esp::Recents.data_dir handles the ESP_DATA_DIR /
+        # MW_DATA_DIR / ~/.config/esp resolution).
+        def default_source_dir
+          ENV['ESP_REFERENCES_DIR'] || File.join(Esp::Recents.data_dir, 'references')
+        end
+
+        def default_db_path
+          File.join(default_source_dir, '.index.sqlite')
+        end
+      end
+
+      attr_reader :db_path, :source_dir
+
+      def initialize(db_path: nil, source_dir: nil)
+        @source_dir = source_dir || self.class.default_source_dir
+        @db_path = db_path || File.join(@source_dir, '.index.sqlite')
+      end
+
+      def db
+        @db ||= SQLite3::Database.new(@db_path).tap { |d| d.results_as_hash = true }
+      end
+
+      def rebuild!
+        FileUtils.mkdir_p(@source_dir)
+        FileUtils.rm_f(@db_path)
+        @db = nil
+        create_schema!
+        sources = Dir["#{@source_dir}/*.esm.json"]
+        raise Esp.t('errors.reference_index.no_sources', dir: @source_dir) if sources.empty?
+
+        sources.sort.each { |path| index_esm(path) }
+        db.execute('ANALYZE')
+        count
+      end
+
+      def count
+        db.execute('SELECT COUNT(*) AS n FROM records').first['n']
+      end
+
+      # Search the index. By default `query` matches as a case-insensitive
+      # substring against BOTH id and name (so "Vivec" surfaces the god,
+      # the city's cells, the region, books, dialogue, …), with exact-id
+      # hits sorted to the top. Pass exact: true for a precise id lookup.
+      # `like` is an explicit SQL LIKE pattern on id; `type` filters by
+      # record type. All filters AND together.
+      def find(query: nil, type: nil, like: nil, exact: false, limit: 100)
+        clauses = []
+        params = []
+        apply_query(clauses, params, query, exact)
+        apply_filter(clauses, params, 'id LIKE ?', like)
+        apply_filter(clauses, params, 'type = ?', type)
+
+        sql = +'SELECT source_esm, record_index, type, id, name FROM records'
+        sql << " WHERE #{clauses.join(' AND ')}" unless clauses.empty?
+        sql << order_clause(query, exact, params)
+        sql << " LIMIT #{limit.to_i}"
+        db.execute(sql, params)
+      end
+
+      # Count of all rows matching the same filters (ignores limit), so
+      # callers can say "showing N of M".
+      def count_matching(query: nil, type: nil, like: nil, exact: false)
+        clauses = []
+        params = []
+        apply_query(clauses, params, query, exact)
+        apply_filter(clauses, params, 'id LIKE ?', like)
+        apply_filter(clauses, params, 'type = ?', type)
+
+        sql = +'SELECT COUNT(*) AS n FROM records'
+        sql << " WHERE #{clauses.join(' AND ')}" unless clauses.empty?
+        db.execute(sql, params).first['n']
+      end
+
+      # Pull the full JSON record for a hit by re-reading the source ESM.
+      def fetch_record(source_esm, record_index)
+        JSON.parse(File.read(File.join(@source_dir, "#{source_esm}.json")))[record_index]
+      end
+
+      # Map a batch of ids → record type via one SQL query against the
+      # case-insensitive id index. Keys come back lowercased; the caller
+      # downcases its own lookups to match. Unknown ids are absent from the
+      # returned hash. Backs the cell-view marker colouring (step 21 slice 3).
+      def types_for(ids)
+        lowered = Array(ids).map { |id| id.to_s.downcase }.uniq
+        return {} if lowered.empty?
+
+        placeholders = (['?'] * lowered.size).join(', ')
+        rows = db.execute(
+          "SELECT LOWER(id) AS id, type FROM records WHERE LOWER(id) IN (#{placeholders})",
+          lowered
+        )
+        # If duplicates exist across masters they're (in practice) the same
+        # type; last-wins is fine.
+        rows.to_h { |r| [r['id'], r['type']] }
+      end
+
+      private
+
+      def apply_query(clauses, params, query, exact)
+        return if query.nil?
+
+        if exact
+          clauses << 'LOWER(id) = LOWER(?)'
+          params << query
+        else
+          term = "%#{query.downcase}%"
+          clauses << '(LOWER(id) LIKE ? OR LOWER(name) LIKE ?)'
+          params.push(term, term)
+        end
+      end
+
+      def apply_filter(clauses, params, clause, value)
+        return if value.nil?
+
+        clauses << clause
+        params << value
+      end
+
+      # Exact-id hits float to the top of a substring search; otherwise
+      # plain alphabetical. Mutates params to bind the ORDER BY placeholder.
+      def order_clause(query, exact, params)
+        if query && !exact
+          params << query
+          ' ORDER BY (LOWER(id) = LOWER(?)) DESC, type, id'
+        else
+          ' ORDER BY source_esm, type, id'
+        end
+      end
+
+      def create_schema!
+        db.execute_batch(<<~SQL)
+          CREATE TABLE records (
+            source_esm   TEXT    NOT NULL,
+            record_index INTEGER NOT NULL,
+            type         TEXT    NOT NULL,
+            id           TEXT,
+            name         TEXT,
+            PRIMARY KEY (source_esm, record_index)
+          );
+          CREATE INDEX idx_records_type   ON records(type);
+          CREATE INDEX idx_records_id_ci  ON records(LOWER(id));
+          CREATE INDEX idx_records_id_lk  ON records(id);
+        SQL
+      end
+
+      def index_esm(path)
+        source_esm = File.basename(path, '.json')
+        data = JSON.parse(File.read(path))
+        db.transaction
+        data.each_with_index do |record, idx|
+          db.execute(
+            'INSERT INTO records (source_esm, record_index, type, id, name) VALUES (?, ?, ?, ?, ?)',
+            [source_esm, idx, record['type'], record['id'], record['name']]
+          )
+        end
+        db.commit
+      end
+    end
+  end
+end

data/lib/esp/mw/scaffolder.rb

@@ -0,0 +1,197 @@
+require 'fileutils'
+require 'json'
+require 'open3'
+
+module Esp
+  module Mw
+    # Scaffolds a fresh mod folder under `mods/<Mod>/`. One TES3 Header
+    # record (pre-filled with sensible defaults), one README, and that's
+    # it — no script / i18n / design subdirectories until the author
+    # actually wants them. Picks the source format from `--format`;
+    # subprocess formats get a `#!` line and exec bit.
+    module Scaffolder
+      DEFAULT_FORMAT = 'json'.freeze
+      SUPPORTED_FORMATS = %w[json rb py js mjs ts].freeze
+      MORROWIND_ESM_SIZE = 79_837_557 # vanilla Steam build; users may need to edit
+      MOD_NAME_RE = /\A[A-Za-z0-9][A-Za-z0-9_-]*\z/
+
+      Result = Struct.new(:mod, :format, :source, :readme, keyword_init: true)
+
+      class << self
+        def create(mod, format: DEFAULT_FORMAT, author: nil, description: nil,
+                   root: Esp::ROOT, force: false)
+          validate!(mod, format)
+          mod_dir = File.join(root, 'mods', mod)
+          if File.exist?(mod_dir) && !force
+            raise ArgumentError, Esp.t('errors.scaffolder.already_exists', mod: mod)
+          end
+
+          FileUtils.mkdir_p(mod_dir)
+          author ||= detect_author
+          description ||= "Morrowind plugin: #{mod}"
+
+          source = write_source(mod_dir, mod, format, author, description)
+          readme = write_readme(mod_dir, mod, author, description)
+          Result.new(mod: mod, format: format, source: source, readme: readme)
+        end
+
+        private
+
+        def validate!(mod, format)
+          unless mod.match?(MOD_NAME_RE)
+            raise ArgumentError, Esp.t('errors.scaffolder.bad_name', mod: mod.inspect)
+          end
+          return if SUPPORTED_FORMATS.include?(format)
+
+          raise ArgumentError, Esp.t('errors.scaffolder.bad_format',
+                                     format: format.inspect, formats: SUPPORTED_FORMATS.join(', '))
+        end
+
+        def detect_author
+          out, status = Open3.capture2('git', 'config', 'user.name')
+          status.success? && !out.strip.empty? ? out.strip : 'mw author'
+        rescue Errno::ENOENT
+          'mw author'
+        end
+
+        def shebang_format?(format)
+          %w[py js mjs].include?(format)
+        end
+
+        def write_source(mod_dir, mod, format, author, description)
+          path = File.join(mod_dir, "#{mod}.#{format}")
+          File.write(path, render_source(format, author, description))
+          File.chmod(0o755, path) if shebang_format?(format)
+          path
+        end
+
+        def write_readme(mod_dir, mod, author, description)
+          path = File.join(mod_dir, 'README.md')
+          File.write(path, render_readme(mod, author, description))
+          path
+        end
+
+        def render_source(format, author, description)
+          case format
+          when 'json'      then json_source(author, description)
+          when 'rb'        then rb_source(author, description)
+          when 'py'        then py_source(author, description)
+          when 'js', 'mjs' then js_source(author, description)
+          when 'ts'        then ts_source(author, description)
+          end
+        end
+
+        def json_source(author, description)
+          "#{JSON.pretty_generate([header_record(author, description)])}\n"
+        end
+
+        def rb_source(author, description)
+          <<~RUBY
+            [
+              {
+                type: 'Header',
+                flags: '',
+                file_type: 'Esp',
+                version: 1.3,
+                author: #{author.inspect},
+                description: #{description.inspect},
+                num_objects: 0,
+                masters: [['Morrowind.esm', #{MORROWIND_ESM_SIZE}]]
+              }
+            ]
+          RUBY
+        end
+
+        def py_source(author, description)
+          <<~PY
+            #!/usr/bin/env python3
+            import json
+
+            records = [
+                {
+                    "type": "Header",
+                    "flags": "",
+                    "file_type": "Esp",
+                    "version": 1.3,
+                    "author": #{author.to_json},
+                    "description": #{description.to_json},
+                    "num_objects": 0,
+                    "masters": [["Morrowind.esm", #{MORROWIND_ESM_SIZE}]],
+                },
+            ]
+
+            print(json.dumps(records))
+          PY
+        end
+
+        def js_source(author, description)
+          <<~JS
+            #!/usr/bin/env node
+            const records = [
+              {
+                type: 'Header',
+                flags: '',
+                file_type: 'Esp',
+                version: 1.3,
+                author: #{author.to_json},
+                description: #{description.to_json},
+                num_objects: 0,
+                masters: [['Morrowind.esm', #{MORROWIND_ESM_SIZE}]]
+              }
+            ];
+            console.log(JSON.stringify(records));
+          JS
+        end
+
+        def ts_source(author, description)
+          <<~TS
+            // Run via the mw loader (deno run --quiet).
+            const records = [
+              {
+                type: 'Header',
+                flags: '',
+                file_type: 'Esp',
+                version: 1.3,
+                author: #{author.to_json},
+                description: #{description.to_json},
+                num_objects: 0,
+                masters: [['Morrowind.esm', #{MORROWIND_ESM_SIZE}]],
+              },
+            ];
+            console.log(JSON.stringify(records));
+          TS
+        end
+
+        def render_readme(mod, author, description)
+          <<~MD
+            # #{mod}
+
+            **Author:** #{author}
+
+            #{description}
+
+            ## Build
+
+            ```sh
+            bin/esp build #{mod}
+            bin/esp install #{mod}
+            ```
+          MD
+        end
+
+        def header_record(author, description)
+          {
+            'type' => 'Header',
+            'flags' => '',
+            'file_type' => 'Esp',
+            'version' => 1.3,
+            'author' => author,
+            'description' => description,
+            'num_objects' => 0,
+            'masters' => [['Morrowind.esm', MORROWIND_ESM_SIZE]]
+          }
+        end
+      end
+    end
+  end
+end

data/lib/esp/mw/script_blob.rb

@@ -0,0 +1,87 @@
+require 'base64'
+require 'zstd-ruby'
+
+module Esp
+  module Mw
+    # Synthesizes the ZSTD-framed blobs that tes3conv expects for a Script
+    # record's SCVR (variable list) and SCDT (bytecode) subrecords.
+    #
+    # Vanilla blobs use ZSTD frames whose blocks may be raw (small/empty
+    # scripts) or compressed-with-LZ77 matches (large scripts). tes3conv
+    # accepts either, so we always emit raw blocks — simpler, smaller code,
+    # and only marginally larger output. Decoding always goes through
+    # libzstd via zstd-ruby so we can extract names from any vanilla blob.
+    #
+    # Raw frame layout:
+    #   4 bytes  ZSTD magic                (28 b5 2f fd)
+    #   1 byte   Frame Header Descriptor   (0x00)
+    #   1 byte   Window Descriptor         (0x58)
+    #   3 bytes  Block header              ((size << 3) | type<<1 | last_block)
+    #   N bytes  Raw payload
+    #
+    # Payload layout (both SCVR and SCDT placeholder):
+    #   4 bytes  LE uint32 — byte length of names section (0 if none)
+    #   N bytes  Null-terminated variable names, concatenated
+    #
+    # Empty vars and the bytecode placeholder both use a 4-byte zero payload
+    # (length prefix = 0). A truly zero-byte payload is rejected by tes3conv
+    # for SCDT and isn't what vanilla emits anywhere.
+    module ScriptBlob
+      ZSTD_MAGIC = "\x28\xb5\x2f\xfd".b.freeze
+      FHD = 0x00
+      WINDOW_DESCRIPTOR = 0x58
+
+      class << self
+        # Returns [base64_blob, variables_length] where variables_length is the
+        # byte count of the names section (matching the SCHD header field).
+        def variables(names)
+          if names.empty?
+            [wrap(("\x00" * 4).b), 0]
+          else
+            names_bytes = names.map { |n| "#{n.b}\x00".b }.join
+            payload = [names_bytes.bytesize].pack('V') + names_bytes
+            [wrap(payload), names_bytes.bytesize]
+          end
+        end
+
+        # SCDT placeholder. OpenMW recompiles bytecode from `text` on load,
+        # so we never emit real bytecode.
+        def empty_bytecode
+          wrap(("\x00" * 4).b)
+        end
+
+        # Decompress any vanilla SCVR blob and split out the variable names.
+        def decode_var_names(b64)
+          payload = decompress(b64)
+          return [] if payload.bytesize <= 4
+
+          names_section = payload.byteslice(4..)
+          names = names_section.split("\x00", -1)
+          names.pop while names.last == '' || names.last.nil?
+          names.map { |n| n.dup.force_encoding(Encoding::ASCII) }
+        end
+
+        # Decompress a blob and return its raw payload (length prefix + names).
+        # Useful for tests that compare semantic content across raw and
+        # compressed framings.
+        def decompress(b64)
+          Zstd.decompress(Base64.decode64(b64))
+        end
+
+        private
+
+        def wrap(payload)
+          raise ArgumentError, 'payload too large for 3-byte block header' if payload.bytesize >= (1 << 21)
+
+          raw = ZSTD_MAGIC + [FHD, WINDOW_DESCRIPTOR].pack('C*') + block_header(payload.bytesize) + payload
+          Base64.strict_encode64(raw)
+        end
+
+        def block_header(payload_len)
+          value = (payload_len << 3) | 1 # last_block=1, type=raw(0)
+          [value].pack('V').byteslice(0, 3)
+        end
+      end
+    end
+  end
+end

data/lib/esp/mw/script_extractor.rb

@@ -0,0 +1,85 @@
+require 'fileutils'
+require 'json'
+
+module Esp
+  module Mw
+    # Inverse of preflight's text_source resolution. Given a JSON-format mod
+    # source whose Script records carry inline `text`, hoist each script
+    # body out to `scripts/<id>.mwscript` and replace the inline fields
+    # (`text`, `bytecode`, `variables`, `header`) with a single
+    # `text_source` pointer. Preflight regenerates everything on build, so
+    # the resulting source round-trips losslessly through `esp build`.
+    #
+    # Skips records that already have `text_source` set — re-running is a
+    # no-op. .rb sources are out of scope; their author owns the layout.
+    module ScriptExtractor
+      SAFE_ID = /\A[A-Za-z0-9_-]+\z/
+      Result = Struct.new(:extracted, :skipped, keyword_init: true)
+
+      class << self
+        def extract!(mod, root: Esp::ROOT)
+          source = Esp::Mw::Loader.resolve(mod, root: root)
+          unless source.end_with?('.json')
+            raise ArgumentError, Esp.t('errors.script_extractor.json_only', file: File.basename(source))
+          end
+
+          records = JSON.parse(File.read(source))
+          source_dir = File.dirname(source)
+          result = Result.new(extracted: [], skipped: [])
+
+          records.each do |record|
+            next unless record['type'] == 'Script'
+
+            process_record!(record, source_dir, result)
+          end
+
+          File.write(source, "#{JSON.pretty_generate(records)}\n")
+          result
+        end
+
+        private
+
+        def process_record!(record, source_dir, result)
+          sid = require_id!(record)
+          return result.skipped << sid if record['text_source']
+
+          require_safe_id!(sid)
+          text = require_text!(record, sid)
+
+          write_script_file(source_dir, sid, text)
+          record['text_source'] = "scripts/#{sid}.mwscript"
+          %w[text bytecode variables header].each { |k| record.delete(k) }
+          result.extracted << sid
+        end
+
+        def require_id!(record)
+          sid = record['id']
+          raise ArgumentError, Esp.t('errors.script_extractor.missing_id') if sid.nil? || sid.empty?
+
+          sid
+        end
+
+        def require_safe_id!(sid)
+          return if sid.match?(SAFE_ID)
+
+          raise ArgumentError, Esp.t('errors.script_extractor.unsafe_id', id: sid.inspect)
+        end
+
+        def require_text!(record, sid)
+          text = record['text']
+          if text.nil? || text.empty?
+            raise ArgumentError, Esp.t('errors.script_extractor.no_text', id: sid.inspect)
+          end
+
+          text
+        end
+
+        def write_script_file(source_dir, sid, text)
+          scripts_dir = File.join(source_dir, 'scripts')
+          FileUtils.mkdir_p(scripts_dir)
+          File.write(File.join(scripts_dir, "#{sid}.mwscript"), text)
+        end
+      end
+    end
+  end
+end

data/lib/esp/mw/tes3conv.rb

@@ -0,0 +1,38 @@
+require 'open3'
+
+module Esp
+  module Mw
+    module Tes3conv
+      BIN = ENV.fetch('TES3CONV', 'tes3conv')
+
+      # The binary isn't on PATH at all.
+      class NotFound < StandardError; end
+
+      # tes3conv ran but rejected the conversion (bad input, missing field, …).
+      # Carries tes3conv's own stderr so the message is actionable.
+      class ConvertFailed < StandardError; end
+
+      class << self
+        def convert(input, output)
+          _out, err, status = Open3.capture3(BIN, '-o', input.to_s, output.to_s)
+          return if status.success?
+
+          raise ConvertFailed, failure_message(err)
+        rescue Errno::ENOENT
+          raise NotFound, Esp.t('errors.tes3conv.not_found', bin: BIN)
+        end
+
+        private
+
+        def failure_message(stderr)
+          msg = stderr.strip
+          msg = Esp.t('errors.tes3conv.failed') if msg.empty?
+          # The single most common authoring gotcha — every TES3 record needs
+          # the field even when empty. Point at the fix rather than the symptom.
+          msg += "\n#{Esp.t('errors.tes3conv.flags_hint')}" if msg.include?('missing field `flags`')
+          msg
+        end
+      end
+    end
+  end
+end